From 7074523b52bc103b32875c415e397278664e2d1c Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Tue, 11 Aug 2026 22:58:50 +0200 Subject: [PATCH 001/166] refactor: establish 0.8 library module boundaries --- crates/pithos_lib/src/adapters/mod.rs | 8 + crates/pithos_lib/src/archive/mod.rs | 370 ++++++++++++++++++++++++++ crates/pithos_lib/src/fs/mod.rs | 96 +++++++ crates/pithos_lib/src/helpers/mod.rs | 9 - crates/pithos_lib/src/io/mod.rs | 4 - crates/pithos_lib/src/lib.rs | 55 +++- 6 files changed, 526 insertions(+), 16 deletions(-) create mode 100644 crates/pithos_lib/src/adapters/mod.rs create mode 100644 crates/pithos_lib/src/archive/mod.rs create mode 100644 crates/pithos_lib/src/fs/mod.rs delete mode 100644 crates/pithos_lib/src/helpers/mod.rs delete mode 100644 crates/pithos_lib/src/io/mod.rs diff --git a/crates/pithos_lib/src/adapters/mod.rs b/crates/pithos_lib/src/adapters/mod.rs new file mode 100644 index 0000000..2af7891 --- /dev/null +++ b/crates/pithos_lib/src/adapters/mod.rs @@ -0,0 +1,8 @@ +//! Presentation adapters for external archive formats. +//! +//! Adapters own presentation-specific parsing, retained-source handling, and error +//! context. Core archive, format, block, and crypto code do not own RO-Crate, ZIP, +//! Crypt4GH, or host filesystem error types. + +pub mod crypt4gh; +pub mod ro_crate; diff --git a/crates/pithos_lib/src/archive/mod.rs b/crates/pithos_lib/src/archive/mod.rs new file mode 100644 index 0000000..20103f8 --- /dev/null +++ b/crates/pithos_lib/src/archive/mod.rs @@ -0,0 +1,370 @@ +//! Validated, immutable archive state. +//! +//! This module deliberately has no filesystem, adapter, or writer-input +//! dependency. `Archive` is the public reader boundary; wire records and +//! recovered access material remain crate-private. + +mod access; +mod append; +mod index; +mod path_validation; +mod planning; +mod reader; +#[cfg(test)] +mod reader_private_tests; +mod snapshot; +mod types; +mod validation; +mod writer; + +pub(crate) use access::{AccessProvenance, ResolvedAccess}; +pub use append::{AppendDurability, AppendObservation, AppendOptions}; +pub(crate) use index::build_effective_index; +pub(crate) use path_validation::{ + validate_new_candidate, validate_symlink_target, validate_wire_map, +}; +pub(crate) use reader::ContentOperationError; +pub use reader::{ + AccessKeys, Archive, ArchiveEntry, ArchiveReference, EntryKind, ExternalBlockResolver, + NoExternalBlocks, OpenLimits, OpenOptions, +}; +pub(crate) use snapshot::AppendSnapshot; +pub use types::{ArchivePath, ExternalLocation}; +pub(crate) use types::{FileId, Span}; +pub(crate) use validation::segment_from_wire; +pub use writer::{ + ArchiveWriter, CdcConfig, CreateError, EntryMetadata, EntryReference, FinishError, + IncompleteWriter, ProcessingOptions, WriteOptions, WriterError, WrittenEntry, +}; + +#[cfg(test)] +mod tests { + use super::access::{AccessProvenance, ResolvedAccess}; + use super::index::build_effective_index; + use super::types::*; + use super::validation::IndexLimits; + use proptest::prelude::*; + use std::sync::Arc; + + fn metadata() -> EntryMetadata { + EntryMetadata { + created: 1, + modified: 2, + permissions: 0o644, + references: Vec::new(), + } + } + + fn descriptor(size: u64) -> BlockDescriptor { + BlockDescriptor { + stored_size: size, + original_size: size, + processing: Processing::from_byte(0).unwrap(), + location: BlockLocation::External(ExternalLocation::new("opaque")), + } + } + + fn segment(entries: Vec) -> ValidatedSegment { + ValidatedSegment { + span: Span::new(100, 10).unwrap(), + parent: None, + entries, + descriptors: Vec::new(), + relationships: Vec::new(), + } + } + + #[test] + fn index_preserves_wire_order_and_component_hierarchy() { + let entries = vec![ + SegmentEntry { + id: FileId(7), + path: ArchivePath::new("a!").unwrap(), + entry: Entry::File(ContentEntry { + metadata: metadata(), + size: 0, + content: ContentState::Available(BlockReferences::new(Vec::new())), + }), + }, + SegmentEntry { + id: FileId(3), + path: ArchivePath::new("a").unwrap(), + entry: Entry::Directory(metadata()), + }, + SegmentEntry { + id: FileId(4), + path: ArchivePath::new("a/child").unwrap(), + entry: Entry::File(ContentEntry { + metadata: metadata(), + size: 0, + content: ContentState::Available(BlockReferences::new(Vec::new())), + }), + }, + ]; + let index = + build_effective_index(&[segment(entries)], 1_000, IndexLimits::default()).unwrap(); + assert_eq!( + index.entries().map(|entry| entry.id.0).collect::>(), + [7, 3, 4] + ); + assert_eq!( + index + .hierarchy() + .map(|entry| entry.path.as_str()) + .collect::>(), + ["a", "a/child", "a!"] + ); + let path = ArchivePath::new("a/child").unwrap(); + assert_eq!(index.entry_at_path(&path).unwrap().id, FileId(4)); + assert_eq!(index.maximum_id(), Some(FileId(7))); + } + + #[test] + fn merge_is_pure_and_earliest_compatible_descriptor_wins() { + let hash = BlockHash([9; 32]); + let older = ValidatedSegment { + span: Span::new(10, 5).unwrap(), + parent: None, + entries: Vec::new(), + descriptors: vec![(hash, descriptor(4))], + relationships: Vec::new(), + }; + let mut newer_descriptor = descriptor(4); + newer_descriptor.stored_size = 99; + let newer = ValidatedSegment { + span: Span::new(20, 5).unwrap(), + parent: Some(older.span), + entries: Vec::new(), + descriptors: vec![(hash, newer_descriptor)], + relationships: Vec::new(), + }; + let older_before = older.clone(); + let newer_before = newer.clone(); + let index = build_effective_index( + &[older.clone(), newer.clone()], + 1_000, + IndexLimits::default(), + ) + .unwrap(); + assert_eq!(index.descriptor(hash).unwrap().stored_size, 4); + assert_eq!(older, older_before); + assert_eq!(newer, newer_before); + + let mut conflicting = newer.clone(); + conflicting.descriptors[0].1.original_size = 5; + let conflicting_before = conflicting.clone(); + assert!( + build_effective_index( + &[older.clone(), conflicting.clone()], + 1_000, + IndexLimits::default() + ) + .is_err() + ); + assert_eq!(older, older_before); + assert_eq!(conflicting, conflicting_before); + } + + #[test] + fn non_winning_compatible_descriptors_still_validate_every_local_range() { + let hash = BlockHash([9; 32]); + let older = ValidatedSegment { + span: Span::new(10, 5).unwrap(), + parent: None, + entries: Vec::new(), + descriptors: vec![(hash, descriptor(4))], + relationships: Vec::new(), + }; + for invalid_span in [Span::new(995, 10).unwrap(), Span::new(20, 5).unwrap()] { + let mut later = descriptor(4); + later.location = BlockLocation::Local(invalid_span); + let newer = ValidatedSegment { + span: Span::new(20, 5).unwrap(), + parent: Some(older.span), + entries: Vec::new(), + descriptors: vec![(hash, later)], + relationships: Vec::new(), + }; + + assert!(matches!( + build_effective_index(&[older.clone(), newer], 1_000, IndexLimits::default()), + Err(crate::error::PithosError::InvalidDirectoryRange { .. }) + )); + } + } + + #[test] + fn unavailable_content_is_visible_but_cannot_plan_reads() { + let entry = SegmentEntry { + id: FileId(1), + path: ArchivePath::new("locked").unwrap(), + entry: Entry::File(ContentEntry { + metadata: metadata(), + size: 10, + content: ContentState::Unavailable, + }), + }; + let index = + build_effective_index(&[segment(vec![entry])], 1_000, IndexLimits::default()).unwrap(); + assert!(matches!( + index.entry(FileId(1)).unwrap().entry, + Entry::File(_) + )); + assert!(matches!( + index.full_file_plan(FileId(1)), + Err(crate::error::PithosError::ContentUnavailable) + )); + } + + #[test] + fn plans_include_whole_intersecting_blocks_and_output_slices() { + let first = BlockHash([1; 32]); + let second = BlockHash([2; 32]); + let mut value = segment(vec![SegmentEntry { + id: FileId(1), + path: ArchivePath::new("data").unwrap(), + entry: Entry::File(ContentEntry { + metadata: metadata(), + size: 10, + content: ContentState::Available(BlockReferences::new(vec![first, second])), + }), + }]); + value.descriptors = vec![(first, descriptor(4)), (second, descriptor(6))]; + let index = build_effective_index(&[value], 1_000, IndexLimits::default()).unwrap(); + let range = ReadRange::new(3..7, 10).unwrap(); + let plan = index.range_plan(FileId(1), range).unwrap(); + assert_eq!(plan.blocks.len(), 2); + assert_eq!(plan.blocks[0].output, 3..4); + assert_eq!(plan.blocks[1].output, 0..3); + } + + #[test] + fn resolved_access_rejects_conflicts_and_keeps_deterministic_provenance() { + let mut access = ResolvedAccess::new(); + let later = AccessProvenance { + segment: 2, + recovery_order: 2, + access_key: 0, + sender_section: 0, + recipient_section: 0, + }; + let earlier = AccessProvenance { + segment: 1, + recovery_order: 1, + access_key: 0, + sender_section: 0, + recipient_section: 0, + }; + access.insert(FileId(2), &[3; 32], later).unwrap(); + access.insert(FileId(2), &[3; 32], earlier).unwrap(); + assert_eq!( + access.key(FileId(2)).map(|key| key.expose_for_protocol()), + Some(&[3; 32]) + ); + assert!(matches!( + access.insert(FileId(2), &[4; 32], later), + Err(crate::error::PithosError::ConflictingRecoveredFileKey) + )); + } + + proptest! { + #[test] + fn path_and_checked_range_properties(parts in proptest::collection::vec("[a-z]{1,8}", 1..8), start in 0u16..128, end in 0u16..128) { + let path = parts.join("/"); + let validated = ArchivePath::new(&path).unwrap(); + prop_assert_eq!(validated.as_str(), path); + let range = ReadRange::new(u64::from(start)..u64::from(end), 127); + prop_assert_eq!(range.is_ok(), start <= end && end <= 127); + } + } + + #[test] + fn checked_spans_reject_overflow_and_symlink_escape() { + assert!(Span::new(u64::MAX, 1).is_err()); + assert!(ArchivePath::new("a//b").is_err()); + let link = SegmentEntry { + id: FileId(0), + path: ArchivePath::new("link").unwrap(), + entry: Entry::Symlink { + metadata: metadata(), + target: Arc::from("../outside"), + }, + }; + assert!( + build_effective_index(&[segment(vec![link])], 1_000, IndexLimits::default()).is_err() + ); + } + + #[test] + fn hierarchy_conflicts_are_order_independent_and_component_aware() { + let file = || { + Entry::File(ContentEntry { + metadata: metadata(), + size: 0, + content: ContentState::Available(BlockReferences::new(Vec::new())), + }) + }; + for reverse in [false, true] { + let mut conflicting = vec![ + SegmentEntry { + id: FileId(1), + path: ArchivePath::new("a").unwrap(), + entry: file(), + }, + SegmentEntry { + id: FileId(2), + path: ArchivePath::new("a/child").unwrap(), + entry: file(), + }, + ]; + if reverse { + conflicting.reverse(); + } + assert!( + build_effective_index(&[segment(conflicting)], 1_000, IndexLimits::default()) + .is_err() + ); + } + let adjacent = vec![ + SegmentEntry { + id: FileId(1), + path: ArchivePath::new("a").unwrap(), + entry: file(), + }, + SegmentEntry { + id: FileId(2), + path: ArchivePath::new("a!").unwrap(), + entry: file(), + }, + ]; + assert!(build_effective_index(&[segment(adjacent)], 1_000, IndexLimits::default()).is_ok()); + } + + proptest! { + #[test] + fn component_ancestor_conflicts_ignore_adjacent_prefixes( + prefix in "[a-z]{1,8}", + child in "[a-z]{1,8}", + reverse in any::(), + ) { + let file = || Entry::File(ContentEntry { + metadata: metadata(), + size: 0, + content: ContentState::Available(BlockReferences::new(Vec::new())), + }); + let mut conflicting = vec![ + SegmentEntry { id: FileId(1), path: ArchivePath::new(&prefix).unwrap(), entry: file() }, + SegmentEntry { id: FileId(2), path: ArchivePath::new(format!("{prefix}/{child}")).unwrap(), entry: file() }, + ]; + if reverse { + conflicting.reverse(); + } + prop_assert!(build_effective_index(&[segment(conflicting)], 1_000, IndexLimits::default()).is_err()); + let adjacent = vec![ + SegmentEntry { id: FileId(1), path: ArchivePath::new(&prefix).unwrap(), entry: file() }, + SegmentEntry { id: FileId(2), path: ArchivePath::new(format!("{prefix}!")).unwrap(), entry: file() }, + ]; + prop_assert!(build_effective_index(&[segment(adjacent)], 1_000, IndexLimits::default()).is_ok()); + } + } +} diff --git a/crates/pithos_lib/src/fs/mod.rs b/crates/pithos_lib/src/fs/mod.rs new file mode 100644 index 0000000..5556e51 --- /dev/null +++ b/crates/pithos_lib/src/fs/mod.rs @@ -0,0 +1,96 @@ +//! Linux filesystem adapters. +//! +//! Ingestion, extraction, append, and reader grants are Linux-only host operations. +//! They keep no-follow traversal, retained source identity, metadata normalization, +//! and filesystem error context out of the core archive and format layers. Extraction +//! stages regular files and publishes without clobbering an existing destination; +//! append and grants use an advisory lock that coordinates only participating writers. + +mod append; +mod extraction; +pub mod ingest; + +use crate::archive::WriterError; +use crate::error::PithosError; +use std::io; +use std::path::PathBuf; +use thiserror::Error; + +pub use append::{append_files, grant_readers}; + +/// Actionable failures from Linux filesystem adapter operations. +/// +/// Each variant retains the attempted operation and relevant host or archive path, +/// and preserves the underlying source error where one exists. This adapter error +/// deliberately remains distinct from core [`PithosError`]. +#[derive(Debug, Error)] +pub enum FsError { + #[error("filesystem {operation} failed for {path}: {source}")] + Host { + operation: &'static str, + path: PathBuf, + #[source] + source: io::Error, + }, + #[error("path is not valid UTF-8: {path}")] + InvalidUtf8Path { path: PathBuf }, + #[error("filesystem {operation} failed while deriving {path}: {source}")] + StripPrefix { + operation: &'static str, + path: PathBuf, + #[source] + source: std::path::StripPrefixError, + }, + #[error( + "filesystem {operation} failed for archive entry {archive_path} in destination {destination}: {source}" + )] + Operation { + operation: &'static str, + archive_path: String, + destination: PathBuf, + #[source] + source: io::Error, + }, + #[error( + "filesystem extraction collision for archive entry {archive_path} in destination {destination}: {reason}" + )] + ExtractionCollision { + archive_path: String, + destination: PathBuf, + reason: &'static str, + }, + #[error( + "archive {operation} failed for archive entry {archive_path} in destination {destination}: {source}" + )] + Archive { + operation: &'static str, + archive_path: String, + destination: PathBuf, + #[source] + source: Box, + }, + #[error("filesystem {operation} failed for {path}: {source}")] + Core { + operation: &'static str, + path: PathBuf, + #[source] + source: PithosError, + }, + #[error("filesystem {operation} failed for archive entry {archive_path}: {source}")] + Writer { + operation: &'static str, + archive_path: String, + #[source] + source: WriterError, + }, + #[error("append source resolves to the archive being modified: {path}")] + AppendSourceIsArchive { path: PathBuf }, + #[error("append requires at least one filesystem input")] + AppendRequiresInput, + #[error("append is locked by another cooperating writer: {path}")] + AppendLocked { path: PathBuf }, + #[error("unsupported filesystem entry {path}: {kind}")] + UnsupportedEntry { path: PathBuf, kind: &'static str }, +} + +pub use extraction::extract; diff --git a/crates/pithos_lib/src/helpers/mod.rs b/crates/pithos_lib/src/helpers/mod.rs deleted file mode 100644 index 438b855..0000000 --- a/crates/pithos_lib/src/helpers/mod.rs +++ /dev/null @@ -1,9 +0,0 @@ -pub(crate) mod archive_path; -pub mod chacha_poly1305; -pub mod crypt4gh; -pub mod directory; -pub mod file_entry_map; -pub mod hash; -pub mod ro_crate; -pub mod x25519_keys; -pub mod zstd; diff --git a/crates/pithos_lib/src/io/mod.rs b/crates/pithos_lib/src/io/mod.rs deleted file mode 100644 index 4e238b2..0000000 --- a/crates/pithos_lib/src/io/mod.rs +++ /dev/null @@ -1,4 +0,0 @@ -pub(crate) mod extraction; -pub mod pithosreader; -pub mod pithoswriter; -pub mod util; diff --git a/crates/pithos_lib/src/lib.rs b/crates/pithos_lib/src/lib.rs index 578decf..b621aa6 100644 --- a/crates/pithos_lib/src/lib.rs +++ b/crates/pithos_lib/src/lib.rs @@ -1,4 +1,53 @@ +//! Pithos archive reading, writing, Linux filesystem operations, and presentation adapters. +//! +//! # Public 0.8 boundary +//! +//! Create archives with [`archive::ArchiveWriter`], then open them through +//! [`archive::Archive`] and an immutable [`source::ArchiveSource`]. Writer construction requires +//! a sender key and one or more recipient public keys; opening encrypted content requires the +//! matching recipient private key in [`archive::AccessKeys`]. +//! +//! Payload integrity is checked when bytes are read, not when the archive is opened. In +//! particular, [`archive::Archive::copy_to`] and [`archive::Archive::copy_range_to`] verify each +//! full block before releasing bytes from that block. A generic sink can retain earlier verified +//! output if a later block fails. For Linux paths, [`fs::extract`] instead stages each regular +//! file and publishes it without replacing an existing destination entry. +//! +//! The compiled examples cover [creation](https://github.com/arunaengine/pithos/tree/main/crates/pithos_lib/examples), +//! listing, range reads, extraction, append and reader grants, RO-Crate conversion, and Crypt4GH +//! export. They use only this public boundary and perform no network operations. +//! +//! The Linux filesystem adapter is intentionally host-specific. Core archive reading and writing +//! remain independent of filesystem traversal and presentation formats. +//! +//! Wire records and compatibility modules are deliberately not part of the public API: +//! +//! ```compile_fail +//! use pithos_lib::model::structs::Directory; +//! ``` +//! +//! ```compile_fail +//! use pithos_lib::helpers::directory::DirectoryBuilder; +//! ``` +//! +//! ```compile_fail +//! use pithos_lib::format::wire::FileHeader; +//! ``` + +pub mod adapters; +pub mod archive; +mod block; +pub mod crypto; pub mod error; -pub mod helpers; -pub mod io; -pub mod model; +mod format; +pub mod fs; + +pub mod source; + +/// Byte-oriented parser entry points for the standalone fuzz targets. +/// +/// This module is intentionally excluded from normal builds and exposes no +/// format records or key material. +#[cfg(feature = "fuzzing")] +#[doc(hidden)] +pub mod fuzzing; From 008de3998caee26c917753c9ea217d6a88173d1d Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Tue, 11 Aug 2026 22:59:17 +0200 Subject: [PATCH 002/166] refactor: centralize private format codec --- crates/pithos_lib/src/error.rs | 173 ++-- crates/pithos_lib/src/format/codec.rs | 803 ++++++++++++++++++ crates/pithos_lib/src/format/directory.rs | 94 ++ crates/pithos_lib/src/format/entries.rs | 188 ++++ crates/pithos_lib/src/format/error.rs | 9 + crates/pithos_lib/src/format/limits.rs | 60 ++ crates/pithos_lib/src/format/mod.rs | 11 + crates/pithos_lib/src/format/wire.rs | 355 ++++++++ crates/pithos_lib/src/helpers/directory.rs | 475 ----------- .../pithos_lib/src/helpers/file_entry_map.rs | 605 ------------- .../pithos_lib/src/model/deserialization.rs | 599 ------------- crates/pithos_lib/src/model/mod.rs | 3 - crates/pithos_lib/src/model/serialization.rs | 264 ------ crates/pithos_lib/src/model/structs.rs | 653 -------------- crates/pithos_lib/tests/marshalling.rs | 736 ---------------- 15 files changed, 1629 insertions(+), 3399 deletions(-) create mode 100644 crates/pithos_lib/src/format/codec.rs create mode 100644 crates/pithos_lib/src/format/directory.rs create mode 100644 crates/pithos_lib/src/format/entries.rs create mode 100644 crates/pithos_lib/src/format/error.rs create mode 100644 crates/pithos_lib/src/format/limits.rs create mode 100644 crates/pithos_lib/src/format/mod.rs create mode 100644 crates/pithos_lib/src/format/wire.rs delete mode 100644 crates/pithos_lib/src/helpers/directory.rs delete mode 100644 crates/pithos_lib/src/helpers/file_entry_map.rs delete mode 100644 crates/pithos_lib/src/model/deserialization.rs delete mode 100644 crates/pithos_lib/src/model/mod.rs delete mode 100644 crates/pithos_lib/src/model/serialization.rs delete mode 100644 crates/pithos_lib/src/model/structs.rs delete mode 100644 crates/pithos_lib/tests/marshalling.rs diff --git a/crates/pithos_lib/src/error.rs b/crates/pithos_lib/src/error.rs index 786cfaf..086cd39 100644 --- a/crates/pithos_lib/src/error.rs +++ b/crates/pithos_lib/src/error.rs @@ -1,41 +1,26 @@ -use crate::helpers::chacha_poly1305::ChaChaPoly1305Error; -use crate::helpers::crypt4gh::Crypt4GHError; -use crate::helpers::x25519_keys::CryptError; -use crate::helpers::zstd::ZstdError; -use crate::model::deserialization::DeserializationError; -use crate::model::serialization::SerializationError; -use rocraters::ro_crate::read::CrateReadError; +use crate::crypto::CryptoError; +use crate::source::SourceError; use std::io; -use std::path::PathBuf; -use std::time::SystemTimeError; use thiserror::Error; -use zip::result::ZipError; + +pub use crate::format::error::SerializationError; +pub use crate::format::limits::DeserializationError; /// Custom top-level error type for all of Pithos #[derive(Error, Debug)] pub enum PithosError { - #[error("IO error: {0}")] + #[error("I/O error: {0}")] Io(#[from] io::Error), - #[error("Conversion error: {0}")] - Conversion(String), - #[error("System time error: {0}")] - SystemTimeError(#[from] SystemTimeError), - #[error("Failed to strip prefix: {0}")] - StripPrefix(#[from] std::path::StripPrefixError), - #[error("Walkdir error: {0}")] - WalkDir(#[from] walkdir::Error), + #[error("archive source error: {0}")] + Source(#[from] SourceError), #[error("FastCDC error: {0}")] FastCDC(#[from] fastcdc::v2020::Error), - #[error("Serialization error: {0:?}")] + #[error("serialization error: {0}")] Serialization(#[from] SerializationError), - #[error("Deserialization error: {0:?}")] + #[error("deserialization error: {0}")] Deserialization(#[from] DeserializationError), #[error("Unsupported file version: supported {supported:#06x}, actual {actual:#06x}")] UnsupportedFileVersion { supported: u16, actual: u16 }, - #[error("Multiple reader keys are not supported")] - UnsupportedMultipleReaderKeys, - #[error("Reference content is not supported for file entry construction")] - UnsupportedReferenceContent, #[error("Invalid directory marker: expected {expected:?}, got {actual:?}")] InvalidDirectoryMarker { expected: [u8; 8], actual: [u8; 8] }, #[error("Directory length mismatch: expected {expected}, got {actual}")] @@ -52,49 +37,68 @@ pub enum PithosError { }, #[error("allocation failed for {field}: {size}")] AllocationFailed { field: &'static str, size: u64 }, - #[error("invalid directory range: {0}")] - InvalidDirectoryRange(String), - #[error("invalid directory chain: {0}")] - InvalidDirectoryChain(String), + #[error("invalid directory range while {operation}")] + InvalidDirectoryRange { operation: &'static str }, + #[error("invalid directory chain while {operation}")] + InvalidDirectoryChain { operation: &'static str }, #[error("Crypt error: {0}")] - Crypt(#[from] CryptError), - #[error("Crypt4GH error: {0}")] - Crypt4GH(#[from] Crypt4GHError), - #[error("Encryption error: {0}")] - Cipher(#[from] ChaChaPoly1305Error), - #[error("Decryption error: {0}")] - Compression(#[from] ZstdError), - #[error("RO-Crate parse or validation error: {0}")] - RoCrate(#[from] CrateReadError), - #[error("ZIP archive error: {0}")] - Zip(#[from] ZipError), - #[error("Invalid RO-Crate source {path}: expected {expected}")] - InvalidRoCrateSource { - path: PathBuf, - expected: &'static str, + Crypt(#[from] CryptoError), + #[error("{operation} failed: {source}")] + Compression { + operation: &'static str, + #[source] + source: io::Error, }, - #[error("RO-Crate metadata file is missing from {0}")] - MissingRoCrateMetadata(PathBuf), - #[error("Unsafe ZIP member path: {0}")] - UnsafeZipPath(String), - #[error("Duplicate ZIP member path after normalization: {0}")] - DuplicateZipPath(String), - #[error("ZIP member path conflicts with a required directory: {0}")] - ZipPathConflict(String), - #[error("Overlapping ZIP members are not supported: {0}")] - OverlappingZipEntries(PathBuf), - #[error("Encrypted ZIP member is not supported: {0}")] - EncryptedZipEntry(String), - #[error("Unsupported ZIP entry type or compression for {0}")] - UnsupportedZipEntry(String), #[error("Invalid block data state: {0}")] InvalidBlockDataState(String), + #[error("content is unavailable with the supplied access keys")] + ContentUnavailable, + #[error("append snapshot does not contain file id {0}")] + SnapshotFileIdNotFound(u64), + #[error("content for snapshot file id {0} is unavailable with the supplied access keys")] + SnapshotContentUnavailable(u64), + #[error("snapshot file id {0} is a directory and has no content key")] + SnapshotDirectoryHasNoContent(u64), + #[error("snapshot file id {0} is a symlink and has no content key")] + SnapshotSymlinkHasNoContent(u64), #[error("external block source required")] ExternalBlockSourceRequired, #[error("external block framing error: {0}")] ExternalBlockFraming(String), #[error("Block hash not found: {0:?}")] BlockHashNotFound([u8; 32]), + #[error("duplicate encoded block hash")] + DuplicateBlockHash, + #[error("duplicate encoded sender key")] + DuplicateSenderKey, + #[error("duplicate encoded recipient key")] + DuplicateRecipientKey, + #[error("duplicate encoded block reference")] + DuplicateBlockReference, + #[error("duplicate encoded recipient file id")] + DuplicateRecipientFileId, + #[error("reserved processing bits are set: {0:#04x}")] + ReservedProcessingBits(u8), + #[error("conflicting relationship definition for id {0}")] + ConflictingRelationshipDefinition(u64), + #[error("unknown relationship id {0}")] + UnknownRelationshipId(u64), + #[error("missing reference target file id {0}")] + MissingReferenceTarget(u64), + #[error("missing block descriptor")] + MissingBlockDescriptor, + #[error( + "accessible file size does not match its block sizes: expected {expected}, actual {actual}" + )] + AccessibleFileSizeMismatch { expected: u64, actual: u64 }, + #[error("conflicting recovered file key")] + ConflictingRecoveredFileKey, + #[error("invalid half-open read range {start}..{end} for file size {file_size}")] + InvalidReadRange { + start: u64, + end: u64, + file_size: u64, + }, #[error("Block size mismatch: expected {expected}, got {actual}")] BlockSizeMismatch { expected: u64, actual: u64 }, #[error("Block hash mismatch: expected {expected:?}, got {actual:?}")] @@ -130,14 +134,55 @@ pub enum PithosError { }, #[error("Invalid symlink entry {path}: {reason}")] InvalidSymlinkEntry { path: String, reason: String }, - #[error("Extraction collision at {path}: {reason}")] - ExtractionCollision { path: String, reason: String }, - #[error("Invalid file type: {0}")] - InvalidFileType(String), #[error("No recipient section found for the provided private key")] NoMatchingRecipient, #[error("Invalid recipient data state: {0}")] InvalidRecipientDataState(String), - #[error("Other error: {0}")] - Other(String), + #[error("archive creation requires at least one recipient")] + WriterRequiresRecipient, + #[error("granting reader access requires at least one file or metadata id")] + GrantRequiresFileId, + #[error("invalid CDC configuration {min_size},{avg_size},{max_size}")] + InvalidCdcConfig { + min_size: usize, + avg_size: usize, + max_size: usize, + }, + #[error("writer is poisoned")] + WriterPoisoned, + #[error("streamed content size overflow")] + WriterSizeOverflow, + #[error("streamed content size mismatch: expected {expected}, actual {actual}")] + WriterExpectedSizeMismatch { expected: u64, actual: u64 }, + #[error("content entry has an unsealed block list")] + WriterUnsealedBlockList, + #[error("recipient record has an unsealed file-key list")] + WriterUnsealedRecipientList, + #[error("directory or symlink has block-key material")] + WriterNoContentHasBlockMaterial, + #[error("append failed after mutation but was rolled back: {source}")] + AppendRolledBack { + #[source] + source: Box, + }, + #[error( + "append failed after mutation and rollback failed (original length {original_len}, observed length {observed_len}): original failure: {source}; rollback failure: {rollback}" + )] + AppendRollbackFailed { + #[source] + source: Box, + rollback: io::Error, + original_len: u64, + observed_len: u64, + }, + #[error("planned IDs are only valid for append writers")] + PlannedIdsRequireAppendWriter, + #[error("grant child must not contain file entries or block descriptors")] + GrantChildContainsContent, + #[error("granting reader access requires an append snapshot")] + GrantRequiresAppendSnapshot, + #[error("file manifest entry has no retained source")] + MissingManifestSource, + #[error("append plan entry count does not match manifest")] + AppendPlanLengthMismatch, } diff --git a/crates/pithos_lib/src/format/codec.rs b/crates/pithos_lib/src/format/codec.rs new file mode 100644 index 0000000..1c78523 --- /dev/null +++ b/crates/pithos_lib/src/format/codec.rs @@ -0,0 +1,803 @@ +use crate::error::PithosError; +use crate::format::entries::WireEntries; +use crate::format::error::SerializationError; +use crate::format::limits::{DeserializationError, DeserializationLimits}; +use crate::format::wire::RecipientKeyList; +use crate::format::wire::*; +use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt}; +use crc32fast::Hasher; +use indexmap::IndexMap; +use integer_encoding::{VarIntReader, VarIntWriter}; +use std::collections::{HashMap, HashSet}; +use std::io::{Cursor, Read, Write}; +use zeroize::Zeroizing; + +const DIRECTORY_MARKER: [u8; 8] = *b"PITHOSDR"; +const MIN_DIRECTORY_LEN: usize = 25; + +pub(crate) fn write_len_prefix( + writer: &mut W, + len: usize, +) -> Result<(), SerializationError> { + writer.write_varint( + u64::try_from(len) + .map_err(|_| SerializationError::Other("length does not fit in u64".to_string()))?, + )?; + Ok(()) +} + +pub(crate) fn encode_string( + writer: &mut W, + value: &str, +) -> Result<(), SerializationError> { + write_len_prefix(writer, value.len())?; + writer.write_all(value.as_bytes())?; + Ok(()) +} + +fn bounded_len(value: u64, limit: u64, field: &'static str) -> Result { + if value > limit { + return Err(DeserializationError::LimitExceeded { + field, + limit, + actual: value, + }); + } + usize::try_from(value).map_err(|_| DeserializationError::InvalidLength) +} + +fn reserve( + output: &mut Vec, + count: usize, + field: &'static str, +) -> Result<(), DeserializationError> { + output + .try_reserve(count) + .map_err(|_| DeserializationError::AllocationFailed { + field, + size: count as u64, + }) +} + +pub(crate) fn decode_string( + reader: &mut R, + limits: &DeserializationLimits, +) -> Result { + let len = bounded_len( + reader.read_varint::()?, + limits.max_string_bytes, + "string", + )?; + let mut bytes = Vec::new(); + reserve(&mut bytes, len, "string")?; + bytes.resize(len, 0); + reader.read_exact(&mut bytes)?; + Ok(String::from_utf8(bytes)?) +} + +pub(crate) fn encode_header( + header: &FileHeader, + writer: &mut W, +) -> Result<(), SerializationError> { + writer.write_all(&header.magic)?; + writer.write_varint(header.version)?; + Ok(()) +} + +pub(crate) fn decode_header(reader: &mut R) -> Result { + let mut magic = [0; 4]; + reader.read_exact(&mut magic)?; + if magic != *b"PITH" { + return Err(DeserializationError::InvalidMarker(format!( + "Read invalid block marker {magic:?}" + ))); + } + Ok(FileHeader { + magic, + version: reader.read_varint()?, + }) +} + +pub(crate) fn encode_block_marker( + header: &BlockHeader, + writer: &mut W, +) -> Result<(), SerializationError> { + writer.write_all(&header.marker)?; + Ok(()) +} + +pub(crate) fn decode_block_marker( + reader: &mut R, +) -> Result { + let mut marker = [0; 4]; + reader.read_exact(&mut marker)?; + if marker != *b"BLCK" { + return Err(DeserializationError::InvalidMarker(format!( + "Read invalid block marker {marker:?}" + ))); + } + Ok(BlockHeader { marker }) +} + +pub(crate) fn encode_flags( + flags: &ProcessingFlags, + writer: &mut W, +) -> Result<(), SerializationError> { + writer.write_all(&[flags.0])?; + Ok(()) +} + +pub(crate) fn decode_flags( + reader: &mut R, +) -> Result { + let mut byte = [0]; + reader.read_exact(&mut byte)?; + let flags = ProcessingFlags::from_byte(byte[0]); + if flags.has_reserved_bits() { + return Err(DeserializationError::InvalidProcessingFlags(byte[0])); + } + Ok(flags) +} + +fn encode_location( + location: &BlockLocation, + writer: &mut W, +) -> Result<(), SerializationError> { + match location { + BlockLocation::Local => writer.write_all(&[0])?, + BlockLocation::External { url } => { + writer.write_all(&[1])?; + encode_string(writer, url)?; + } + } + Ok(()) +} + +fn decode_location( + reader: &mut R, + limits: &DeserializationLimits, +) -> Result { + let mut tag = [0]; + reader.read_exact(&mut tag)?; + match tag[0] { + 0 => Ok(BlockLocation::Local), + 1 => Ok(BlockLocation::External { + url: decode_string(reader, limits)?, + }), + value => Err(DeserializationError::InvalidEnumValue(value)), + } +} + +pub(crate) fn encode_block_index_entry( + entry: &BlockIndexEntry, + writer: &mut W, +) -> Result<(), SerializationError> { + writer.write_varint(entry.offset)?; + writer.write_varint(entry.stored_size)?; + writer.write_varint(entry.original_size)?; + encode_flags(&entry.flags, writer)?; + encode_location(&entry.location, writer) +} + +pub(crate) fn decode_block_index_entry( + reader: &mut R, + limits: &DeserializationLimits, +) -> Result { + Ok(BlockIndexEntry { + offset: reader.read_varint()?, + stored_size: reader.read_varint()?, + original_size: reader.read_varint()?, + flags: decode_flags(reader)?, + location: decode_location(reader, limits)?, + }) +} + +fn encode_file_type( + file_type: &FileType, + writer: &mut W, +) -> Result<(), SerializationError> { + writer.write_all(&[*file_type as u8])?; + Ok(()) +} + +pub(crate) fn decode_file_type(reader: &mut R) -> Result { + let mut tag = [0]; + reader.read_exact(&mut tag)?; + match tag[0] { + 0 => Ok(FileType::Directory), + 1 => Ok(FileType::Data), + 2 => Ok(FileType::Metadata), + 3 => Ok(FileType::Symlink), + value => Err(DeserializationError::InvalidEnumValue(value)), + } +} + +fn encode_reference( + reference: &Reference, + writer: &mut W, +) -> Result<(), SerializationError> { + writer.write_varint(reference.target_file_id)?; + writer.write_varint(reference.relationship)?; + Ok(()) +} + +fn decode_reference(reader: &mut R) -> Result { + Ok(Reference { + target_file_id: reader.read_varint()?, + relationship: reader.read_varint()?, + }) +} + +fn encode_block_data( + data: &BlockDataState, + writer: &mut W, +) -> Result<(), SerializationError> { + match data { + BlockDataState::Encrypted(bytes) => { + writer.write_all(&[0])?; + write_len_prefix(writer, bytes.len())?; + writer.write_all(bytes)?; + } + BlockDataState::Decrypted(entries) => { + writer.write_all(&[1])?; + write_len_prefix(writer, entries.len())?; + for (hash, key) in entries.iter() { + writer.write_all(hash)?; + writer.write_all(key)?; + } + } + } + Ok(()) +} + +fn decode_block_data( + reader: &mut R, + limits: &DeserializationLimits, +) -> Result { + let mut tag = [0]; + reader.read_exact(&mut tag)?; + match tag[0] { + 0 => { + let len = bounded_len( + reader.read_varint()?, + limits.max_opaque_bytes, + "encrypted block", + )?; + let mut bytes = Vec::new(); + reserve(&mut bytes, len, "encrypted block")?; + bytes.resize(len, 0); + reader.read_exact(&mut bytes)?; + Ok(BlockDataState::Encrypted(bytes)) + } + 1 => Ok(BlockDataState::Decrypted( + decode_decrypted_block_list_reader(reader, limits)?, + )), + value => Err(DeserializationError::InvalidEnumValue(value)), + } +} + +fn encode_file_entry( + entry: &FileEntry, + writer: &mut W, +) -> Result<(), SerializationError> { + encode_file_type(&entry.file_type, writer)?; + encode_block_data(&entry.block_data, writer)?; + writer.write_varint(entry.created)?; + writer.write_varint(entry.modified)?; + writer.write_varint(entry.file_size)?; + writer.write_varint(entry.permissions)?; + write_len_prefix(writer, entry.references.len())?; + for reference in &entry.references { + encode_reference(reference, writer)?; + } + match &entry.symlink_target { + Some(target) => { + writer.write_all(&[1])?; + encode_string(writer, target)?; + } + None => writer.write_all(&[0])?, + } + Ok(()) +} + +fn decode_file_entry( + reader: &mut R, + limits: &DeserializationLimits, + remaining_references: &mut u64, +) -> Result { + let file_type = decode_file_type(reader)?; + let block_data = decode_block_data(reader, limits)?; + let created = reader.read_varint()?; + let modified = reader.read_varint()?; + let file_size = reader.read_varint()?; + let permissions = reader.read_varint()?; + let count = bounded_len(reader.read_varint()?, *remaining_references, "references")?; + *remaining_references -= count as u64; + let mut references = Vec::new(); + reserve(&mut references, count, "references")?; + for _ in 0..count { + references.push(decode_reference(reader)?); + } + let mut tag = [0]; + reader.read_exact(&mut tag)?; + let symlink_target = match tag[0] { + 0 => None, + 1 => Some(decode_string(reader, limits)?), + _ => return Err(DeserializationError::InvalidOption), + }; + Ok(FileEntry { + file_type, + block_data, + created, + modified, + file_size, + permissions, + references, + symlink_target, + }) +} + +fn encode_recipient_data( + data: &RecipientData, + writer: &mut W, +) -> Result<(), SerializationError> { + match data { + RecipientData::Encrypted(bytes) => { + writer.write_all(&[0])?; + write_len_prefix(writer, bytes.len())?; + writer.write_all(bytes)?; + } + RecipientData::Decrypted(entries) => { + writer.write_all(&[1])?; + encode_decrypted_recipient_list(entries, writer)?; + } + } + Ok(()) +} + +fn decode_recipient_data( + reader: &mut R, + limits: &DeserializationLimits, +) -> Result { + let mut tag = [0]; + reader.read_exact(&mut tag)?; + match tag[0] { + 0 => { + let len = bounded_len( + reader.read_varint()?, + limits.max_opaque_bytes, + "encrypted recipient data", + )?; + let mut bytes = Vec::new(); + reserve(&mut bytes, len, "encrypted recipient data")?; + bytes.resize(len, 0); + reader.read_exact(&mut bytes)?; + Ok(RecipientData::Encrypted(bytes)) + } + 1 => Ok(RecipientData::Decrypted( + decode_decrypted_recipient_list_reader(reader, limits)?, + )), + value => Err(DeserializationError::InvalidEnumValue(value)), + } +} + +fn encode_encryption_section( + section: &EncryptionSection, + writer: &mut W, +) -> Result<(), SerializationError> { + write_len_prefix(writer, section.recipients.len())?; + for (key, recipient) in §ion.recipients { + writer.write_all(key)?; + encode_recipient_data(&recipient.recipient_data, writer)?; + } + Ok(()) +} + +fn decode_encryption_section( + reader: &mut R, + limits: &DeserializationLimits, +) -> Result { + let count = bounded_len( + reader.read_varint()?, + limits.max_collection_entries, + "recipients", + )?; + let mut recipients = IndexMap::new(); + for _ in 0..count { + let mut key = [0; 32]; + reader.read_exact(&mut key)?; + if recipients.contains_key(&key) { + return Err(DeserializationError::DuplicateRecipientKey); + } + recipients.insert( + key, + RecipientSection { + recipient_data: decode_recipient_data(reader, limits)?, + }, + ); + } + Ok(EncryptionSection { recipients }) +} + +pub(crate) fn encode_directory( + directory: &Directory, + writer: &mut W, +) -> Result<(), SerializationError> { + writer.write_all(&directory.identifier)?; + match directory.parent_directory_offset { + Some((start, len)) => { + writer.write_all(&[1])?; + writer.write_varint(start)?; + writer.write_varint(len)?; + } + None => writer.write_all(&[0])?, + } + write_len_prefix(writer, directory.files.len())?; + for (id, path, entry) in directory.files.iter() { + writer.write_varint::(id)?; + encode_string(writer, path)?; + encode_file_entry(entry, writer)?; + } + write_len_prefix(writer, directory.blocks.len())?; + for (hash, entry) in &directory.blocks { + writer.write_all(hash)?; + encode_block_index_entry(entry, writer)?; + } + write_len_prefix(writer, directory.relations.len())?; + for (id, name) in &directory.relations { + writer.write_varint(*id)?; + encode_string(writer, name)?; + } + write_len_prefix(writer, directory.encryption.len())?; + for (key, section) in &directory.encryption { + writer.write_all(key)?; + encode_encryption_section(section, writer)?; + } + writer.write_u64::(directory.dir_len)?; + writer.write_u32::(directory.crc32)?; + Ok(()) +} + +pub(crate) fn decode_directory( + reader: &mut R, + limits: &DeserializationLimits, +) -> Result { + let mut identifier = [0; 8]; + reader.read_exact(&mut identifier)?; + if identifier != DIRECTORY_MARKER { + return Err(PithosError::InvalidDirectoryMarker { + expected: DIRECTORY_MARKER, + actual: identifier, + }); + } + let mut tag = [0]; + reader.read_exact(&mut tag)?; + let parent_directory_offset = match tag[0] { + 0 => None, + 1 => Some((reader.read_varint()?, reader.read_varint()?)), + _ => return Err(DeserializationError::InvalidOption.into()), + }; + let file_count = bounded_len(reader.read_varint()?, limits.max_file_entries, "files")?; + let mut files = WireEntries::new(); + let mut remaining_references = limits.max_references; + for _ in 0..file_count { + let id = reader.read_varint()?; + let path = decode_string(reader, limits)?; + files.insert( + id, + path, + decode_file_entry(reader, limits, &mut remaining_references)?, + )?; + } + let block_count = bounded_len( + reader.read_varint()?, + limits.max_block_descriptors, + "blocks", + )?; + let mut blocks = IndexMap::new(); + for _ in 0..block_count { + let mut hash = [0; 32]; + reader.read_exact(&mut hash)?; + if blocks.contains_key(&hash) { + return Err(PithosError::DuplicateBlockHash); + } + blocks.insert(hash, decode_block_index_entry(reader, limits)?); + } + let relation_count = bounded_len(reader.read_varint()?, limits.max_relationships, "relations")?; + let mut relations = Vec::new(); + reserve(&mut relations, relation_count, "relations")?; + let mut relation_ids = HashSet::new(); + for _ in 0..relation_count { + let id = reader.read_varint()?; + let name = decode_string(reader, limits)?; + if !relation_ids.insert(id) { + return Err(PithosError::ConflictingRelationshipDefinition(id)); + } + relations.push((id, name)); + } + let encryption_count = bounded_len( + reader.read_varint()?, + limits.max_collection_entries, + "encryption", + )?; + let mut encryption = IndexMap::new(); + for _ in 0..encryption_count { + let mut key = [0; 32]; + reader.read_exact(&mut key)?; + if encryption.contains_key(&key) { + return Err(PithosError::DuplicateSenderKey); + } + encryption.insert(key, decode_encryption_section(reader, limits)?); + } + let directory = Directory { + identifier, + parent_directory_offset, + files, + blocks, + relations, + encryption, + dir_len: reader.read_u64::()?, + crc32: reader.read_u32::()?, + }; + crate::archive::validate_wire_map(&directory.files)?; + Ok(directory) +} + +pub(crate) fn decode_complete_directory( + bytes: &[u8], + limits: &DeserializationLimits, +) -> Result { + if bytes.len() < MIN_DIRECTORY_LEN { + return Err(PithosError::DirectoryLengthMismatch { + expected: MIN_DIRECTORY_LEN as u64, + actual: bytes.len() as u64, + }); + } + let actual_marker: [u8; 8] = bytes[..8] + .try_into() + .expect("checked directory minimum length"); + if actual_marker != DIRECTORY_MARKER { + return Err(PithosError::InvalidDirectoryMarker { + expected: DIRECTORY_MARKER, + actual: actual_marker, + }); + } + let encoded_len = u64::from_be_bytes( + bytes[bytes.len() - 12..bytes.len() - 4] + .try_into() + .expect("checked directory footer"), + ); + if encoded_len != bytes.len() as u64 { + return Err(PithosError::DirectoryLengthMismatch { + expected: bytes.len() as u64, + actual: encoded_len, + }); + } + let encoded_crc = u32::from_be_bytes( + bytes[bytes.len() - 4..] + .try_into() + .expect("checked directory footer"), + ); + let computed_crc = crc32fast::hash(&bytes[..bytes.len() - 4]); + if encoded_crc != computed_crc { + return Err(PithosError::DirectoryChecksumMismatch { + expected: computed_crc, + actual: encoded_crc, + }); + } + let mut reader = Cursor::new(bytes); + let directory = decode_directory(&mut reader, limits)?; + if reader.position() != bytes.len() as u64 { + return Err(PithosError::DirectoryConsumptionMismatch { + expected: bytes.len() as u64, + actual: reader.position(), + }); + } + Ok(directory) +} + +pub(crate) fn update_directory_len(directory: &mut Directory) -> Result<(), SerializationError> { + let mut bytes = Vec::new(); + encode_directory(directory, &mut bytes)?; + directory.dir_len = u64::try_from(bytes.len()) + .map_err(|_| SerializationError::Other("length does not fit in u64".to_string()))?; + Ok(()) +} + +pub(crate) fn update_directory_crc(directory: &mut Directory) -> Result<(), SerializationError> { + let mut bytes = Vec::new(); + encode_directory(directory, &mut bytes)?; + let mut hasher = Hasher::new(); + hasher.update(&bytes[..bytes.len() - 4]); + directory.crc32 = hasher.finalize(); + Ok(()) +} + +pub(crate) fn encode_decrypted_block_list( + entries: &[BlockDataEntry], + writer: &mut W, +) -> Result<(), SerializationError> { + write_len_prefix(writer, entries.len())?; + for (hash, key) in entries { + writer.write_all(hash)?; + writer.write_all(key)?; + } + Ok(()) +} + +pub(crate) fn decode_decrypted_block_list_reader( + reader: &mut R, + limits: &DeserializationLimits, +) -> Result>, DeserializationError> { + let count = bounded_len( + reader.read_varint()?, + limits.max_collection_entries, + "block index", + )?; + let mut entries = Zeroizing::new(Vec::new()); + reserve(&mut entries, count, "block index")?; + let mut keys = HashMap::with_capacity(count); + for _ in 0..count { + let mut hash = [0; 32]; + reader.read_exact(&mut hash)?; + let mut key = [0; 32]; + reader.read_exact(&mut key)?; + if keys + .insert(hash, key) + .is_some_and(|existing| existing != key) + { + return Err(DeserializationError::DuplicateBlockReference); + } + entries.push((hash, key)); + } + Ok(entries) +} + +pub(crate) fn decode_decrypted_block_list( + bytes: &[u8], + limits: &DeserializationLimits, +) -> Result>, DeserializationError> { + let mut reader = Cursor::new(bytes); + let entries = decode_decrypted_block_list_reader(&mut reader, limits)?; + if reader.position() != bytes.len() as u64 { + return Err(DeserializationError::InvalidLength); + } + Ok(entries) +} + +pub(crate) fn encode_decrypted_recipient_list( + entries: &[(u64, [u8; 32])], + writer: &mut W, +) -> Result<(), SerializationError> { + write_len_prefix(writer, entries.len())?; + for (id, key) in entries { + writer.write_varint(*id)?; + writer.write_all(key)?; + } + Ok(()) +} + +pub(crate) fn decode_decrypted_recipient_list_reader( + reader: &mut R, + limits: &DeserializationLimits, +) -> Result { + let count = bounded_len( + reader.read_varint()?, + limits.max_collection_entries, + "recipient keys", + )?; + let mut entries = Zeroizing::new(Vec::new()); + reserve(&mut entries, count, "recipient keys")?; + let mut ids = HashSet::with_capacity(count); + for _ in 0..count { + let id = reader.read_varint()?; + entries.push((id, [0; 32])); + reader.read_exact(&mut entries.last_mut().expect("just pushed file key").1)?; + if !ids.insert(id) { + return Err(DeserializationError::DuplicateRecipientFileId); + } + } + Ok(entries) +} + +pub(crate) fn decode_decrypted_recipient_list( + bytes: &[u8], + limits: &DeserializationLimits, +) -> Result { + let mut reader = Cursor::new(bytes); + let entries = decode_decrypted_recipient_list_reader(&mut reader, limits)?; + if reader.position() != bytes.len() as u64 { + return Err(DeserializationError::InvalidLength); + } + Ok(entries) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn complete_directory_rejects_bad_footer_and_trailing_bytes() { + let bytes = vec![ + 0x50, 0x49, 0x54, 0x48, 0x4f, 0x53, 0x44, 0x52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0x19, 0xb1, 0x67, 0x40, 0x81, + ]; + assert!(decode_complete_directory(&bytes, &DeserializationLimits::default()).is_ok()); + let mut trailing = bytes.clone(); + trailing.push(0); + assert!(matches!( + decode_complete_directory(&trailing, &DeserializationLimits::default()), + Err(PithosError::DirectoryLengthMismatch { .. }) + )); + let mut corrupt = bytes; + corrupt[0] = b'X'; + assert!(matches!( + decode_complete_directory(&corrupt, &DeserializationLimits::default()), + Err(PithosError::InvalidDirectoryMarker { .. }) + )); + } + + #[test] + fn header_and_block_markers_reject_wrong_tags() { + assert!(matches!( + decode_header(&mut &b"POT\x80"[..]), + Err(DeserializationError::InvalidMarker(_)) + )); + assert!(matches!( + decode_block_marker(&mut &b"BLOK"[..]), + Err(DeserializationError::InvalidMarker(_)) + )); + } + + #[test] + fn authenticated_lists_require_exact_consumption_and_unique_keys() { + let limits = DeserializationLimits::default(); + assert!(matches!( + decode_decrypted_block_list(&[0, 0], &limits), + Err(DeserializationError::InvalidLength) + )); + let mut duplicate = vec![2]; + duplicate.extend_from_slice(&[1; 32]); + duplicate.extend_from_slice(&[2; 32]); + duplicate.extend_from_slice(&[1; 32]); + duplicate.extend_from_slice(&[3; 32]); + assert!(matches!( + decode_decrypted_block_list(&duplicate, &limits), + Err(DeserializationError::DuplicateBlockReference) + )); + let mut repeated = vec![2]; + repeated.extend_from_slice(&[1; 32]); + repeated.extend_from_slice(&[2; 32]); + repeated.extend_from_slice(&[1; 32]); + repeated.extend_from_slice(&[2; 32]); + assert_eq!( + decode_decrypted_block_list(&repeated, &limits) + .unwrap() + .len(), + 2 + ); + } + + #[test] + fn directory_rejects_duplicate_relationship_definitions() { + let mut directory = Directory { + identifier: DIRECTORY_MARKER, + parent_directory_offset: None, + blocks: IndexMap::new(), + files: WireEntries::new(), + relations: vec![(7, "same".to_owned()), (7, "same".to_owned())], + encryption: IndexMap::new(), + dir_len: 0, + crc32: 0, + }; + update_directory_len(&mut directory).unwrap(); + update_directory_crc(&mut directory).unwrap(); + let mut bytes = Vec::new(); + encode_directory(&directory, &mut bytes).unwrap(); + + assert!(matches!( + decode_complete_directory(&bytes, &DeserializationLimits::default()), + Err(PithosError::ConflictingRelationshipDefinition(7)) + )); + } +} diff --git a/crates/pithos_lib/src/format/directory.rs b/crates/pithos_lib/src/format/directory.rs new file mode 100644 index 0000000..1b2abe1 --- /dev/null +++ b/crates/pithos_lib/src/format/directory.rs @@ -0,0 +1,94 @@ +use crate::error::PithosError; +use crate::format::entries::WireEntries; +use crate::format::wire::{BlockDataState, Directory, EncryptionSection, FileEntry, FileType}; + +impl Directory { + pub(crate) fn new( + parent_directory_offset: Option<(u64, u64)>, + files: WireEntries, + encryption: indexmap::IndexMap<[u8; 32], EncryptionSection>, + ) -> Self { + Self { + identifier: *b"PITHOSDR", + parent_directory_offset, + files, + blocks: indexmap::IndexMap::new(), + relations: vec![ + (0, "Describes".into()), + (1, "Annotates".into()), + (2, "Derived_From".into()), + (3, "Source_Of".into()), + (4, "Previous_Version".into()), + (5, "Next_Version".into()), + (6, "Part_of".into()), + (7, "Contains".into()), + (8, "Input_To".into()), + (9, "Output_From".into()), + ], + encryption, + dir_len: 0, + crc32: 0, + } + } + + pub(crate) fn validate_references_and_accessible_blocks_with( + &self, + has_entry: impl Fn(u64) -> bool, + has_relationship: impl Fn(u64) -> bool, + block_size: impl Fn([u8; 32]) -> Option, + ) -> Result<(), PithosError> { + let relationship_ids = self + .relations + .iter() + .map(|(id, _)| *id) + .collect::>(); + for (_, _, file) in self.files.iter() { + for reference in &file.references { + if !relationship_ids.contains(&reference.relationship) + && !has_relationship(reference.relationship) + { + return Err(PithosError::UnknownRelationshipId(reference.relationship)); + } + if !has_entry(reference.target_file_id) { + return Err(PithosError::MissingReferenceTarget( + reference.target_file_id, + )); + } + } + if !matches!(file.file_type, FileType::Data | FileType::Metadata) { + continue; + } + if let BlockDataState::Decrypted(references) = &file.block_data { + crate::format::wire::validate_unique_block_references(references)?; + let actual = references.iter().try_fold(0u64, |total, (hash, _)| { + let original_size = + block_size(*hash).ok_or(PithosError::MissingBlockDescriptor)?; + total.checked_add(original_size).ok_or({ + PithosError::AccessibleFileSizeMismatch { + expected: file.file_size, + actual: u64::MAX, + } + }) + })?; + if actual != file.file_size { + return Err(PithosError::AccessibleFileSizeMismatch { + expected: file.file_size, + actual, + }); + } + } + } + Ok(()) + } + + #[tracing::instrument(level = "trace", skip(self))] + pub fn next_free_file_index(&self) -> Result { + self.files + .next_free_id(self.parent_directory_offset.is_some()) + } + + #[tracing::instrument(level = "trace", skip(self, file_id))] + pub fn get_file_by_id(&self, file_id: u64) -> Option<&FileEntry> { + self.files.get_by_id(file_id) + } +} diff --git a/crates/pithos_lib/src/format/entries.rs b/crates/pithos_lib/src/format/entries.rs new file mode 100644 index 0000000..4ed32f2 --- /dev/null +++ b/crates/pithos_lib/src/format/entries.rs @@ -0,0 +1,188 @@ +use crate::error::PithosError; +use crate::format::wire::FileEntry; +use indexmap::IndexMap; +use std::cmp::Ordering; +use std::collections::BTreeMap; +use std::ops::Bound::Excluded; +use std::sync::Arc; + +#[derive(Clone, Debug, Eq, PartialEq)] +struct OrderedPath(Arc); + +impl Ord for OrderedPath { + fn cmp(&self, other: &Self) -> Ordering { + self.0.split('/').cmp(other.0.split('/')) + } +} + +impl PartialOrd for OrderedPath { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct WireEntry { + id: u64, + path: Arc, + entry: FileEntry, +} + +/// Format-owned file records with stable wire insertion order and private indexes. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct WireEntries { + by_id: IndexMap, + by_path: IndexMap, usize>, + ordered_paths: BTreeMap, + entries: Vec, + maximum_id: u64, +} + +impl WireEntries { + pub(crate) fn new() -> Self { + Self::with_maximum_id(0) + } + + pub(crate) fn with_maximum_id(maximum_id: u64) -> Self { + Self { + by_id: IndexMap::new(), + by_path: IndexMap::new(), + ordered_paths: BTreeMap::new(), + entries: Vec::new(), + maximum_id, + } + } + + pub(crate) fn len(&self) -> usize { + self.entries.len() + } + + pub(crate) fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + pub(crate) fn get_by_id(&self, id: u64) -> Option<&FileEntry> { + self.by_id + .get(&id) + .and_then(|index| self.entries.get(*index)) + .map(|entry| &entry.entry) + } + + pub(crate) fn get_by_path(&self, path: &str) -> Option<&FileEntry> { + self.by_path + .get(path) + .and_then(|index| self.entries.get(*index)) + .map(|entry| &entry.entry) + } + + pub(crate) fn first_path_after(&self, path: &str) -> Option<&str> { + let query = OrderedPath(Arc::from(path)); + self.ordered_paths + .range((Excluded(query), std::ops::Bound::Unbounded)) + .next() + .map(|(path, _)| path.0.as_ref()) + } + + pub(crate) fn iter(&self) -> impl ExactSizeIterator + '_ { + self.entries + .iter() + .map(|entry| (entry.id, entry.path.as_ref(), &entry.entry)) + } + + pub(crate) fn iter_ordered(&self) -> impl Iterator + '_ { + self.ordered_paths + .iter() + .map(|(path, index)| (path.0.as_ref(), &self.entries[*index].entry)) + } + + pub(crate) fn try_for_each_mut( + &mut self, + mut operation: impl FnMut(u64, &mut FileEntry) -> Result<(), E>, + ) -> Result<(), E> { + for entry in &mut self.entries { + operation(entry.id, &mut entry.entry)?; + } + Ok(()) + } + + pub(crate) fn next_free_id(&self, has_parent: bool) -> Result { + if self.maximum_id == 0 && self.entries.is_empty() { + Ok(u64::from(has_parent)) + } else { + self.maximum_id + .checked_add(1) + .ok_or(PithosError::FileIdExhausted) + } + } + + pub(crate) fn prevalidate_insert(&self, id: u64, path: &str) -> Result<(), PithosError> { + if self.by_id.contains_key(&id) { + return Err(PithosError::DuplicateFileId(format!( + "File id already occupied: {id}" + ))); + } + if self.by_path.contains_key(path) { + return Err(PithosError::PathOccupied(format!( + "File path already occupied: {path}" + ))); + } + Ok(()) + } + + pub(crate) fn reserve_one(&mut self) -> Result<(), PithosError> { + let size = self + .entries + .len() + .checked_add(1) + .ok_or(PithosError::AllocationFailed { + field: "file entries", + size: u64::MAX, + })?; + self.by_id + .try_reserve(1) + .map_err(|_| allocation_failed("file entry ids", size))?; + self.by_path + .try_reserve(1) + .map_err(|_| allocation_failed("file entry paths", size))?; + self.entries + .try_reserve(1) + .map_err(|_| allocation_failed("file entries", size)) + } + + /// Insert only after validation and reservation of the vector-backed indexes. + pub(crate) fn insert_prepared(&mut self, id: u64, path: impl Into>, entry: FileEntry) { + let path = path.into(); + debug_assert!(self.prevalidate_insert(id, &path).is_ok()); + let ordered = OrderedPath(Arc::clone(&path)); + let index = self.entries.len(); + self.entries.push(WireEntry { + id, + path: Arc::clone(&path), + entry, + }); + self.by_id.insert(id, index); + self.by_path.insert(path, index); + self.ordered_paths.insert(ordered, index); + self.maximum_id = self.maximum_id.max(id); + } + + pub(crate) fn insert( + &mut self, + id: u64, + path: impl Into>, + entry: FileEntry, + ) -> Result<(), PithosError> { + let path = path.into(); + self.prevalidate_insert(id, &path)?; + self.reserve_one()?; + self.insert_prepared(id, path, entry); + Ok(()) + } +} + +fn allocation_failed(field: &'static str, size: usize) -> PithosError { + PithosError::AllocationFailed { + field, + size: u64::try_from(size).unwrap_or(u64::MAX), + } +} diff --git a/crates/pithos_lib/src/format/error.rs b/crates/pithos_lib/src/format/error.rs new file mode 100644 index 0000000..f11fef1 --- /dev/null +++ b/crates/pithos_lib/src/format/error.rs @@ -0,0 +1,9 @@ +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum SerializationError { + #[error("IoError error: {0}")] + IoError(#[from] std::io::Error), + #[error("Serialization error: {0}")] + Other(String), +} diff --git a/crates/pithos_lib/src/format/limits.rs b/crates/pithos_lib/src/format/limits.rs new file mode 100644 index 0000000..b8c99eb --- /dev/null +++ b/crates/pithos_lib/src/format/limits.rs @@ -0,0 +1,60 @@ +use std::io::Error as IoError; +use std::string::FromUtf8Error; +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum DeserializationError { + #[error("I/O error: {0}")] + Io(#[from] IoError), + #[error("UTF-8 decoding error: {0}")] + Utf8(#[from] FromUtf8Error), + #[error("Invalid enum value: {0}")] + InvalidEnumValue(u8), + #[error("Invalid marker: {0}")] + InvalidMarker(String), + #[error("Invalid option")] + InvalidOption, + #[error("Invalid length")] + InvalidLength, + #[error("reserved processing bits are set: {0:#04x}")] + InvalidProcessingFlags(u8), + #[error("duplicate encoded recipient key")] + DuplicateRecipientKey, + #[error("duplicate encoded block reference")] + DuplicateBlockReference, + #[error("duplicate encoded recipient file id")] + DuplicateRecipientFileId, + #[error("{field} exceeds limit {limit}: {actual}")] + LimitExceeded { + field: &'static str, + limit: u64, + actual: u64, + }, + #[error("allocation failed for {field}: {size}")] + AllocationFailed { field: &'static str, size: u64 }, +} + +#[derive(Clone, Copy, Debug)] +pub struct DeserializationLimits { + pub max_string_bytes: u64, + pub max_opaque_bytes: u64, + pub max_collection_entries: u64, + pub max_file_entries: u64, + pub max_block_descriptors: u64, + pub max_references: u64, + pub max_relationships: u64, +} + +impl Default for DeserializationLimits { + fn default() -> Self { + Self { + max_string_bytes: 1024 * 1024, + max_opaque_bytes: 64 * 1024 * 1024, + max_collection_entries: 1_000_000, + max_file_entries: 1_000_000, + max_block_descriptors: 1_000_000, + max_references: 1_000_000, + max_relationships: 1_000_000, + } + } +} diff --git a/crates/pithos_lib/src/format/mod.rs b/crates/pithos_lib/src/format/mod.rs new file mode 100644 index 0000000..d309a26 --- /dev/null +++ b/crates/pithos_lib/src/format/mod.rs @@ -0,0 +1,11 @@ +//! Private current-format byte grammar. +//! +//! This module deliberately contains no archive policy, filesystem, adapter, or +//! transport concerns. + +pub(crate) mod codec; +pub(crate) mod directory; +pub(crate) mod entries; +pub(crate) mod error; +pub(crate) mod limits; +pub(crate) mod wire; diff --git a/crates/pithos_lib/src/format/wire.rs b/crates/pithos_lib/src/format/wire.rs new file mode 100644 index 0000000..cba672d --- /dev/null +++ b/crates/pithos_lib/src/format/wire.rs @@ -0,0 +1,355 @@ +use crate::crypto::{self, FileKey, SharedSecret}; +use std::fmt::{Display, Formatter}; +use zeroize::Zeroizing; + +use crate::error::PithosError; +use crate::format::entries::WireEntries; +use indexmap::IndexMap; +use x25519_dalek::PublicKey; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FileHeader { + pub magic: [u8; 4], // MUST be b"PITH" + pub version: u16, // Format version (e.g., 0x0100 for 1.0) +} + +impl Default for FileHeader { + fn default() -> Self { + FileHeader { + magic: *b"PITH", + version: Self::SUPPORTED_VERSION, + } + } +} + +impl FileHeader { + pub const SUPPORTED_VERSION: u16 = 0x0100; +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BlockHeader { + pub marker: [u8; 4], // MUST be b"BLCK" +} + +impl Default for BlockHeader { + fn default() -> Self { + BlockHeader { marker: *b"BLCK" } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ProcessingFlags(pub u8); + +impl ProcessingFlags { + // Compression level can be set from 0-7 with the first three bits + // 0 = Uncompressed + // 7 = Highest compression + const COMPRESSION_MASK: u8 = 0b0000_0111; + + // Encryption is indicated with the 4th bit + const ENCRYPTION_MASK: u8 = 0b0000_1000; + pub const RESERVED_MASK: u8 = 0b1111_0000; + + pub fn new(encrypted: bool, compression_level: Option) -> Self { + // Init + let mut flags = ProcessingFlags(0b0); + + // Set compression level + match compression_level { + Some(level) => { + if level > Self::COMPRESSION_MASK { + flags.set_compression_level(Self::COMPRESSION_MASK) // Cap at maximum + } else { + flags.set_compression_level(level) + } + } + None => flags.set_compression_level(3), // Default + } + + // Set encryption + flags.set_encryption(encrypted); + flags + } + + pub fn from_byte(byte: u8) -> Self { + ProcessingFlags(byte) + } + + pub fn has_reserved_bits(&self) -> bool { + self.0 & Self::RESERVED_MASK != 0 + } + + pub fn set_encryption(&mut self, encrypted: bool) { + if encrypted { + self.0 |= Self::ENCRYPTION_MASK; // Set bit for encryption + } else { + self.0 &= !Self::ENCRYPTION_MASK; // Clear bit for encryption + } + } + + // New function to check if encryption is enabled + pub fn is_encrypted(&self) -> bool { + (self.0 & Self::ENCRYPTION_MASK) != 0 + } + + pub fn set_compression_level(&mut self, mut compression_level: u8) { + // Sanitize + compression_level = if compression_level > Self::COMPRESSION_MASK { + Self::COMPRESSION_MASK + } else { + compression_level + }; + + // Only use the lowest 3 bits (0-7) + self.0 = (self.0 & !Self::COMPRESSION_MASK) | (compression_level & Self::COMPRESSION_MASK); + } + + pub fn get_compression_level(&self) -> u8 { + self.0 & Self::COMPRESSION_MASK + } +} + +impl Default for ProcessingFlags { + fn default() -> Self { + ProcessingFlags::new(true, None) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum BlockLocation { + Local, // Block data at specified offset in this file + External { url: String }, // URL to external storage +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BlockIndexEntry { + pub offset: u64, // varint + pub stored_size: u64, // varint + pub original_size: u64, // varint + pub flags: ProcessingFlags, // Compression, encryption settings + pub location: BlockLocation, // Where block data resides +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Directory { + pub identifier: [u8; 8], // MUST be b"PITHOSDR" + pub parent_directory_offset: Option<(u64, u64)>, // (start, len) varint + pub blocks: IndexMap<[u8; 32], BlockIndexEntry>, // Blocks in this segment + pub files: WireEntries, // Files in this segment + pub relations: Vec<(u64, String)>, // Relation idx, relationname/id + pub encryption: IndexMap<[u8; 32], EncryptionSection>, // (Sender's X25519 public key, section with recipients) + pub dir_len: u64, + pub crc32: u32, // CRC32 of all preceding fields +} + +#[repr(u8)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum FileType { + Directory = 0, + Data = 1, + Metadata = 2, + Symlink = 3, + // 4-255 reserved +} + +/// A block's content hash and the key used to encrypt its content. +pub type BlockDataEntry = ([u8; 32], [u8; 32]); +pub(crate) type RecipientKeyList = Zeroizing>; + +/// Transient writer/reader wire state. Decrypted block keys are crate-private. +#[derive(Clone, PartialEq, Eq)] +pub(crate) enum BlockDataState { + Encrypted(Vec), // Chacha + nonce (Random key) + Decrypted(Zeroizing>), // BLAKE3 hash / Shake256 hash +} + +impl std::fmt::Debug for BlockDataState { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Encrypted(bytes) => formatter + .debug_tuple("Encrypted") + .field(&format_args!("{} bytes", bytes.len())) + .finish(), + Self::Decrypted(entries) => formatter + .debug_tuple("Decrypted") + .field(&format_args!("{} block keys [REDACTED]", entries.len())) + .finish(), + } + } +} + +impl BlockDataState { + pub(crate) fn encrypt_with_nonce( + &mut self, + key: &FileKey, + nonce: [u8; 12], + ) -> Result<(), PithosError> { + match &self { + BlockDataState::Encrypted(_) => { + return Err(PithosError::InvalidBlockDataState( + "Block already encrypted.".to_string(), + )); + } + BlockDataState::Decrypted(entries) => { + let mut data_bytes = Zeroizing::new(Vec::with_capacity(1 + entries.len() * 64)); + crate::format::codec::encode_decrypted_block_list(entries, &mut *data_bytes)?; + let encrypted_data = + crypto::seal_file_block_list_with_nonce(key, &data_bytes, nonce)?; + + *self = BlockDataState::Encrypted(encrypted_data) + } + }; + + Ok(()) + } +} + +pub(crate) fn validate_unique_block_references( + entries: &[BlockDataEntry], +) -> Result<(), PithosError> { + let mut keys = std::collections::HashMap::with_capacity(entries.len()); + for (hash, key) in entries { + if keys + .insert(*hash, *key) + .is_some_and(|existing| existing != *key) + { + return Err(PithosError::DuplicateBlockReference); + } + } + Ok(()) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FileEntry { + pub file_type: FileType, + pub(crate) block_data: BlockDataState, + pub created: u64, + pub modified: u64, + pub file_size: u64, + pub permissions: u32, + pub references: Vec, + pub symlink_target: Option, // Target path for symlinks +} + +impl Display for FileEntry { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(&format!("{:<12} {:?}\n", "Type:", self.file_type))?; + match &self.block_data { + BlockDataState::Encrypted(_) => f.write_str("Blocks: Encrypted\n")?, + BlockDataState::Decrypted(_) => f.write_str("Blocks: Decrypted\n")?, + } + f.write_str(&format!("{:<12} {}\n", "Created:", self.created))?; + f.write_str(&format!("{:<12} {}\n", "Modified:", self.modified))?; + f.write_str(&format!("{:<12} {}\n", "Size:", self.file_size))?; + f.write_str(&format!("{:<12} {:o}\n", "Permissions:", self.permissions))?; + f.write_str(&format!("{:<12} {:?}\n", "References:", self.references))?; + + if let Some(target) = &self.symlink_target { + f.write_str(&format!("{:<12} {target}\n", "Target:"))?; + } + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Reference { + pub target_file_id: u64, // varint + pub relationship: u64, // varint +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EncryptionSection { + // Recipient's X25519 public key, Recipient file list + pub recipients: IndexMap<[u8; 32], RecipientSection>, +} + +impl EncryptionSection { + #[tracing::instrument(level = "trace", skip(recipient_pubkeys))] + pub fn new(recipient_pubkeys: &[PublicKey]) -> Self { + EncryptionSection { + recipients: IndexMap::from_iter( + recipient_pubkeys + .iter() + .map(|key| { + ( + key.to_bytes(), + RecipientSection { + recipient_data: RecipientData::Decrypted( + Zeroizing::new(Vec::new()), + ), + }, + ) + }) + .collect::>(), + ), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RecipientSection { + pub(crate) recipient_data: RecipientData, // Encrypted FileKeyEntry list +} + +/// Transient writer/reader wire state. Recovered file keys are crate-private. +#[derive(Clone, PartialEq, Eq)] +pub(crate) enum RecipientData { + Encrypted(Vec), // Chacha + nonce (Shared key PrivKey Writer <--> PubKey Reader) + Decrypted(RecipientKeyList), // Fileindex / Random key to decrypt BlockDataState +} + +impl std::fmt::Debug for RecipientData { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Encrypted(bytes) => formatter + .debug_tuple("Encrypted") + .field(&format_args!("{} bytes", bytes.len())) + .finish(), + Self::Decrypted(entries) => formatter + .debug_tuple("Decrypted") + .field(&format_args!("{} file keys [REDACTED]", entries.len())) + .finish(), + } + } +} + +impl RecipientData { + pub(crate) fn encrypt_with_secret_and_nonce( + &mut self, + shared_key: SharedSecret, + nonce: [u8; 12], + ) -> Result<(), PithosError> { + match &self { + RecipientData::Encrypted(_) => { + return Err(PithosError::InvalidRecipientDataState( + "Recipient data already encrypted".to_string(), + )); + } + RecipientData::Decrypted(entries) => { + let mut data_bytes = Zeroizing::new(Vec::with_capacity(1 + entries.len() * 40)); + crate::format::codec::encode_decrypted_recipient_list(entries, &mut *data_bytes)?; + + let encrypted_data = + crypto::wrap_recipient_list_with_nonce(&shared_key, &data_bytes, nonce)?; + + *self = RecipientData::Encrypted(encrypted_data) + } + }; + + Ok(()) + } +} + +#[cfg(test)] +mod zeroization_tests { + use super::*; + use zeroize::ZeroizeOnDrop; + + fn assert_zeroize_on_drop() {} + + #[test] + fn transient_plaintext_key_lists_have_drop_zeroization_contracts() { + assert_zeroize_on_drop::>>(); + assert_zeroize_on_drop::>>(); + } +} diff --git a/crates/pithos_lib/src/helpers/directory.rs b/crates/pithos_lib/src/helpers/directory.rs deleted file mode 100644 index a4cfd98..0000000 --- a/crates/pithos_lib/src/helpers/directory.rs +++ /dev/null @@ -1,475 +0,0 @@ -use crate::error::PithosError; -use crate::helpers::file_entry_map::{FileEntryMap, Key, KeyQuery}; -use crate::model::structs::{RecipientData, RecipientSection}; -use crate::model::{ - deserialization::DeserializationLimits, - serialization::SerializationError, - structs::{BlockIndexEntry, Directory, EncryptionSection, FileEntry}, -}; -use crc32fast::Hasher; -use indexmap::IndexMap; -use indexmap::map::Entry; -use x25519_dalek::{PublicKey, StaticSecret}; - -pub struct DirectoryBuilder { - identifier: [u8; 8], - parent_directory_offset: Option<(u64, u64)>, - files: FileEntryMap, - blocks: IndexMap<[u8; 32], BlockIndexEntry>, - relations: Vec<(u64, String)>, - encryption: IndexMap<[u8; 32], EncryptionSection>, -} - -impl Default for DirectoryBuilder { - fn default() -> Self { - Self::new() - } -} - -impl DirectoryBuilder { - #[tracing::instrument(level = "trace", skip())] - pub fn new() -> Self { - DirectoryBuilder { - identifier: *b"PITHOSDR", - parent_directory_offset: None, - files: FileEntryMap::new(), - blocks: IndexMap::new(), - relations: Self::default_relations(), - encryption: IndexMap::new(), - } - } - - fn default_relations() -> Vec<(u64, String)> { - Vec::from_iter([ - (0, "Describes".to_string()), - (1, "Annotates".to_string()), - (2, "Derived_From".to_string()), - (3, "Source_Of".to_string()), - (4, "Previous_Version".to_string()), - (5, "Next_Version".to_string()), - (6, "Part_of".to_string()), - (7, "Contains".to_string()), - (8, "Input_To".to_string()), - (9, "Output_From".to_string()), - ]) - } - - #[tracing::instrument(level = "trace", skip(self, offset))] - pub fn parent_directory_offset(mut self, offset: Option<(u64, u64)>) -> Self { - self.parent_directory_offset = offset; - self - } - - #[tracing::instrument(level = "trace", skip(self, files))] - pub fn files(mut self, files: FileEntryMap) -> Self { - self.files = files; - self - } - - #[tracing::instrument(level = "trace", skip(self, blocks))] - pub fn blocks(mut self, blocks: IndexMap<[u8; 32], BlockIndexEntry>) -> Self { - self.blocks = blocks; - self - } - - #[tracing::instrument(level = "trace", skip(self, relations))] - pub fn set_relations(mut self, relations: Vec<(u64, String)>) -> Self { - self.relations = relations; - self - } - - #[tracing::instrument(level = "trace", skip(self, relations))] - pub fn add_relations(mut self, relations: Vec<(u64, String)>) -> Self { - self.relations.extend(relations); - self - } - - #[tracing::instrument(level = "trace", skip(self, encryption))] - pub fn encryption(mut self, encryption: IndexMap<[u8; 32], EncryptionSection>) -> Self { - self.encryption = encryption; - self - } - - #[tracing::instrument(level = "trace", skip(self))] - pub fn build(self) -> Result { - let mut directory = Directory { - identifier: self.identifier, - parent_directory_offset: self.parent_directory_offset, - files: self.files, - blocks: self.blocks, - relations: self.relations, - encryption: self.encryption, - dir_len: 0, - crc32: 0, - }; - directory.update_len()?; - directory.update_crc32()?; - Ok(directory) - } -} - -impl Directory { - #[tracing::instrument(level = "trace", skip())] - pub fn new() -> Result { - DirectoryBuilder::new().build() - } - - #[tracing::instrument(level = "trace", skip(self, newer_directory))] - pub fn merge(&mut self, newer_directory: Directory) -> Result<(), PithosError> { - // Append files (also checks for duplicate file_id and path duplicates) - self.add_files_to_index(newer_directory.files)?; - - // Append blocks - self.add_blocks_to_index(newer_directory.blocks)?; - - // Append relations - self.add_relation_definitions(newer_directory.relations)?; - - // Merge encryption sections - // - Merge sections that can be decrypted - // - Drop encryption sections which already exist ??? - for (section_key, new_section) in newer_directory.encryption { - match self.encryption.entry(section_key) { - Entry::Occupied(ref mut entry) => { - // Encryption section does not exist -> merge decrypted recipients - for (recipient_key, new_recipient) in new_section.recipients { - match entry.get_mut().recipients.get_mut(&recipient_key) { - Some(existing_recipient) => { - merge_recipient_data( - &mut existing_recipient.recipient_data, - new_recipient.recipient_data, - ); - } - None => { - entry - .get_mut() - .recipients - .insert(recipient_key, new_recipient); - } - } - } - } - Entry::Vacant(entry) => { - // Encryption section does not exist -> insert - entry.insert(new_section); - return Ok(()); - } - } - } - - Ok(()) - } - - #[tracing::instrument(level = "trace", skip(self, hash))] - pub fn block_hash_exists(&mut self, hash: &blake3::Hash) -> Option { - if let Entry::Occupied(entry) = self.blocks.entry(*hash.as_bytes()) { - Some(entry.get().clone()) - } else { - None - } - } - - #[tracing::instrument(level = "trace", skip(self, block_index_entries))] - pub fn add_blocks_to_index( - &mut self, - block_index_entries: IndexMap<[u8; 32], BlockIndexEntry>, - ) -> Result<(), PithosError> { - for (hash, block) in block_index_entries { - self.add_block_to_index(hash, block)?; - } - Ok(()) - } - - #[tracing::instrument(level = "trace", skip(self, block_hash, block_entry))] - pub fn add_block_to_index( - &mut self, - block_hash: [u8; 32], - block_entry: BlockIndexEntry, - ) -> Result<(), PithosError> { - match self.blocks.entry(block_hash) { - Entry::Occupied(existing) => { - if existing.get().original_size != block_entry.original_size { - return Err(PithosError::BlockIndexConflict { - hash: block_hash, - existing_original_size: existing.get().original_size, - new_original_size: block_entry.original_size, - }); - } - } - Entry::Vacant(entry) => { - entry.insert(block_entry); - } - } - Ok(()) - } - - #[tracing::instrument(level = "trace", skip(self, path, file_entry))] - pub fn add_file(&mut self, path: &str, file_entry: &FileEntry) -> Result<(), PithosError> { - let key = Key::new( - self.files - .next_free_id(self.parent_directory_offset.is_some())?, - path.to_owned(), - ); - self.files.insert(key, file_entry.clone()) - } - - #[tracing::instrument(level = "trace", skip(self, files))] - pub fn add_files(&mut self, files: Vec<(&str, &FileEntry)>) -> Result<(), PithosError> { - for (path, file_entry) in files { - self.add_file(path, file_entry)? - } - Ok(()) - } - - #[tracing::instrument(level = "trace", skip(self, file_entry))] - pub fn add_file_to_index( - &mut self, - key: &Key, - file_entry: &FileEntry, - ) -> Result<(), PithosError> { - self.files.insert(key.clone(), file_entry.clone()) - } - - #[tracing::instrument(level = "trace", skip(self, file_entries))] - pub fn add_files_to_index(&mut self, file_entries: FileEntryMap) -> Result<(), PithosError> { - for (id, path, file) in &file_entries { - let key = Key::new(id, path); - self.add_file_to_index(&key, file)? - } - Ok(()) - } - - #[tracing::instrument(level = "trace", skip(self, writer_key, reader_key, file_entry))] - pub fn add_file_to_recipient( - &mut self, - writer_key: &StaticSecret, - reader_key: &PublicKey, - file_entry: (u64, [u8; 32]), // (file index, encryption key) - ) -> Result<(), PithosError> { - let _: () = match self.encryption.entry(*writer_key.as_bytes()) { - Entry::Occupied(ref mut entry) => { - match entry.get_mut().recipients.entry(reader_key.to_bytes()) { - Entry::Occupied(ref mut entry) => { - entry.get_mut().add_file_to_recipient(file_entry)? - } - Entry::Vacant(vacant) => { - vacant.insert(RecipientSection { - recipient_data: RecipientData::Decrypted(vec![]), - }); - } - } - } - Entry::Vacant(entry) => { - entry.insert(EncryptionSection { - recipients: IndexMap::from_iter([( - reader_key.to_bytes(), - RecipientSection { - recipient_data: RecipientData::Decrypted(vec![]), - }, - )]), - }); - } - }; - Ok(()) - } - - #[tracing::instrument(level = "trace", skip(self, entry))] - pub fn add_file_to_all_recipients(&mut self, entry: (u64, [u8; 32])) { - for (_, e_section) in self.encryption.iter_mut() { - for (_, r_section) in e_section.recipients.iter_mut() { - match r_section.recipient_data { - RecipientData::Encrypted(_) => { - // Won't add files to encrypted sections ¯\_(ツ)_/¯ - } - RecipientData::Decrypted(ref mut entries) => { - entries.push(entry); - } - } - } - } - } - - #[tracing::instrument(level = "trace", skip(self, relation))] - pub fn add_relation_definition(&mut self, relation: (u64, String)) -> Result<(), PithosError> { - for existing_relation in &self.relations { - if relation == *existing_relation { - // Same relation already exists - continue; - } - - if existing_relation.0 == relation.0 && existing_relation.1 != relation.1 { - // Try to add existing relation id with different semantic - return Err(PithosError::RelationIdOccupied(relation.0)); - } - } - - self.relations.push(relation); - Ok(()) - } - - #[tracing::instrument(level = "trace", skip(self, relations))] - pub fn add_relation_definitions( - &mut self, - relations: Vec<(u64, String)>, - ) -> Result<(), PithosError> { - for relation in relations { - self.add_relation_definition(relation)?; - } - Ok(()) - } - - #[tracing::instrument(level = "trace", skip(self))] - pub fn next_free_file_index(&self) -> Result { - self.files - .next_free_id(self.parent_directory_offset.is_some()) - /* - let current_id = self.files.get_current_max_id(); - if current_id == 0 { - 0 - } else { - current_id + 1 - } - */ - } - - #[tracing::instrument(level = "trace", skip(self, path))] - pub fn get_file_by_path(&self, path: &str) -> Option<&FileEntry> { - self.files.get(&KeyQuery::Path(path.to_owned())) - } - - #[tracing::instrument(level = "trace", skip(self, file_id))] - pub fn get_file_by_id(&self, file_id: u64) -> Option<&FileEntry> { - self.files.get(&KeyQuery::Id(file_id)) - } - - // Checks only decrypted recipient sections - #[tracing::instrument(level = "trace", skip(self, file_id))] - pub fn get_file_encryption_key(&self, file_id: u64) -> Option<[u8; 32]> { - for (_, e_section) in &self.encryption { - for (_, r_section) in &e_section.recipients { - if let RecipientData::Decrypted(entries) = &r_section.recipient_data - && let Some((_, key)) = entries.iter().find(|(k, _)| *k == file_id) - { - return Some(*key); - } - } - } - None - } - - #[tracing::instrument(level = "trace", skip(self, writer_key))] - pub fn encrypt_recipients(&mut self, writer_key: &StaticSecret) -> Result<(), PithosError> { - let writer_pubkey = PublicKey::from(writer_key); - for (sender_pubkey, e_section) in self.encryption.iter_mut() { - if writer_pubkey.as_bytes() == sender_pubkey { - for (recipient_pubkey, r_section) in e_section.recipients.iter_mut() { - let reader_pubkey = PublicKey::from(*recipient_pubkey); - let shared_key = writer_key.diffie_hellman(&reader_pubkey); - r_section.recipient_data.encrypt(&shared_key)?; - } - } - } - - Ok(()) - } - - #[tracing::instrument(level = "trace", skip(self, reader_key))] - pub fn decrypt_recipient( - &mut self, - reader_key: &StaticSecret, - ) -> Result, PithosError> { - self.decrypt_recipient_with_limits(reader_key, &DeserializationLimits::default()) - } - - #[tracing::instrument(level = "trace", skip(self, reader_key, limits))] - pub fn decrypt_recipient_with_limits( - &mut self, - reader_key: &StaticSecret, - limits: &DeserializationLimits, - ) -> Result, PithosError> { - // Store for decrypted sections - let mut available_file_indices = Vec::<(u64, [u8; 32])>::new(); - let reader_pubkey = PublicKey::from(reader_key); - - // Check if key is sender key -> Decrypt complete encryption section - if let Entry::Occupied(mut entry) = self.encryption.entry(*reader_pubkey.as_bytes()) { - for (key, r_section) in entry.get_mut().recipients.iter_mut() { - let shared_key = reader_key.diffie_hellman(&PublicKey::from(*key)); - match &r_section.recipient_data { - RecipientData::Encrypted(_) => { - let entries = r_section - .recipient_data - .decrypt_with_limits(&shared_key, limits)?; - available_file_indices.extend(entries); - } - RecipientData::Decrypted(entries) => { - available_file_indices.extend(entries); - } - } - } - } - - // Iterate available encryption sections as the key still could be used as recipient in other sections - for (sender_pubkey, e_section) in self.encryption.iter_mut() { - let sender_pubkey = PublicKey::from(*sender_pubkey); - - // Skip section if reader key is from sender -> Already decrypted above - if sender_pubkey == reader_pubkey { - continue; - } - - let shared_key = reader_key.diffie_hellman(&sender_pubkey); - - match e_section.recipients.entry(*reader_pubkey.as_bytes()) { - Entry::Occupied(ref mut entry) => match &entry.get().recipient_data { - RecipientData::Encrypted(_) => { - let entries = entry - .get_mut() - .recipient_data - .decrypt_with_limits(&shared_key, limits)?; - available_file_indices.extend(entries); - } - RecipientData::Decrypted(entries) => { - available_file_indices.extend(entries); - } - }, - Entry::Vacant(_) => { - // Recipient does not exist in encryption section - } - } - } - - Ok(available_file_indices) - } - - #[tracing::instrument(level = "trace", skip(self))] - pub fn update_len(&mut self) -> Result<(), SerializationError> { - let mut buf = Vec::new(); - self.serialize(&mut buf)?; - self.dir_len = u64::try_from(buf.len()) - .map_err(|_| SerializationError::Other("length does not fit in u64".to_string()))?; - Ok(()) - } - - #[tracing::instrument(level = "trace", skip(self))] - pub fn update_crc32(&mut self) -> Result<(), SerializationError> { - let mut buf = Vec::new(); - self.serialize(&mut buf)?; - - // Calculate CRC32 checksum - let mut hasher = Hasher::new(); - hasher.update(&buf[..buf.len() - 4]); - self.crc32 = hasher.finalize(); - - Ok(()) - } -} - -#[tracing::instrument(level = "trace", skip(existing, new))] -fn merge_recipient_data(existing: &mut RecipientData, new: RecipientData) { - if let (RecipientData::Decrypted(existing_files), RecipientData::Decrypted(new_files)) = - (existing, new) - { - existing_files.extend(new_files); - } -} diff --git a/crates/pithos_lib/src/helpers/file_entry_map.rs b/crates/pithos_lib/src/helpers/file_entry_map.rs deleted file mode 100644 index 4bc0531..0000000 --- a/crates/pithos_lib/src/helpers/file_entry_map.rs +++ /dev/null @@ -1,605 +0,0 @@ -use crate::error::PithosError; -use crate::model::structs::FileEntry; -use indexmap::IndexMap; -use indexmap::map::Entry; -use std::cmp::Ordering; -use std::collections::BTreeMap; -use std::fmt::{Debug, Display, Formatter}; -use std::ops::Bound::{Excluded, Unbounded}; -use std::sync::Arc; - -/// Values used as keys in a Map, e.g. `HashMap`. -#[derive(Clone, Debug, Eq, Hash, PartialEq)] -pub struct Key { - id: u64, - path: Arc, -} - -impl Key { - pub fn new(id: u64, path: impl Into) -> Key { - Key { - id, - path: Arc::from(path.into()), - } - } - - pub fn id(&self) -> u64 { - self.id - } - - pub fn id_mut(&mut self) -> &mut u64 { - &mut self.id - } - - pub fn id_query(&self) -> KeyQuery { - KeyQuery::Id(self.id) - } - - pub fn path(&self) -> &str { - &self.path - } - - pub fn path_query(&self) -> KeyQuery { - KeyQuery::Path(self.path.to_string()) - } -} - -/// An archive path ordered by slash-delimited components rather than raw bytes. -#[derive(Clone, Debug, Eq, PartialEq)] -struct ArchivePathKey(Arc); - -impl Ord for ArchivePathKey { - fn cmp(&self, other: &Self) -> Ordering { - self.0.split('/').cmp(other.0.split('/')) - } -} - -impl PartialOrd for ArchivePathKey { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -/// Different ways to look up a value in a Map keyed by `Key`. -/// -/// Each variant must constitute a unique index. -#[derive(Clone, Debug, Eq, Hash, PartialEq)] -pub enum KeyQuery { - Id(u64), - Path(String), -} - -impl Display for KeyQuery { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - match &self { - KeyQuery::Id(id) => f.write_str(&format!("key query for id {}", id)), - KeyQuery::Path(path) => f.write_str(&format!("key query for path {}", path)), - } - } -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct FileEntryMap { - id_map: IndexMap, - path_map: IndexMap, usize>, - ordered_path_map: BTreeMap, - values: Vec, - current_max_id: u64, -} - -impl FileEntryMap { - pub fn new() -> Self { - Self { - id_map: IndexMap::new(), - path_map: IndexMap::new(), - ordered_path_map: BTreeMap::new(), - values: Vec::new(), - current_max_id: 0, - } - } - - pub fn new_with_max(max_id: u64) -> Self { - Self { - id_map: IndexMap::new(), - path_map: IndexMap::new(), - ordered_path_map: BTreeMap::new(), - values: Vec::new(), - current_max_id: max_id, - } - } - - pub fn get_ids_ref(&self) -> &IndexMap { - &self.id_map - } - - pub fn get_paths_ref(&self) -> impl Iterator { - self.path_map.keys().map(|path| path.as_ref()) - } - - pub fn get_id_by_path(&self, path: &str) -> Option { - if let Some(idx) = self.path_map.get(path) { - if let Some((id, _)) = self.id_map.get_index(*idx) { - Some(*id) - } else { - None - } - } else { - None - } - } - - pub(crate) fn get_by_path(&self, path: &str) -> Option<&FileEntry> { - self.path_map - .get(path) - .and_then(|idx| self.values.get(*idx)) - } - - pub(crate) fn first_path_after(&self, path: &str) -> Option<&str> { - let query = self - .path_map - .get_key_value(path) - .map(|(stored, _)| ArchivePathKey(Arc::clone(stored))) - .unwrap_or_else(|| ArchivePathKey(Arc::from(path))); - self.ordered_path_map - .range((Excluded(query), Unbounded)) - .next() - .map(|(path, _)| path.0.as_ref()) - } - - pub(crate) fn iter_ordered(&self) -> impl Iterator { - self.ordered_path_map.iter().map(|(path, idx)| { - ( - path.0.as_ref(), - self.values - .get(*idx) - .expect("ordered path index must reference an entry"), - ) - }) - } - - pub fn get_path_by_id(&self, id: &u64) -> Option<&str> { - if let Some(idx) = self.id_map.get(id) { - if let Some((path, _)) = self.path_map.get_index(*idx) { - Some(path) - } else { - None - } - } else { - None - } - } - - pub fn extend(&mut self, other: FileEntryMap) -> Result<(), PithosError> { - for (id, path, entry) in other { - self.insert(Key::new(id, path), entry)? - } - Ok(()) - } - - pub fn retain_mut(&mut self, mut f: F) -> Result<(), PithosError> - where - F: FnMut(u64, &str, &mut FileEntry) -> bool, - { - let current_max_id = self.current_max_id; - let existing = std::mem::take(self); - let mut retained = FileEntryMap::new_with_max(current_max_id); - - for (id, path, mut entry) in existing { - if f(id, &path, &mut entry) { - retained.insert(Key::new(id, path), entry)?; - } - } - - *self = retained; - Ok(()) - } - - /// Returns an iterator over references to the entries in the map. - /// - /// The iterator yields tuples of `(id, path, &FileEntry)`. - /// - /// # Examples - /// - /// ```ignore - /// let map = FileEntryMap::new(); - /// // ... insert some entries ... - /// - /// for (id, path, entry) in map.iter() { - /// println!("ID: {}, Path: {}, Entry: {:?}", id, path, entry); - /// } - /// ``` - pub fn iter(&self) -> Iter<'_> { - Iter { - file_entry_map: self, - pos: 0, - } - } - - /// Returns an iterator over mutable references to the entries in the map. - /// - /// The iterator yields tuples of `(id, path, &mut FileEntry)`. - /// - /// # Examples - /// - /// ```ignore - /// let mut map = FileEntryMap::new(); - /// // ... insert some entries ... - /// - /// for (id, path, entry) in map.iter_mut() { - /// // Modify the entry - /// entry.some_field = new_value; - /// println!("ID: {}, Path: {}, Modified Entry: {:?}", id, path, entry); - /// } - /// ``` - pub fn iter_mut(&mut self) -> IterMut<'_> { - IterMut { - id_map: &self.id_map, - path_map: &self.path_map, - values: self.values.iter_mut(), - pos: 0, - } - } - - /// Returns the number of entries in the map. - pub fn len(&self) -> usize { - self.values.len() - } - - /// Returns `true` if the map is empty. - pub fn is_empty(&self) -> bool { - self.values.is_empty() - } - - pub fn insert(&mut self, key: Key, value: FileEntry) -> Result<(), PithosError> { - // Check if id is already occupied - match self.id_map.entry(key.id) { - Entry::Occupied(_) => Err(PithosError::DuplicateFileId(format!( - "File id already occupied: {}", - key.id - ))), - Entry::Vacant(vacant_id_entry) => { - // Also check if path is occupied - let path = key.path; - let ordered_path = ArchivePathKey(path.clone()); - let vacant_path_entry = match self.path_map.entry(path) { - Entry::Occupied(entry) => { - return Err(PithosError::PathOccupied(format!( - "File path already occupied: {}", - entry.key() - ))); - } - Entry::Vacant(entry) => entry, - }; - - // Insert into maps and vec - self.values.push(value.clone()); - let idx = self.values.len() - 1; - vacant_id_entry.insert(idx); - vacant_path_entry.insert(idx); - self.ordered_path_map.insert(ordered_path, idx); - - // Set current max - if key.id > self.current_max_id { - self.current_max_id = key.id; - } - - Ok(()) - } - } - } - - pub fn get(&self, kq: &KeyQuery) -> Option<&FileEntry> { - let idx = match kq { - KeyQuery::Id(file_id) => self.id_map.get(file_id), - KeyQuery::Path(file_path) => self.path_map.get(file_path.as_str()), - }; - - match idx { - Some(idx) => self.values.get(*idx), - None => None, - } - } - - pub fn get_entry(&self, kq: &KeyQuery) -> Option<(Key, &FileEntry)> { - let entry = self.get(kq)?; - - match kq { - KeyQuery::Id(id) => { - if let Some(path) = self.get_path_by_id(id) { - let key = Key::new(*id, path); - Some((key, entry)) - } else { - None - } - } - KeyQuery::Path(path) => { - if let Some(id) = self.get_id_by_path(path) { - Some((Key::new(id, path.clone()), entry)) - } else { - None - } - } - } - } - - pub fn get_current_max_id(&self) -> u64 { - self.current_max_id - } - - pub fn next_free_id(&self, has_parent: bool) -> Result { - if self.current_max_id == 0 && self.values.is_empty() && has_parent { - Ok(1) - } else if self.current_max_id == 0 && self.values.is_empty() { - Ok(0) - } else { - self.current_max_id - .checked_add(1) - .ok_or(PithosError::FileIdExhausted) - } - } -} - -impl Default for FileEntryMap { - fn default() -> Self { - Self::new() - } -} - -/// Iterator that yields references to (id, path, FileEntry) tuples -pub struct Iter<'a> { - file_entry_map: &'a FileEntryMap, - pos: usize, -} - -impl<'a> Iterator for Iter<'a> { - type Item = (u64, &'a str, &'a FileEntry); - - fn next(&mut self) -> Option { - if self.pos >= self.file_entry_map.values.len() { - return None; - } - - // Get the id and path from their respective maps at this position - let (id, _) = self.file_entry_map.id_map.get_index(self.pos)?; - let (path, _) = self.file_entry_map.path_map.get_index(self.pos)?; - let value = self.file_entry_map.values.get(self.pos)?; - - self.pos += 1; - - Some((*id, path.as_ref(), value)) - } - - fn size_hint(&self) -> (usize, Option) { - let remaining = self.file_entry_map.values.len().saturating_sub(self.pos); - (remaining, Some(remaining)) - } -} - -impl<'a> ExactSizeIterator for Iter<'a> {} - -/// Iterator that yields mutable references to (id, path, FileEntry) tuples -pub struct IterMut<'a> { - id_map: &'a IndexMap, - path_map: &'a IndexMap, usize>, - values: std::slice::IterMut<'a, FileEntry>, - pos: usize, -} - -impl<'a> Iterator for IterMut<'a> { - type Item = (u64, &'a str, &'a mut FileEntry); - - fn next(&mut self) -> Option { - let value = self.values.next()?; - - // Get the id and path from their respective maps at this position - let (id, _) = self.id_map.get_index(self.pos)?; - let (path, _) = self.path_map.get_index(self.pos)?; - - self.pos += 1; - - Some((*id, path.as_ref(), value)) - } - - fn size_hint(&self) -> (usize, Option) { - self.values.size_hint() - } -} - -impl<'a> ExactSizeIterator for IterMut<'a> {} - -/// Iterator that yields owned (id, path, FileEntry) tuples -pub struct IntoIter { - id_map: IndexMap, - path_map: IndexMap, usize>, - values: Vec, - pos: usize, -} - -impl Iterator for IntoIter { - type Item = (u64, String, FileEntry); - - fn next(&mut self) -> Option { - if self.pos >= self.values.len() { - return None; - } - - let (id, _) = self.id_map.get_index(self.pos)?; - let (path, _) = self.path_map.get_index(self.pos)?; - let value = self.values.get(self.pos)?.clone(); - - self.pos += 1; - - Some((*id, path.to_string(), value)) - } - - fn size_hint(&self) -> (usize, Option) { - let remaining = self.values.len().saturating_sub(self.pos); - (remaining, Some(remaining)) - } -} - -impl ExactSizeIterator for IntoIter {} - -// Implement IntoIterator for owned FileEntryMap -impl IntoIterator for FileEntryMap { - type Item = (u64, String, FileEntry); - type IntoIter = IntoIter; - - fn into_iter(self) -> Self::IntoIter { - IntoIter { - id_map: self.id_map, - path_map: self.path_map, - values: self.values, - pos: 0, - } - } -} - -// Implement IntoIterator for &FileEntryMap -impl<'a> IntoIterator for &'a FileEntryMap { - type Item = (u64, &'a str, &'a FileEntry); - type IntoIter = Iter<'a>; - - fn into_iter(self) -> Self::IntoIter { - self.iter() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::model::structs::{BlockDataState, Directory, FileType}; - use indexmap::IndexMap; - - fn entry() -> FileEntry { - FileEntry { - file_type: FileType::Data, - block_data: BlockDataState::Decrypted(vec![]), - created: 0, - modified: 0, - file_size: 0, - permissions: 0o644, - references: vec![], - symlink_target: None, - } - } - - fn assert_indexes_consistent(map: &FileEntryMap) { - assert_eq!(map.id_map.len(), map.values.len()); - assert_eq!(map.path_map.len(), map.values.len()); - assert_eq!(map.ordered_path_map.len(), map.values.len()); - for (index, ((id, id_index), (path, path_index))) in - map.id_map.iter().zip(map.path_map.iter()).enumerate() - { - assert_eq!(*id, map.get_id_by_path(path).unwrap()); - assert_eq!(index, *id_index); - assert_eq!(index, *path_index); - } - for ordered_index in map.ordered_path_map.values() { - assert!(map.values.get(*ordered_index).is_some()); - } - } - - #[test] - fn file_entry_map_indexes_survive_clone_extend_and_retain() { - let mut map = FileEntryMap::new(); - for (id, path) in [(4, "a!"), (2, "a/child"), (7, "a")] { - map.insert(Key::new(id, path), entry()).unwrap(); - } - assert_indexes_consistent(&map); - - let clone = map.clone(); - assert_indexes_consistent(&clone); - - let mut extended = FileEntryMap::new(); - extended.extend(clone).unwrap(); - assert_indexes_consistent(&extended); - extended.retain_mut(|id, _, _| id != 2).unwrap(); - assert_indexes_consistent(&extended); - assert_eq!( - extended - .iter() - .map(|(id, path, _)| (id, path)) - .collect::>(), - [(4, "a!"), (7, "a")] - ); - } - - #[test] - fn file_entry_map_duplicate_errors_do_not_mutate_indexes() { - let mut map = FileEntryMap::new(); - map.insert(Key::new(1, "first"), entry()).unwrap(); - let expected = map.clone(); - - assert!(matches!( - map.insert(Key::new(1, "second"), entry()), - Err(PithosError::DuplicateFileId(_)) - )); - assert_eq!(map, expected); - assert!(matches!( - map.insert(Key::new(2, "first"), entry()), - Err(PithosError::PathOccupied(_)) - )); - assert_eq!(map, expected); - assert_indexes_consistent(&map); - } - - #[test] - fn file_entry_map_preserves_insertion_order_and_shares_path_storage() { - let mut map = FileEntryMap::new(); - for (id, path) in [(3, "third"), (1, "first"), (2, "second")] { - map.insert(Key::new(id, path), entry()).unwrap(); - } - - assert_eq!( - map.iter() - .map(|(id, path, _)| (id, path)) - .collect::>(), - [(3, "third"), (1, "first"), (2, "second")] - ); - let directory = Directory { - identifier: *b"PITHOSDR", - parent_directory_offset: None, - files: map.clone(), - blocks: IndexMap::new(), - relations: vec![], - encryption: IndexMap::new(), - dir_len: 0, - crc32: 0, - }; - let mut serialized = Vec::new(); - directory.serialize(&mut serialized).unwrap(); - let path_position = |path: &[u8]| { - serialized - .windows(path.len()) - .position(|window| window == path) - .unwrap() - }; - assert!( - path_position(b"\x05third") < path_position(b"\x05first") - && path_position(b"\x05first") < path_position(b"\x06second") - ); - - assert_eq!( - map.clone() - .into_iter() - .map(|(id, path, _)| (id, path)) - .collect::>(), - [ - (3, "third".to_string()), - (1, "first".to_string()), - (2, "second".to_string()) - ] - ); - for path in map.path_map.keys() { - let ordered = map - .ordered_path_map - .keys() - .find(|ordered| ordered.0 == *path) - .unwrap(); - assert!(Arc::ptr_eq(path, &ordered.0)); - assert!(map.get_by_path(path).is_some()); - assert!(map.get_id_by_path(path).is_some()); - } - } -} diff --git a/crates/pithos_lib/src/model/deserialization.rs b/crates/pithos_lib/src/model/deserialization.rs deleted file mode 100644 index a748167..0000000 --- a/crates/pithos_lib/src/model/deserialization.rs +++ /dev/null @@ -1,599 +0,0 @@ -// Deserialization implementations for PITHOS structs and enums. -// Extracted from helpers/structs.rs for modularity. -// -// - Varint decoding uses integer_encoding::VarIntReader -// - Multi-byte values use big-endian decoding -// - Strings: UTF-8 with varint length prefix -// - Error handling via DeserializationError - -use crate::error::PithosError; -use crate::helpers::archive_path::{validate_entry, validate_hierarchy}; -use crate::helpers::file_entry_map::{FileEntryMap, Key}; -use crate::model::structs::*; -use byteorder::{BigEndian, ReadBytesExt}; -use indexmap::IndexMap; -use integer_encoding::VarIntReader; -use std::io::{Error as IoError, Read}; -use std::string::FromUtf8Error; -use thiserror::Error; - -/// Custom error type for deserialization operations -#[derive(Error, Debug)] -pub enum DeserializationError { - /// I/O error during deserialization - #[error("I/O error: {0}")] - Io(#[from] IoError), - /// UTF-8 decoding error - #[error("UTF-8 decoding error: {0}")] - Utf8(#[from] FromUtf8Error), - /// Invalid enum value encountered - #[error("Invalid enum value: {0}")] - InvalidEnumValue(u8), - #[error("Invalid marker: {0}")] - InvalidMarker(String), - /// Invalid option encountered - #[error("Invalid option")] - InvalidOption, - /// Invalid length encountered - #[error("Invalid length")] - InvalidLength, - #[error("{field} exceeds limit {limit}: {actual}")] - LimitExceeded { - field: &'static str, - limit: u64, - actual: u64, - }, - #[error("allocation failed for {field}: {size}")] - AllocationFailed { field: &'static str, size: u64 }, -} - -#[derive(Clone, Copy, Debug)] -pub struct DeserializationLimits { - pub max_string_bytes: u64, - pub max_opaque_bytes: u64, - pub max_collection_entries: u64, -} - -impl Default for DeserializationLimits { - fn default() -> Self { - Self { - max_string_bytes: 1024 * 1024, - max_opaque_bytes: 64 * 1024 * 1024, - max_collection_entries: 1_000_000, - } - } -} - -fn bounded_len(value: u64, limit: u64, field: &'static str) -> Result { - if value > limit { - return Err(DeserializationError::LimitExceeded { - field, - limit, - actual: value, - }); - } - usize::try_from(value).map_err(|_| DeserializationError::InvalidLength) -} - -fn reserve( - vec: &mut Vec, - count: usize, - field: &'static str, -) -> Result<(), DeserializationError> { - vec.try_reserve(count) - .map_err(|_| DeserializationError::AllocationFailed { - field, - size: count as u64, - }) -} - -// Helper: decode string (UTF-8 with varint length prefix) -#[tracing::instrument(level = "trace", skip(reader))] -pub fn decode_string(reader: &mut R) -> Result { - decode_string_with_limits(reader, &DeserializationLimits::default()) -} - -pub fn decode_string_with_limits( - reader: &mut R, - limits: &DeserializationLimits, -) -> Result { - let len = bounded_len( - reader.read_varint::()?, - limits.max_string_bytes, - "string", - )?; - let mut buf = Vec::new(); - reserve(&mut buf, len, "string")?; - buf.resize(len, 0); - reader.read_exact(&mut buf)?; - Ok(String::from_utf8(buf)?) -} - -// FileHeader -impl FileHeader { - const FILE_HEADER_MARKER: [u8; 4] = *b"PITH"; - - #[tracing::instrument(level = "trace", skip(reader))] - pub fn deserialize(reader: &mut R) -> Result { - let mut marker = [0u8; 4]; - reader.read_exact(&mut marker)?; - - // Validate correct file header marker - if marker != Self::FILE_HEADER_MARKER { - return Err(DeserializationError::InvalidMarker(format!( - "Read invalid block marker {marker:?}" - ))); - } - - let version: u16 = reader.read_varint()?; - Ok(FileHeader { - magic: marker, - version, - }) - } -} - -// BlockHeader -impl BlockHeader { - const BLOCK_HEADER_MARKER: [u8; 4] = *b"BLCK"; - - #[tracing::instrument(level = "trace", skip(reader))] - pub fn deserialize(reader: &mut R) -> Result { - let mut marker = [0u8; 4]; - reader.read_exact(&mut marker)?; - - if marker != Self::BLOCK_HEADER_MARKER { - return Err(DeserializationError::InvalidMarker(format!( - "Read invalid block marker {marker:?}" - ))); - } - - Ok(BlockHeader { marker }) - } -} - -// ProcessingFlags -impl ProcessingFlags { - #[tracing::instrument(level = "trace", skip(reader))] - pub fn deserialize(reader: &mut R) -> Result { - let mut buf = [0u8; 1]; - reader.read_exact(&mut buf)?; - Ok(ProcessingFlags(buf[0])) - } -} - -// BlockLocation -impl BlockLocation { - #[tracing::instrument(level = "trace", skip(reader))] - pub fn deserialize(reader: &mut R) -> Result { - Self::deserialize_with_limits(reader, &DeserializationLimits::default()) - } - - pub fn deserialize_with_limits( - reader: &mut R, - limits: &DeserializationLimits, - ) -> Result { - let mut tag = [0u8; 1]; - reader.read_exact(&mut tag)?; - match tag[0] { - 0 => Ok(BlockLocation::Local), - 1 => { - let url = decode_string_with_limits(reader, limits)?; - Ok(BlockLocation::External { url }) - } - v => Err(DeserializationError::InvalidEnumValue(v)), - } - } -} - -// BlockIndexEntry -impl BlockIndexEntry { - #[tracing::instrument(level = "trace", skip(reader))] - pub fn deserialize(reader: &mut R) -> Result { - Self::deserialize_with_limits(reader, &DeserializationLimits::default()) - } - - pub fn deserialize_with_limits( - reader: &mut R, - limits: &DeserializationLimits, - ) -> Result { - let offset: u64 = reader.read_varint()?; - let stored_size: u64 = reader.read_varint()?; - let original_size: u64 = reader.read_varint()?; - let flags = ProcessingFlags::deserialize(reader)?; - let location = BlockLocation::deserialize_with_limits(reader, limits)?; - Ok(BlockIndexEntry { - offset, - stored_size, - original_size, - flags, - location, - }) - } -} - -// Directory -impl Directory { - pub(crate) const DIRECTORY_MARKER: [u8; 8] = *b"PITHOSDR"; - - #[tracing::instrument(level = "trace", skip(reader))] - pub fn deserialize(reader: &mut R) -> Result { - Self::deserialize_with_limits(reader, &DeserializationLimits::default()) - } - - pub fn deserialize_with_limits( - reader: &mut R, - limits: &DeserializationLimits, - ) -> Result { - // Read static directory identifier - let mut identifier = [0u8; 8]; - reader.read_exact(&mut identifier)?; - if identifier != Self::DIRECTORY_MARKER { - return Err(PithosError::InvalidDirectoryMarker { - expected: Self::DIRECTORY_MARKER, - actual: identifier, - }); - } - - // Read parent directory offset - let mut tag = [0u8; 1]; - reader.read_exact(&mut tag)?; - let parent_directory_offset = match tag[0] { - 0 => None, - 1 => { - let start = reader.read_varint::()?; - let len = reader.read_varint::()?; - Some((start, len)) - } - _ => return Err(PithosError::from(DeserializationError::InvalidOption)), - }; - - // Read file entries - let mut files = FileEntryMap::new(); - let files_len = bounded_len( - reader.read_varint::()?, - limits.max_collection_entries, - "files", - )?; - for _ in 0..files_len { - let id = reader.read_varint::()?; - let path = decode_string_with_limits(reader, limits)?; - let entry = FileEntry::deserialize_with_limits(reader, limits)?; - validate_entry(&path, &entry)?; - files.insert(Key::new(id, path), entry)?; - } - validate_hierarchy(&files)?; - - let blocks_len = bounded_len( - reader.read_varint::()?, - limits.max_collection_entries, - "blocks", - )?; - let mut blocks = IndexMap::new(); - for _ in 0..blocks_len { - let mut hash = [0u8; 32]; - reader.read_exact(&mut hash)?; - let block = BlockIndexEntry::deserialize_with_limits(reader, limits)?; - blocks.insert(hash, block); - } - - let relations_len = bounded_len( - reader.read_varint::()?, - limits.max_collection_entries, - "relations", - )?; - let mut relations = Vec::new(); - reserve(&mut relations, relations_len, "relations")?; - for _ in 0..relations_len { - let idx = reader.read_varint::()?; - let name = decode_string_with_limits(reader, limits)?; - relations.push((idx, name)); - } - - let encryption_len = bounded_len( - reader.read_varint::()?, - limits.max_collection_entries, - "encryption", - )?; - let mut encryption = IndexMap::new(); - for _ in 0..encryption_len { - let mut key = [0u8; 32]; - reader.read_exact(&mut key)?; - encryption.insert( - key, - EncryptionSection::deserialize_with_limits(reader, limits)?, - ); - } - - let dir_len = reader.read_u64::()?; - let crc32 = reader.read_u32::()?; - - Ok(Directory { - identifier, - parent_directory_offset, - files, - blocks, - relations, - encryption, - dir_len, - crc32, - }) - } -} - -// FileType -impl FileType { - #[tracing::instrument(level = "trace", skip(reader))] - pub fn deserialize(reader: &mut R) -> Result { - let mut buf = [0u8; 1]; - reader.read_exact(&mut buf)?; - match buf[0] { - 0 => Ok(FileType::Directory), - 1 => Ok(FileType::Data), - 2 => Ok(FileType::Metadata), - 3 => Ok(FileType::Symlink), - v => Err(DeserializationError::InvalidEnumValue(v)), - } - } -} - -// BlockDataState -impl BlockDataState { - #[tracing::instrument(level = "trace", skip(reader))] - pub fn deserialize(reader: &mut R) -> Result { - Self::deserialize_with_limits(reader, &DeserializationLimits::default()) - } - - pub fn deserialize_with_limits( - reader: &mut R, - limits: &DeserializationLimits, - ) -> Result { - let mut tag = [0u8; 1]; - reader.read_exact(&mut tag)?; - match tag[0] { - 0 => { - let len = bounded_len( - reader.read_varint::()?, - limits.max_opaque_bytes, - "encrypted block", - )?; - let mut buf = Vec::new(); - reserve(&mut buf, len, "encrypted block")?; - buf.resize(len, 0); - reader.read_exact(&mut buf)?; - Ok(BlockDataState::Encrypted(buf)) - } - 1 => { - let list_len = bounded_len( - reader.read_varint::()?, - limits.max_collection_entries, - "block keys", - )?; - let mut list = Vec::new(); - reserve(&mut list, list_len, "block keys")?; - for _ in 0..list_len { - let mut hash = [0u8; 32]; - reader.read_exact(&mut hash)?; - let mut key = [0u8; 32]; - reader.read_exact(&mut key)?; - list.push((hash, key)); - } - Ok(BlockDataState::Decrypted(list)) - } - v => Err(DeserializationError::InvalidEnumValue(v)), - } - } - - #[tracing::instrument(level = "trace", skip(self, reader))] - pub fn deserialize_block_index( - &self, - reader: &mut R, - ) -> Result, DeserializationError> { - self.deserialize_block_index_with_limits(reader, &DeserializationLimits::default()) - } - - pub fn deserialize_block_index_with_limits( - &self, - reader: &mut R, - limits: &DeserializationLimits, - ) -> Result, DeserializationError> { - let list_len = bounded_len( - reader.read_varint::()?, - limits.max_collection_entries, - "block index", - )?; - let mut list = Vec::new(); - reserve(&mut list, list_len, "block index")?; - for _ in 0..list_len { - let mut hash = [0u8; 32]; - reader.read_exact(&mut hash)?; - let mut key = [0u8; 32]; - reader.read_exact(&mut key)?; - list.push((hash, key)); - } - Ok(list) - } -} - -// FileEntry -impl FileEntry { - #[tracing::instrument(level = "trace", skip(reader))] - pub fn deserialize(reader: &mut R) -> Result { - Self::deserialize_with_limits(reader, &DeserializationLimits::default()) - } - - pub fn deserialize_with_limits( - reader: &mut R, - limits: &DeserializationLimits, - ) -> Result { - let file_type = FileType::deserialize(reader)?; - let block_data = BlockDataState::deserialize_with_limits(reader, limits)?; - let created: u64 = reader.read_varint()?; - let modified: u64 = reader.read_varint()?; - let file_size: u64 = reader.read_varint()?; - let permissions: u32 = reader.read_varint()?; - let refs_len = bounded_len( - reader.read_varint::()?, - limits.max_collection_entries, - "references", - )?; - let mut references = Vec::new(); - reserve(&mut references, refs_len, "references")?; - for _ in 0..refs_len { - references.push(Reference::deserialize(reader)?); - } - let mut tag = [0u8; 1]; - reader.read_exact(&mut tag)?; - let symlink_target = match tag[0] { - 0 => None, - 1 => Some(decode_string_with_limits(reader, limits)?), - _ => return Err(DeserializationError::InvalidOption), - }; - Ok(FileEntry { - file_type, - block_data, - created, - modified, - file_size, - permissions, - references, - symlink_target, - }) - } -} - -// Reference -impl Reference { - #[tracing::instrument(level = "trace", skip(reader))] - pub fn deserialize(reader: &mut R) -> Result { - let target_file_id: u64 = reader.read_varint()?; - let relationship: u64 = reader.read_varint()?; - Ok(Reference { - target_file_id, - relationship, - }) - } -} - -// EncryptionSection -impl EncryptionSection { - #[tracing::instrument(level = "trace", skip(reader))] - pub fn deserialize(reader: &mut R) -> Result { - Self::deserialize_with_limits(reader, &DeserializationLimits::default()) - } - - pub fn deserialize_with_limits( - reader: &mut R, - limits: &DeserializationLimits, - ) -> Result { - let recipients_len = bounded_len( - reader.read_varint::()?, - limits.max_collection_entries, - "recipients", - )?; - let mut recipients = IndexMap::new(); - for _ in 0..recipients_len { - let mut key = [0u8; 32]; - reader.read_exact(&mut key)?; - recipients.insert( - key, - RecipientSection::deserialize_with_limits(reader, limits)?, - ); - } - Ok(EncryptionSection { recipients }) - } -} - -// RecipientData -impl RecipientData { - #[tracing::instrument(level = "trace", skip(reader))] - pub fn deserialize(reader: &mut R) -> Result { - Self::deserialize_with_limits(reader, &DeserializationLimits::default()) - } - - pub fn deserialize_with_limits( - reader: &mut R, - limits: &DeserializationLimits, - ) -> Result { - let mut tag = [0u8; 1]; - reader.read_exact(&mut tag)?; - - match tag[0] { - 0 => { - let len = bounded_len( - reader.read_varint::()?, - limits.max_opaque_bytes, - "encrypted recipient data", - )?; - let mut buf = Vec::new(); - reserve(&mut buf, len, "encrypted recipient data")?; - buf.resize(len, 0); - reader.read_exact(&mut buf)?; - Ok(RecipientData::Encrypted(buf)) - } - 1 => { - let list_len = bounded_len( - reader.read_varint::()?, - limits.max_collection_entries, - "recipient keys", - )?; - let mut list = Vec::new(); - reserve(&mut list, list_len, "recipient keys")?; - for _ in 0..list_len { - let idx: u64 = reader.read_varint()?; - let mut key = [0u8; 32]; - reader.read_exact(&mut key)?; - list.push((idx, key)); - } - Ok(RecipientData::Decrypted(list)) - } - v => Err(DeserializationError::InvalidEnumValue(v)), - } - } - - #[tracing::instrument(level = "trace", skip(self, reader))] - pub fn deserialize_decrypted_list( - &self, - reader: &mut R, - ) -> Result, DeserializationError> { - self.deserialize_decrypted_list_with_limits(reader, &DeserializationLimits::default()) - } - - pub fn deserialize_decrypted_list_with_limits( - &self, - reader: &mut R, - limits: &DeserializationLimits, - ) -> Result, DeserializationError> { - let list_len = bounded_len( - reader.read_varint::()?, - limits.max_collection_entries, - "recipient keys", - )?; - let mut list = Vec::new(); - reserve(&mut list, list_len, "recipient keys")?; - for _ in 0..list_len { - let idx: u64 = reader.read_varint()?; - let mut key = [0u8; 32]; - reader.read_exact(&mut key)?; - list.push((idx, key)); - } - - Ok(list) - } -} - -// RecipientSection -impl RecipientSection { - #[tracing::instrument(level = "trace", skip(reader))] - pub fn deserialize(reader: &mut R) -> Result { - Self::deserialize_with_limits(reader, &DeserializationLimits::default()) - } - - pub fn deserialize_with_limits( - reader: &mut R, - limits: &DeserializationLimits, - ) -> Result { - let recipient_data = RecipientData::deserialize_with_limits(reader, limits)?; - Ok(RecipientSection { recipient_data }) - } -} diff --git a/crates/pithos_lib/src/model/mod.rs b/crates/pithos_lib/src/model/mod.rs deleted file mode 100644 index be4f8a4..0000000 --- a/crates/pithos_lib/src/model/mod.rs +++ /dev/null @@ -1,3 +0,0 @@ -pub mod deserialization; -pub mod serialization; -pub mod structs; diff --git a/crates/pithos_lib/src/model/serialization.rs b/crates/pithos_lib/src/model/serialization.rs deleted file mode 100644 index 17ba762..0000000 --- a/crates/pithos_lib/src/model/serialization.rs +++ /dev/null @@ -1,264 +0,0 @@ -// Manual serialization utilities for PITHOS spec structs. -// No external libraries are used. All encoding is performed manually. -// - Varint encoding: unsigned LEB128 -// - Strings: UTF-8 with varint length prefix -// - Multi-byte values: big-endian -// - CRC32: not implemented here (use a separate utility) -// - Only serialization to bytes is implemented (no deserialization) - -use crate::model::structs::*; -use byteorder::{BigEndian, WriteBytesExt}; -use integer_encoding::VarIntWriter; -use std::io::Write; -use thiserror::Error; - -// Helper: error type for serialization -#[derive(Error, Debug)] -pub enum SerializationError { - #[error("IoError error: {0}")] - IoError(#[from] std::io::Error), - #[error("Serialization error: {0}")] - Other(String), -} - -pub(crate) fn write_len_prefix( - writer: &mut W, - len: usize, -) -> Result<(), SerializationError> { - let len = u64::try_from(len) - .map_err(|_| SerializationError::Other("length does not fit in u64".to_string()))?; - writer.write_varint(len)?; - Ok(()) -} - -// Helper: encode string (UTF-8 with varint length prefix) -#[tracing::instrument(level = "trace", skip(writer, s))] -pub fn encode_string(writer: &mut W, s: &str) -> Result<(), SerializationError> { - write_len_prefix(writer, s.len())?; - writer.write_all(s.as_bytes())?; - Ok(()) -} - -impl FileHeader { - #[tracing::instrument(level = "trace", skip(self, writer))] - pub fn serialize(&self, writer: &mut W) -> Result<(), SerializationError> { - writer.write_all(&self.magic)?; - writer.write_varint(self.version)?; - Ok(()) - } - - #[tracing::instrument(level = "trace", skip(self))] - pub fn serialize_to_bytes(&self) -> Result, SerializationError> { - let mut buf = Vec::new(); - self.serialize(&mut buf)?; - Ok(buf) - } -} - -impl BlockHeader { - #[tracing::instrument(level = "trace", skip(self, writer))] - pub fn serialize(&self, writer: &mut W) -> Result<(), SerializationError> { - writer.write_all(&self.marker)?; - Ok(()) - } -} - -impl ProcessingFlags { - #[tracing::instrument(level = "trace", skip(self, writer))] - pub fn serialize(&self, writer: &mut W) -> Result<(), SerializationError> { - writer.write_all(&[self.0])?; - Ok(()) - } -} - -impl BlockLocation { - #[tracing::instrument(level = "trace", skip(self, writer))] - pub fn serialize(&self, writer: &mut W) -> Result<(), SerializationError> { - match self { - BlockLocation::Local => { - writer.write_all(&[0u8])?; - } - BlockLocation::External { url } => { - writer.write_all(&[1u8])?; - encode_string(writer, url)?; - } - } - Ok(()) - } -} - -impl BlockIndexEntry { - #[tracing::instrument(level = "trace", skip(self, writer))] - pub fn serialize(&self, writer: &mut W) -> Result<(), SerializationError> { - writer.write_varint(self.offset)?; - writer.write_varint(self.stored_size)?; - writer.write_varint(self.original_size)?; - self.flags.serialize(writer)?; - self.location.serialize(writer)?; - Ok(()) - } -} - -impl Directory { - #[tracing::instrument(level = "trace", skip(self, writer))] - pub fn serialize(&self, writer: &mut W) -> Result<(), SerializationError> { - // Write static identifier - writer.write_all(&self.identifier)?; - - // Write parent directory offset, if present - match &self.parent_directory_offset { - Some((start, len)) => { - writer.write_all(&[1u8])?; - writer.write_varint(*start)?; - writer.write_varint(*len)?; - } - None => writer.write_all(&[0u8])?, - } - - // Write file entries - write_len_prefix(writer, self.files.len())?; - for (id, path, file) in &self.files { - writer.write_varint::(id)?; - encode_string(writer, path)?; - file.serialize(writer)?; - } - write_len_prefix(writer, self.blocks.len())?; - for (hash, block) in &self.blocks { - writer.write_all(hash)?; - block.serialize(writer)?; - } - write_len_prefix(writer, self.relations.len())?; - for (idx, name) in &self.relations { - writer.write_varint(*idx)?; - encode_string(writer, name)?; - } - write_len_prefix(writer, self.encryption.len())?; - for (key, enc) in &self.encryption { - writer.write_all(key)?; - enc.serialize(writer)?; - } - - writer.write_u64::(self.dir_len)?; // Actually writes the 8 bytes - writer.write_u32::(self.crc32)?; // Actually writes the 4 bytes - Ok(()) - } -} - -impl FileType { - #[tracing::instrument(level = "trace", skip(self, writer))] - pub fn serialize(&self, writer: &mut W) -> Result<(), SerializationError> { - writer.write_all(&[*self as u8])?; - Ok(()) - } -} - -impl BlockDataState { - #[tracing::instrument(level = "trace", skip(self, writer))] - pub fn serialize(&self, writer: &mut W) -> Result<(), SerializationError> { - match self { - BlockDataState::Encrypted(data) => { - writer.write_all(&[0u8])?; - write_len_prefix(writer, data.len())?; - writer.write_all(data)?; - } - BlockDataState::Decrypted(list) => { - writer.write_all(&[1u8])?; - write_len_prefix(writer, list.len())?; - for (hash, key) in list { - writer.write_all(hash)?; - writer.write_all(key)?; - } - } - } - Ok(()) - } - - #[tracing::instrument(level = "trace", skip(self))] - pub fn serialize_to_bytes(&self) -> Result, SerializationError> { - let mut buf = Vec::new(); - self.serialize(&mut buf)?; - Ok(buf) - } -} - -impl FileEntry { - #[tracing::instrument(level = "trace", skip(self, writer))] - pub fn serialize(&self, writer: &mut W) -> Result<(), SerializationError> { - self.file_type.serialize(writer)?; - self.block_data.serialize(writer)?; - writer.write_varint(self.created)?; - writer.write_varint(self.modified)?; - writer.write_varint(self.file_size)?; - writer.write_varint(self.permissions)?; - write_len_prefix(writer, self.references.len())?; - for r in &self.references { - r.serialize(writer)?; - } - match &self.symlink_target { - Some(target) => { - writer.write_all(&[1u8])?; - encode_string(writer, target)?; - } - None => writer.write_all(&[0u8])?, - } - Ok(()) - } - - #[tracing::instrument(level = "trace", skip(self))] - pub fn serialize_to_bytes(&self) -> Result, SerializationError> { - let mut buf = Vec::new(); - self.serialize(&mut buf)?; - Ok(buf) - } -} - -impl Reference { - #[tracing::instrument(level = "trace", skip(self, writer))] - pub fn serialize(&self, writer: &mut W) -> Result<(), SerializationError> { - writer.write_varint(self.target_file_id)?; - writer.write_varint(self.relationship)?; - Ok(()) - } -} - -impl EncryptionSection { - #[tracing::instrument(level = "trace", skip(self, writer))] - pub fn serialize(&self, writer: &mut W) -> Result<(), SerializationError> { - write_len_prefix(writer, self.recipients.len())?; - for (key, recipient) in &self.recipients { - writer.write_all(key)?; - recipient.serialize(writer)?; - } - Ok(()) - } -} - -impl RecipientData { - #[tracing::instrument(level = "trace", skip(self, writer))] - pub fn serialize(&self, writer: &mut W) -> Result<(), SerializationError> { - match self { - RecipientData::Encrypted(data) => { - writer.write_all(&[0u8])?; - write_len_prefix(writer, data.len())?; - writer.write_all(data)?; - } - RecipientData::Decrypted(list) => { - writer.write_all(&[1u8])?; - write_len_prefix(writer, list.len())?; - for (idx, hash) in list { - writer.write_varint(*idx)?; - writer.write_all(hash)?; - } - } - } - Ok(()) - } -} - -impl RecipientSection { - #[tracing::instrument(level = "trace", skip(self, writer))] - pub fn serialize(&self, writer: &mut W) -> Result<(), SerializationError> { - self.recipient_data.serialize(writer)?; - Ok(()) - } -} diff --git a/crates/pithos_lib/src/model/structs.rs b/crates/pithos_lib/src/model/structs.rs deleted file mode 100644 index 1ce8f1e..0000000 --- a/crates/pithos_lib/src/model/structs.rs +++ /dev/null @@ -1,653 +0,0 @@ -use crate::helpers::chacha_poly1305::{decrypt_chunk, encrypt_chunk}; -use crate::io::pithoswriter::Content; -use crate::io::util::{current_timestamp, get_symlink_target}; -use std::fmt::{Display, Formatter}; - -use crate::error::PithosError; -use crate::helpers::file_entry_map::{FileEntryMap, Key}; -use crate::model::deserialization::DeserializationLimits; -use crate::model::serialization::write_len_prefix; -use indexmap::IndexMap; -use integer_encoding::VarIntWriter; -use std::fs::{Metadata, symlink_metadata}; -use std::io::Write; -use std::os::unix::fs::PermissionsExt; -use std::time::SystemTime; -use x25519_dalek::{PublicKey, SharedSecret}; - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct FileHeader { - pub magic: [u8; 4], // MUST be b"PITH" - pub version: u16, // Format version (e.g., 0x0100 for 1.0) -} - -impl Default for FileHeader { - fn default() -> Self { - FileHeader { - magic: *b"PITH", - version: Self::SUPPORTED_VERSION, - } - } -} - -impl FileHeader { - pub const SUPPORTED_VERSION: u16 = 0x0100; -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct BlockHeader { - pub marker: [u8; 4], // MUST be b"BLCK" -} - -impl Default for BlockHeader { - fn default() -> Self { - BlockHeader { marker: *b"BLCK" } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct ProcessingFlags(pub u8); - -impl ProcessingFlags { - // Compression level can be set from 0-7 with the first three bits - // 0 = Uncompressed - // 7 = Highest compression - const COMPRESSION_MASK: u8 = 0b0000_0111; - - // Encryption is indicated with the 4th bit - const ENCRYPTION_MASK: u8 = 0b0000_1000; - - pub fn new(encrypted: bool, compression_level: Option) -> Self { - // Init - let mut flags = ProcessingFlags(0b0); - - // Set compression level - match compression_level { - Some(level) => { - if level > Self::COMPRESSION_MASK { - flags.set_compression_level(Self::COMPRESSION_MASK) // Cap at maximum - } else { - flags.set_compression_level(level) - } - } - None => flags.set_compression_level(3), // Default - } - - // Set encryption - flags.set_encryption(encrypted); - flags - } - - pub fn from_byte(byte: u8) -> Self { - // Init with byte - let mut flags = ProcessingFlags(byte); - // Clear unused bits - flags.0 &= !(Self::COMPRESSION_MASK | Self::ENCRYPTION_MASK); - flags - } - - pub fn set_encryption(&mut self, encrypted: bool) { - if encrypted { - self.0 |= Self::ENCRYPTION_MASK; // Set bit for encryption - } else { - self.0 &= !Self::ENCRYPTION_MASK; // Clear bit for encryption - } - } - - // New function to check if encryption is enabled - pub fn is_encrypted(&self) -> bool { - (self.0 & Self::ENCRYPTION_MASK) != 0 - } - - pub fn set_compression_level(&mut self, mut compression_level: u8) { - // Sanitize - compression_level = if compression_level > Self::COMPRESSION_MASK { - Self::COMPRESSION_MASK - } else { - compression_level - }; - - // Only use the lowest 3 bits (0-7) - self.0 = (self.0 & !Self::COMPRESSION_MASK) | (compression_level & Self::COMPRESSION_MASK); - } - - pub fn get_compression_level(&self) -> u8 { - self.0 & Self::COMPRESSION_MASK - } -} - -impl Default for ProcessingFlags { - fn default() -> Self { - ProcessingFlags::new(true, None) - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum Credentials { - UserPass((String, String)), // (Username, Password) -> Basic Auth - Token(String), // Token -> Bearer Auth -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum BlockLocation { - Local, // Block data at specified offset in this file - External { url: String }, // URL to external storage -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct BlockIndexEntry { - pub offset: u64, // varint - pub stored_size: u64, // varint - pub original_size: u64, // varint - pub flags: ProcessingFlags, // Compression, encryption settings - pub location: BlockLocation, // Where block data resides -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct Directory { - pub identifier: [u8; 8], // MUST be b"PITHOSDR" - pub parent_directory_offset: Option<(u64, u64)>, // (start, len) varint - pub blocks: IndexMap<[u8; 32], BlockIndexEntry>, // Blocks in this segment - pub files: FileEntryMap, // Files in this segment - pub relations: Vec<(u64, String)>, // Relation idx, relationname/id - pub encryption: IndexMap<[u8; 32], EncryptionSection>, // (Sender's X25519 public key, section with recipients) - pub dir_len: u64, - pub crc32: u32, // CRC32 of all preceding fields -} - -#[repr(u8)] -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -pub enum FileType { - Directory = 0, - Data = 1, - Metadata = 2, - Symlink = 3, - // 4-255 reserved -} - -impl TryFrom<&Metadata> for FileType { - type Error = PithosError; - - fn try_from(value: &Metadata) -> Result { - Ok(if value.is_file() { - FileType::Data - } else if value.is_dir() { - FileType::Directory - } else if value.is_symlink() { - FileType::Symlink - } else { - return Err(PithosError::Conversion(format!( - "Invalid input file type: {:?}", - value.file_type() - ))); - }) - } -} - -/// A block's content hash and the key used to encrypt its content. -pub type BlockDataEntry = ([u8; 32], [u8; 32]); - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum BlockDataState { - Encrypted(Vec), // Chacha + nonce (Random key) - Decrypted(Vec), // BLAKE3 hash / Shake256 hash -} - -impl BlockDataState { - #[tracing::instrument(level = "trace", skip(self, key))] - pub fn encrypt(&mut self, key: [u8; 32]) -> Result<(), PithosError> { - match &self { - BlockDataState::Encrypted(_) => { - return Err(PithosError::InvalidBlockDataState( - "Block already encrypted.".to_string(), - )); - } - BlockDataState::Decrypted(entries) => { - let mut data_bytes = Vec::new(); - write_len_prefix(&mut data_bytes, entries.len())?; - for (hash, key) in entries { - data_bytes.write_all(hash)?; - data_bytes.write_all(key)?; - } - let encrypted_data = encrypt_chunk(data_bytes.as_slice(), b"", &key)?; - - *self = BlockDataState::Encrypted(encrypted_data.to_vec()) - } - }; - - Ok(()) - } - - #[tracing::instrument(level = "trace", skip(self, key))] - pub fn decrypt(&mut self, key: &[u8; 32]) -> Result<(), PithosError> { - self.decrypt_with_limits(key, &DeserializationLimits::default()) - } - - #[tracing::instrument(level = "trace", skip(self, key, limits))] - pub fn decrypt_with_limits( - &mut self, - key: &[u8; 32], - limits: &DeserializationLimits, - ) -> Result<(), PithosError> { - match &self { - BlockDataState::Encrypted(data) => { - let decrypted_bytes = decrypt_chunk(data, key)?; - let block_data_entries = self - .deserialize_block_index_with_limits(&mut decrypted_bytes.as_slice(), limits)?; - - *self = BlockDataState::Decrypted(block_data_entries); - } - BlockDataState::Decrypted(_) => { - // Nothing to do - } - } - - Ok(()) - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct FileEntry { - pub file_type: FileType, - pub block_data: BlockDataState, - pub created: u64, - pub modified: u64, - pub file_size: u64, - pub permissions: u32, - pub references: Vec, - pub symlink_target: Option, // Target path for symlinks -} - -impl Display for FileEntry { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.write_str(&format!("{:<12} {:?}\n", "Type:", self.file_type))?; - match &self.block_data { - BlockDataState::Encrypted(_) => f.write_str("Blocks: Encrypted\n")?, - BlockDataState::Decrypted(_) => f.write_str("Blocks: Decrypted\n")?, - } - f.write_str(&format!("{:<12} {}\n", "Created:", self.created))?; - f.write_str(&format!("{:<12} {}\n", "Modified:", self.modified))?; - f.write_str(&format!("{:<12} {}\n", "Size:", self.file_size))?; - f.write_str(&format!("{:<12} {:o}\n", "Permissions:", self.permissions))?; - f.write_str(&format!("{:<12} {:?}\n", "References:", self.references))?; - - if let Some(target) = &self.symlink_target { - f.write_str(&format!("{:<12} {target}\n", "Target:"))?; - } - Ok(()) - } -} - -impl FileEntry { - #[tracing::instrument(level = "trace", skip(file_type, disk_path, metadata))] - pub fn new( - file_type: FileType, - disk_path: &str, - metadata: &Metadata, - ) -> Result { - Ok(FileEntry { - //file_id: file_id.unwrap_or(0), - //path: pithos_path.to_string(), - file_type, - block_data: BlockDataState::Decrypted(vec![]), - created: metadata - .created() - .unwrap_or(SystemTime::UNIX_EPOCH) - .duration_since(SystemTime::UNIX_EPOCH)? - .as_secs(), - modified: metadata - .modified() - .unwrap_or(SystemTime::UNIX_EPOCH) - .duration_since(SystemTime::UNIX_EPOCH)? - .as_secs(), - file_size: metadata.len(), - permissions: metadata.permissions().mode(), - references: vec![], - symlink_target: if file_type == FileType::Symlink { - Some(std::fs::read_link(disk_path)?.to_string_lossy().to_string()) - } else { - None - }, - }) - } - - #[tracing::instrument( - level = "trace", - skip( - file_type, - disk_path, - created, - modified, - permissions, - references, - file_size - ) - )] - pub fn new_ext( - file_type: FileType, - disk_path: &str, - created: Option, - modified: Option, - permissions: Option, - references: Vec, - file_size: Option, - ) -> Result { - Ok(FileEntry { - file_type, - block_data: BlockDataState::Decrypted(vec![]), - created: created - .unwrap_or(SystemTime::UNIX_EPOCH) - .duration_since(SystemTime::UNIX_EPOCH)? - .as_secs(), - modified: modified - .unwrap_or(SystemTime::UNIX_EPOCH) - .duration_since(SystemTime::UNIX_EPOCH)? - .as_secs(), - file_size: file_size.unwrap_or(0), - permissions: permissions.unwrap_or(0o644), - references, - symlink_target: if file_type == FileType::Symlink { - Some(std::fs::read_link(disk_path)?.to_string_lossy().to_string()) - } else { - None - }, - }) - } - - pub(crate) fn new_from_archive( - file_type: FileType, - file_size: u64, - created: u64, - modified: u64, - permissions: u32, - symlink_target: Option, - ) -> Self { - Self { - file_type, - block_data: BlockDataState::Decrypted(Vec::new()), - created, - modified, - file_size, - permissions, - references: Vec::new(), - symlink_target, - } - } - - #[tracing::instrument(level = "trace", skip(file_type, content))] - pub fn new_from_content(file_type: FileType, content: &Content) -> Result { - Ok(match content { - Content::File(disk_path) => { - let file_metadata = symlink_metadata(disk_path)?; - FileEntry::new(file_type, disk_path, &file_metadata)? - } - Content::Raw(raw_content) => { - let current_timestamp = current_timestamp()?; - FileEntry { - //file_id: file_id.unwrap_or(0), - //path: pithos_path.to_string(), - file_type, - block_data: BlockDataState::Decrypted(vec![]), - created: current_timestamp, - modified: current_timestamp, - file_size: raw_content.len() as u64, - permissions: 0o644, - references: vec![], - symlink_target: None, - } - } - Content::Reference(_) => { - return Err(PithosError::UnsupportedReferenceContent); - } - }) - } - - #[tracing::instrument(level = "trace", skip(file, metadata))] - pub fn new_from_file(file: &std::fs::File, metadata: Metadata) -> Result { - // Evaluate file type and create file entry - let file_type = FileType::try_from(&metadata)?; - Ok(FileEntry { - file_type, - block_data: BlockDataState::Decrypted(Vec::new()), - created: metadata - .created() - .unwrap_or(SystemTime::UNIX_EPOCH) - .duration_since(SystemTime::UNIX_EPOCH)? - .as_secs(), - modified: metadata - .modified() - .unwrap_or(SystemTime::UNIX_EPOCH) - .duration_since(SystemTime::UNIX_EPOCH)? - .as_secs(), - file_size: metadata.len(), - permissions: metadata.permissions().mode(), - references: vec![], - symlink_target: if file_type == FileType::Symlink { - Some(get_symlink_target(file)?) - } else { - None - }, - }) - } - - #[tracing::instrument(level = "trace", skip(other_file_id, other_file_entry, meta_length))] - pub fn meta_from( - other_file_id: u64, - other_file_entry: &FileEntry, - meta_length: u64, - ) -> Result { - Ok(FileEntry { - //file_id: other_file.file_id - 1, - //path: format!("{}.meta", other_file.path), - file_type: FileType::Metadata, - block_data: BlockDataState::Decrypted(Vec::new()), - created: current_timestamp()?, // Just inherit from other file? - modified: current_timestamp()?, // Just inherit from other file? - file_size: meta_length, - permissions: other_file_entry.permissions, - references: vec![Reference { - target_file_id: other_file_id, - relationship: 0, - }], - symlink_target: None, - }) - } - - /* - #[tracing::instrument(level = "trace", skip(self, file_id))] - pub fn set_file_id(&mut self, file_id: u64) { - self.file_id = file_id; - } - */ - - #[tracing::instrument(level = "trace", skip(self, entry))] - pub fn add_block_data(&mut self, entry: BlockDataEntry) -> Result<(), PithosError> { - match self.block_data { - BlockDataState::Encrypted(_) => { - return Err(PithosError::InvalidBlockDataState( - "Block data already/still encrypted".to_string(), - )); - } - BlockDataState::Decrypted(ref mut entries) => { - entries.push(entry); - } - } - Ok(()) - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct Reference { - pub target_file_id: u64, // varint - pub relationship: u64, // varint -} - -impl TryFrom<(u64, &mut FileEntry)> for Reference { - type Error = PithosError; - - fn try_from(value: (u64, &mut FileEntry)) -> Result { - let (target_file_id, file_entry) = value; - Ok(Reference { - target_file_id, - relationship: match file_entry.file_type { - FileType::Directory => 6, // PART_OF - FileType::Data => 7, // DERIVED_FROM - FileType::Metadata => 0, // DESCRIBES - FileType::Symlink => 3, // SOURCE - }, - }) - } -} - -impl TryFrom<(&Key, &mut FileEntry)> for Reference { - type Error = PithosError; - - fn try_from(value: (&Key, &mut FileEntry)) -> Result { - let (file_entry_key, file_entry) = value; - Ok(Reference { - target_file_id: file_entry_key.id(), - relationship: match file_entry.file_type { - FileType::Directory => 6, // PART_OF - FileType::Data => 7, // DERIVED_FROM - FileType::Metadata => 0, // DESCRIBES - FileType::Symlink => 3, // SOURCE - }, - }) - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct EncryptionSection { - // Recipient's X25519 public key, Recipient file list - pub recipients: IndexMap<[u8; 32], RecipientSection>, -} - -impl EncryptionSection { - #[tracing::instrument(level = "trace", skip(recipient_pubkeys))] - pub fn new(recipient_pubkeys: &[PublicKey]) -> Self { - EncryptionSection { - recipients: IndexMap::from_iter( - recipient_pubkeys - .iter() - .map(|key| { - ( - key.to_bytes(), - RecipientSection { - recipient_data: RecipientData::Decrypted(Vec::new()), - }, - ) - }) - .collect::>(), - ), - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct RecipientSection { - pub recipient_data: RecipientData, // Encrypted FileKeyEntry list -} - -impl RecipientSection { - #[tracing::instrument(level = "trace", skip(self, entry))] - pub fn add_file_to_recipient(&mut self, entry: (u64, [u8; 32])) -> Result<(), PithosError> { - match self.recipient_data { - RecipientData::Encrypted(_) => Err(PithosError::InvalidRecipientDataState( - "Cannot add file entry to encrypted recipient data".to_string(), - )), - RecipientData::Decrypted(ref mut entries) => { - let _: () = entries.push(entry); - Ok(()) - } - } - } - - #[tracing::instrument(level = "trace", skip(self, shared_key))] - pub fn encrypt(&mut self, shared_key: SharedSecret) -> Result<(), PithosError> { - match &self.recipient_data { - RecipientData::Encrypted(_) => { - return Err(PithosError::InvalidRecipientDataState( - "Recipient data already encrypted".to_string(), - )); - } - RecipientData::Decrypted(entries) => { - let mut data_bytes = Vec::new(); - write_len_prefix(&mut data_bytes, entries.len())?; - for (idx, key) in entries { - data_bytes.write_varint(*idx)?; - data_bytes.write_all(key)?; - } - - let encrypted_data = - encrypt_chunk(data_bytes.as_slice(), b"", shared_key.as_bytes())?; - - self.recipient_data = RecipientData::Encrypted(encrypted_data.to_vec()) - } - }; - - Ok(()) - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum RecipientData { - Encrypted(Vec), // Chacha + nonce (Shared key PrivKey Writer <--> PubKey Reader) - Decrypted(Vec<(u64, [u8; 32])>), // Fileindex / Random key to decrypt BlockDataState -} - -impl RecipientData { - #[tracing::instrument(level = "trace", skip(self, shared_key))] - pub fn encrypt(&mut self, shared_key: &SharedSecret) -> Result<(), PithosError> { - match &self { - RecipientData::Encrypted(_) => { - return Err(PithosError::InvalidRecipientDataState( - "Recipient data already encrypted".to_string(), - )); - } - RecipientData::Decrypted(entries) => { - let mut data_bytes = Vec::new(); - write_len_prefix(&mut data_bytes, entries.len())?; - for (idx, key) in entries { - data_bytes.write_varint(*idx)?; - data_bytes.write_all(key)?; - } - - let encrypted_data = - encrypt_chunk(data_bytes.as_slice(), b"", shared_key.as_bytes())?; - - *self = RecipientData::Encrypted(encrypted_data.to_vec()) - } - }; - - Ok(()) - } - - #[tracing::instrument(level = "trace", skip(self, shared_key))] - pub fn decrypt( - &mut self, - shared_key: &SharedSecret, - ) -> Result, PithosError> { - self.decrypt_with_limits(shared_key, &DeserializationLimits::default()) - } - - #[tracing::instrument(level = "trace", skip(self, shared_key, limits))] - pub fn decrypt_with_limits( - &mut self, - shared_key: &SharedSecret, - limits: &DeserializationLimits, - ) -> Result, PithosError> { - let entries = match &self { - RecipientData::Decrypted(entries) => entries.clone(), - RecipientData::Encrypted(enc_data) => { - let dec_data = decrypt_chunk(enc_data, shared_key.as_bytes())?; - let entries = - self.deserialize_decrypted_list_with_limits(&mut dec_data.as_slice(), limits)?; - - *self = RecipientData::Decrypted(entries.clone()); - entries - } - }; - - Ok(entries) - } -} diff --git a/crates/pithos_lib/tests/marshalling.rs b/crates/pithos_lib/tests/marshalling.rs deleted file mode 100644 index bc9578c..0000000 --- a/crates/pithos_lib/tests/marshalling.rs +++ /dev/null @@ -1,736 +0,0 @@ -use indexmap::IndexMap; -use pithos_lib::error::PithosError; -use pithos_lib::helpers::directory::DirectoryBuilder; -use pithos_lib::helpers::file_entry_map::{FileEntryMap, Key}; -use pithos_lib::helpers::x25519_keys::generate_private_key; -use pithos_lib::model::deserialization::{DeserializationError, DeserializationLimits}; -use pithos_lib::model::serialization::encode_string; -use pithos_lib::model::structs::*; -use std::io::Cursor; -use x25519_dalek::PublicKey; - -#[test] -fn robust_overlong_zero_file_count_is_accepted() { - let mut bytes = hex_bytes("504954484f53445200000000000000000000000019b1674081"); - bytes.splice(9..10, [0x80, 0x00]); - let length = bytes.len() as u64; - let footer_start = bytes.len() - 12; - bytes[footer_start..footer_start + 8].copy_from_slice(&length.to_be_bytes()); - let crc_start = bytes.len() - 4; - let crc = crc32fast::hash(&bytes[..crc_start]); - bytes[crc_start..].copy_from_slice(&crc.to_be_bytes()); - - let directory = Directory::deserialize(&mut Cursor::new(bytes)).unwrap(); - assert!(directory.files.is_empty()); -} - -#[test] -fn robust_typed_narrow_overflow_is_rejected() { - let mut bytes = b"PITH".to_vec(); - use integer_encoding::VarIntWriter; - bytes.write_varint(u64::from(u16::MAX) + 1).unwrap(); - - assert!(FileHeader::deserialize(&mut Cursor::new(bytes)).is_err()); -} - -#[test] -fn robust_u64_tenth_byte_overflow_is_rejected() { - let bytes = [0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x02]; - - assert!(BlockIndexEntry::deserialize(&mut Cursor::new(bytes)).is_err()); -} - -#[test] -fn robust_unterminated_varint_is_rejected() { - let bytes = [b'P', b'I', b'T', b'H', 0x80]; - - assert!(FileHeader::deserialize(&mut Cursor::new(bytes)).is_err()); -} - -#[test] -fn robust_maximum_typed_values_roundtrip() { - let header = FileHeader { - magic: *b"PITH", - version: u16::MAX, - }; - let mut header_bytes = Vec::new(); - header.serialize(&mut header_bytes).unwrap(); - assert_eq!( - FileHeader::deserialize(&mut Cursor::new(header_bytes)).unwrap(), - header - ); - - let entry = FileEntry { - file_type: FileType::Data, - block_data: BlockDataState::Decrypted(vec![]), - created: u64::MAX, - modified: u64::MAX, - file_size: u64::MAX, - permissions: u32::MAX, - references: vec![], - symlink_target: None, - }; - let mut entry_bytes = Vec::new(); - entry.serialize(&mut entry_bytes).unwrap(); - assert_eq!( - FileEntry::deserialize(&mut Cursor::new(entry_bytes)).unwrap(), - entry - ); -} - -#[test] -fn robust_over_policy_file_count_is_rejected_before_entry_parsing() { - let mut bytes = hex_bytes("504954484f53445200"); - use integer_encoding::VarIntWriter; - bytes.write_varint(1_000_001u64).unwrap(); - - let error = Directory::deserialize(&mut Cursor::new(bytes)).unwrap_err(); - assert!(error.to_string().contains("limit")); -} - -#[test] -fn robust_custom_collection_limit_applies_to_blocks() { - let directory = Directory { - identifier: *b"PITHOSDR", - parent_directory_offset: None, - files: FileEntryMap::new(), - blocks: IndexMap::from_iter([( - [1u8; 32], - BlockIndexEntry { - offset: 0, - stored_size: 0, - original_size: 0, - flags: ProcessingFlags(0), - location: BlockLocation::Local, - }, - )]), - relations: vec![], - encryption: IndexMap::new(), - dir_len: 0, - crc32: 0, - }; - let mut bytes = Vec::new(); - directory.serialize(&mut bytes).unwrap(); - let limits = DeserializationLimits { - max_collection_entries: 0, - ..DeserializationLimits::default() - }; - - let error = Directory::deserialize_with_limits(&mut Cursor::new(bytes), &limits).unwrap_err(); - assert!(error.to_string().contains("blocks")); -} - -#[test] -fn test_varint_encoding() { - use integer_encoding::VarIntWriter; - let mut buf = Vec::new(); - buf.write_varint(0u64).unwrap(); - assert_eq!(buf, vec![0x00]); - buf.clear(); - buf.write_varint(127u64).unwrap(); - assert_eq!(buf, vec![0x7F]); - buf.clear(); - buf.write_varint(128u64).unwrap(); - assert_eq!(buf, vec![0x80, 0x01]); - buf.clear(); - buf.write_varint(300u64).unwrap(); - assert_eq!(buf, vec![0xAC, 0x02]); -} - -#[test] -fn test_string_encoding() { - let s = "hello"; - let mut buf = Vec::new(); - encode_string(&mut buf, s).unwrap(); - assert_eq!(buf, vec![5, b'h', b'e', b'l', b'l', b'o']); -} - -#[test] -fn file_header_roundtrip() { - let original = FileHeader::default(); - let mut buf = Vec::new(); - original.serialize(&mut buf).unwrap(); - let decoded = FileHeader::deserialize(&mut Cursor::new(buf)).unwrap(); - assert_eq!(original, decoded); -} - -#[test] -fn block_header_roundtrip() { - let original = BlockHeader { marker: *b"BLCK" }; - let mut buf = Vec::new(); - original.serialize(&mut buf).unwrap(); - let decoded = BlockHeader::deserialize(&mut Cursor::new(buf)).unwrap(); - assert_eq!(original, decoded); -} - -#[test] -fn processing_flags_roundtrip() { - let original = ProcessingFlags(0xAB); - let mut buf = Vec::new(); - original.serialize(&mut buf).unwrap(); - let decoded = ProcessingFlags::deserialize(&mut Cursor::new(buf)).unwrap(); - assert_eq!(original, decoded); -} - -#[test] -fn block_location_roundtrip() { - let local = BlockLocation::Local; - let external = BlockLocation::External { - url: "https://example.com".to_string(), - }; - - let mut buf = Vec::new(); - local.serialize(&mut buf).unwrap(); - let decoded = BlockLocation::deserialize(&mut Cursor::new(&buf)).unwrap(); - assert_eq!(local, decoded); - - buf.clear(); - external.serialize(&mut buf).unwrap(); - let decoded = BlockLocation::deserialize(&mut Cursor::new(&buf)).unwrap(); - assert_eq!(external, decoded); -} - -#[test] -fn block_index_entry_roundtrip() { - let original = BlockIndexEntry { - offset: 100, - stored_size: 200, - original_size: 300, - flags: ProcessingFlags(0x01), - location: BlockLocation::External { - url: "url".to_string(), - }, - }; - let mut buf = Vec::new(); - original.serialize(&mut buf).unwrap(); - let decoded = BlockIndexEntry::deserialize(&mut Cursor::new(buf)).unwrap(); - assert_eq!(original, decoded); -} - -#[test] -fn directory_roundtrip() { - let file_entry = FileEntry { - file_type: FileType::Data, - block_data: BlockDataState::Decrypted(vec![([1u8; 32], [2u8; 32])]), - created: 123, - modified: 456, - file_size: 789, - permissions: 0o644, - references: vec![Reference { - target_file_id: 2, - relationship: 3, - }], - symlink_target: None, - }; - let block_index = BlockIndexEntry { - offset: 4, - stored_size: 5, - original_size: 6, - flags: ProcessingFlags(7), - location: BlockLocation::Local, - }; - let enc_section = IndexMap::from_iter([( - [4u8; 32], - EncryptionSection { - recipients: IndexMap::from_iter([( - [5u8; 32], - RecipientSection { - recipient_data: RecipientData::Decrypted(vec![(6, [7u8; 32])]), - }, - )]), - }, - )]); - - let mut files = FileEntryMap::new(); - files.insert(Key::new(0, "file.txt"), file_entry).unwrap(); - - let original = Directory { - identifier: *b"PITHOSDR", - parent_directory_offset: None, - files, - blocks: IndexMap::from_iter([([1u8; 32], block_index)]), - relations: vec![(1, "rel".to_string())], - encryption: enc_section, - dir_len: 12345, - crc32: 67890, - }; - - let mut buf = Vec::new(); - original.serialize(&mut buf).unwrap(); - let decoded = Directory::deserialize(&mut Cursor::new(buf)).unwrap(); - assert_eq!(original, decoded); -} - -#[test] -fn directory_integrity_crc_matches_serialized_prefix() { - let file_entry = FileEntry { - file_type: FileType::Data, - block_data: BlockDataState::Decrypted(vec![]), - created: 0, - modified: 0, - file_size: 0, - permissions: 0o644, - references: vec![], - symlink_target: None, - }; - let block_index = BlockIndexEntry { - offset: 0, - stored_size: 0, - original_size: 0, - flags: ProcessingFlags(0), - location: BlockLocation::Local, - }; - let mut files = FileEntryMap::new(); - files.insert(Key::new(0, "file.txt"), file_entry).unwrap(); - let blocks = IndexMap::from_iter([([1u8; 32], block_index)]); - - let directory = DirectoryBuilder::new() - .files(files) - .blocks(blocks) - .build() - .unwrap(); - let mut serialized = Vec::new(); - directory.serialize(&mut serialized).unwrap(); - assert_eq!( - directory.crc32, - crc32fast::hash(&serialized[..serialized.len() - 4]) - ); -} - -fn assert_builder_directory_integrity(directory: &Directory) { - let mut serialized = Vec::new(); - directory.serialize(&mut serialized).unwrap(); - - assert_eq!(directory.dir_len, serialized.len() as u64); - assert_eq!( - u64::from_be_bytes( - serialized[serialized.len() - 12..serialized.len() - 4] - .try_into() - .unwrap() - ), - directory.dir_len - ); - assert_eq!( - directory.crc32, - crc32fast::hash(&serialized[..serialized.len() - 4]) - ); -} - -#[test] -fn directory_builder_sets_integrity_for_empty_directory() { - let directory = DirectoryBuilder::new() - .set_relations(vec![]) - .build() - .unwrap(); - - assert!(directory.files.is_empty()); - assert!(directory.relations.is_empty()); - assert_builder_directory_integrity(&directory); -} - -#[test] -fn directory_builder_sets_integrity_for_default_relations() { - let directory = DirectoryBuilder::new().build().unwrap(); - - assert_eq!(directory.relations.len(), 10); - assert_builder_directory_integrity(&directory); -} - -#[test] -fn directory_builder_sets_integrity_for_populated_directory() { - let mut files = FileEntryMap::new(); - files - .insert(Key::new(7, "file.txt"), plain_entry(None)) - .unwrap(); - let blocks = IndexMap::from_iter([( - [1u8; 32], - BlockIndexEntry { - offset: 4, - stored_size: 5, - original_size: 6, - flags: ProcessingFlags(0), - location: BlockLocation::Local, - }, - )]); - let directory = DirectoryBuilder::new() - .files(files) - .blocks(blocks) - .set_relations(vec![(42, "Related".into())]) - .build() - .unwrap(); - - assert_eq!(directory.files.len(), 1); - assert_eq!(directory.blocks.len(), 1); - assert_eq!(directory.relations, vec![(42, "Related".into())]); - assert_builder_directory_integrity(&directory); -} - -#[test] -fn directory_builder_sets_integrity_for_encrypted_directory() { - let sender_key = generate_private_key().unwrap(); - let recipient_key = generate_private_key().unwrap(); - let mut recipient_data = RecipientData::Decrypted(vec![(7, [3u8; 32])]); - recipient_data - .encrypt(&sender_key.diffie_hellman(&PublicKey::from(&recipient_key))) - .unwrap(); - let directory = DirectoryBuilder::new() - .encryption(IndexMap::from_iter([( - PublicKey::from(&sender_key).to_bytes(), - EncryptionSection { - recipients: IndexMap::from_iter([( - PublicKey::from(&recipient_key).to_bytes(), - RecipientSection { recipient_data }, - )]), - }, - )])) - .build() - .unwrap(); - - assert!(matches!( - directory - .encryption - .values() - .next() - .unwrap() - .recipients - .values() - .next() - .unwrap() - .recipient_data, - RecipientData::Encrypted(_) - )); - assert_builder_directory_integrity(&directory); -} - -#[test] -fn directory_integrity_minimal_vector_matches_specification() { - let directory = Directory { - identifier: *b"PITHOSDR", - parent_directory_offset: None, - files: FileEntryMap::new(), - blocks: IndexMap::new(), - relations: vec![], - encryption: IndexMap::new(), - dir_len: 25, - crc32: 0, - }; - let mut prefix = Vec::new(); - directory.serialize(&mut prefix).unwrap(); - let crc = crc32fast::hash(&prefix[..prefix.len() - 4]); - let mut complete = prefix.clone(); - let crc_start = complete.len() - 4; - complete[crc_start..].copy_from_slice(&crc.to_be_bytes()); - - assert_eq!(crc, 0xb1674081); - assert_eq!( - complete, - hex_bytes("504954484f53445200000000000000000000000019b1674081") - ); -} - -#[test] -fn directory_integrity_deserialization_rejects_invalid_marker() { - let mut bytes = hex_bytes("504954484f53445200000000000000000000000019b1674081"); - bytes[..8].copy_from_slice(b"INVALID!"); - let error = Directory::deserialize(&mut Cursor::new(bytes)).unwrap_err(); - - assert!(matches!(error, PithosError::InvalidDirectoryMarker { .. })); -} - -fn hex_bytes(value: &str) -> Vec { - value - .as_bytes() - .chunks_exact(2) - .map(|pair| u8::from_str_radix(std::str::from_utf8(pair).unwrap(), 16).unwrap()) - .collect() -} - -fn serialized_directory(path: &str, entry: FileEntry) -> Vec { - let mut files = FileEntryMap::new(); - files.insert(Key::new(0, path), entry).unwrap(); - let directory = DirectoryBuilder::new().files(files).build().unwrap(); - let mut bytes = Vec::new(); - directory.serialize(&mut bytes).unwrap(); - bytes -} - -fn plain_entry(target: Option<&str>) -> FileEntry { - FileEntry { - file_type: FileType::Data, - block_data: BlockDataState::Decrypted(vec![]), - created: 0, - modified: 0, - file_size: 0, - permissions: 0o644, - references: vec![], - symlink_target: target.map(str::to_owned), - } -} - -#[test] -fn directory_deserialization_rejects_unsafe_entry_paths() { - for path in ["../outside", "/absolute", "nested//file", "C:/file"] { - assert!( - Directory::deserialize(&mut Cursor::new(serialized_directory( - path, - plain_entry(None) - ))) - .is_err() - ); - } -} - -#[test] -fn directory_deserialization_rejects_missing_inconsistent_symlink_targets() { - let mut missing = plain_entry(None); - missing.file_type = FileType::Symlink; - assert!( - Directory::deserialize(&mut Cursor::new(serialized_directory("link", missing))).is_err() - ); - assert!( - Directory::deserialize(&mut Cursor::new(serialized_directory( - "file", - plain_entry(Some("target")) - ))) - .is_err() - ); - let mut blocks = plain_entry(Some("target")); - blocks.file_type = FileType::Symlink; - blocks.block_data = BlockDataState::Decrypted(vec![([0; 32], [0; 32])]); - assert!( - Directory::deserialize(&mut Cursor::new(serialized_directory("link", blocks))).is_err() - ); - let mut encrypted = plain_entry(Some("target")); - encrypted.file_type = FileType::Symlink; - encrypted.block_data = BlockDataState::Encrypted(vec![]); - assert!( - Directory::deserialize(&mut Cursor::new(serialized_directory("link", encrypted))).is_err() - ); -} - -#[test] -fn directory_deserialization_rejects_unsafe_symlink_targets() { - let mut link = plain_entry(Some("../outside")); - link.file_type = FileType::Symlink; - assert!(Directory::deserialize(&mut Cursor::new(serialized_directory("link", link))).is_err()); -} - -#[test] -fn file_type_roundtrip() { - for ft in [ - FileType::Data, - FileType::Metadata, - FileType::Directory, - FileType::Symlink, - ] { - let mut buf = Vec::new(); - ft.serialize(&mut buf).unwrap(); - let decoded = FileType::deserialize(&mut Cursor::new(buf)).unwrap(); - assert_eq!(ft, decoded); - } -} - -#[test] -fn block_data_state_roundtrip() { - let decrypted = - BlockDataState::Decrypted(vec![([42u8; 32], [1u8; 32]), ([43u8; 32], [2u8; 32])]); - let encrypted = BlockDataState::Encrypted(vec![1, 2, 3, 4, 5]); - for bds in [decrypted, encrypted] { - let mut buf = Vec::new(); - bds.serialize(&mut buf).unwrap(); - let decoded = BlockDataState::deserialize(&mut Cursor::new(buf)).unwrap(); - assert_eq!(bds, decoded); - } -} - -#[test] -fn file_entry_roundtrip() { - let original = FileEntry { - //file_id: 123, - //path: "foo/bar.txt".to_string(), - file_type: FileType::Symlink, - block_data: BlockDataState::Decrypted(vec![([1u8; 32], [2u8; 32])]), - created: 111, - modified: 222, - file_size: 333, - permissions: 444, - references: vec![Reference { - target_file_id: 555, - relationship: 666, - }], - symlink_target: None, - }; - let mut buf = Vec::new(); - original.serialize(&mut buf).unwrap(); - let decoded = FileEntry::deserialize(&mut Cursor::new(buf)).unwrap(); - assert_eq!(original, decoded); -} - -#[test] -fn reference_roundtrip() { - let original = Reference { - target_file_id: 1, - relationship: 2, - }; - let mut buf = Vec::new(); - original.serialize(&mut buf).unwrap(); - let decoded = Reference::deserialize(&mut Cursor::new(buf)).unwrap(); - assert_eq!(original, decoded); -} - -#[test] -fn encryption_section_roundtrip() { - let recipient = RecipientSection { - recipient_data: RecipientData::Decrypted(vec![(2, [3u8; 32])]), - }; - let original = EncryptionSection { - recipients: IndexMap::from_iter([([1u8; 32], recipient)]), - }; - let mut buf = Vec::new(); - original.serialize(&mut buf).unwrap(); - let decoded = EncryptionSection::deserialize(&mut Cursor::new(buf)).unwrap(); - assert_eq!(original, decoded); -} - -#[test] -fn recipient_data_roundtrip() { - let sender_key = generate_private_key().unwrap(); - let recipient_key = generate_private_key().unwrap(); - let shared_key = sender_key.diffie_hellman(&PublicKey::from(&recipient_key)); - let rand_key = generate_private_key().unwrap(); - - // Create RecipientData - let entry = (1u64, rand_key.to_bytes()); - let mut recipient_data = RecipientData::Decrypted(vec![entry]); - - // Encrypt RecipientData - recipient_data.encrypt(&shared_key).unwrap(); - assert!(matches!(recipient_data, RecipientData::Encrypted { .. })); - - //Decrypt RecipientData - recipient_data.decrypt(&shared_key).unwrap(); - match recipient_data { - RecipientData::Encrypted(_) => panic!("Invalid recipient data state"), - RecipientData::Decrypted(entries) => { - assert_eq!(entries, vec![entry]) - } - } -} - -#[test] -fn block_data_state_decrypt_with_limits_rejects_oversized_block_index() { - let key = [7u8; 32]; - let mut block_data = - BlockDataState::Decrypted(vec![([1u8; 32], [2u8; 32]), ([3u8; 32], [4u8; 32])]); - block_data.encrypt(key).unwrap(); - let limits = DeserializationLimits { - max_collection_entries: 1, - ..DeserializationLimits::default() - }; - - let error = block_data.decrypt_with_limits(&key, &limits).unwrap_err(); - assert!(matches!( - error, - PithosError::Deserialization(DeserializationError::LimitExceeded { - field: "block index", - limit: 1, - actual: 2, - }) - )); -} - -#[test] -fn recipient_data_decrypt_with_limits_rejects_oversized_recipient_keys() { - let sender_key = generate_private_key().unwrap(); - let recipient_key = generate_private_key().unwrap(); - let shared_key = sender_key.diffie_hellman(&PublicKey::from(&recipient_key)); - let mut recipient_data = RecipientData::Decrypted(vec![(1, [2u8; 32]), (3, [4u8; 32])]); - recipient_data.encrypt(&shared_key).unwrap(); - let limits = DeserializationLimits { - max_collection_entries: 1, - ..DeserializationLimits::default() - }; - - let error = recipient_data - .decrypt_with_limits(&shared_key, &limits) - .unwrap_err(); - assert!(matches!( - error, - PithosError::Deserialization(DeserializationError::LimitExceeded { - field: "recipient keys", - limit: 1, - actual: 2, - }) - )); -} - -#[test] -fn directory_decrypt_with_limits_recipient_path_propagates_recipient_limit() { - let sender_key = generate_private_key().unwrap(); - let recipient_key = generate_private_key().unwrap(); - let shared_key = sender_key.diffie_hellman(&PublicKey::from(&recipient_key)); - let mut recipient_data = RecipientData::Decrypted(vec![(1, [2u8; 32]), (3, [4u8; 32])]); - recipient_data.encrypt(&shared_key).unwrap(); - let mut directory = Directory { - identifier: *b"PITHOSDR", - parent_directory_offset: None, - files: FileEntryMap::new(), - blocks: IndexMap::new(), - relations: vec![], - encryption: IndexMap::from_iter([( - PublicKey::from(&sender_key).to_bytes(), - EncryptionSection { - recipients: IndexMap::from_iter([( - PublicKey::from(&recipient_key).to_bytes(), - RecipientSection { recipient_data }, - )]), - }, - )]), - dir_len: 0, - crc32: 0, - }; - let limits = DeserializationLimits { - max_collection_entries: 1, - ..DeserializationLimits::default() - }; - - let error = directory - .decrypt_recipient_with_limits(&sender_key, &limits) - .unwrap_err(); - assert!(matches!( - error, - PithosError::Deserialization(DeserializationError::LimitExceeded { - field: "recipient keys", - limit: 1, - actual: 2, - }) - )); -} - -#[test] -fn decrypt_with_limits_default_wrappers_decrypt_small_payloads() { - let block_key = [7u8; 32]; - let mut block_data = BlockDataState::Decrypted(vec![([1u8; 32], [2u8; 32])]); - block_data.encrypt(block_key).unwrap(); - block_data.decrypt(&block_key).unwrap(); - assert!(matches!(block_data, BlockDataState::Decrypted(entries) if entries.len() == 1)); - - let sender_key = generate_private_key().unwrap(); - let recipient_key = generate_private_key().unwrap(); - let shared_key = sender_key.diffie_hellman(&PublicKey::from(&recipient_key)); - let mut recipient_data = RecipientData::Decrypted(vec![(1, [2u8; 32])]); - recipient_data.encrypt(&shared_key).unwrap(); - recipient_data.decrypt(&shared_key).unwrap(); - assert!(matches!(recipient_data, RecipientData::Decrypted(entries) if entries.len() == 1)); -} - -#[test] -fn recipient_section_roundtrip() { - let original = RecipientSection { - recipient_data: RecipientData::Decrypted(vec![(2, [3u8; 32])]), - }; - let mut buf = Vec::new(); - original.serialize(&mut buf).unwrap(); - let decoded = RecipientSection::deserialize(&mut Cursor::new(buf)).unwrap(); - assert_eq!(original, decoded); -} From 74ef34c451785314f826e1eec9c8153095c403a8 Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Tue, 11 Aug 2026 22:59:35 +0200 Subject: [PATCH 003/166] test: add format fuzz harnesses --- crates/pithos_lib/src/fuzzing.rs | 95 + deny.toml | 3 +- fuzz/Cargo.lock | 2082 +++++++++++++++++ fuzz/Cargo.toml | 43 + fuzz/README.md | 39 + .../chain_validation/fixed-short-archive | 1 + .../corpus/crypt4gh_decode/fixed-short-header | 1 + fuzz/corpus/directory_decode/directory-marker | 1 + .../minimal-complete-directory | Bin 0 -> 25 bytes fuzz/corpus/header_decode/file-header-marker | 1 + fuzz/fuzz_targets/chain_validation.rs | 8 + fuzz/fuzz_targets/crypt4gh_decode.rs | 8 + fuzz/fuzz_targets/directory_decode.rs | 8 + fuzz/fuzz_targets/header_decode.rs | 8 + 14 files changed, 2297 insertions(+), 1 deletion(-) create mode 100644 crates/pithos_lib/src/fuzzing.rs create mode 100644 fuzz/Cargo.lock create mode 100644 fuzz/Cargo.toml create mode 100644 fuzz/README.md create mode 100644 fuzz/corpus/chain_validation/fixed-short-archive create mode 100644 fuzz/corpus/crypt4gh_decode/fixed-short-header create mode 100644 fuzz/corpus/directory_decode/directory-marker create mode 100644 fuzz/corpus/directory_decode/minimal-complete-directory create mode 100644 fuzz/corpus/header_decode/file-header-marker create mode 100644 fuzz/fuzz_targets/chain_validation.rs create mode 100644 fuzz/fuzz_targets/crypt4gh_decode.rs create mode 100644 fuzz/fuzz_targets/directory_decode.rs create mode 100644 fuzz/fuzz_targets/header_decode.rs diff --git a/crates/pithos_lib/src/fuzzing.rs b/crates/pithos_lib/src/fuzzing.rs new file mode 100644 index 0000000..c77eb8b --- /dev/null +++ b/crates/pithos_lib/src/fuzzing.rs @@ -0,0 +1,95 @@ +//! Narrow byte-oriented entry points for standalone fuzzing. +//! +//! This module is feature-gated so normal library consumers cannot depend on +//! raw format parsing as an API. + +use crate::archive::{Archive, OpenOptions}; +use crate::error::PithosError; +use crate::format::limits::DeserializationLimits; +use crate::source::MemorySource; +use std::io::Cursor; +use std::sync::Arc; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum FuzzErrorCategory { + Archive, + Authentication, + Deserialization, + Limit, + Source, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum FuzzOutcome { + Accepted, + Rejected(FuzzErrorCategory), +} + +pub fn decode_header(data: &[u8]) -> FuzzOutcome { + match crate::format::codec::decode_header(&mut Cursor::new(data)) { + Ok(_) => FuzzOutcome::Accepted, + Err(_) => FuzzOutcome::Rejected(FuzzErrorCategory::Deserialization), + } +} + +pub fn decode_directory(data: &[u8]) -> FuzzOutcome { + match crate::format::codec::decode_complete_directory(data, &DeserializationLimits::default()) { + Ok(_) => FuzzOutcome::Accepted, + Err(error) => FuzzOutcome::Rejected(classify_archive_error(&error)), + } +} + +pub fn validate_archive(data: &[u8]) -> FuzzOutcome { + match Archive::open( + MemorySource::new(Arc::<[u8]>::from(data)), + OpenOptions::default(), + ) { + Ok(_) => FuzzOutcome::Accepted, + Err(error) => FuzzOutcome::Rejected(classify_archive_error(&error)), + } +} + +/// Exercises the private Crypt4GH header parser without exposing its records. +pub fn decode_crypt4gh_header(data: &[u8]) -> FuzzOutcome { + match crate::adapters::crypt4gh::Crypt4GHHeader::try_from(data) { + Ok(_) => FuzzOutcome::Accepted, + Err(_) => FuzzOutcome::Rejected(FuzzErrorCategory::Deserialization), + } +} + +fn classify_archive_error(error: &PithosError) -> FuzzErrorCategory { + match error { + PithosError::Crypt(_) => FuzzErrorCategory::Authentication, + PithosError::Deserialization(_) => FuzzErrorCategory::Deserialization, + PithosError::LimitExceeded { .. } | PithosError::AllocationFailed { .. } => { + FuzzErrorCategory::Limit + } + PithosError::Source(_) => FuzzErrorCategory::Source, + _ => FuzzErrorCategory::Archive, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const MINIMAL_DIRECTORY: [u8; 25] = [ + 0x50, 0x49, 0x54, 0x48, 0x4f, 0x53, 0x44, 0x52, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x19, 0xb1, 0x67, 0x40, 0x81, + ]; + + #[test] + fn directory_fuzz_boundary_validates_complete_framing() { + assert_eq!(decode_directory(&MINIMAL_DIRECTORY), FuzzOutcome::Accepted); + let mut corrupt_crc = MINIMAL_DIRECTORY; + *corrupt_crc.last_mut().unwrap() ^= 1; + assert!(matches!( + decode_directory(&corrupt_crc), + FuzzOutcome::Rejected(FuzzErrorCategory::Archive) + )); + assert!(matches!( + decode_directory(&MINIMAL_DIRECTORY[..MINIMAL_DIRECTORY.len() - 12]), + FuzzOutcome::Rejected(FuzzErrorCategory::Archive) + )); + } +} diff --git a/deny.toml b/deny.toml index d03b311..d604bfd 100644 --- a/deny.toml +++ b/deny.toml @@ -106,7 +106,8 @@ confidence-threshold = 0.8 exceptions = [ { allow = ["MPL-2.0"], crate = "option-ext" }, { allow = ["Zlib"], crate = "zlib-rs" }, - { allow = ["Apache-2.0 WITH LLVM-exception"], crate = "winx" } + { allow = ["Apache-2.0 WITH LLVM-exception"], crate = "winx" }, + { allow = ["NCSA"], crate = "libfuzzer-sys" } ] # Some crates don't have (easily) machine readable licensing information, diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock new file mode 100644 index 0000000..2fde604 --- /dev/null +++ b/fuzz/Cargo.lock @@ -0,0 +1,2082 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aead" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1973cfbc1a2daf9cf550e74e1f088c28e7f7d8c1e1418fb6c9dc5184b7e84c99" +dependencies = [ + "crypto-common 0.2.2", + "inout", +] + +[[package]] +name = "ambient-authority" +version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9d4ee0d472d1cd2e28c97dfa124b3d8d992e10eb0a035f33f5d12e3a177ba3b" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" + +[[package]] +name = "arrayref" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76a2e8124351fda1ef8aaaa3bbd7ebbcb486bbcd4225aca0aa0d84bb2db8fecb" + +[[package]] +name = "arrayvec" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56" + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64ct" +version = "1.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "blake2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +dependencies = [ + "digest 0.10.7", +] + +[[package]] +name = "blake3" +version = "1.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0aa83c34e62843d924f905e0f5c866eb1dd6545fc4d719e803d9ba6030371fce" +dependencies = [ + "arrayref", + "arrayvec", + "cc", + "cfg-if", + "constant_time_eq", + "cpufeatures", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "block-buffer" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cap-primitives" +version = "4.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdadbd7c002d3a484b35243669abdae85a0ebaded5a61117169dc3400f9a7ff0" +dependencies = [ + "ambient-authority", + "fs-set-times", + "io-extras", + "io-lifetimes 3.0.1", + "ipnet", + "maybe-owned", + "rustix", + "rustix-linux-procfs", + "windows-sys 0.61.2", + "winx", +] + +[[package]] +name = "cap-std" +version = "4.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7281235d6e96d3544ca18bba9049be92f4190f8d923e3caef1b5f66cfa752608" +dependencies = [ + "cap-primitives", + "io-extras", + "io-lifetimes 3.0.1", + "rustix", +] + +[[package]] +name = "cc" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89588d05638b5b4594a3348a2d6c20277e43a7f5c5202b05cc56888475a47b8" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", + "rand_core", +] + +[[package]] +name = "chacha20poly1305" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b89e1c441e926b9c82a8d023f6e1b7ae0adcfaa7d621814e4d60789bac751cb" +dependencies = [ + "aead", + "chacha20", + "cipher", + "poly1305", +] + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-link", +] + +[[package]] +name = "cipher" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" +dependencies = [ + "block-buffer 0.12.1", + "crypto-common 0.2.2", + "inout", +] + +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + +[[package]] +name = "constant_time_eq" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "crypto-common" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" +dependencies = [ + "getrandom 0.4.3", + "hybrid-array", + "rand_core", +] + +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + +[[package]] +name = "curve25519-dalek" +version = "5.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5eed333089e2e1c1ac8c6c0398e5e2497b4c9926ca6d0365ed1e099afa5bc23" +dependencies = [ + "cfg-if", + "cpufeatures", + "curve25519-dalek-derive", + "fiat-crypto", + "rustc_version", + "subtle", + "zeroize", +] + +[[package]] +name = "curve25519-dalek-derive" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "der" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a69dedd701da44b0536442edf09c81a64b0ab97a7a4a5e3d1971f00027cbc63d" +dependencies = [ + "const-oid", + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "deranged" +version = "0.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "crypto-common 0.1.7", + "subtle", +] + +[[package]] +name = "digest" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" +dependencies = [ + "block-buffer 0.12.1", + "crypto-common 0.2.2", +] + +[[package]] +name = "dirs" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.61.2", +] + +[[package]] +name = "displaydoc" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ac70aa55017e108007fbaf5aa0f54b021c98f92ff8af59d42eda9da96e3dd4f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "fastcdc" +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77af40d8a8dadb92dc178569a5f5edb5f3056e98255c2de48ab5d59a52892e0c" + +[[package]] +name = "fiat-crypto" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64cd1e32ddd350061ae6edb1b082d7c54915b5c672c389143b9a63403a109f24" + +[[package]] +name = "find-msvc-tools" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" + +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fs-set-times" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94e7099f6313ecacbe1256e8ff9d617b75d1bcb16a6fddef94866d225a01a14a" +dependencies = [ + "io-lifetimes 2.0.4", + "rustix", + "windows-sys 0.59.0", +] + +[[package]] +name = "futures-channel" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" + +[[package]] +name = "futures-io" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a" + +[[package]] +name = "futures-sink" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" + +[[package]] +name = "futures-task" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" + +[[package]] +name = "futures-util" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +dependencies = [ + "futures-core", + "futures-io", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", + "rand_core", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "http" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6970f50e31d6fc17d3fa27329444bfa74e196cf62e95052a3f6fee181dba6425" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "hybrid-array" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" +dependencies = [ + "typenum", +] + +[[package]] +name = "hyper" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "inout" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4250ce6452e92010fdf7268ccc5d14faa80bb12fc741938534c58f16804e03c7" +dependencies = [ + "hybrid-array", +] + +[[package]] +name = "integer-encoding" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14c00403deb17c3221a1fe4fb571b9ed0370b3dcd116553c77fa294a3d918699" + +[[package]] +name = "io-extras" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20fd6de4ccfcc187e38bc21cfa543cb5a302cb86a8b114eb7f0bf0dc9f8ac00f" +dependencies = [ + "io-lifetimes 3.0.1", + "windows-sys 0.60.2", +] + +[[package]] +name = "io-lifetimes" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06432fb54d3be7964ecd3649233cddf80db2832f47fec34c01f65b3d9d774983" + +[[package]] +name = "io-lifetimes" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f0fb0570afe1fed943c5c3d4102d5358592d8625fda6a0007fdbe65a92fba96" + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b44bfcdb3f8d5837a46dae1ca9660a837176eee74a28b229bc626816589102" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "keccak" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e24a010dd405bd7ed803e5253182815b41bf2e6a80cc3bfc066658e03a198aa" +dependencies = [ + "cfg-if", + "cpufeatures", +] + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libfuzzer-sys" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9fd2f41a1cba099f79a0b6b6c35656cf7c03351a7bae8ff0f28f25270f929d2" +dependencies = [ + "arbitrary", + "cc", +] + +[[package]] +name = "libredox" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c943259e342f1e06ff2da7a83eabdfe7f92ce10262688dbf1895ff0b3e6e4652" +dependencies = [ + "libc", +] + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "maybe-owned" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4facc753ae494aeb6e3c22f839b158aebd4f9270f55cd3c79906c45476c47ab4" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "num-conv" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + +[[package]] +name = "pem-rfc7468" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6305423e0e7738146434843d1694d621cce767262b2a86910beab705e4493d9" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pithos-fuzz" +version = "0.0.0" +dependencies = [ + "libfuzzer-sys", + "pithos_lib", +] + +[[package]] +name = "pithos_lib" +version = "0.8.0" +dependencies = [ + "blake2", + "blake3", + "byteorder", + "cap-std", + "chacha20poly1305", + "crc32fast", + "digest 0.11.3", + "fastcdc", + "indexmap", + "integer-encoding", + "libc", + "pkcs8", + "ro-crate-rs", + "rustix", + "shake", + "thiserror", + "tracing", + "x25519-dalek", + "zeroize", + "zip", + "zstd", +] + +[[package]] +name = "pkcs8" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7" +dependencies = [ + "der", + "spki", +] + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "poly1305" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e2d0073b297041425c7c3df6eb4792d598a15323fe63346852b092eca02904c" +dependencies = [ + "cpufeatures", + "universal-hash", +] + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "redox_users" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" +dependencies = [ + "getrandom 0.2.17", + "libredox", + "thiserror", +] + +[[package]] +name = "reqwest" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "sync_wrapper", + "tokio", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "ro-crate-rs" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "878d87f91b1966f167b36964b4a3b9b891a420c7e08c82167d79a54d2db6ee48" +dependencies = [ + "chrono", + "dirs", + "log", + "rand", + "reqwest", + "serde", + "serde_json", + "thiserror", + "url", + "uuid", + "walkdir", + "zip", +] + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustix-linux-procfs" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fc84bf7e9aa16c4f2c758f27412dc9841341e16aa682d9c7ac308fe3ee12056" +dependencies = [ + "once_cell", + "rustix", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "shake" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09057cb2149ad4cbd2da1e26b351f9a4c354219421229c69c3063e6f61947c4a" +dependencies = [ + "digest 0.11.3", + "keccak", + "sponge-cursor", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "spki" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d9efca8738c78ee9484207732f728b1ef517bbb1833d6fc0879ca898a522f6f" +dependencies = [ + "base64ct", + "der", +] + +[[package]] +name = "sponge-cursor" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a0219bd7d979d58245a4f41f695e1ac9f8befdffadd7f61f1bae9e39abc6620" + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "thiserror" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09a43598840e33d5b0331f38c5e30d13bb11c11210a4b58f0d9b18a5a5eefcd9" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43cbfe0cf76104d42a574802844187e84a305e531ed54455f11fbde0f10541cd" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "time" +version = "0.3.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e1d5e639ff6bab73cb6885cc7e7b1de96c3f32c68ec55f3952614bec1092244" +dependencies = [ + "deranged", + "js-sys", + "num-conv", + "powerfmt", + "serde_core", + "time-core", +] + +[[package]] +name = "time-core" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "libc", + "mio", + "pin-project-lite", + "socket2", + "windows-sys 0.61.2", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typed-path" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "universal-hash" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4987bdc12753382e0bec4a65c50738ffaabc998b9cdd1f952fb5f39b0048a96" +dependencies = [ + "crypto-common 0.2.2", + "ctutils", +] + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "uuid" +version = "1.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" +dependencies = [ + "getrandom 0.4.3", + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasm-bindgen" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b067c0c11094aef6b7a801c1e34a26affafdf3d051dba08456b868789aaf9a4" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c62df1340f32221cb9c54d6a27b030e3dba64361d4a95bed55f9aacb44da291d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167ce5e579f6bcf889c4f7175a8a5a585de84e8ff93976ce393efa5f2837aab1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3997c7839262f4ef12cf90b818d6340c18e80f263f1a94bf157d0ec4420380e" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1b4cb0cc549fcf58d7dfc081778139b3d283a081644e833e84682ad71cea24" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.103" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8622dcb61c0bcc9fffa6938bed81210af2da9a7e4a1a834b2e37a59b6dfb6141" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "windows-core" +version = "0.62.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" +dependencies = [ + "windows-implement", + "windows-interface", + "windows-link", + "windows-result", + "windows-strings", +] + +[[package]] +name = "windows-implement" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-interface" +version = "0.59.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-result" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-strings" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm 0.52.6", + "windows_aarch64_msvc 0.52.6", + "windows_i686_gnu 0.52.6", + "windows_i686_gnullvm 0.52.6", + "windows_i686_msvc 0.52.6", + "windows_x86_64_gnu 0.52.6", + "windows_x86_64_gnullvm 0.52.6", + "windows_x86_64_msvc 0.52.6", +] + +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "winx" +version = "0.36.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f3fd376f71958b862e7afb20cfe5a22830e1963462f3a17f49d82a6c1d1f42d" +dependencies = [ + "bitflags", + "windows-sys 0.59.0", +] + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "x25519-dalek" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e8131a03190127fb2263afc72b322ecadae46b6ff8c6f399ff5d02f5559af6" +dependencies = [ + "curve25519-dalek", + "getrandom 0.4.3", + "rand_core", + "zeroize", +] + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zip" +version = "8.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" +dependencies = [ + "crc32fast", + "flate2", + "indexmap", + "memchr", + "time", + "typed-path", + "zopfli", +] + +[[package]] +name = "zlib-rs" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b142a20ec14a91d5bc708c1dc21b080c550113d8aa77afa29635673a65dd02c5" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml new file mode 100644 index 0000000..c3c3a14 --- /dev/null +++ b/fuzz/Cargo.toml @@ -0,0 +1,43 @@ +[package] +name = "pithos-fuzz" +version = "0.0.0" +publish = false +edition = "2024" +license = "MIT OR Apache-2.0" + +[workspace] + +[package.metadata] +cargo-fuzz = true + +[dependencies] +libfuzzer-sys = "0.4.12" +pithos_lib = { path = "../crates/pithos_lib", features = ["fuzzing"] } + +[[bin]] +name = "header_decode" +path = "fuzz_targets/header_decode.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "directory_decode" +path = "fuzz_targets/directory_decode.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "crypt4gh_decode" +path = "fuzz_targets/crypt4gh_decode.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "chain_validation" +path = "fuzz_targets/chain_validation.rs" +test = false +doc = false +bench = false diff --git a/fuzz/README.md b/fuzz/README.md new file mode 100644 index 0000000..a23f3d3 --- /dev/null +++ b/fuzz/README.md @@ -0,0 +1,39 @@ +# Pithos Fuzzing + +This guide is for maintainers running the standalone format fuzzing targets. It is a conventional non-workspace `cargo-fuzz` package with its own `Cargo.lock`; do not add it to the root workspace or use the workspace lockfile for fuzz jobs. + +Install exactly `cargo-fuzz 0.13.2` and a Rust nightly toolchain with the platform C/C++ linker: + +```bash +cargo +nightly install cargo-fuzz --version 0.13.2 --locked +cargo +nightly fuzz build +``` + +Run commands from the repository root. Committed seed corpora are under `fuzz/corpus/` and are +small format-marker cases plus fixed short-input regressions. The +`header_decode/file-header-marker` seed is exactly the four ASCII bytes `PITH`. The directory +corpus includes the valid 25-byte minimal complete directory so mutations can reach +footer, checksum, consumption, and body parsing rather than stopping at the minimum-length check. +Crash artifacts and generated build output belong under `fuzz/artifacts/` and `fuzz/target/` and +are ignored by Git. Keep any minimized regression that fixes a bug in `fuzz/corpus//`. + +Release-candidate smoke limits are 60 seconds, five seconds per input, and 1024 MiB RSS: + +```bash +cargo +nightly fuzz run header_decode -- -max_total_time=60 -timeout=5 -rss_limit_mb=1024 +cargo +nightly fuzz run directory_decode -- -max_total_time=60 -timeout=5 -rss_limit_mb=1024 +cargo +nightly fuzz run crypt4gh_decode -- -max_total_time=60 -timeout=5 -rss_limit_mb=1024 +cargo +nightly fuzz run chain_validation -- -max_total_time=60 -timeout=5 -rss_limit_mb=1024 +``` + +The root dependency policy does not include this package. Run its policy check separately after +generating or updating `fuzz/Cargo.lock`: + +```bash +cargo deny --manifest-path fuzz/Cargo.toml --config deny.toml --locked check +``` + +Targets receive only bytes through the feature-gated `pithos_lib::fuzzing` harness and reduce +results to an error/success category. They do not expose wire records, decrypted metadata, or +secrets. The directory target validates complete marker, length, checksum, and parser-consumption +framing rather than invoking the raw directory body decoder. diff --git a/fuzz/corpus/chain_validation/fixed-short-archive b/fuzz/corpus/chain_validation/fixed-short-archive new file mode 100644 index 0000000..b5fc0e0 --- /dev/null +++ b/fuzz/corpus/chain_validation/fixed-short-archive @@ -0,0 +1 @@ +PITH diff --git a/fuzz/corpus/crypt4gh_decode/fixed-short-header b/fuzz/corpus/crypt4gh_decode/fixed-short-header new file mode 100644 index 0000000..a3c9f11 --- /dev/null +++ b/fuzz/corpus/crypt4gh_decode/fixed-short-header @@ -0,0 +1 @@ +crypt4gh diff --git a/fuzz/corpus/directory_decode/directory-marker b/fuzz/corpus/directory_decode/directory-marker new file mode 100644 index 0000000..556d8db --- /dev/null +++ b/fuzz/corpus/directory_decode/directory-marker @@ -0,0 +1 @@ +PITHOSDR diff --git a/fuzz/corpus/directory_decode/minimal-complete-directory b/fuzz/corpus/directory_decode/minimal-complete-directory new file mode 100644 index 0000000000000000000000000000000000000000..33059e6fae1e2c2830c19afda995cde5571d2fce GIT binary patch literal 25 WcmWIW4Ds*}b_rrY0+Jik9U1{Sy98hW literal 0 HcmV?d00001 diff --git a/fuzz/corpus/header_decode/file-header-marker b/fuzz/corpus/header_decode/file-header-marker new file mode 100644 index 0000000..b5fc0e0 --- /dev/null +++ b/fuzz/corpus/header_decode/file-header-marker @@ -0,0 +1 @@ +PITH diff --git a/fuzz/fuzz_targets/chain_validation.rs b/fuzz/fuzz_targets/chain_validation.rs new file mode 100644 index 0000000..f5931a5 --- /dev/null +++ b/fuzz/fuzz_targets/chain_validation.rs @@ -0,0 +1,8 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + let _category = pithos_lib::fuzzing::validate_archive(data); + std::hint::black_box(_category); +}); diff --git a/fuzz/fuzz_targets/crypt4gh_decode.rs b/fuzz/fuzz_targets/crypt4gh_decode.rs new file mode 100644 index 0000000..803d656 --- /dev/null +++ b/fuzz/fuzz_targets/crypt4gh_decode.rs @@ -0,0 +1,8 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; +use pithos_lib::fuzzing::decode_crypt4gh_header; + +fuzz_target!(|data: &[u8]| { + std::hint::black_box(decode_crypt4gh_header(data)); +}); diff --git a/fuzz/fuzz_targets/directory_decode.rs b/fuzz/fuzz_targets/directory_decode.rs new file mode 100644 index 0000000..884bcab --- /dev/null +++ b/fuzz/fuzz_targets/directory_decode.rs @@ -0,0 +1,8 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + let _category = pithos_lib::fuzzing::decode_directory(data); + std::hint::black_box(_category); +}); diff --git a/fuzz/fuzz_targets/header_decode.rs b/fuzz/fuzz_targets/header_decode.rs new file mode 100644 index 0000000..dc30ff3 --- /dev/null +++ b/fuzz/fuzz_targets/header_decode.rs @@ -0,0 +1,8 @@ +#![no_main] + +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + let _category = pithos_lib::fuzzing::decode_header(data); + std::hint::black_box(_category); +}); From 26d8cf7687bfd6626376567525e8ea810975e4b8 Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Tue, 11 Aug 2026 22:59:45 +0200 Subject: [PATCH 004/166] refactor: add validated immutable archive index --- crates/pithos_lib/src/archive/access.rs | 92 ++++ crates/pithos_lib/src/archive/index.rs | 283 +++++++++++ crates/pithos_lib/src/archive/planning.rs | 85 ++++ crates/pithos_lib/src/archive/types.rs | 307 ++++++++++++ crates/pithos_lib/src/archive/validation.rs | 298 ++++++++++++ crates/pithos_lib/src/helpers/archive_path.rs | 444 ------------------ 6 files changed, 1065 insertions(+), 444 deletions(-) create mode 100644 crates/pithos_lib/src/archive/access.rs create mode 100644 crates/pithos_lib/src/archive/index.rs create mode 100644 crates/pithos_lib/src/archive/planning.rs create mode 100644 crates/pithos_lib/src/archive/types.rs create mode 100644 crates/pithos_lib/src/archive/validation.rs delete mode 100644 crates/pithos_lib/src/helpers/archive_path.rs diff --git a/crates/pithos_lib/src/archive/access.rs b/crates/pithos_lib/src/archive/access.rs new file mode 100644 index 0000000..ab00908 --- /dev/null +++ b/crates/pithos_lib/src/archive/access.rs @@ -0,0 +1,92 @@ +use crate::archive::types::FileId; +use crate::crypto::{BlockKey, FileKey}; +use crate::error::PithosError; +use std::collections::BTreeMap; + +/// Recovered file secrets have one crate-private owner and never enter an index. +pub(crate) struct ResolvedAccess { + keys: BTreeMap, + block_keys: BTreeMap>, + provenance: BTreeMap, +} + +#[derive(Clone, Copy, Eq, Ord, PartialEq, PartialOrd)] +pub(crate) struct AccessProvenance { + pub(crate) segment: usize, + pub(crate) recovery_order: usize, + pub(crate) access_key: usize, + pub(crate) sender_section: usize, + pub(crate) recipient_section: usize, +} + +impl ResolvedAccess { + pub(crate) fn new() -> Self { + Self { + keys: BTreeMap::new(), + block_keys: BTreeMap::new(), + provenance: BTreeMap::new(), + } + } + + pub(crate) fn insert( + &mut self, + id: FileId, + key: &[u8; 32], + provenance: AccessProvenance, + ) -> Result<(), PithosError> { + match self.keys.get(&id) { + Some(existing) if existing.expose_for_protocol() != key => { + Err(PithosError::ConflictingRecoveredFileKey) + } + Some(_) => { + if provenance < self.provenance[&id] { + self.provenance.insert(id, provenance); + } + Ok(()) + } + None => { + self.keys.insert(id, FileKey::from_protocol(key)); + self.provenance.insert(id, provenance); + Ok(()) + } + } + } + + pub(crate) fn key(&self, id: FileId) -> Option<&FileKey> { + self.keys.get(&id) + } + + pub(crate) fn provenance(&self, id: FileId) -> Option { + self.provenance.get(&id).copied() + } + + pub(crate) fn with_file_key( + &self, + id: FileId, + operation: impl FnOnce(&FileKey) -> T, + ) -> Option { + self.keys.get(&id).map(operation) + } + + pub(crate) fn insert_block_keys<'a>( + &mut self, + id: FileId, + entries: impl IntoIterator, + ) { + self.block_keys.insert( + id, + entries + .into_iter() + .map(|(hash, key)| (hash, BlockKey::from_protocol(key))) + .collect(), + ); + } + + pub(crate) fn block_key( + &self, + id: FileId, + hash: crate::archive::types::BlockHash, + ) -> Option<&BlockKey> { + self.block_keys.get(&id).and_then(|keys| keys.get(&hash)) + } +} diff --git a/crates/pithos_lib/src/archive/index.rs b/crates/pithos_lib/src/archive/index.rs new file mode 100644 index 0000000..045d1ab --- /dev/null +++ b/crates/pithos_lib/src/archive/index.rs @@ -0,0 +1,283 @@ +use crate::archive::planning::{ReadPlan, full_file_plan, range_plan}; +use crate::archive::types::{ + ArchivePath, BlockDescriptor, BlockHash, ContentState, Entry, FileId, ReadRange, RelationId, + SegmentEntry, Span, ValidatedSegment, +}; +use crate::archive::validation::{ + IndexLimits, validate_aggregate, validate_entry, validate_hierarchy, +}; +use crate::error::PithosError; +use indexmap::IndexMap; +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::sync::Arc; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct IndexedEntry { + pub(crate) id: FileId, + pub(crate) path: ArchivePath, + pub(crate) entry: Entry, +} + +/// Immutable effective archive state. Its private fields contain no secrets. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ArchiveIndex { + entries: Vec, + by_id: BTreeMap, + by_path: HashMap, usize>, + hierarchy: BTreeMap, + descriptors: IndexMap, + relationships: BTreeMap>, + segment_spans: Vec, + maximum_id: Option, +} + +impl ArchiveIndex { + pub(crate) fn entries(&self) -> impl ExactSizeIterator { + self.entries.iter() + } + + pub(crate) fn entry(&self, id: FileId) -> Option<&IndexedEntry> { + self.by_id + .get(&id) + .and_then(|index| self.entries.get(*index)) + } + + pub(crate) fn entry_at_path(&self, path: &ArchivePath) -> Option<&IndexedEntry> { + self.by_path + .get(path.as_str()) + .and_then(|index| self.entries.get(*index)) + } + + #[cfg(test)] + pub(crate) fn hierarchy(&self) -> impl Iterator { + self.hierarchy + .values() + .filter_map(|index| self.entries.get(*index)) + } + + pub(crate) fn descriptor(&self, hash: BlockHash) -> Option<&BlockDescriptor> { + self.descriptors.get(&hash) + } + + pub(crate) fn relationships(&self) -> impl Iterator { + self.relationships + .iter() + .map(|(id, name)| (*id, name.as_ref())) + } + + pub(crate) fn relationship(&self, id: RelationId) -> Option<&str> { + self.relationships.get(&id).map(Arc::as_ref) + } + + #[cfg(test)] + pub(crate) fn maximum_id(&self) -> Option { + self.maximum_id + } + + /// Transfers only the effective state needed to form an append snapshot. + pub(crate) fn into_append_snapshot_parts( + self, + ) -> ( + Vec, + IndexMap, + Option, + ) { + (self.entries, self.descriptors, self.maximum_id) + } + + pub(crate) fn full_file_plan(&self, id: FileId) -> Result { + full_file_plan(self, id) + } + + pub(crate) fn range_plan(&self, id: FileId, range: ReadRange) -> Result { + range_plan(self, id, range) + } +} + +/// Builds a new effective index. Inputs are only borrowed and are unchanged on every error. +pub(crate) fn build_effective_index( + segments: &[ValidatedSegment], + archive_len: u64, + limits: IndexLimits, +) -> Result { + validate_aggregate(segments, limits)?; + let mut entries = Vec::new(); + let mut by_id = BTreeMap::new(); + let mut by_path = HashMap::new(); + let mut hierarchy = BTreeMap::new(); + let mut descriptors: IndexMap = IndexMap::new(); + let mut relationships: BTreeMap> = BTreeMap::new(); + let mut segment_spans = BTreeSet::new(); + + for segment in segments { + validate_segment_chain(segment, &segment_spans)?; + segment_spans.insert(segment.span); + for (id, name) in &segment.relationships { + match relationships.get(id) { + Some(existing) if existing.as_ref() != name.as_ref() => { + return Err(PithosError::ConflictingRelationshipDefinition(id.0)); + } + Some(_) => {} + None => { + relationships.insert(*id, Arc::clone(name)); + } + } + } + for (hash, descriptor) in &segment.descriptors { + if let Some(existing) = descriptors.get(hash) { + // The current format defines compatibility by identity and plaintext size only. + if existing.original_size != descriptor.original_size { + return Err(PithosError::BlockIndexConflict { + hash: hash.0, + existing_original_size: existing.original_size, + new_original_size: descriptor.original_size, + }); + } + } else { + descriptors.insert(*hash, descriptor.clone()); + } + } + for SegmentEntry { id, path, entry } in &segment.entries { + validate_entry(path, entry)?; + if by_id.contains_key(id) { + return Err(PithosError::DuplicateFileId(format!( + "File id already occupied: {}", + id.0 + ))); + } + if by_path.contains_key(path.as_str()) { + return Err(PithosError::PathOccupied(format!( + "File path already occupied: {}", + path.as_str() + ))); + } + let index = entries.len(); + entries.push(IndexedEntry { + id: *id, + path: path.clone(), + entry: entry.clone(), + }); + by_id.insert(*id, index); + by_path.insert(Arc::from(path.as_str()), index); + hierarchy.insert(path.clone(), index); + } + } + + validate_hierarchy(&entries, &by_path)?; + let segment_spans = segment_spans.into_iter().collect::>(); + for segment in segments { + for (_, descriptor) in &segment.descriptors { + validate_descriptor(descriptor, archive_len, &segment_spans)?; + } + } + validate_references_and_content(&entries, &by_id, &relationships, &descriptors)?; + let maximum_id = entries.iter().map(|entry| entry.id).max(); + Ok(ArchiveIndex { + entries, + by_id, + by_path, + hierarchy, + descriptors, + relationships, + segment_spans, + maximum_id, + }) +} + +fn validate_segment_chain( + segment: &ValidatedSegment, + earlier: &BTreeSet, +) -> Result<(), PithosError> { + if let Some(parent) = segment.parent { + if !earlier.contains(&parent) || parent.end() > segment.span.start() { + return Err(PithosError::InvalidDirectoryChain { + operation: "validate parent ordering", + }); + } + } else if !earlier.is_empty() { + return Err(PithosError::InvalidDirectoryChain { + operation: "validate root segment", + }); + } + let previous_overlaps = earlier + .range(..segment.span) + .next_back() + .is_some_and(|span| span.overlaps(segment.span)); + let next_overlaps = earlier + .range(segment.span..) + .next() + .is_some_and(|span| span.overlaps(segment.span)); + if previous_overlaps || next_overlaps { + return Err(PithosError::InvalidDirectoryChain { + operation: "validate segment overlap", + }); + } + Ok(()) +} + +fn validate_descriptor( + descriptor: &BlockDescriptor, + archive_len: u64, + directory_spans: &[Span], +) -> Result<(), PithosError> { + if let crate::archive::types::BlockLocation::Local(span) = descriptor.location { + if span.end() > archive_len { + return Err(PithosError::InvalidDirectoryRange { + operation: "validate block range", + }); + } + let directory_index = + directory_spans.partition_point(|directory| directory.end() <= span.start()); + if directory_spans + .get(directory_index) + .is_some_and(|directory| span.overlaps(*directory)) + { + return Err(PithosError::InvalidDirectoryRange { + operation: "validate block overlap", + }); + } + } + Ok(()) +} + +fn validate_references_and_content( + entries: &[IndexedEntry], + by_id: &BTreeMap, + relationships: &BTreeMap>, + descriptors: &IndexMap, +) -> Result<(), PithosError> { + for indexed in entries { + for reference in &indexed.entry.metadata().references { + if !relationships.contains_key(&reference.relationship) { + return Err(PithosError::UnknownRelationshipId(reference.relationship.0)); + } + if !by_id.contains_key(&reference.target) { + return Err(PithosError::MissingReferenceTarget(reference.target.0)); + } + } + let Some(content) = indexed.entry.content() else { + continue; + }; + let ContentState::Available(blocks) = &content.content else { + continue; + }; + let actual = blocks.iter().try_fold(0u64, |total, hash| { + let descriptor = descriptors + .get(&hash) + .ok_or(PithosError::MissingBlockDescriptor)?; + total.checked_add(descriptor.original_size).ok_or( + PithosError::AccessibleFileSizeMismatch { + expected: content.size, + actual: u64::MAX, + }, + ) + })?; + if actual != content.size { + return Err(PithosError::AccessibleFileSizeMismatch { + expected: content.size, + actual, + }); + } + } + Ok(()) +} diff --git a/crates/pithos_lib/src/archive/planning.rs b/crates/pithos_lib/src/archive/planning.rs new file mode 100644 index 0000000..1e9ea07 --- /dev/null +++ b/crates/pithos_lib/src/archive/planning.rs @@ -0,0 +1,85 @@ +use crate::archive::index::ArchiveIndex; +use crate::archive::types::{BlockDescriptor, BlockHash, ContentState, Entry, FileId, ReadRange}; +use crate::error::PithosError; +use std::ops::Range; + +/// A block must be completely verified before its `output` slice is released. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct PlannedBlock { + pub(crate) hash: BlockHash, + pub(crate) descriptor: BlockDescriptor, + pub(crate) output: Range, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ReadPlan { + pub(crate) file_size: u64, + pub(crate) blocks: Vec, +} + +pub(crate) fn full_file_plan(index: &ArchiveIndex, id: FileId) -> Result { + let content = content_entry(index, id)?; + let range = ReadRange::new(0..content.size, content.size)?; + range_plan(index, id, range) +} + +pub(crate) fn range_plan( + index: &ArchiveIndex, + id: FileId, + range: ReadRange, +) -> Result { + let content = content_entry(index, id)?; + let ContentState::Available(references) = &content.content else { + return Err(PithosError::ContentUnavailable); + }; + let mut blocks = Vec::new(); + let mut cursor = 0u64; + for hash in references.iter() { + let descriptor = index + .descriptor(hash) + .ok_or(PithosError::MissingBlockDescriptor)?; + let end = cursor.checked_add(descriptor.original_size).ok_or( + PithosError::AccessibleFileSizeMismatch { + expected: content.size, + actual: u64::MAX, + }, + )?; + if end > range.start() && cursor < range.end() { + let output_start = range.start().saturating_sub(cursor); + let output_end = range.end().min(end) - cursor; + blocks.push(PlannedBlock { + hash, + descriptor: descriptor.clone(), + output: usize::try_from(output_start).map_err(|_| { + PithosError::InvalidDirectoryRange { + operation: "convert range index", + } + })?..usize::try_from(output_end).map_err(|_| { + PithosError::InvalidDirectoryRange { + operation: "convert range index", + } + })?, + }); + } + cursor = end; + } + Ok(ReadPlan { + file_size: content.size, + blocks, + }) +} + +fn content_entry( + index: &ArchiveIndex, + id: FileId, +) -> Result<&crate::archive::types::ContentEntry, PithosError> { + index + .entry(id) + .and_then(|entry| match &entry.entry { + Entry::File(content) | Entry::Metadata(content) => Some(content), + Entry::Directory(_) | Entry::Symlink { .. } => None, + }) + .ok_or_else(|| { + PithosError::InvalidBlockDataState("only data/metadata entries have content".into()) + }) +} diff --git a/crates/pithos_lib/src/archive/types.rs b/crates/pithos_lib/src/archive/types.rs new file mode 100644 index 0000000..7eb7572 --- /dev/null +++ b/crates/pithos_lib/src/archive/types.rs @@ -0,0 +1,307 @@ +use crate::error::PithosError; +use std::cmp::Ordering; +use std::fmt::{Debug, Formatter}; +use std::ops::Range; +use std::sync::Arc; + +#[derive(Clone, Eq, Hash, PartialEq)] +pub struct ArchivePath(Arc); + +impl ArchivePath { + pub fn new(path: impl AsRef) -> Result { + let path = path.as_ref(); + if path.is_empty() { + return Err(invalid_path(path, "path is empty")); + } + if path.contains('\0') { + return Err(invalid_path(path, "NUL is not allowed")); + } + if path.contains('\\') { + return Err(invalid_path(path, "backslash is not allowed")); + } + if path.starts_with('/') || path.ends_with('/') { + return Err(invalid_path(path, "path must not start or end with /")); + } + if path.as_bytes().get(1) == Some(&b':') { + return Err(invalid_path(path, "drive forms are not allowed")); + } + if path + .split('/') + .any(|part| part.is_empty() || part == "." || part == "..") + { + return Err(invalid_path( + path, + "empty or dot path components are not allowed", + )); + } + Ok(Self(Arc::from(path))) + } + + pub fn as_str(&self) -> &str { + &self.0 + } + + pub(crate) fn is_ancestor_of(&self, other: &Self) -> bool { + other + .as_str() + .strip_prefix(self.as_str()) + .is_some_and(|suffix| suffix.starts_with('/')) + } +} + +impl Debug for ArchivePath { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_tuple("ArchivePath").field(&self.as_str()).finish() + } +} + +impl Ord for ArchivePath { + fn cmp(&self, other: &Self) -> Ordering { + self.as_str().split('/').cmp(other.as_str().split('/')) + } +} + +impl PartialOrd for ArchivePath { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +fn invalid_path(path: &str, reason: impl Into) -> PithosError { + PithosError::InvalidArchivePath { + path: path.to_owned(), + reason: reason.into(), + } +} + +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub(crate) struct FileId(pub(crate) u64); + +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub(crate) struct RelationId(pub(crate) u64); + +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub(crate) struct BlockHash(pub(crate) [u8; 32]); + +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub(crate) struct Span { + start: u64, + end: u64, +} + +impl Span { + pub(crate) fn new(start: u64, len: u64) -> Result { + let end = start + .checked_add(len) + .ok_or(PithosError::InvalidDirectoryRange { + operation: "construct span", + })?; + Ok(Self { start, end }) + } + + pub(crate) fn start(self) -> u64 { + self.start + } + + pub(crate) fn end(self) -> u64 { + self.end + } + + pub(crate) fn len(self) -> u64 { + self.end - self.start + } + + pub(crate) fn overlaps(self, other: Self) -> bool { + self.start < other.end && other.start < self.end + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct ReadRange { + start: u64, + end: u64, +} + +impl ReadRange { + pub(crate) fn new(range: Range, file_size: u64) -> Result { + if range.start > range.end || range.end > file_size { + return Err(PithosError::InvalidReadRange { + start: range.start, + end: range.end, + file_size, + }); + } + Ok(Self { + start: range.start, + end: range.end, + }) + } + + pub(crate) fn start(self) -> u64 { + self.start + } + + pub(crate) fn end(self) -> u64 { + self.end + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct Compression(u8); + +impl Compression { + pub(crate) fn new(level: u8) -> Result { + if level <= 7 { + Ok(Self(level)) + } else { + Err(PithosError::ReservedProcessingBits(level)) + } + } + + pub(crate) fn level(self) -> u8 { + self.0 + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct Processing { + compression: Compression, + encrypted: bool, +} + +impl Processing { + pub(crate) fn from_byte(value: u8) -> Result { + if value & 0xf0 != 0 { + return Err(PithosError::ReservedProcessingBits(value)); + } + Ok(Self { + compression: Compression::new(value & 0x07)?, + encrypted: value & 0x08 != 0, + }) + } + + pub(crate) fn to_byte(self) -> u8 { + self.compression.level() | (u8::from(self.encrypted) << 3) + } +} + +#[derive(Clone, Eq, PartialEq)] +pub struct ExternalLocation(Arc); + +impl ExternalLocation { + pub(crate) fn new(value: impl AsRef) -> Self { + Self(Arc::from(value.as_ref())) + } + + /// The archive-owned opaque location passed to an external resolver. + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl Debug for ExternalLocation { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str("ExternalLocation(..)") + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum BlockLocation { + Local(Span), + External(ExternalLocation), +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct BlockDescriptor { + pub(crate) stored_size: u64, + pub(crate) original_size: u64, + pub(crate) processing: Processing, + pub(crate) location: BlockLocation, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct Reference { + pub(crate) target: FileId, + pub(crate) relationship: RelationId, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct EntryMetadata { + pub(crate) created: u64, + pub(crate) modified: u64, + pub(crate) permissions: u32, + pub(crate) references: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct BlockReferences(Vec); + +impl BlockReferences { + pub(crate) fn new(blocks: Vec) -> Self { + Self(blocks) + } + + pub(crate) fn iter(&self) -> impl ExactSizeIterator + '_ { + self.0.iter().copied() + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum ContentState { + Available(BlockReferences), + Unavailable, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ContentEntry { + pub(crate) metadata: EntryMetadata, + pub(crate) size: u64, + pub(crate) content: ContentState, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum Entry { + File(ContentEntry), + Metadata(ContentEntry), + Directory(EntryMetadata), + Symlink { + metadata: EntryMetadata, + target: Arc, + }, +} + +impl Entry { + pub(crate) fn metadata(&self) -> &EntryMetadata { + match self { + Self::File(content) | Self::Metadata(content) => &content.metadata, + Self::Directory(metadata) | Self::Symlink { metadata, .. } => metadata, + } + } + + pub(crate) fn content(&self) -> Option<&ContentEntry> { + match self { + Self::File(content) | Self::Metadata(content) => Some(content), + Self::Directory(_) | Self::Symlink { .. } => None, + } + } + + pub(crate) fn is_directory(&self) -> bool { + matches!(self, Self::Directory(_)) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct SegmentEntry { + pub(crate) id: FileId, + pub(crate) path: ArchivePath, + pub(crate) entry: Entry, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ValidatedSegment { + pub(crate) span: Span, + pub(crate) parent: Option, + pub(crate) entries: Vec, + pub(crate) descriptors: Vec<(BlockHash, BlockDescriptor)>, + pub(crate) relationships: Vec<(RelationId, Arc)>, +} diff --git a/crates/pithos_lib/src/archive/validation.rs b/crates/pithos_lib/src/archive/validation.rs new file mode 100644 index 0000000..9e2b6a4 --- /dev/null +++ b/crates/pithos_lib/src/archive/validation.rs @@ -0,0 +1,298 @@ +use crate::archive::types::{ + ArchivePath, BlockDescriptor, BlockHash, BlockLocation, ContentEntry, ContentState, Entry, + EntryMetadata, ExternalLocation, FileId, Processing, Reference, RelationId, SegmentEntry, Span, + ValidatedSegment, +}; +use crate::error::PithosError; +use crate::format::wire::{ + BlockDataState, BlockLocation as WireBlockLocation, Directory, FileType, +}; +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::sync::Arc; + +#[derive(Clone, Copy, Debug)] +pub(crate) struct IndexLimits { + pub(crate) max_entries: u64, + pub(crate) max_descriptors: u64, + pub(crate) max_references: u64, + pub(crate) max_relationships: u64, + pub(crate) max_segments: u64, +} + +impl Default for IndexLimits { + fn default() -> Self { + Self { + max_entries: 1_000_000, + max_descriptors: 1_000_000, + max_references: 1_000_000, + max_relationships: 1_000_000, + max_segments: 1025, + } + } +} + +pub(crate) fn segment_from_wire( + directory: &Directory, + span: Span, + parent: Option, +) -> Result { + let mut entries = Vec::new(); + let mut entry_ids = HashSet::new(); + for (id, path, wire) in directory.files.iter() { + if !entry_ids.insert(id) { + return Err(PithosError::DuplicateFileId(format!( + "File id already occupied: {id}" + ))); + } + entries.push(SegmentEntry { + id: FileId(id), + path: ArchivePath::new(path)?, + entry: entry_from_wire(wire)?, + }); + } + let mut descriptors = Vec::new(); + for (hash, wire) in &directory.blocks { + let stored_size_with_marker = + wire.stored_size + .checked_add(4) + .ok_or(PithosError::InvalidDirectoryRange { + operation: "validate block range", + })?; + let location = match &wire.location { + WireBlockLocation::Local => { + BlockLocation::Local(Span::new(wire.offset, stored_size_with_marker)?) + } + WireBlockLocation::External { url } => { + BlockLocation::External(ExternalLocation::new(url)) + } + }; + descriptors.push(( + BlockHash(*hash), + BlockDescriptor { + stored_size: wire.stored_size, + original_size: wire.original_size, + processing: Processing::from_byte(wire.flags.0)?, + location, + }, + )); + } + let mut relationships = Vec::new(); + let mut relation_names: BTreeMap = BTreeMap::new(); + for (id, name) in &directory.relations { + match relation_names.get(id) { + Some(existing) if existing != name => { + return Err(PithosError::ConflictingRelationshipDefinition(*id)); + } + Some(_) => {} + None => { + relation_names.insert(*id, name.clone()); + relationships.push((RelationId(*id), Arc::from(name.as_str()))); + } + } + } + Ok(ValidatedSegment { + span, + parent, + entries, + descriptors, + relationships, + }) +} + +fn entry_from_wire(wire: &crate::format::wire::FileEntry) -> Result { + let metadata = EntryMetadata { + created: wire.created, + modified: wire.modified, + permissions: wire.permissions, + references: wire + .references + .iter() + .map(|reference| Reference { + target: FileId(reference.target_file_id), + relationship: RelationId(reference.relationship), + }) + .collect(), + }; + let content = || match &wire.block_data { + BlockDataState::Decrypted(blocks) => { + ContentState::Available(crate::archive::types::BlockReferences::new( + blocks.iter().map(|(hash, _)| BlockHash(*hash)).collect(), + )) + } + BlockDataState::Encrypted(_) => ContentState::Unavailable, + }; + match wire.file_type { + FileType::Data => Ok(Entry::File(ContentEntry { + metadata, + size: wire.file_size, + content: content(), + })), + FileType::Metadata => Ok(Entry::Metadata(ContentEntry { + metadata, + size: wire.file_size, + content: content(), + })), + FileType::Directory => { + require_no_content(wire, "directory")?; + if wire.symlink_target.is_some() { + return Err(PithosError::InvalidSymlinkEntry { + path: "directory".into(), + reason: "non-symlink has a target".into(), + }); + } + Ok(Entry::Directory(metadata)) + } + FileType::Symlink => { + require_no_content(wire, "symlink")?; + let target = + wire.symlink_target + .as_deref() + .ok_or_else(|| PithosError::InvalidSymlinkEntry { + path: "symlink".into(), + reason: "missing target".into(), + })?; + Ok(Entry::Symlink { + metadata, + target: Arc::from(target), + }) + } + } +} + +fn require_no_content( + entry: &crate::format::wire::FileEntry, + kind: &str, +) -> Result<(), PithosError> { + match &entry.block_data { + BlockDataState::Decrypted(blocks) if blocks.is_empty() => Ok(()), + BlockDataState::Decrypted(_) => Err(PithosError::InvalidBlockDataState(format!( + "{kind} has block references" + ))), + BlockDataState::Encrypted(_) => Err(PithosError::InvalidBlockDataState(format!( + "{kind} has encrypted content" + ))), + } +} + +pub(crate) fn validate_entry(path: &ArchivePath, entry: &Entry) -> Result<(), PithosError> { + if let Entry::Symlink { target, .. } = entry { + validate_symlink_target(path, target)?; + } + Ok(()) +} + +fn validate_symlink_target(path: &ArchivePath, target: &str) -> Result<(), PithosError> { + if target.is_empty() + || target.contains('\0') + || target.contains('\\') + || target.starts_with('/') + || target.as_bytes().get(1) == Some(&b':') + { + return Err(invalid_target(path, target, "invalid target")); + } + let mut depth = path.as_str().split('/').count().saturating_sub(1); + for component in target.split('/') { + if component.is_empty() || component == "." { + return Err(invalid_target(path, target, "empty or dot component")); + } + if component == ".." { + if depth == 0 { + return Err(invalid_target(path, target, "target escapes archive root")); + } + depth -= 1; + } else { + depth += 1; + } + } + Ok(()) +} + +fn invalid_target(path: &ArchivePath, target: &str, reason: &str) -> PithosError { + PithosError::InvalidSymlinkTarget { + path: path.as_str().into(), + target: target.into(), + reason: reason.into(), + } +} + +pub(crate) fn validate_hierarchy( + entries: &[crate::archive::index::IndexedEntry], + exact_paths: &HashMap, usize>, +) -> Result<(), PithosError> { + for entry in entries { + let path = entry.path.as_str(); + // Every hierarchy conflict has a direct slash-delimited ancestor. Looking + // up only those prefixes avoids scanning lexically preceding siblings. + for (offset, _) in path.match_indices('/') { + let ancestor = &path[..offset]; + if let Some(ancestor_index) = exact_paths.get(ancestor) + && !entries[*ancestor_index].entry.is_directory() + { + return Err(PithosError::InvalidArchivePath { + path: path.into(), + reason: format!("file entry {ancestor} is an ancestor"), + }); + } + } + } + Ok(()) +} + +pub(crate) fn validate_aggregate( + segments: &[ValidatedSegment], + limits: IndexLimits, +) -> Result<(), PithosError> { + if segments.len() as u64 > limits.max_segments { + return Err(PithosError::LimitExceeded { + field: "directory segments", + limit: limits.max_segments, + actual: segments.len() as u64, + }); + } + let (entries, descriptors, references, relationships) = segments.iter().try_fold( + (0u64, 0u64, 0u64, 0u64), + |(entries, descriptors, references, relationships), segment| { + let entry_count = u64::try_from(segment.entries.len()).unwrap_or(u64::MAX); + let descriptor_count = u64::try_from(segment.descriptors.len()).unwrap_or(u64::MAX); + let reference_count = segment + .entries + .iter() + .try_fold(0u64, |total, entry| { + total.checked_add( + u64::try_from(entry.entry.metadata().references.len()).unwrap_or(u64::MAX), + ) + }) + .ok_or(PithosError::LimitExceeded { + field: "references", + limit: limits.max_references, + actual: u64::MAX, + })?; + Ok::<_, PithosError>(( + entries.saturating_add(entry_count), + descriptors.saturating_add(descriptor_count), + references.saturating_add(reference_count), + relationships + .saturating_add(u64::try_from(segment.relationships.len()).unwrap_or(u64::MAX)), + )) + }, + )?; + for (field, actual, limit) in [ + ("entries", entries, limits.max_entries), + ("block descriptors", descriptors, limits.max_descriptors), + ("references", references, limits.max_references), + ( + "relationship definitions", + relationships, + limits.max_relationships, + ), + ] { + if actual > limit { + return Err(PithosError::LimitExceeded { + field, + limit, + actual, + }); + } + } + Ok(()) +} diff --git a/crates/pithos_lib/src/helpers/archive_path.rs b/crates/pithos_lib/src/helpers/archive_path.rs deleted file mode 100644 index b838ac4..0000000 --- a/crates/pithos_lib/src/helpers/archive_path.rs +++ /dev/null @@ -1,444 +0,0 @@ -use crate::error::PithosError; -use crate::helpers::file_entry_map::FileEntryMap; -use crate::model::structs::{BlockDataState, FileEntry, FileType}; - -fn invalid_path(path: &str, reason: impl Into) -> PithosError { - PithosError::InvalidArchivePath { - path: path.to_string(), - reason: reason.into(), - } -} - -pub(crate) fn validate_entry_path(path: &str) -> Result<(), PithosError> { - if path.is_empty() { - return Err(invalid_path(path, "path is empty")); - } - if path.contains('\0') { - return Err(invalid_path(path, "NUL is not allowed")); - } - if path.contains('\\') { - return Err(invalid_path(path, "backslash is not allowed")); - } - if path.starts_with('/') || path.ends_with('/') { - return Err(invalid_path(path, "path must not start or end with /")); - } - if path.as_bytes().get(1) == Some(&b':') { - return Err(invalid_path(path, "drive forms are not allowed")); - } - for component in path.split('/') { - if component.is_empty() { - return Err(invalid_path(path, "empty path components are not allowed")); - } - if component == "." || component == ".." { - return Err(invalid_path(path, "dot components are not allowed")); - } - } - Ok(()) -} - -pub(crate) fn validate_symlink_target(path: &str, target: &str) -> Result<(), PithosError> { - validate_entry_path(path).map_err(|error| PithosError::InvalidSymlinkTarget { - path: path.to_string(), - target: target.to_string(), - reason: error.to_string(), - })?; - if target.is_empty() { - return Err(PithosError::InvalidSymlinkTarget { - path: path.to_string(), - target: target.to_string(), - reason: "target is empty".into(), - }); - } - if target.contains('\0') || target.contains('\\') { - return Err(PithosError::InvalidSymlinkTarget { - path: path.to_string(), - target: target.to_string(), - reason: "invalid separator or NUL".into(), - }); - } - if target.starts_with('/') || target.as_bytes().get(1) == Some(&b':') { - return Err(PithosError::InvalidSymlinkTarget { - path: path.to_string(), - target: target.to_string(), - reason: "absolute or drive target".into(), - }); - } - let mut depth = path.split('/').count() - 1; - for component in target.split('/') { - if component.is_empty() || component == "." { - return Err(PithosError::InvalidSymlinkTarget { - path: path.to_string(), - target: target.to_string(), - reason: "empty or dot component".into(), - }); - } - if component == ".." { - if depth == 0 { - return Err(PithosError::InvalidSymlinkTarget { - path: path.to_string(), - target: target.to_string(), - reason: "target escapes archive root".into(), - }); - } - depth -= 1; - } else { - depth += 1; - } - } - Ok(()) -} - -pub(crate) fn validate_entry(path: &str, entry: &FileEntry) -> Result<(), PithosError> { - validate_entry_path(path)?; - match (&entry.file_type, &entry.symlink_target, &entry.block_data) { - (FileType::Symlink, Some(target), BlockDataState::Decrypted(blocks)) - if blocks.is_empty() => - { - validate_symlink_target(path, target) - } - (FileType::Symlink, None, _) => Err(PithosError::InvalidSymlinkEntry { - path: path.into(), - reason: "missing target".into(), - }), - (FileType::Symlink, Some(_), BlockDataState::Decrypted(blocks)) if !blocks.is_empty() => { - Err(PithosError::InvalidSymlinkEntry { - path: path.into(), - reason: "symlink has block references".into(), - }) - } - (FileType::Symlink, Some(_), BlockDataState::Encrypted(_)) => { - Err(PithosError::InvalidSymlinkEntry { - path: path.into(), - reason: "encrypted symlink block data".into(), - }) - } - (_, Some(_), _) => Err(PithosError::InvalidSymlinkEntry { - path: path.into(), - reason: "non-symlink has a target".into(), - }), - _ => Ok(()), - } -} - -fn validate_candidate_hierarchy( - map: &FileEntryMap, - path: &str, - entry: &FileEntry, -) -> Result<(), PithosError> { - for (index, _) in path.match_indices('/') { - let ancestor = &path[..index]; - if map - .get_by_path(ancestor) - .is_some_and(|existing| existing.file_type != FileType::Directory) - { - return Err(PithosError::InvalidArchivePath { - path: path.into(), - reason: format!("file entry {ancestor} is an ancestor"), - }); - } - } - - if entry.file_type != FileType::Directory - && let Some(successor) = map.first_path_after(path) - && successor.starts_with(path) - && successor.as_bytes().get(path.len()) == Some(&b'/') - { - return Err(PithosError::InvalidArchivePath { - path: path.into(), - reason: format!("entry is an ancestor of {successor}"), - }); - } - - Ok(()) -} - -pub(crate) fn validate_existing_candidate( - map: &FileEntryMap, - path: &str, - entry: &FileEntry, -) -> Result<(), PithosError> { - validate_entry(path, entry)?; - validate_candidate_hierarchy(map, path, entry) -} - -pub(crate) fn validate_new_candidate( - map: &FileEntryMap, - path: &str, - entry: &FileEntry, -) -> Result<(), PithosError> { - validate_entry(path, entry)?; - if map.get_by_path(path).is_some() { - return Err(PithosError::PathOccupied(format!( - "File path already occupied: {path}" - ))); - } - validate_candidate_hierarchy(map, path, entry) -} - -pub(crate) fn validate_map(map: &FileEntryMap) -> Result<(), PithosError> { - for (_, path, entry) in map { - validate_entry(path, entry)?; - } - - validate_hierarchy(map) -} - -pub(crate) fn validate_hierarchy(map: &FileEntryMap) -> Result<(), PithosError> { - let mut entries = map.iter_ordered(); - let Some((mut previous_path, mut previous_entry)) = entries.next() else { - return Ok(()); - }; - - for (path, entry) in entries { - if previous_entry.file_type != FileType::Directory - && path.starts_with(previous_path) - && path.as_bytes().get(previous_path.len()) == Some(&b'/') - { - return Err(PithosError::InvalidArchivePath { - path: previous_path.into(), - reason: format!("entry is an ancestor of {path}"), - }); - } - - previous_path = path; - previous_entry = entry; - } - - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::helpers::file_entry_map::Key; - - fn entry(file_type: FileType, target: Option<&str>, blocks: BlockDataState) -> FileEntry { - FileEntry { - file_type, - block_data: blocks, - created: 0, - modified: 0, - file_size: 0, - permissions: 0o644, - references: vec![], - symlink_target: target.map(str::to_owned), - } - } - - #[test] - fn archive_path_valid_entry_corpus() { - for path in ["file", "nested/file", "ユニコード/file"] { - assert!(validate_entry_path(path).is_ok()); - } - } - #[test] - fn archive_path_invalid_entry_corpus() { - for path in [ - "", - "/file", - "file/", - "a//b", - "a/./b", - "a/../b", - "a\\b", - "C:", - "C:/x", - "\\\\server\\x", - "a\0b", - ] { - assert!(validate_entry_path(path).is_err(), "{path:?}"); - } - } - #[test] - fn archive_path_symlink_target_corpus() { - for target in ["target", "nested/target", "../target", "../dangling"] { - assert!(validate_symlink_target("nested/link", target).is_ok()); - } - for target in [ - "", - ".", - "target/.", - "target//child", - "target/", - "a\0b", - "a\\b", - "/absolute", - "C:", - "C:/target", - "\\\\server\\target", - "../target", - "../../target", - ] { - assert!( - validate_symlink_target("link", target).is_err(), - "{target:?}" - ); - } - assert!(validate_symlink_target("", "target").is_err()); - assert!(validate_symlink_target("bad//link", "target").is_err()); - assert!(validate_symlink_target("bad/../link", "target").is_err()); - } - - #[test] - fn archive_path_entry_invariants_corpus() { - assert!( - validate_entry( - "link", - &entry(FileType::Symlink, None, BlockDataState::Decrypted(vec![])) - ) - .is_err() - ); - assert!( - validate_entry( - "file", - &entry( - FileType::Data, - Some("target"), - BlockDataState::Decrypted(vec![]) - ) - ) - .is_err() - ); - assert!( - validate_entry( - "link", - &entry( - FileType::Symlink, - Some("target"), - BlockDataState::Decrypted(vec![([0; 32], [0; 32])]) - ) - ) - .is_err() - ); - assert!( - validate_entry( - "link", - &entry( - FileType::Symlink, - Some("target"), - BlockDataState::Encrypted(vec![]) - ) - ) - .is_err() - ); - } - - #[test] - fn archive_path_candidate_and_map_conflicts_are_order_independent() { - let file = entry(FileType::Data, None, BlockDataState::Decrypted(vec![])); - let link = entry( - FileType::Symlink, - Some("target"), - BlockDataState::Decrypted(vec![]), - ); - let directory = entry(FileType::Directory, None, BlockDataState::Decrypted(vec![])); - - for (ancestor_path, ancestor) in [("a", file.clone()), ("a", link.clone())] { - for order in [0, 1] { - let mut map = FileEntryMap::new(); - if order == 0 { - map.insert(Key::new(0, ancestor_path), ancestor.clone()) - .unwrap(); - map.insert(Key::new(1, "a/child"), file.clone()).unwrap(); - } else { - map.insert(Key::new(0, "a/child"), file.clone()).unwrap(); - map.insert(Key::new(1, ancestor_path), ancestor.clone()) - .unwrap(); - } - assert!(validate_map(&map).is_err()); - } - } - - for order in [0, 1] { - let mut map = FileEntryMap::new(); - if order == 0 { - map.insert(Key::new(0, "a/child"), file.clone()).unwrap(); - map.insert(Key::new(1, "a"), file.clone()).unwrap(); - } else { - map.insert(Key::new(0, "a"), file.clone()).unwrap(); - map.insert(Key::new(1, "a/child"), file.clone()).unwrap(); - } - assert!(validate_map(&map).is_err()); - } - - for order in [0, 1] { - let mut map = FileEntryMap::new(); - if order == 0 { - map.insert(Key::new(0, "a"), directory.clone()).unwrap(); - map.insert(Key::new(1, "a/child"), file.clone()).unwrap(); - } else { - map.insert(Key::new(0, "a/child"), file.clone()).unwrap(); - map.insert(Key::new(1, "a"), directory.clone()).unwrap(); - } - assert!(validate_map(&map).is_ok()); - } - } - - #[test] - fn archive_path_candidate_validation_handles_component_boundaries_and_depth() { - let file = entry(FileType::Data, None, BlockDataState::Decrypted(vec![])); - let directory = entry(FileType::Directory, None, BlockDataState::Decrypted(vec![])); - - let mut map = FileEntryMap::new(); - map.insert(Key::new(0, "a"), file.clone()).unwrap(); - assert!(validate_new_candidate(&map, "a/child", &file).is_err()); - - let mut map = FileEntryMap::new(); - map.insert(Key::new(0, "a/child"), file.clone()).unwrap(); - assert!(validate_new_candidate(&map, "a", &file).is_err()); - - let mut map = FileEntryMap::new(); - map.insert(Key::new(0, "a"), directory.clone()).unwrap(); - assert!(validate_new_candidate(&map, "a/child", &file).is_ok()); - - let mut map = FileEntryMap::new(); - for (id, path) in [(0, "ab"), (1, "a!"), (2, "a.b"), (3, "a/child")] { - map.insert(Key::new(id, path), file.clone()).unwrap(); - } - assert!(validate_new_candidate(&map, "a", &file).is_err()); - assert!(validate_new_candidate(&map, "a!x", &file).is_ok()); - assert!(validate_new_candidate(&map, "a.bx", &file).is_ok()); - assert!(validate_new_candidate(&map, "abx", &file).is_ok()); - - let mut map = FileEntryMap::new(); - map.insert(Key::new(0, "ユニコード"), file.clone()).unwrap(); - assert!(validate_new_candidate(&map, "ユニコード/子", &file).is_err()); - - let deep = (0..64) - .map(|part| format!("part{part}")) - .collect::>(); - let ancestor = deep.join("/"); - let descendant = format!("{ancestor}/leaf"); - let mut map = FileEntryMap::new(); - map.insert(Key::new(0, descendant), file.clone()).unwrap(); - assert!(validate_new_candidate(&map, &ancestor, &file).is_err()); - } - - #[test] - fn archive_path_existing_and_new_candidate_exact_path_semantics() { - let file = entry(FileType::Data, None, BlockDataState::Decrypted(vec![])); - let mut map = FileEntryMap::new(); - map.insert(Key::new(0, "occupied"), file.clone()).unwrap(); - - assert!(validate_existing_candidate(&map, "occupied", &file).is_ok()); - assert!(matches!( - validate_new_candidate(&map, "occupied", &file), - Err(PithosError::PathOccupied(message)) if message == "File path already occupied: occupied" - )); - } - - #[test] - fn archive_path_component_order_keeps_descendants_adjacent() { - let file = entry(FileType::Data, None, BlockDataState::Decrypted(vec![])); - let mut map = FileEntryMap::new(); - for (id, path) in [(0, "a"), (1, "a!"), (2, "a/child")] { - map.insert(Key::new(id, path), file.clone()).unwrap(); - } - - assert_eq!( - map.iter_ordered().map(|(path, _)| path).collect::>(), - ["a", "a/child", "a!"] - ); - assert!(validate_map(&map).is_err()); - } -} From 72d67461ea50337ffdb7e701cffef4c389771bab Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Tue, 11 Aug 2026 22:59:57 +0200 Subject: [PATCH 005/166] refactor: isolate crypto and block processing --- crates/pithos_lib/src/block.rs | 261 ++++++++++++ crates/pithos_lib/src/crypto.rs | 388 ++++++++++++++++++ .../pithos_lib/src/helpers/chacha_poly1305.rs | 69 ---- crates/pithos_lib/src/helpers/hash.rs | 45 -- crates/pithos_lib/src/helpers/x25519_keys.rs | 291 ------------- crates/pithos_lib/src/helpers/zstd.rs | 69 ---- crates/pithos_lib/tests/chacha_poly1305.rs | 48 --- crates/pithos_lib/tests/crypto.rs | 63 +++ crates/pithos_lib/tests/x25519.rs | 156 ------- 9 files changed, 712 insertions(+), 678 deletions(-) create mode 100644 crates/pithos_lib/src/block.rs create mode 100644 crates/pithos_lib/src/crypto.rs delete mode 100644 crates/pithos_lib/src/helpers/chacha_poly1305.rs delete mode 100644 crates/pithos_lib/src/helpers/hash.rs delete mode 100644 crates/pithos_lib/src/helpers/x25519_keys.rs delete mode 100644 crates/pithos_lib/src/helpers/zstd.rs delete mode 100644 crates/pithos_lib/tests/chacha_poly1305.rs create mode 100644 crates/pithos_lib/tests/crypto.rs delete mode 100644 crates/pithos_lib/tests/x25519.rs diff --git a/crates/pithos_lib/src/block.rs b/crates/pithos_lib/src/block.rs new file mode 100644 index 0000000..5720cd0 --- /dev/null +++ b/crates/pithos_lib/src/block.rs @@ -0,0 +1,261 @@ +//! Pure current-protocol block transformation and verification. + +use crate::crypto::{self, BlockKey}; +use crate::error::PithosError; +use crate::format::wire::{BlockIndexEntry, ProcessingFlags}; +use std::fmt; +use zeroize::Zeroizing; +use zstd::bulk; + +#[derive(Clone, Copy, Debug)] +pub(crate) struct Limits { + pub max_stored_bytes: u64, + pub max_decoded_bytes: u64, +} + +pub struct EncodedBlock { + pub(crate) stored: Zeroizing>, + pub(crate) hash: [u8; 32], + pub(crate) key: BlockKey, + pub(crate) flags: ProcessingFlags, +} + +impl fmt::Debug for EncodedBlock { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("EncodedBlock") + .field("stored_len", &self.stored.len()) + .field("hash", &self.hash) + .field("flags", &self.flags) + .finish_non_exhaustive() + } +} + +pub fn encode( + plaintext: &[u8], + requested_flags: ProcessingFlags, + nonce: [u8; 12], +) -> Result { + let hash = crypto::block_hash(plaintext); + let key = crypto::derive_block_key(plaintext); + let mut flags = requested_flags; + let compression_level = zstd_level(flags); + let mut stored = Zeroizing::new( + if compression_level > 0 && probe_ratio(plaintext, compression_level)? < 0.85 { + compress(plaintext, compression_level)? + } else { + flags.set_compression_level(0); + plaintext.to_vec() + }, + ); + if flags.is_encrypted() { + stored = Zeroizing::new(crypto::seal_block_with_nonce(&key, &stored, nonce)?); + } + Ok(EncodedBlock { + stored, + hash, + key, + flags, + }) +} + +pub(crate) fn verify( + stored: impl Into>>, + key: &BlockKey, + expected_hash: [u8; 32], + meta: &BlockIndexEntry, + limits: Limits, +) -> Result>, PithosError> { + let stored = stored.into(); + if meta.stored_size > limits.max_stored_bytes { + return Err(PithosError::LimitExceeded { + field: "stored block", + limit: limits.max_stored_bytes, + actual: meta.stored_size, + }); + } + if meta.original_size > limits.max_decoded_bytes { + return Err(PithosError::LimitExceeded { + field: "decoded block", + limit: limits.max_decoded_bytes, + actual: meta.original_size, + }); + } + if stored.len() as u64 != meta.stored_size { + return Err(PithosError::BlockSizeMismatch { + expected: meta.stored_size, + actual: stored.len() as u64, + }); + } + let mut plaintext = if meta.flags.is_encrypted() { + crypto::open_block(key, &stored)? + } else { + stored + }; + if meta.flags.get_compression_level() > 0 { + plaintext = Zeroizing::new(decompress(&plaintext, meta.original_size)?); + } + if plaintext.len() as u64 != meta.original_size { + return Err(PithosError::BlockSizeMismatch { + expected: meta.original_size, + actual: plaintext.len() as u64, + }); + } + let actual_hash = crypto::block_hash(&plaintext); + if actual_hash != expected_hash { + return Err(PithosError::BlockHashMismatch { + expected: expected_hash, + actual: actual_hash, + }); + } + Ok(plaintext) +} + +pub(crate) fn zstd_level(flags: ProcessingFlags) -> i32 { + match flags.get_compression_level() { + 0 => 0, + 1 => 1, + 2 => 4, + 3 => 8, + 4 => 11, + 5 => 15, + 6 => 18, + _ => 22, + } +} + +fn probe_ratio(input: &[u8], level: i32) -> Result { + if input.is_empty() { + return Ok(1.0); + } + let sample = &input[..input.len().min(4096)]; + let compressed = Zeroizing::new(compress(sample, level)?); + Ok(compressed.len() as f64 / sample.len() as f64) +} + +fn compress(input: &[u8], level: i32) -> Result, PithosError> { + bulk::compress(input, level).map_err(|source| PithosError::Compression { + operation: "compress block", + source, + }) +} + +fn decompress(input: &[u8], expected_size: u64) -> Result, PithosError> { + let size = usize::try_from(expected_size).map_err(|_| PithosError::InvalidDirectoryRange { + operation: "convert decoded block size", + })?; + bulk::decompress(input, size).map_err(|source| PithosError::Compression { + operation: "decompress block", + source, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::format::wire::BlockLocation; + + fn descriptor(encoded: &EncodedBlock, original_size: usize) -> BlockIndexEntry { + BlockIndexEntry { + offset: 0, + stored_size: encoded.stored.len() as u64, + original_size: original_size as u64, + flags: encoded.flags, + location: BlockLocation::Local, + } + } + + #[test] + fn controlled_nonce_vector_round_trips_without_changing_the_protocol_shape() { + let plain = b"current protocol vector"; + let encoded = encode(plain, ProcessingFlags::new(true, Some(2)), [7; 12]).unwrap(); + assert_eq!(encoded.hash, *blake3::hash(plain).as_bytes()); + assert_eq!(&encoded.stored[..12], &[7; 12]); + assert_eq!( + &*verify( + encoded.stored, + &encoded.key, + encoded.hash, + &descriptor( + &encode(plain, ProcessingFlags::new(true, Some(2)), [7; 12]).unwrap(), + plain.len() + ), + Limits { + max_stored_bytes: 1024, + max_decoded_bytes: 1024 + }, + ) + .unwrap(), + plain + ); + } + + #[test] + fn corruption_never_returns_unverified_plaintext() { + let plain = b"compressible ".repeat(1024); + let encoded = encode(&plain, ProcessingFlags::new(true, Some(3)), [9; 12]).unwrap(); + let meta = descriptor(&encoded, plain.len()); + for byte in [0, 12, encoded.stored.len() - 1] { + let mut corrupt = encoded.stored.clone(); + corrupt[byte] ^= 1; + assert!( + verify( + corrupt, + &BlockKey::from_bytes(*encoded.key.expose_for_protocol()), + encoded.hash, + &meta, + Limits { + max_stored_bytes: 1024 * 1024, + max_decoded_bytes: 1024 * 1024 + }, + ) + .is_err() + ); + } + let compressed = encode(&plain, ProcessingFlags::new(false, Some(3)), [0; 12]).unwrap(); + let compressed_meta = descriptor(&compressed, plain.len()); + let mut corrupt_compressed = compressed.stored.clone(); + corrupt_compressed[0] ^= 1; + assert!( + verify( + corrupt_compressed, + &compressed.key, + compressed.hash, + &compressed_meta, + Limits { + max_stored_bytes: 1024 * 1024, + max_decoded_bytes: 1024 * 1024 + }, + ) + .is_err() + ); + let mut wrong_size = meta.clone(); + wrong_size.original_size -= 1; + assert!( + verify( + encoded.stored.clone(), + &BlockKey::from_bytes(*encoded.key.expose_for_protocol()), + encoded.hash, + &wrong_size, + Limits { + max_stored_bytes: 1024 * 1024, + max_decoded_bytes: 1024 * 1024 + }, + ) + .is_err() + ); + assert!( + verify( + encoded.stored, + &encoded.key, + [0; 32], + &meta, + Limits { + max_stored_bytes: 1024 * 1024, + max_decoded_bytes: 1024 * 1024 + }, + ) + .is_err() + ); + } +} diff --git a/crates/pithos_lib/src/crypto.rs b/crates/pithos_lib/src/crypto.rs new file mode 100644 index 0000000..97d1e14 --- /dev/null +++ b/crates/pithos_lib/src/crypto.rs @@ -0,0 +1,388 @@ +//! Current Pithos cryptographic roles and protocol operations. +//! +//! The byte formats deliberately remain in the established protocol: every sealed value is +//! `nonce || ciphertext || tag` and always uses empty associated data. + +use chacha20poly1305::{ + ChaCha20Poly1305, KeyInit, Nonce, + aead::{Aead, Generate, Payload}, +}; +use digest::{ExtendableOutput, Update, XofReader}; +use pkcs8::der::EncodePem; +use pkcs8::der::pem::PemLabel; +use pkcs8::{ + Document, LineEnding, ObjectIdentifier, PrivateKeyInfoRef, SecretDocument, + SubjectPublicKeyInfoRef, der, +}; +use std::fmt; +use std::str::FromStr; +use thiserror::Error; +use x25519_dalek::{PublicKey as DalekPublicKey, StaticSecret}; +use zeroize::{Zeroize, ZeroizeOnDrop, Zeroizing}; + +const X25519_PKCS8_DER_HEADER: [u8; 16] = [ + 0x30, 0x2E, 0x02, 0x01, 0x00, 0x30, 0x05, 0x06, 0x03, 0x2B, 0x65, 0x6E, 0x04, 0x22, 0x04, 0x20, +]; +const X25519_PUBLIC_KEY_DER_HEADER: [u8; 12] = [ + 0x30, 0x2A, 0x30, 0x05, 0x06, 0x03, 0x2B, 0x65, 0x6E, 0x03, 0x21, 0x00, +]; + +#[derive(Debug, Error)] +pub enum CryptoError { + #[error("invalid key document")] + InvalidKeyDocument, + #[error("invalid private key")] + InvalidPrivateKey, + #[error("invalid public key")] + InvalidPublicKey, + #[error("X25519 public key does not produce a contributory shared secret")] + NonContributoryPublicKey, + #[error("key document encoding failed")] + KeyDocumentEncoding, + #[error("encrypted payload is too short")] + EncryptedPayloadTooShort, + #[error("cipher initialization failed")] + CipherInitialization, + #[error("payload authentication failed")] + AuthenticationFailed, +} + +/// An owned X25519 private key. Its debug representation and errors never include bytes. +pub struct PrivateKey([u8; 32]); + +/// An X25519 public key suitable for a Pithos recipient or sender identity. +#[derive(Clone, Copy, PartialEq, Eq, Hash)] +pub struct PublicKey([u8; 32]); + +macro_rules! secret_role { + ($name:ident) => { + pub(crate) struct $name([u8; 32]); + + impl $name { + // SharedSecret does not construct directly, but this uniform role macro + // supplies the constructor used by FileKey and BlockKey. + #[allow(dead_code)] + pub(crate) fn from_bytes(bytes: [u8; 32]) -> Self { + Self(bytes) + } + + // SharedSecret is recovered only through derive_shared, while FileKey + // and BlockKey need this wire-to-owned-key conversion. + #[allow(dead_code)] + pub(crate) fn from_protocol(bytes: &[u8; 32]) -> Self { + Self(*bytes) + } + + pub(crate) fn expose_for_protocol(&self) -> &[u8; 32] { + &self.0 + } + } + + impl Drop for $name { + fn drop(&mut self) { + self.0.zeroize(); + } + } + + impl ZeroizeOnDrop for $name {} + + impl fmt::Debug for $name { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(concat!(stringify!($name), "([REDACTED])")) + } + } + }; +} + +secret_role!(FileKey); +secret_role!(BlockKey); +secret_role!(SharedSecret); + +impl PrivateKey { + pub fn generate() -> Self { + Self(StaticSecret::random().to_bytes()) + } + + /// Duplicate a secret only when a distinct owner is required by an API boundary. + pub fn duplicate(&self) -> Self { + Self(self.0) + } + + pub fn public_key(&self) -> PublicKey { + PublicKey(DalekPublicKey::from(&StaticSecret::from(self.0)).to_bytes()) + } + + pub fn from_private_pem_bytes(pem: &[u8]) -> Result { + // `SecretDocument` owns the decoded DER; no ordinary String retains private PEM bytes. + let pem = std::str::from_utf8(pem).map_err(|_| CryptoError::InvalidKeyDocument)?; + let (label, document) = + SecretDocument::from_pem(pem).map_err(|_| CryptoError::InvalidKeyDocument)?; + PrivateKeyInfoRef::validate_pem_label(label) + .map_err(|_| CryptoError::InvalidKeyDocument)?; + let bytes = document.as_bytes(); + if bytes.len() != 48 || bytes[..16] != X25519_PKCS8_DER_HEADER { + return Err(CryptoError::InvalidPrivateKey); + } + let mut secret = Self([0; 32]); + secret.0.copy_from_slice(&bytes[16..]); + Ok(secret) + } + + pub fn to_private_pem_bytes(&self) -> Result>, CryptoError> { + let mut private_key = Zeroizing::new([0u8; 34]); + private_key[0] = 0x04; + private_key[1] = 0x20; + private_key[2..].copy_from_slice(&self.0); + let private_key = der::asn1::OctetStringRef::new(&private_key[..]) + .map_err(|_| CryptoError::KeyDocumentEncoding)?; + let info = PrivateKeyInfoRef { + algorithm: pkcs8::AlgorithmIdentifierRef { + oid: ObjectIdentifier::from_str("1.3.101.110").expect("X25519 OID is valid"), + parameters: None, + }, + private_key, + public_key: None, + }; + let document = + SecretDocument::encode_msg(&info).map_err(|_| CryptoError::KeyDocumentEncoding)?; + document + .to_pem(PrivateKeyInfoRef::PEM_LABEL, LineEnding::LF) + .map(|pem| Zeroizing::new(pem.as_bytes().to_vec())) + .map_err(|_| CryptoError::KeyDocumentEncoding) + } + + pub(crate) fn as_dalek_static_secret(&self) -> StaticSecret { + StaticSecret::from(self.0) + } + + pub(crate) fn into_dalek_static_secret(self) -> StaticSecret { + StaticSecret::from(self.0) + } + + #[cfg(test)] + pub(crate) fn from_dalek_static_secret(key: &StaticSecret) -> Self { + Self(key.to_bytes()) + } +} + +impl Drop for PrivateKey { + fn drop(&mut self) { + self.0.zeroize(); + } +} + +impl ZeroizeOnDrop for PrivateKey {} + +impl fmt::Debug for PrivateKey { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("PrivateKey([REDACTED])") + } +} + +impl PublicKey { + pub fn from_public_pem_bytes(pem: &[u8]) -> Result { + let pem = std::str::from_utf8(pem).map_err(|_| CryptoError::InvalidKeyDocument)?; + let (label, document) = + Document::from_pem(pem).map_err(|_| CryptoError::InvalidKeyDocument)?; + SubjectPublicKeyInfoRef::validate_pem_label(label) + .map_err(|_| CryptoError::InvalidKeyDocument)?; + let bytes = document.as_bytes(); + if bytes.len() != 44 || bytes[..12] != X25519_PUBLIC_KEY_DER_HEADER { + return Err(CryptoError::InvalidPublicKey); + } + let mut key = [0; 32]; + key.copy_from_slice(&bytes[12..]); + validate_contributory_public_key(&key)?; + Ok(Self(key)) + } + + pub fn to_public_pem_bytes(&self) -> Result, CryptoError> { + let info = SubjectPublicKeyInfoRef { + algorithm: pkcs8::AlgorithmIdentifierRef { + oid: ObjectIdentifier::new_unwrap("1.3.101.110"), + parameters: None, + }, + subject_public_key: der::asn1::BitStringRef::from_bytes(&self.0) + .map_err(|_| CryptoError::KeyDocumentEncoding)?, + }; + info.to_pem(LineEnding::LF) + .map(|pem| pem.into_bytes()) + .map_err(|_| CryptoError::KeyDocumentEncoding) + } + + pub(crate) fn into_dalek_public_key(self) -> DalekPublicKey { + DalekPublicKey::from(self.0) + } +} + +impl fmt::Debug for PublicKey { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str("PublicKey(..)") + } +} + +pub fn generate_private_key() -> PrivateKey { + PrivateKey::generate() +} + +pub fn parse_private_pem(pem: &[u8]) -> Result { + PrivateKey::from_private_pem_bytes(pem) +} + +pub fn parse_public_pem(pem: &[u8]) -> Result { + PublicKey::from_public_pem_bytes(pem) +} + +pub fn serialize_public_pem(key: &PublicKey) -> Result, CryptoError> { + key.to_public_pem_bytes() +} + +pub(crate) fn derive_shared( + private: &[u8; 32], + public: &[u8; 32], +) -> Result { + let private = StaticSecret::from(*private); + let shared = private.diffie_hellman(&DalekPublicKey::from(*public)); + if !shared.was_contributory() { + return Err(CryptoError::NonContributoryPublicKey); + } + Ok(SharedSecret(shared.to_bytes())) +} + +fn validate_contributory_public_key(public: &[u8; 32]) -> Result<(), CryptoError> { + // Low-order public keys produce the identity for every clamped X25519 scalar. + let probe = StaticSecret::from([0; 32]).diffie_hellman(&DalekPublicKey::from(*public)); + if probe.was_contributory() { + Ok(()) + } else { + Err(CryptoError::NonContributoryPublicKey) + } +} + +pub(crate) fn derive_block_key(plaintext: &[u8]) -> BlockKey { + let mut shake = shake::Shake256::default(); + shake.update(plaintext); + let mut key = BlockKey([0; 32]); + shake.finalize_xof().read(&mut key.0); + key +} + +pub(crate) fn block_hash(plaintext: &[u8]) -> [u8; 32] { + *blake3::hash(plaintext).as_bytes() +} + +pub(crate) fn seal_block_with_nonce( + key: &BlockKey, + plaintext: &[u8], + nonce: [u8; 12], +) -> Result, CryptoError> { + seal_empty_aad(key.expose_for_protocol(), plaintext, nonce) +} + +pub(crate) fn open_block( + key: &BlockKey, + payload: &[u8], +) -> Result>, CryptoError> { + open_empty_aad(key.expose_for_protocol(), payload) +} + +pub(crate) fn seal_file_block_list_with_nonce( + key: &FileKey, + plaintext: &[u8], + nonce: [u8; 12], +) -> Result, CryptoError> { + seal_empty_aad(key.expose_for_protocol(), plaintext, nonce) +} + +pub(crate) fn open_file_block_list( + key: &FileKey, + payload: &[u8], +) -> Result>, CryptoError> { + open_empty_aad(key.expose_for_protocol(), payload) +} + +pub(crate) fn wrap_recipient_list_with_nonce( + key: &SharedSecret, + plaintext: &[u8], + nonce: [u8; 12], +) -> Result, CryptoError> { + seal_empty_aad(key.expose_for_protocol(), plaintext, nonce) +} + +pub(crate) fn unwrap_recipient_list( + key: &SharedSecret, + payload: &[u8], +) -> Result>, CryptoError> { + open_empty_aad(key.expose_for_protocol(), payload) +} + +pub(crate) fn random_nonce() -> [u8; 12] { + Nonce::generate().into() +} + +pub(crate) fn seal_crypt4gh_payload( + key: &[u8; 32], + plaintext: &[u8], +) -> Result, CryptoError> { + seal_empty_aad(key, plaintext, random_nonce()) +} + +fn seal_empty_aad( + key: &[u8; 32], + plaintext: &[u8], + nonce: [u8; 12], +) -> Result, CryptoError> { + let cipher = + ChaCha20Poly1305::new_from_slice(key).map_err(|_| CryptoError::CipherInitialization)?; + let ciphertext = cipher + .encrypt( + &Nonce::from(nonce), + Payload { + msg: plaintext, + aad: b"", + }, + ) + .map_err(|_| CryptoError::AuthenticationFailed)?; + let mut payload = Vec::with_capacity(12 + ciphertext.len()); + payload.extend_from_slice(&nonce); + payload.extend_from_slice(&ciphertext); + Ok(payload) +} + +fn open_empty_aad(key: &[u8; 32], payload: &[u8]) -> Result>, CryptoError> { + if payload.len() < 15 { + return Err(CryptoError::EncryptedPayloadTooShort); + } + let (nonce, ciphertext) = payload.split_at(12); + let cipher = + ChaCha20Poly1305::new_from_slice(key).map_err(|_| CryptoError::CipherInitialization)?; + let nonce: [u8; 12] = nonce + .try_into() + .map_err(|_| CryptoError::EncryptedPayloadTooShort)?; + cipher + .decrypt( + &Nonce::from(nonce), + Payload { + msg: ciphertext, + aad: b"", + }, + ) + .map(Zeroizing::new) + .map_err(|_| CryptoError::AuthenticationFailed) +} + +#[cfg(test)] +mod zeroization_tests { + use super::*; + + fn assert_zeroize_on_drop() {} + + #[test] + fn owned_secret_roles_have_drop_zeroization_contracts() { + assert_zeroize_on_drop::(); + assert_zeroize_on_drop::(); + assert_zeroize_on_drop::(); + assert_zeroize_on_drop::(); + assert_zeroize_on_drop::>>(); + } +} diff --git a/crates/pithos_lib/src/helpers/chacha_poly1305.rs b/crates/pithos_lib/src/helpers/chacha_poly1305.rs deleted file mode 100644 index 5dd9a6a..0000000 --- a/crates/pithos_lib/src/helpers/chacha_poly1305.rs +++ /dev/null @@ -1,69 +0,0 @@ -use chacha20poly1305::{ - ChaCha20Poly1305, KeyInit, Nonce, - aead::{Aead, Generate, Payload}, -}; -use thiserror::Error; - -#[derive(Error, Debug)] -pub enum ChaChaPoly1305Error { - #[error("Message too large: {0}")] - MessageTooLarge(usize), - #[error("Unable to initialize cipher from key: {0}")] - CipherInitError(String), - #[error("Unable to encrypt chunk: {0}")] - EncryptionError(String), - #[error("Unable to decrypt chunk: {0}")] - DecryptionError(String), - #[error("Invalid nonce size: {0}")] - InvalidNonce(usize), - #[error("Unexpected chunk size < 15: {0}")] - ChunkTooSmall(usize), -} - -#[tracing::instrument(level = "trace", skip(msg, aad, enc))] -pub fn encrypt_chunk(msg: &[u8], aad: &[u8], enc: &[u8]) -> Result, ChaChaPoly1305Error> { - let nonce = Nonce::generate(); - let payload = Payload { msg, aad }; - let cipher = ChaCha20Poly1305::new_from_slice(enc) - .map_err(|e| ChaChaPoly1305Error::CipherInitError(e.to_string()))?; - - // Encrypt chunk - let result = cipher - .encrypt(&nonce, payload) - .map_err(|e| ChaChaPoly1305Error::EncryptionError(e.to_string()))?; - - // Gather nonce, encrypted chunk and aad together - let mut bytes = Vec::new(); - bytes.extend(nonce.as_slice()); - bytes.extend(result.as_slice()); - bytes.extend(aad); - - Ok(bytes) -} - -#[tracing::instrument(level = "trace", skip(chunk, decryption_key))] -pub fn decrypt_chunk( - chunk: &[u8], - decryption_key: &[u8; 32], -) -> Result, ChaChaPoly1305Error> { - if chunk.len() < 15 { - return Err(ChaChaPoly1305Error::ChunkTooSmall(chunk.len())); - } - - let (nonce_slice, data) = chunk.split_at(12); - - let nonce = <&Nonce>::try_from(nonce_slice) - .map_err(|_| ChaChaPoly1305Error::InvalidNonce(nonce_slice.len()))?; - - let cipher = ChaCha20Poly1305::new_from_slice(decryption_key) - .map_err(|e| ChaChaPoly1305Error::CipherInitError(e.to_string()))?; - - let payload = Payload { - msg: data, - aad: b"", - }; - - cipher - .decrypt(nonce, payload) - .map_err(|e| ChaChaPoly1305Error::DecryptionError(e.to_string())) -} diff --git a/crates/pithos_lib/src/helpers/hash.rs b/crates/pithos_lib/src/helpers/hash.rs deleted file mode 100644 index f67b073..0000000 --- a/crates/pithos_lib/src/helpers/hash.rs +++ /dev/null @@ -1,45 +0,0 @@ -use digest::{ExtendableOutput, Update, XofReader}; -use shake::Shake256; - -pub struct Hasher { - blake3: blake3::Hasher, - shake256: Shake256, -} - -pub struct Hashes { - pub blake3: blake3::Hash, - pub shake256: [u8; 32], -} - -impl Default for Hasher { - fn default() -> Self { - Self::new() - } -} - -impl Hasher { - #[tracing::instrument(level = "trace", skip())] - pub fn new() -> Self { - Hasher { - blake3: blake3::Hasher::new(), - shake256: Shake256::default(), - } - } - - #[tracing::instrument(level = "trace", skip(self, bytes))] - pub fn update(&mut self, bytes: &[u8]) { - self.blake3.update(bytes); - self.shake256.update(bytes); - } - - #[tracing::instrument(level = "trace", skip(self))] - pub fn finalize(&mut self) -> Hashes { - let mut key_buf = [0u8; 32]; - self.shake256.clone().finalize_xof().read(&mut key_buf); - - Hashes { - blake3: self.blake3.finalize(), - shake256: key_buf, - } - } -} diff --git a/crates/pithos_lib/src/helpers/x25519_keys.rs b/crates/pithos_lib/src/helpers/x25519_keys.rs deleted file mode 100644 index 90633d0..0000000 --- a/crates/pithos_lib/src/helpers/x25519_keys.rs +++ /dev/null @@ -1,291 +0,0 @@ -//! X25519 Key Management Module -//! -//! This module provides functionality for generating, parsing, and converting X25519 keys -//! between x25519-dalek types and PEM-encoded PKCS#8 format. -//! -//! # Examples -//! -//! ```rust -//! use pithos_lib::helpers::x25519_keys::*; -//! -//! // Generate a new private key -//! let private_key = generate_private_key().unwrap(); -//! let public_key = x25519_dalek::PublicKey::from(&private_key); -//! -//! // Convert to PEM format -//! let private_pem = private_key_to_pem_bytes(&private_key).unwrap(); -//! let public_pem = public_key_to_pem_bytes(&public_key).unwrap(); -//! -//! // Parse from PEM format -//! let parsed_private = private_key_from_pem_bytes(&private_pem).unwrap(); -//! let parsed_public = public_key_from_pem_bytes(&public_pem).unwrap(); -//! ``` - -use pkcs8::der::EncodePem; -use pkcs8::der::pem::PemLabel; -use pkcs8::{ - Document, LineEnding, ObjectIdentifier, PrivateKeyInfoRef, SecretDocument, - SubjectPublicKeyInfoRef, der, -}; -use std::str::FromStr; -use thiserror::Error; -use x25519_dalek::{PublicKey, StaticSecret}; - -const X25519_PKCS8_DER_HEADER: [u8; 16] = [ - 0x30, 0x2E, // SEQUENCE (30), length 46 (2E) - 0x02, 0x01, 0x00, // INTEGER (02), length (01) version 0 (00) - 0x30, 0x05, // SEQUENCE (30), for algorithm identifier, length 5 (05) - 0x06, 0x03, 0x2B, 0x65, 0x6E, // OID for X25519 (1.3.101.110) - 0x04, 0x22, // OCTET STRING, length 34 (0x22) - 0x04, 0x20, // Inner OCTET STRING, length 32 (raw key follows) -]; - -const X25519_PUBLIC_KEY_DER_HEADER: [u8; 12] = [ - 0x30, 0x2A, // SEQUENCE, length 42 - 0x30, 0x05, // SEQUENCE for algorithm identifier - 0x06, 0x03, 0x2B, 0x65, 0x6E, // OID for X25519 (1.3.101.110) - 0x03, 0x21, // BIT STRING, length 33 - 0x00, // Unused bits (always 0) -]; - -/// Custom error type for cryptographic operations -#[derive(Error, Debug)] -pub enum CryptError { - /// Invalid PEM format or structure - #[error("Invalid PEM format: {0}")] - InvalidPemFormat(String), - /// Invalid private key data - #[error("Invalid private key: {0}")] - InvalidPrivateKey(String), - /// Invalid public key data - #[error("Invalid public key: {0}")] - InvalidPublicKey(String), - /// PKCS#8 encoding/decoding error - #[error("PKCS#8 error: {0}")] - Pkcs8Error(String), - /// Key generation failure - #[error("Key generation error: {0}")] - KeyGenerationError(String), -} - -/// Generate a new X25519 private key using cryptographically secure randomness -/// -/// # Returns -/// -/// Returns a `StaticSecret` that can be used for X25519 key agreement operations. -/// -/// # Examples -/// -/// ```rust -/// use pithos_lib::helpers::x25519_keys::generate_private_key; -/// let private_key = generate_private_key().unwrap(); -/// let public_key = x25519_dalek::PublicKey::from(&private_key); -/// ``` -#[tracing::instrument(level = "trace", skip())] -pub fn generate_private_key() -> Result { - let secret = StaticSecret::random(); - Ok(secret) -} - -/// Derive a shared secret using X25519 key agreement -/// -/// # Arguments -/// -/// * `private_key` - The local party's X25519 private key (`StaticSecret`) -/// * `public_key` - The remote party's X25519 public key (`PublicKey`) -/// -/// # Returns -/// -/// Returns a 32-byte shared secret as a `[u8;32]`. -/// -/// # Examples -/// -/// ```rust -/// use pithos_lib::helpers::x25519_keys::{generate_private_key, derive_shared_key}; -/// use x25519_dalek::PublicKey; -/// -/// let private_a = generate_private_key().unwrap(); -/// let public_a = PublicKey::from(&private_a); -/// let private_b = generate_private_key().unwrap(); -/// let public_b = PublicKey::from(&private_b); -/// -/// let shared_ab = derive_shared_key(&private_a, &public_b); -/// let shared_ba = derive_shared_key(&private_b, &public_a); -/// assert_eq!(shared_ab, shared_ba); -/// ``` -#[tracing::instrument(level = "trace", skip(private_key, public_key))] -pub fn derive_shared_key(private_key: &StaticSecret, public_key: &PublicKey) -> [u8; 32] { - let shared = private_key.diffie_hellman(public_key); - shared.to_bytes() -} - -/// Convert an X25519 private key to PEM-encoded PKCS#8 bytes -/// -/// # Arguments -/// -/// * `key` - The StaticSecret to convert -/// -/// # Returns -/// -/// Returns a Vec containing the PEM-encoded private key data. -/// -/// # Examples -/// -/// ```rust -/// use pithos_lib::helpers::x25519_keys::*; -/// let private_key = generate_private_key().unwrap(); -/// let pem_bytes = private_key_to_pem_bytes(&private_key).unwrap(); -/// println!("{}", String::from_utf8_lossy(&pem_bytes)); -/// ``` -#[tracing::instrument(level = "trace", skip(key))] -pub fn private_key_to_pem_bytes(key: &StaticSecret) -> Result, CryptError> { - // Serialize private key as nested OCTET STRING - let mut private_key = [0u8; 34]; - private_key[0] = 0x04; - private_key[1] = 0x20; - private_key[2..].copy_from_slice(key.as_bytes()); - - let private_key_octets = der::asn1::OctetStringRef::new(&private_key) - .map_err(|e| CryptError::Pkcs8Error(e.to_string()))?; - let private_key_info = PrivateKeyInfoRef { - algorithm: pkcs8::AlgorithmIdentifierRef { - oid: ObjectIdentifier::from_str("1.3.101.110").unwrap(), - parameters: None, // X25519 has no parameters - }, - private_key: private_key_octets, - public_key: None, - }; - - let secret_document = SecretDocument::encode_msg(&private_key_info) - .map_err(|e| CryptError::Pkcs8Error(e.to_string()))?; - - let pem = secret_document - .to_pem(PrivateKeyInfoRef::PEM_LABEL, LineEnding::LF) - .map_err(|e| CryptError::Pkcs8Error(e.to_string()))?; - Ok(pem.as_bytes().to_vec()) -} - -/// Parse an X25519 private key from PEM-encoded PKCS#8 bytes -/// -/// # Arguments -/// -/// * `pem_data` - The PEM-encoded private key bytes -/// -/// # Returns -/// -/// Returns a StaticSecret parsed from the PEM data. -/// -/// # Examples -/// -/// ```rust -/// use pithos_lib::helpers::x25519_keys::*; -/// let private_key = generate_private_key().unwrap(); -/// let pem_bytes = private_key_to_pem_bytes(&private_key).unwrap(); -/// let private_key = private_key_from_pem_bytes(&pem_bytes).unwrap(); -/// ``` -#[tracing::instrument(level = "trace", skip(pem_data))] -pub fn private_key_from_pem_bytes(pem_data: &[u8]) -> Result { - let pem_str = std::str::from_utf8(pem_data) - .map_err(|e| CryptError::InvalidPemFormat(format!("Invalid UTF-8: {e}")))?; - - let (label, doc) = SecretDocument::from_pem(pem_str) - .map_err(|e| CryptError::InvalidPemFormat(e.to_string()))?; - PrivateKeyInfoRef::validate_pem_label(label) - .map_err(|e| CryptError::InvalidPemFormat(e.to_string()))?; - - let key_bytes = match doc.as_bytes() { - bytes if bytes.len() == 48 && bytes[..16] == X25519_PKCS8_DER_HEADER => { - let mut key = [0u8; 32]; - key.copy_from_slice(&bytes[16..]); - key - } - _ => { - return Err(CryptError::InvalidPrivateKey( - "Invalid private key".to_string(), - )); - } - }; - - Ok(StaticSecret::from(key_bytes)) -} - -/// Convert an X25519 public key to PEM-encoded SubjectPublicKeyInfo bytes -/// -/// # Arguments -/// -/// * `key` - The PublicKey to convert -/// -/// # Returns -/// -/// Returns a Vec containing the PEM-encoded public key data. -/// -/// # Examples -/// -/// ```rust -/// use pithos_lib::helpers::x25519_keys::*; -/// let private_key = generate_private_key().unwrap(); -/// let public_key = x25519_dalek::PublicKey::from(&private_key); -/// let pem_bytes = public_key_to_pem_bytes(&public_key).unwrap(); -/// ``` -#[tracing::instrument(level = "trace", skip(key))] -pub fn public_key_to_pem_bytes(key: &PublicKey) -> Result, CryptError> { - let key_bytes = key.as_bytes(); - - // Create SubjectPublicKeyInfo - let public_key_info = SubjectPublicKeyInfoRef { - algorithm: pkcs8::AlgorithmIdentifierRef { - oid: ObjectIdentifier::new_unwrap("1.3.101.110"), // X25519 OID - parameters: None, - }, - subject_public_key: der::asn1::BitStringRef::from_bytes(key_bytes) - .map_err(|e| CryptError::Pkcs8Error(e.to_string()))?, - }; - - let pem_string = public_key_info - .to_pem(LineEnding::LF) - .map_err(|e| CryptError::Pkcs8Error(e.to_string()))?; - - Ok(pem_string.into_bytes()) -} - -/// Parse an X25519 public key from PEM-encoded SubjectPublicKeyInfo bytes -/// -/// # Arguments -/// -/// * `pem_data` - The PEM-encoded public key bytes -/// -/// # Returns -/// -/// Returns a PublicKey parsed from the PEM data. -/// -/// # Examples -/// -/// ```rust -/// use pithos_lib::helpers::x25519_keys::*; -/// let private_key = generate_private_key().unwrap(); -/// let public_key = x25519_dalek::PublicKey::from(&private_key); -/// let pem_bytes = public_key_to_pem_bytes(&public_key).unwrap(); -/// let public_key = public_key_from_pem_bytes(&pem_bytes).unwrap(); -/// ``` -#[tracing::instrument(level = "trace", skip(pem_data))] -pub fn public_key_from_pem_bytes(pem_data: &[u8]) -> Result { - let pem_str = std::str::from_utf8(pem_data) - .map_err(|e| CryptError::InvalidPemFormat(format!("Invalid UTF-8: {e}")))?; - - let (label, doc) = - Document::from_pem(pem_str).map_err(|e| CryptError::InvalidPemFormat(e.to_string()))?; - - SubjectPublicKeyInfoRef::validate_pem_label(label) - .map_err(|e| CryptError::InvalidPemFormat(e.to_string()))?; - - let public_key_bytes = match doc.as_bytes() { - bytes if bytes.len() == 44 && bytes[..12] == X25519_PUBLIC_KEY_DER_HEADER => { - let mut key = [0u8; 32]; - key.copy_from_slice(&bytes[12..]); - key - } - _ => return Err(CryptError::InvalidPublicKey("Invalid format".to_string())), - }; - - Ok(PublicKey::from(public_key_bytes)) -} diff --git a/crates/pithos_lib/src/helpers/zstd.rs b/crates/pithos_lib/src/helpers/zstd.rs deleted file mode 100644 index abe2c1c..0000000 --- a/crates/pithos_lib/src/helpers/zstd.rs +++ /dev/null @@ -1,69 +0,0 @@ -use crate::model::structs::ProcessingFlags; -use thiserror::Error; -use zstd::bulk; - -/// Custom error type for zstd compression operations -#[derive(Error, Debug)] -pub enum ZstdError { - /// Invalid input data - #[error("Invalid input data: {0}")] - InvalidInput(String), - /// Compression failure - #[error("Compression error: {0}")] - CompressionError(String), - /// Compression failure - #[error("Decompression error: {0}")] - DecompressionError(String), - #[error("decompressed size does not fit platform: {0}")] - SizeOverflow(u64), -} - -const _ZSTD_MAGIC_NUMBER: u32 = 0xFD2FB528; // 4 Bytes, little-endian format - -#[tracing::instrument(level = "trace", skip(flags))] -pub fn map_to_zstd_level(flags: &ProcessingFlags) -> i32 { - match flags.get_compression_level() { - 0 => 0, - 1 => 1, - 2 => 4, - 3 => 8, - 4 => 11, - 5 => 15, - 6 => 18, - 7 => 22, - _ => 22, // Everything above is just max compression - } -} - -/// Compresses a chunk of data with zstd and returns the compression ratio. -#[tracing::instrument(level = "trace", skip(input))] -pub fn probe_compression_ratio(input: &[u8], level: Option) -> Result { - if input.is_empty() { - return Ok(1.0); - } - - let orig_len = input.len(); - let probe_size = if orig_len > 4096 { 4096 } else { orig_len }; - let compressed = if orig_len > 4096 { - bulk::compress(&input[..probe_size], level.unwrap_or_default()) - .map_err(|e| ZstdError::CompressionError(e.to_string()))? - } else { - bulk::compress(input, level.unwrap_or_default()) - .map_err(|e| ZstdError::CompressionError(e.to_string()))? - }; - - Ok(compressed.len() as f64 / probe_size as f64) -} - -#[tracing::instrument(level = "trace", skip(input, level))] -pub fn compress_data(input: &[u8], level: Option) -> Result, ZstdError> { - bulk::compress(input, level.unwrap_or_default()) - .map_err(|e| ZstdError::CompressionError(e.to_string())) -} - -#[tracing::instrument(level = "trace", skip(input, decompressed_size))] -pub fn decompress_data(input: &[u8], decompressed_size: u64) -> Result, ZstdError> { - let size = usize::try_from(decompressed_size) - .map_err(|_| ZstdError::SizeOverflow(decompressed_size))?; - bulk::decompress(input, size).map_err(|e| ZstdError::DecompressionError(e.to_string())) -} diff --git a/crates/pithos_lib/tests/chacha_poly1305.rs b/crates/pithos_lib/tests/chacha_poly1305.rs deleted file mode 100644 index ee6359a..0000000 --- a/crates/pithos_lib/tests/chacha_poly1305.rs +++ /dev/null @@ -1,48 +0,0 @@ -use pithos_lib::helpers::chacha_poly1305::*; - -#[test] -fn test_encrypt_and_decrypt_chunk_roundtrip() { - let key = [42u8; 32]; - let msg = b"hello world"; - let aad = b""; - - let encrypted = encrypt_chunk(msg, aad, &key).expect("encryption failed"); - let decrypted = decrypt_chunk(&encrypted, &key).expect("decryption failed"); - - // The decrypted message should match the original - assert_eq!(&decrypted[..msg.len()], msg); -} - -#[test] -fn test_decrypt_chunk_too_small() { - let key = [42u8; 32]; - let chunk = [1u8; 10]; // less than 15 bytes - - let result = decrypt_chunk(&chunk, &key); - assert!(matches!(result, Err(ChaChaPoly1305Error::ChunkTooSmall(_)))); -} - -#[test] -fn test_decrypt_chunk_invalid_nonce() { - let key = [42u8; 32]; - let chunk = [1u8; 20]; - // Make nonce slice not 12 bytes - let result = decrypt_chunk(&chunk[..5], &key); - assert!( - matches!(result, Err(ChaChaPoly1305Error::ChunkTooSmall(_))) - || matches!(result, Err(ChaChaPoly1305Error::InvalidNonce(_))) - ); -} - -#[test] -fn test_encrypt_chunk_invalid_key() { - let msg = b"hello"; - let aad = b""; - let bad_key = [1u8; 10]; // too short - - let result = encrypt_chunk(msg, aad, &bad_key); - assert!(matches!( - result, - Err(ChaChaPoly1305Error::CipherInitError(_)) - )); -} diff --git a/crates/pithos_lib/tests/crypto.rs b/crates/pithos_lib/tests/crypto.rs new file mode 100644 index 0000000..5a8c5d6 --- /dev/null +++ b/crates/pithos_lib/tests/crypto.rs @@ -0,0 +1,63 @@ +use pithos_lib::crypto::{ + CryptoError, generate_private_key, parse_private_pem, parse_public_pem, serialize_public_pem, +}; +use pkcs8::{Document, LineEnding}; + +#[test] +fn curated_pem_api_preserves_fixture_bytes_and_redacts_private_debug() { + let private_pem = std::fs::read("tests/data/keys/sender_private.pem").unwrap(); + let private = parse_private_pem(&private_pem).unwrap(); + assert_eq!(&*private.to_private_pem_bytes().unwrap(), &private_pem); + assert!(!format!("{private:?}").contains("0f")); + + let public_pem = std::fs::read("tests/data/keys/sender_public.pem").unwrap(); + let public = parse_public_pem(&public_pem).unwrap(); + assert_eq!(serialize_public_pem(&public).unwrap(), public_pem); + assert_eq!( + private.public_key().to_public_pem_bytes().unwrap(), + public_pem + ); +} + +#[test] +fn generated_private_keys_are_owned_and_have_public_identities() { + let first = generate_private_key(); + let second = generate_private_key(); + assert_ne!( + first.public_key().to_public_pem_bytes().unwrap(), + second.public_key().to_public_pem_bytes().unwrap() + ); +} + +#[test] +fn malformed_pem_is_rejected_by_both_curated_parsers() { + let malformed = b"this is not valid PEM data"; + assert!(matches!( + parse_private_pem(malformed), + Err(CryptoError::InvalidKeyDocument) + )); + assert!(matches!( + parse_public_pem(malformed), + Err(CryptoError::InvalidKeyDocument) + )); +} + +#[test] +fn invalid_utf8_private_pem_is_rejected() { + assert!(matches!( + parse_private_pem(&[0xff, 0xfe, 0xfd]), + Err(CryptoError::InvalidKeyDocument) + )); +} + +#[test] +fn non_contributory_x25519_public_keys_are_rejected_at_every_public_boundary() { + let public_pem = std::fs::read_to_string("tests/data/keys/sender_public.pem").unwrap(); + let (_, document) = Document::from_pem(&public_pem).unwrap(); + let mut der = document.as_bytes().to_vec(); + der[12..].fill(0); + let document = Document::try_from(der.as_slice()).unwrap(); + let zero_public_pem = document.to_pem("PUBLIC KEY", LineEnding::LF).unwrap(); + + assert!(parse_public_pem(zero_public_pem.as_bytes()).is_err()); +} diff --git a/crates/pithos_lib/tests/x25519.rs b/crates/pithos_lib/tests/x25519.rs deleted file mode 100644 index 372ef2c..0000000 --- a/crates/pithos_lib/tests/x25519.rs +++ /dev/null @@ -1,156 +0,0 @@ -use pithos_lib::helpers::x25519_keys::*; -use std::fs; -use x25519_dalek::PublicKey; - -#[test] -fn test_generate_private_key() { - let key = generate_private_key().expect("Failed to generate private key"); - // Verify we can derive a public key - let _public_key = PublicKey::from(&key); -} - -#[test] -fn test_derive_shared_key() { - let private_a = generate_private_key().unwrap(); - let public_a = PublicKey::from(&private_a); - let private_b = generate_private_key().unwrap(); - let public_b = PublicKey::from(&private_b); - - let shared_ab = derive_shared_key(&private_a, &public_b); - let shared_ba = derive_shared_key(&private_b, &public_a); - - // The shared secrets should be equal - assert_eq!(shared_ab, shared_ba); - - // The shared secret should be 32 bytes - assert_eq!(shared_ab.len(), 32); -} - -#[test] -fn test_private_key_round_trip() { - let original_key = generate_private_key().expect("Failed to generate key"); - - // Convert to PEM - let pem_bytes = private_key_to_pem_bytes(&original_key).expect("Failed to convert to PEM"); - - // Parse back from PEM - let parsed_key = private_key_from_pem_bytes(&pem_bytes).expect("Failed to parse from PEM"); - - // Verify keys are equivalent by comparing derived public keys - let original_public = PublicKey::from(&original_key); - let parsed_public = PublicKey::from(&parsed_key); - - assert_eq!(original_public.as_bytes(), parsed_public.as_bytes()); -} - -#[test] -fn test_public_key_round_trip() { - let private_key = generate_private_key().expect("Failed to generate key"); - let original_public = PublicKey::from(&private_key); - - // Convert to PEM - let pem_bytes = public_key_to_pem_bytes(&original_public).expect("Failed to convert to PEM"); - - // Parse back from PEM - let parsed_public = public_key_from_pem_bytes(&pem_bytes).expect("Failed to parse from PEM"); - - assert_eq!(original_public.as_bytes(), parsed_public.as_bytes()); -} - -#[test] -fn test_fixture_pem_bytes_stable() { - let private_pem = fs::read("tests/data/keys/sender_private.pem").unwrap(); - let private_key = private_key_from_pem_bytes(&private_pem).unwrap(); - assert_eq!(private_key_to_pem_bytes(&private_key).unwrap(), private_pem); - - let public_pem = fs::read("tests/data/keys/sender_public.pem").unwrap(); - let public_key = public_key_from_pem_bytes(&public_pem).unwrap(); - assert_eq!(public_key_to_pem_bytes(&public_key).unwrap(), public_pem); -} - -#[test] -fn test_pem_format_validation() { - let private_key = generate_private_key().expect("Failed to generate key"); - let pem_bytes = private_key_to_pem_bytes(&private_key).expect("Failed to convert"); - let pem_str = String::from_utf8(pem_bytes).expect("Invalid UTF-8"); - - assert!(pem_str.contains("-----BEGIN PRIVATE KEY-----")); - assert!(pem_str.contains("-----END PRIVATE KEY-----")); -} - -#[test] -fn test_public_pem_format_validation() { - let private_key = generate_private_key().expect("Failed to generate key"); - let public_key = PublicKey::from(&private_key); - let pem_bytes = public_key_to_pem_bytes(&public_key).expect("Failed to convert"); - let pem_str = String::from_utf8(pem_bytes).expect("Invalid UTF-8"); - - assert!(pem_str.contains("-----BEGIN PUBLIC KEY-----")); - assert!(pem_str.contains("-----END PUBLIC KEY-----")); -} - -#[test] -fn test_invalid_pem_format() { - let invalid_pem = b"this is not valid PEM data"; - let result = private_key_from_pem_bytes(invalid_pem); - assert!(result.is_err()); - - let result = public_key_from_pem_bytes(invalid_pem); - assert!(result.is_err()); -} - -#[test] -fn test_invalid_utf8() { - let invalid_utf8 = vec![0xFF, 0xFE, 0xFD]; - let result = private_key_from_pem_bytes(&invalid_utf8); - assert!(matches!(result, Err(CryptError::InvalidPemFormat(_)))); -} - -#[test] -fn test_multiple_key_generation_uniqueness() { - let key1 = generate_private_key().expect("Failed to generate key1"); - let key2 = generate_private_key().expect("Failed to generate key2"); - - let public1 = PublicKey::from(&key1); - let public2 = PublicKey::from(&key2); - - // Keys should be different - assert_ne!(public1.as_bytes(), public2.as_bytes()); -} - -#[test] -fn test_error_display() { - let error = CryptError::InvalidPemFormat("test message".to_string()); - assert_eq!(format!("{error}"), "Invalid PEM format: test message"); - - let error = CryptError::KeyGenerationError("generation failed".to_string()); - assert_eq!( - format!("{error}"), - "Key generation error: generation failed" - ); -} - -// Property-based testing -#[test] -fn test_many_round_trips() { - for _ in 0..100 { - let private_key = generate_private_key().expect("Failed to generate key"); - let public_key = PublicKey::from(&private_key); - - // Test private key round trip - let private_pem = - private_key_to_pem_bytes(&private_key).expect("Failed to encode private key"); - - let parsed_private = - private_key_from_pem_bytes(&private_pem).expect("Failed to parse private key"); - let parsed_public_from_private = PublicKey::from(&parsed_private); - - // Test public key round trip - let public_pem = public_key_to_pem_bytes(&public_key).expect("Failed to encode public key"); - let parsed_public = - public_key_from_pem_bytes(&public_pem).expect("Failed to parse public key"); - - assert_eq!(public_key.as_bytes(), parsed_public_from_private.as_bytes()); - assert_eq!(public_key.as_bytes(), parsed_public.as_bytes()); - } -} From 9325f7d52ac2c0b270d9dce3c8e29f817960e034 Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Tue, 11 Aug 2026 23:00:15 +0200 Subject: [PATCH 006/166] refactor: replace reader with positioned sources --- crates/pithos_lib/src/archive/reader.rs | 763 +++++++ .../src/archive/reader_private_tests.rs | 677 ++++++ crates/pithos_lib/src/io/pithosreader.rs | 790 ------- crates/pithos_lib/src/source.rs | 294 +++ crates/pithos_lib/tests/access.rs | 23 + crates/pithos_lib/tests/block_pipeline.rs | 35 + crates/pithos_lib/tests/common/append.rs | 177 ++ crates/pithos_lib/tests/common/archive.rs | 12 + crates/pithos_lib/tests/common/fixtures.rs | 29 + crates/pithos_lib/tests/common/keys.rs | 17 + crates/pithos_lib/tests/common/mod.rs | 19 +- crates/pithos_lib/tests/common/ro_crate.rs | 136 ++ crates/pithos_lib/tests/common/util.rs | 166 -- crates/pithos_lib/tests/common/writer.rs | 9 + .../tests/data/t8.shakespeare.sample.txt | 2000 ----------------- crates/pithos_lib/tests/format_validation.rs | 160 ++ crates/pithos_lib/tests/reader.rs | 1570 +------------ crates/pithos_lib/tests/robustness.rs | 58 - crates/pithos_lib/tests/source_equivalence.rs | 226 ++ 19 files changed, 2684 insertions(+), 4477 deletions(-) create mode 100644 crates/pithos_lib/src/archive/reader.rs create mode 100644 crates/pithos_lib/src/archive/reader_private_tests.rs delete mode 100644 crates/pithos_lib/src/io/pithosreader.rs create mode 100644 crates/pithos_lib/src/source.rs create mode 100644 crates/pithos_lib/tests/access.rs create mode 100644 crates/pithos_lib/tests/block_pipeline.rs create mode 100644 crates/pithos_lib/tests/common/append.rs create mode 100644 crates/pithos_lib/tests/common/archive.rs create mode 100644 crates/pithos_lib/tests/common/fixtures.rs create mode 100644 crates/pithos_lib/tests/common/keys.rs create mode 100644 crates/pithos_lib/tests/common/ro_crate.rs delete mode 100644 crates/pithos_lib/tests/common/util.rs create mode 100644 crates/pithos_lib/tests/common/writer.rs delete mode 100644 crates/pithos_lib/tests/data/t8.shakespeare.sample.txt create mode 100644 crates/pithos_lib/tests/format_validation.rs delete mode 100644 crates/pithos_lib/tests/robustness.rs create mode 100644 crates/pithos_lib/tests/source_equivalence.rs diff --git a/crates/pithos_lib/src/archive/reader.rs b/crates/pithos_lib/src/archive/reader.rs new file mode 100644 index 0000000..6140c8b --- /dev/null +++ b/crates/pithos_lib/src/archive/reader.rs @@ -0,0 +1,763 @@ +use super::{ + AccessProvenance, AppendSnapshot, FileId, ResolvedAccess, Span, build_effective_index, + segment_from_wire, +}; +use crate::archive::index::ArchiveIndex; +use crate::archive::types::{ + ArchivePath, BlockHash, BlockLocation, ContentState, Entry, ExternalLocation, ReadRange, + ValidatedSegment, +}; +use crate::archive::validation::IndexLimits; +use crate::block; +use crate::crypto::{self, FileKey, PrivateKey}; +use crate::error::PithosError; +use crate::format::limits::DeserializationLimits; +use crate::format::wire::{BlockDataState, BlockIndexEntry, Directory, FileHeader}; +use crate::source::ArchiveSource; +use std::collections::HashSet; +use std::io::Write; + +/// Distinguishes archive failures from a presentation callback failure without +/// making the archive core depend on the callback's error type. +pub(crate) enum ContentOperationError { + Core(PithosError), + Callback(E), +} +use std::ops::Range; +use x25519_dalek::PublicKey as DalekPublicKey; +use zeroize::Zeroizing; + +#[derive(Default)] +struct DecodedDirectoryCounts { + entries: u64, + descriptors: u64, + references: u64, + relationships: u64, +} + +impl DecodedDirectoryCounts { + fn record(&mut self, directory: &Directory) { + self.entries += directory.files.len() as u64; + self.descriptors += directory.blocks.len() as u64; + self.references += directory + .files + .iter() + .map(|(_, _, file)| file.references.len() as u64) + .sum::(); + self.relationships += directory.relations.len() as u64; + } +} + +/// Limits enforced before directory data is retained or expensive metadata work begins. +#[derive(Clone, Copy, Debug)] +pub struct OpenLimits { + pub max_directory_bytes: u64, + pub max_total_directory_bytes: u64, + pub max_parent_directories: u64, + pub max_entries: u64, + pub max_descriptors: u64, + pub max_references: u64, + pub max_relationships: u64, + pub max_accessible_block_references: u64, + pub max_opaque_metadata_bytes: u64, + pub max_stored_block_bytes: u64, + pub max_decoded_block_bytes: u64, +} + +impl Default for OpenLimits { + fn default() -> Self { + Self { + max_directory_bytes: 64 * 1024 * 1024, + max_total_directory_bytes: 256 * 1024 * 1024, + max_parent_directories: 1024, + max_entries: 1_000_000, + max_descriptors: 1_000_000, + max_references: 1_000_000, + max_relationships: 1_000_000, + max_accessible_block_references: 1_000_000, + max_opaque_metadata_bytes: 64 * 1024 * 1024, + max_stored_block_bytes: 64 * 1024 * 1024, + max_decoded_block_bytes: 64 * 1024 * 1024, + } + } +} + +/// A finite ordered set of recipient keys used while opening encrypted metadata. +#[derive(Default)] +#[allow(clippy::vec_box)] // Stable allocations prevent stale secret copies when the key list grows. +pub struct AccessKeys(Vec>); + +impl AccessKeys { + pub fn new() -> Self { + Self::default() + } + + pub fn with_key(mut self, key: PrivateKey) -> Self { + self.0.push(Box::new(key)); + self + } + + pub fn push(&mut self, key: PrivateKey) { + self.0.push(Box::new(key)); + } +} + +/// Resolves an opaque external block location to exactly one framed `BLCK` value. +pub trait ExternalBlockResolver { + fn resolve( + &self, + location: &ExternalLocation, + expected_len: u64, + max_response_size: u64, + ) -> Result, PithosError>; +} + +/// The default resolver rejects external block reads. +#[derive(Clone, Copy, Debug, Default)] +pub struct NoExternalBlocks; + +impl ExternalBlockResolver for NoExternalBlocks { + fn resolve( + &self, + _location: &ExternalLocation, + _expected_len: u64, + _max_response_size: u64, + ) -> Result, PithosError> { + Err(PithosError::ExternalBlockSourceRequired) + } +} + +/// Options consumed by the single validated archive open boundary. +pub struct OpenOptions { + limits: OpenLimits, + keys: AccessKeys, + external: E, +} + +impl Default for OpenOptions { + fn default() -> Self { + Self { + limits: OpenLimits::default(), + keys: AccessKeys::default(), + external: NoExternalBlocks, + } + } +} + +impl OpenOptions { + pub fn with_limits(mut self, limits: OpenLimits) -> Self { + self.limits = limits; + self + } + + pub fn with_access_keys(mut self, keys: AccessKeys) -> Self { + self.keys = keys; + self + } + + pub fn with_external_resolver(self, external: T) -> OpenOptions { + OpenOptions { + limits: self.limits, + keys: self.keys, + external, + } + } +} + +/// Public, secret-free entry shape returned by an opened archive. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum EntryKind { + File { size: u64, available: bool }, + Metadata { size: u64, available: bool }, + Directory, + Symlink { target: String }, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ArchiveEntry { + pub id: u64, + pub path: String, + pub kind: EntryKind, + pub created: u64, + pub modified: u64, + pub permissions: u32, + pub references: Vec, +} + +/// A secret-free resolved relationship attached to an immutable archive entry. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct ArchiveReference { + pub target_id: u64, + pub relationship: String, +} + +/// Immutable validated archive state. Payload bytes are deliberately verified lazily. +/// +/// `copy_to` and `copy_range_to` verify a complete block before writing any bytes +/// from that block. A generic sink can therefore contain earlier verified blocks +/// when a later block fails. Filesystem extraction is provided by [`crate::fs::extract`], +/// which stages each regular-file output before publication. +pub struct Archive { + source: S, + external: E, + archive_len: u64, + terminal_directory: Span, + index: ArchiveIndex, + segments: Vec, + index_limits: IndexLimits, + access: ResolvedAccess, + access_keys: AccessKeys, + limits: OpenLimits, +} + +impl Archive +where + S: ArchiveSource, + E: ExternalBlockResolver, +{ + /// Opens, frames, validates, resolves access metadata, and indexes an archive. + pub fn open(source: S, options: OpenOptions) -> Result { + let archive_len = source.len()?; + let mut header = [0; 6]; + source.read_exact_at(0, &mut header)?; + let header = crate::format::codec::decode_header(&mut header.as_slice())?; + if header.version != FileHeader::SUPPORTED_VERSION { + return Err(PithosError::UnsupportedFileVersion { + supported: FileHeader::SUPPORTED_VERSION, + actual: header.version, + }); + } + + let (terminal_start, terminal_len) = terminal_span(&source, archive_len, options.limits)?; + let mut raw = Vec::new(); + let mut total_directory_bytes = 0u64; + let mut next = Some((terminal_start, terminal_len)); + let mut child_start = archive_len; + let mut visited = HashSet::new(); + let mut decoded = DecodedDirectoryCounts::default(); + while let Some((start, len)) = next { + validate_directory_len(len, options.limits)?; + if raw.len() as u64 > options.limits.max_parent_directories { + return Err(PithosError::LimitExceeded { + field: "parent directories", + limit: options.limits.max_parent_directories, + actual: raw.len() as u64, + }); + } + let span = Span::new(start, len)?; + if span.end() > child_start || !visited.insert((start, len)) { + return Err(PithosError::InvalidDirectoryChain { + operation: "validate parent ordering", + }); + } + total_directory_bytes = + total_directory_bytes + .checked_add(len) + .ok_or(PithosError::LimitExceeded { + field: "total directory bytes", + limit: options.limits.max_total_directory_bytes, + actual: u64::MAX, + })?; + if total_directory_bytes > options.limits.max_total_directory_bytes { + return Err(PithosError::LimitExceeded { + field: "total directory bytes", + limit: options.limits.max_total_directory_bytes, + actual: total_directory_bytes, + }); + } + let bytes = Zeroizing::new(read_source(&source, start, len, "directory")?); + let directory = crate::format::codec::decode_complete_directory( + &bytes, + &remaining_deserialization_limits(options.limits, &decoded), + )?; + decoded.record(&directory); + next = directory.parent_directory_offset; + child_start = start; + raw.push((directory, span)); + } + raw.reverse(); + + let mut access = ResolvedAccess::new(); + let mut recovery_order = 0usize; + for (segment, (directory, _)) in raw.iter().enumerate() { + resolve_recipients( + directory, + &options.keys, + segment, + &mut recovery_order, + &mut access, + options.limits, + )?; + } + + let mut segments: Vec = Vec::new(); + let mut accessible_references = 0u64; + for (segment_index, (directory, span)) in raw.into_iter().enumerate() { + let mut directory = directory; + resolve_block_lists( + &mut directory, + &mut access, + &mut accessible_references, + options.limits, + )?; + let parent = segment_index + .checked_sub(1) + .map(|index| segments[index].span); + segments.push(segment_from_wire(&directory, span, parent)?); + } + let index_limits = IndexLimits { + max_entries: options.limits.max_entries, + max_descriptors: options.limits.max_descriptors, + max_references: options.limits.max_references, + max_relationships: options.limits.max_relationships, + max_segments: options.limits.max_parent_directories.saturating_add(1), + }; + let index = build_effective_index(&segments, archive_len, index_limits)?; + + Ok(Self { + source, + external: options.external, + archive_len, + terminal_directory: Span::new(terminal_start, terminal_len)?, + index, + segments, + index_limits, + access, + access_keys: options.keys, + limits: options.limits, + }) + } + + pub fn entries(&self) -> impl ExactSizeIterator + '_ { + self.index + .entries() + .map(|entry| archive_entry(&self.index, entry)) + } + + /// Consumes the reader and transfers its validated state to append/grant planning. + pub(crate) fn into_append_snapshot(self) -> AppendSnapshot { + let Self { + archive_len, + terminal_directory, + index, + segments, + index_limits, + access, + .. + } = self; + AppendSnapshot::new( + archive_len, + terminal_directory, + index, + segments, + index_limits, + access, + ) + } + + pub fn entry(&self, path: &str) -> Result, PithosError> { + let path = ArchivePath::new(path)?; + Ok(self + .index + .entry_at_path(&path) + .map(|entry| archive_entry(&self.index, entry))) + } + + pub fn copy_to(&self, path: &str, sink: &mut W) -> Result<(), PithosError> { + let (id, _) = self.content_id(path)?; + self.copy_plan(id, self.index.full_file_plan(id)?, sink) + } + + pub fn copy_range_to( + &self, + path: &str, + range: Range, + sink: &mut W, + ) -> Result<(), PithosError> { + let (id, size) = self.content_id(path)?; + let range = ReadRange::new(range, size)?; + self.copy_plan(id, self.index.range_plan(id, range)?, sink) + } + + pub(crate) fn with_crypt4gh_content( + &self, + path: &str, + operation: impl FnOnce(FileId, &PrivateKey, &FileKey) -> Result, + ) -> Result> { + let (id, _) = self.content_id(path).map_err(ContentOperationError::Core)?; + let key = self + .access + .key(id) + .ok_or(PithosError::ContentUnavailable) + .map_err(ContentOperationError::Core)?; + let provenance = self + .access + .provenance(id) + .ok_or(PithosError::ContentUnavailable) + .map_err(ContentOperationError::Core)?; + let reader = self + .access_keys + .0 + .get(provenance.access_key) + .ok_or(PithosError::ContentUnavailable) + .map_err(ContentOperationError::Core)?; + operation(id, reader, key).map_err(ContentOperationError::Callback) + } + + pub(crate) fn for_each_verified_file_block( + &self, + id: FileId, + mut operation: impl FnMut(Zeroizing>) -> Result<(), CallbackError>, + ) -> Result<(), ContentOperationError> { + let plan = self + .index + .full_file_plan(id) + .map_err(ContentOperationError::Core)?; + for block in plan.blocks { + let plaintext = self + .verified_block(id, &block) + .map_err(ContentOperationError::Core)?; + operation(plaintext).map_err(ContentOperationError::Callback)?; + } + Ok(()) + } + + fn content_id(&self, path: &str) -> Result<(FileId, u64), PithosError> { + let path = ArchivePath::new(path)?; + let entry = self + .index + .entry_at_path(&path) + .ok_or_else(|| PithosError::FileNotFound(path.as_str().to_owned()))?; + let content = entry.entry.content().ok_or_else(|| { + PithosError::InvalidBlockDataState("only data/metadata entries have content".into()) + })?; + Ok((entry.id, content.size)) + } + + fn copy_plan( + &self, + id: FileId, + plan: super::planning::ReadPlan, + sink: &mut W, + ) -> Result<(), PithosError> { + for block in plan.blocks { + let plaintext = self.verified_block(id, &block)?; + sink.write_all(&plaintext[block.output])?; + } + Ok(()) + } + + fn verified_block( + &self, + id: FileId, + planned: &super::planning::PlannedBlock, + ) -> Result>, PithosError> { + if planned.descriptor.stored_size > self.limits.max_stored_block_bytes { + return Err(PithosError::LimitExceeded { + field: "stored block", + limit: self.limits.max_stored_block_bytes, + actual: planned.descriptor.stored_size, + }); + } + if planned.descriptor.original_size > self.limits.max_decoded_block_bytes { + return Err(PithosError::LimitExceeded { + field: "decoded block", + limit: self.limits.max_decoded_block_bytes, + actual: planned.descriptor.original_size, + }); + } + let stored = match &planned.descriptor.location { + BlockLocation::Local(span) => { + let mut marker = [0; 4]; + self.source.read_exact_at(span.start(), &mut marker)?; + crate::format::codec::decode_block_marker(&mut marker.as_slice())?; + read_source( + &self.source, + span.start() + 4, + planned.descriptor.stored_size, + "block", + )? + } + BlockLocation::External(location) => { + let expected_len = + planned + .descriptor + .stored_size + .checked_add(4) + .ok_or_else(|| { + PithosError::ExternalBlockFraming("response size overflow".into()) + })?; + let response = self.external.resolve( + location, + expected_len, + self.limits + .max_stored_block_bytes + .checked_add(4) + .ok_or_else(|| { + PithosError::ExternalBlockFraming("response policy overflow".into()) + })?, + )?; + if response.len() as u64 != expected_len { + return Err(PithosError::ExternalBlockFraming( + "response does not match expected size".into(), + )); + } + let (mut marker, stored) = response.split_at(4); + crate::format::codec::decode_block_marker(&mut marker)?; + stored.to_vec() + } + }; + let meta = BlockIndexEntry { + offset: 0, + stored_size: planned.descriptor.stored_size, + original_size: planned.descriptor.original_size, + flags: crate::format::wire::ProcessingFlags::from_byte( + planned.descriptor.processing.to_byte(), + ), + location: crate::format::wire::BlockLocation::Local, + }; + let key = self + .access + .block_key(id, planned.hash) + .ok_or(PithosError::ContentUnavailable)?; + block::verify( + stored, + key, + planned.hash.0, + &meta, + block::Limits { + max_stored_bytes: self.limits.max_stored_block_bytes, + max_decoded_bytes: self.limits.max_decoded_block_bytes, + }, + ) + } +} + +fn entry_kind(entry: &Entry) -> EntryKind { + match entry { + Entry::File(content) => EntryKind::File { + size: content.size, + available: matches!(content.content, ContentState::Available(_)), + }, + Entry::Metadata(content) => EntryKind::Metadata { + size: content.size, + available: matches!(content.content, ContentState::Available(_)), + }, + Entry::Directory(_) => EntryKind::Directory, + Entry::Symlink { target, .. } => EntryKind::Symlink { + target: target.to_string(), + }, + } +} + +fn archive_entry(index: &ArchiveIndex, entry: &super::index::IndexedEntry) -> ArchiveEntry { + let metadata = entry.entry.metadata(); + ArchiveEntry { + id: entry.id.0, + path: entry.path.as_str().to_owned(), + kind: entry_kind(&entry.entry), + created: metadata.created, + modified: metadata.modified, + permissions: metadata.permissions, + references: metadata + .references + .iter() + .map(|reference| ArchiveReference { + target_id: reference.target.0, + relationship: index + .relationship(reference.relationship) + .map(str::to_owned) + .unwrap_or_else(|| format!("unknown:{}", reference.relationship.0)), + }) + .collect(), + } +} + +fn deserialization_limits(limits: OpenLimits) -> DeserializationLimits { + DeserializationLimits { + max_collection_entries: limits.max_entries, + max_file_entries: limits.max_entries, + max_block_descriptors: limits.max_descriptors, + max_references: limits.max_references, + max_relationships: limits.max_relationships, + max_opaque_bytes: limits.max_opaque_metadata_bytes, + ..DeserializationLimits::default() + } +} + +fn remaining_deserialization_limits( + limits: OpenLimits, + decoded: &DecodedDirectoryCounts, +) -> DeserializationLimits { + let mut remaining = deserialization_limits(limits); + remaining.max_file_entries = remaining.max_file_entries.saturating_sub(decoded.entries); + remaining.max_block_descriptors = remaining + .max_block_descriptors + .saturating_sub(decoded.descriptors); + remaining.max_references = remaining.max_references.saturating_sub(decoded.references); + remaining.max_relationships = remaining + .max_relationships + .saturating_sub(decoded.relationships); + remaining +} + +fn terminal_span( + source: &S, + archive_len: u64, + limits: OpenLimits, +) -> Result<(u64, u64), PithosError> { + if archive_len < 12 { + return Err(PithosError::InvalidDirectoryRange { + operation: "read directory footer", + }); + } + let mut footer = [0; 12]; + source.read_exact_at(archive_len - 12, &mut footer)?; + let len = u64::from_be_bytes(footer[..8].try_into().expect("fixed footer length")); + validate_directory_len(len, limits)?; + let start = archive_len + .checked_sub(len) + .ok_or(PithosError::InvalidDirectoryRange { + operation: "validate terminal directory", + })?; + Ok((start, len)) +} + +fn validate_directory_len(len: u64, limits: OpenLimits) -> Result<(), PithosError> { + if len < 25 { + return Err(PithosError::DirectoryLengthMismatch { + expected: 25, + actual: len, + }); + } + if len > limits.max_directory_bytes { + return Err(PithosError::LimitExceeded { + field: "directory", + limit: limits.max_directory_bytes, + actual: len, + }); + } + Ok(()) +} + +fn read_source( + source: &S, + offset: u64, + len: u64, + field: &'static str, +) -> Result, PithosError> { + let len = usize::try_from(len) + .map_err(|_| PithosError::InvalidDirectoryRange { operation: field })?; + let mut bytes = Vec::new(); + bytes + .try_reserve_exact(len) + .map_err(|_| PithosError::AllocationFailed { + field, + size: len as u64, + })?; + bytes.resize(len, 0); + source.read_exact_at(offset, &mut bytes)?; + Ok(bytes) +} + +fn resolve_recipients( + directory: &Directory, + keys: &AccessKeys, + segment: usize, + recovery_order: &mut usize, + access: &mut ResolvedAccess, + limits: OpenLimits, +) -> Result<(), PithosError> { + let decoded_limits = deserialization_limits(limits); + for (access_key, key) in keys.0.iter().enumerate() { + let secret = key.as_dalek_static_secret(); + let recipient = DalekPublicKey::from(&secret).to_bytes(); + for (sender_section, (sender, section)) in directory.encryption.iter().enumerate() { + let candidates: Vec<(&[u8; 32], &crate::format::wire::RecipientData)> = + if sender == &recipient { + section + .recipients + .iter() + .map(|(recipient, section)| (recipient, §ion.recipient_data)) + .collect() + } else { + section + .recipients + .get(&recipient) + .map(|section| vec![(sender, §ion.recipient_data)]) + .unwrap_or_default() + }; + for (recipient_section, (peer, data)) in candidates.into_iter().enumerate() { + let shared = crypto::derive_shared(secret.as_bytes(), peer)?; + let entries = match data { + crate::format::wire::RecipientData::Encrypted(bytes) => { + let plaintext = crypto::unwrap_recipient_list(&shared, bytes)?; + crate::format::codec::decode_decrypted_recipient_list( + &plaintext, + &decoded_limits, + )? + } + crate::format::wire::RecipientData::Decrypted(entries) => entries.clone(), + }; + for (file_id, file_key) in entries.iter() { + access.insert( + FileId(*file_id), + file_key, + AccessProvenance { + segment, + recovery_order: *recovery_order, + access_key, + sender_section, + recipient_section, + }, + )?; + *recovery_order = recovery_order.saturating_add(1); + } + } + } + } + Ok(()) +} + +fn resolve_block_lists( + directory: &mut Directory, + access: &mut ResolvedAccess, + reference_count: &mut u64, + limits: OpenLimits, +) -> Result<(), PithosError> { + directory.files.try_for_each_mut(|id, file| { + let Some(file_key) = access.key(FileId(id)) else { + return Ok(()); + }; + let BlockDataState::Encrypted(bytes) = &file.block_data else { + return Ok(()); + }; + let mut decoded_limits = deserialization_limits(limits); + decoded_limits.max_collection_entries = limits + .max_accessible_block_references + .saturating_sub(*reference_count); + let plaintext = crypto::open_file_block_list(file_key, bytes)?; + let entries = + crate::format::codec::decode_decrypted_block_list(&plaintext, &decoded_limits)?; + *reference_count = reference_count.checked_add(entries.len() as u64).ok_or( + PithosError::LimitExceeded { + field: "accessible block references", + limit: limits.max_accessible_block_references, + actual: u64::MAX, + }, + )?; + if *reference_count > limits.max_accessible_block_references { + return Err(PithosError::LimitExceeded { + field: "accessible block references", + limit: limits.max_accessible_block_references, + actual: *reference_count, + }); + } + access.insert_block_keys( + FileId(id), + entries.iter().map(|(hash, key)| (BlockHash(*hash), key)), + ); + file.block_data = BlockDataState::Decrypted(entries); + Ok(()) + }) +} diff --git a/crates/pithos_lib/src/archive/reader_private_tests.rs b/crates/pithos_lib/src/archive/reader_private_tests.rs new file mode 100644 index 0000000..9fcccef --- /dev/null +++ b/crates/pithos_lib/src/archive/reader_private_tests.rs @@ -0,0 +1,677 @@ +use super::reader::{ + AccessKeys, Archive, EntryKind, ExternalBlockResolver, OpenLimits, OpenOptions, +}; +use crate::archive::{ + ArchivePath, ArchiveWriter, CdcConfig, EntryMetadata, ProcessingOptions, WriteOptions, +}; +use crate::crypto::{self, PrivateKey, PublicKey}; +use crate::error::PithosError; +use crate::format::limits::DeserializationLimits; +use crate::format::wire::{ + BlockDataState, BlockLocation, Directory, EncryptionSection, RecipientData, RecipientSection, +}; +use crate::source::{ArchiveSource, MemorySource, SourceError}; +use indexmap::IndexMap; +use std::fs::File; +use std::io::{Cursor, Write}; +use std::ops::Range; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::sync::Mutex; +use std::sync::atomic::{AtomicUsize, Ordering}; +use x25519_dalek::{PublicKey as DalekPublicKey, StaticSecret}; + +fn private(name: &str) -> PrivateKey { + crate::crypto::parse_private_pem( + &std::fs::read(format!("tests/data/keys/{name}_private.pem")).unwrap(), + ) + .unwrap() +} + +fn public(name: &str) -> PublicKey { + crate::crypto::parse_public_pem( + &std::fs::read(format!("tests/data/keys/{name}_public.pem")).unwrap(), + ) + .unwrap() +} + +fn fixture() -> (tempfile::TempDir, std::path::PathBuf) { + fixture_with("archive reader private fixture", 0) +} + +fn fixture_with(content: &str, compression_level: u8) -> (tempfile::TempDir, PathBuf) { + fixture_with_encryption(content, compression_level, true) +} + +fn fixture_with_encryption( + content: &str, + compression_level: u8, + encrypt: bool, +) -> (tempfile::TempDir, PathBuf) { + fixture_with_options(content, compression_level, encrypt, None) +} + +fn fixture_with_options( + content: &str, + compression_level: u8, + encrypt: bool, + cdc: Option, +) -> (tempfile::TempDir, PathBuf) { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("archive.pith"); + let options = WriteOptions::new(private("sender"), vec![public("recipient1")]); + let options = match cdc { + Some(cdc) => options.with_cdc(cdc), + None => options, + }; + let mut writer = ArchiveWriter::create(File::create(&path).unwrap(), options).unwrap(); + writer + .add_file( + ArchivePath::new("data").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(encrypt, compression_level).unwrap(), + Some(content.len() as u64), + Cursor::new(content.as_bytes()), + ) + .unwrap(); + writer.finish().unwrap(); + (temp, path) +} + +fn fixture_entries(entries: &[(&str, &str)]) -> (tempfile::TempDir, PathBuf) { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("entries.pith"); + let mut writer = ArchiveWriter::create( + File::create(&path).unwrap(), + WriteOptions::new(private("sender"), vec![public("recipient1")]), + ) + .unwrap(); + for (path, content) in entries { + writer + .add_file( + ArchivePath::new(*path).unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(true, 0).unwrap(), + Some(content.len() as u64), + Cursor::new(content.as_bytes()), + ) + .unwrap(); + } + writer.finish().unwrap(); + (temp, path) +} + +fn open_path(path: &Path, keys: AccessKeys) -> Archive { + Archive::open( + MemorySource::new(Arc::<[u8]>::from(std::fs::read(path).unwrap())), + OpenOptions::default().with_access_keys(keys), + ) + .unwrap() +} + +fn directory_bounds(bytes: &[u8]) -> (usize, usize) { + let len = u64::from_be_bytes(bytes[bytes.len() - 12..bytes.len() - 4].try_into().unwrap()); + let len = usize::try_from(len).unwrap(); + (bytes.len() - len, len) +} + +fn rewrite_terminal_directory(path: &Path, mutate: impl FnOnce(&mut Directory)) { + let mut archive = std::fs::read(path).unwrap(); + let (start, _) = directory_bounds(&archive); + let mut directory = crate::format::codec::decode_directory( + &mut Cursor::new(&archive[start..]), + &DeserializationLimits::default(), + ) + .unwrap(); + mutate(&mut directory); + crate::format::codec::update_directory_len(&mut directory).unwrap(); + crate::format::codec::update_directory_crc(&mut directory).unwrap(); + let mut replacement = Vec::new(); + crate::format::codec::encode_directory(&directory, &mut replacement).unwrap(); + archive.truncate(start); + archive.extend_from_slice(&replacement); + std::fs::write(path, archive).unwrap(); +} + +fn first_block(path: &Path) -> (u64, u64) { + let bytes = std::fs::read(path).unwrap(); + let (start, _) = directory_bounds(&bytes); + let directory = crate::format::codec::decode_directory( + &mut Cursor::new(&bytes[start..]), + &DeserializationLimits::default(), + ) + .unwrap(); + let block = directory.blocks.first().unwrap().1; + (block.offset, block.stored_size) +} + +fn corrupt_payload(path: &Path, byte: usize) { + let (offset, stored) = first_block(path); + assert!(byte < stored as usize); + let mut bytes = std::fs::read(path).unwrap(); + bytes[offset as usize + 4 + byte] ^= 1; + std::fs::write(path, bytes).unwrap(); +} + +fn assert_copy_failure_without_sink(path: &Path) { + let archive = open_path(path, AccessKeys::new().with_key(private("recipient1"))); + let mut sink = RecordingSink(Vec::new()); + assert!(archive.copy_to("data", &mut sink).is_err()); + assert!(sink.0.is_empty()); +} + +struct RecordingSink(Vec); + +impl Write for RecordingSink { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +#[derive(Clone)] +struct CountingResolver { + response: Arc<[u8]>, + calls: Arc, + expected: Arc>>, +} + +impl ExternalBlockResolver for CountingResolver { + fn resolve( + &self, + _location: &super::types::ExternalLocation, + expected_len: u64, + max_response_size: u64, + ) -> Result, PithosError> { + self.calls.fetch_add(1, Ordering::Relaxed); + self.expected + .lock() + .unwrap() + .push((expected_len, max_response_size)); + Ok(self.response.to_vec()) + } +} + +fn as_external(path: &Path) -> Vec { + let bytes = std::fs::read(path).unwrap(); + let (offset, stored) = first_block(path); + let response = bytes[offset as usize..offset as usize + 4 + stored as usize].to_vec(); + rewrite_terminal_directory(path, |directory| { + let block = directory.blocks.first_mut().unwrap().1; + block.location = BlockLocation::External { + url: "test:external".into(), + }; + block.offset = u64::MAX; + }); + response +} + +struct CountingSource { + bytes: Arc<[u8]>, + reads: Arc, +} +impl ArchiveSource for CountingSource { + fn len(&self) -> Result { + Ok(self.bytes.len() as u64) + } + fn read_exact_at(&self, offset: u64, output: &mut [u8]) -> Result<(), SourceError> { + self.reads.fetch_add(1, Ordering::Relaxed); + let start = usize::try_from(offset).map_err(|_| SourceError::RangeOverflow { + offset, + length: output.len(), + })?; + let input = + self.bytes + .get(start..start + output.len()) + .ok_or(SourceError::UnexpectedEof { + offset, + expected: output.len(), + actual: self.bytes.len().saturating_sub(start), + })?; + output.copy_from_slice(input); + Ok(()) + } +} + +#[test] +fn archive_open_copy_range_unavailable_and_limits_are_boundary_checked() { + let (_temp, path) = fixture(); + let bytes = Arc::<[u8]>::from(std::fs::read(path).unwrap()); + let reads = Arc::new(AtomicUsize::new(0)); + let archive = Archive::open( + CountingSource { + bytes: Arc::clone(&bytes), + reads: Arc::clone(&reads), + }, + OpenOptions::default().with_access_keys(AccessKeys::new().with_key(private("recipient1"))), + ) + .unwrap(); + let mut output = Vec::new(); + archive.copy_to("data", &mut output).unwrap(); + assert_eq!(output, b"archive reader private fixture"); + let before = reads.load(Ordering::Relaxed); + archive + .copy_range_to("data", 0..0, &mut Vec::new()) + .unwrap(); + assert_eq!(reads.load(Ordering::Relaxed), before); + assert!( + archive + .copy_range_to("data", 99..100, &mut Vec::new()) + .is_err() + ); + assert_eq!(reads.load(Ordering::Relaxed), before); + let unavailable = Archive::open(MemorySource::new(bytes), OpenOptions::default()).unwrap(); + assert!(matches!( + unavailable.entries().next().unwrap().kind, + EntryKind::File { + available: false, + .. + } + )); +} + +#[test] +fn archive_opens_terminal_entries_preserving_entry_order_and_unavailable_entries() { + let (_temporary, path) = fixture_entries(&[("first", "one"), ("second", "two")]); + let archive = open_path(&path, AccessKeys::new().with_key(private("recipient1"))); + let entries = archive.entries().collect::>(); + assert_eq!( + entries + .iter() + .map(|entry| entry.path.as_str()) + .collect::>(), + ["first", "second"] + ); + assert_eq!(entries[0].permissions, 0o644); + assert_eq!(entries[0].created, 0); + assert_eq!(entries[0].modified, 0); + assert!(matches!( + entries[0].kind, + EntryKind::File { + available: true, + .. + } + )); + + let unavailable = open_path(&path, AccessKeys::new()); + assert!(matches!( + unavailable.entries().next().unwrap().kind, + EntryKind::File { + available: false, + .. + } + )); +} + +#[test] +fn archive_distinguishes_missing_and_wrong_keys_from_matching_metadata_corruption() { + let (_temporary, path) = fixture(); + for keys in [ + AccessKeys::new(), + AccessKeys::new().with_key(private("recipient2")), + ] { + let archive = open_path(&path, keys); + assert!(matches!( + archive.copy_to("data", &mut Vec::new()), + Err(PithosError::ContentUnavailable) + )); + } + + let (_temporary, envelope) = fixture(); + rewrite_terminal_directory(&envelope, |directory| { + let recipient = directory + .encryption + .values_mut() + .next() + .unwrap() + .recipients + .values_mut() + .next() + .unwrap(); + if let RecipientData::Encrypted(bytes) = &mut recipient.recipient_data { + bytes[0] ^= 1; + } + }); + assert!(matches!( + Archive::open( + MemorySource::new(Arc::<[u8]>::from(std::fs::read(envelope).unwrap())), + OpenOptions::default() + .with_access_keys(AccessKeys::new().with_key(private("recipient1"))) + ), + Err(PithosError::Crypt(_)) + )); + + let (_temporary, blocks) = fixture(); + rewrite_terminal_directory(&blocks, |directory| { + directory + .files + .try_for_each_mut(|_, file| { + if let BlockDataState::Encrypted(bytes) = &mut file.block_data { + bytes[0] ^= 1; + } + Ok::<_, ()>(()) + }) + .unwrap(); + }); + assert!(matches!( + Archive::open( + MemorySource::new(Arc::<[u8]>::from(std::fs::read(blocks).unwrap())), + OpenOptions::default() + .with_access_keys(AccessKeys::new().with_key(private("recipient1"))) + ), + Err(PithosError::Crypt(_)) + )); +} + +#[test] +fn archive_rejects_conflicting_recovered_file_keys_at_open() { + let (_temporary, path) = fixture(); + let reader = private("recipient1").into_dalek_static_secret(); + rewrite_terminal_directory(&path, |directory| { + let sender = StaticSecret::from([9; 32]); + let recipient = DalekPublicKey::from(&reader); + let shared = sender.diffie_hellman(&recipient); + let mut plaintext = vec![1, 0]; + plaintext.extend_from_slice(&[0x77; 32]); + let encrypted = crypto::seal_crypt4gh_payload(shared.as_bytes(), &plaintext).unwrap(); + directory.encryption.insert( + DalekPublicKey::from(&sender).to_bytes(), + EncryptionSection { + recipients: IndexMap::from_iter([( + recipient.to_bytes(), + RecipientSection { + recipient_data: RecipientData::Encrypted(encrypted), + }, + )]), + }, + ); + }); + assert!(matches!( + Archive::open( + MemorySource::new(Arc::<[u8]>::from(std::fs::read(path).unwrap())), + OpenOptions::default() + .with_access_keys(AccessKeys::new().with_key(private("recipient1"))) + ), + Err(PithosError::ConflictingRecoveredFileKey) + )); +} + +#[test] +fn archive_ranges_are_half_open_and_invalid_ranges_do_not_acquire_payload() { + let (_temporary, path) = fixture_with("0123456789abcdef", 0); + let bytes = Arc::<[u8]>::from(std::fs::read(path).unwrap()); + let reads = Arc::new(AtomicUsize::new(0)); + let archive = Archive::open( + CountingSource { + bytes, + reads: Arc::clone(&reads), + }, + OpenOptions::default().with_access_keys(AccessKeys::new().with_key(private("recipient1"))), + ) + .unwrap(); + for (range, expected) in [ + (0..4, b"0123".as_slice()), + (4..12, b"456789ab".as_slice()), + (12..16, b"cdef".as_slice()), + (16..16, b"".as_slice()), + ] { + let mut sink = Vec::new(); + archive.copy_range_to("data", range, &mut sink).unwrap(); + assert_eq!(sink, expected); + } + let before = reads.load(Ordering::Relaxed); + for range in [Range { start: 8, end: 7 }, 17..17, 0..17] { + assert!(matches!( + archive.copy_range_to("data", range, &mut Vec::new()), + Err(PithosError::InvalidReadRange { .. }) + )); + } + assert_eq!(reads.load(Ordering::Relaxed), before); +} + +#[test] +fn archive_ranges_verify_and_slice_every_intersecting_multiblock_payload() { + let content = (0..512) + .map(|index| format!("{index:08x}-unique-cdc-payload-")) + .collect::(); + let (_temporary, path) = fixture_with_options( + &content, + 0, + true, + Some(CdcConfig::new(64, 256, 1024).unwrap()), + ); + let archive = open_path(&path, AccessKeys::new().with_key(private("recipient1"))); + let ranges = [ + 0..31, + 91..511, + (content.len() - 71) as u64..content.len() as u64, + ]; + for range in ranges { + let mut output = Vec::new(); + archive + .copy_range_to("data", range.clone(), &mut output) + .unwrap(); + assert_eq!( + output, + content.as_bytes()[range.start as usize..range.end as usize] + ); + } +} + +#[test] +fn archive_rejects_bad_nonce_tag_and_ciphertext_before_sink_output() { + for byte in [0, 12, 28] { + let (_temporary, path) = fixture_with( + "encrypted payload long enough for every corruption position", + 0, + ); + corrupt_payload(&path, byte); + assert_copy_failure_without_sink(&path); + } +} + +#[test] +fn archive_rejects_bad_compressed_payload_before_sink_output() { + let (_temporary, path) = + fixture_with_encryption(&"repeated block payload ".repeat(512), 3, false); + corrupt_payload(&path, 0); + let archive = open_path(&path, AccessKeys::new().with_key(private("recipient1"))); + let mut sink = RecordingSink(Vec::new()); + let error = archive.copy_to("data", &mut sink).unwrap_err(); + assert!(sink.0.is_empty()); + assert!(error.to_string().contains("decompress block")); + assert!(std::error::Error::source(&error).is_some()); +} + +#[test] +fn archive_rejects_descriptor_size_mismatches_before_sink_output() { + let (_temporary, path) = fixture(); + rewrite_terminal_directory(&path, |directory| { + let block = directory.blocks.first_mut().unwrap().1; + block.original_size += 1; + directory + .files + .try_for_each_mut(|_, file| { + file.file_size += 1; + Ok::<_, ()>(()) + }) + .unwrap(); + }); + let archive = open_path(&path, AccessKeys::new().with_key(private("recipient1"))); + let mut sink = RecordingSink(Vec::new()); + assert!(archive.copy_to("data", &mut sink).is_err()); + assert!(sink.0.is_empty()); +} + +#[test] +fn archive_rejects_plaintext_hash_mismatches_before_sink_output() { + let (_temporary, hash) = fixture_with_encryption("plaintext hash verification", 0, false); + corrupt_payload(&hash, 0); + let archive = open_path(&hash, AccessKeys::new().with_key(private("recipient1"))); + let mut sink = RecordingSink(Vec::new()); + assert!(matches!( + archive.copy_to("data", &mut sink), + Err(PithosError::BlockHashMismatch { .. }) + )); + assert!(sink.0.is_empty()); +} + +#[test] +fn archive_rejects_stored_size_failures_before_sink_output() { + let (_temporary, stored) = fixture(); + rewrite_terminal_directory(&stored, |directory| { + directory.blocks.first_mut().unwrap().1.stored_size -= 1 + }); + assert_copy_failure_without_sink(&stored); +} + +#[test] +fn archive_external_blocks_validate_exact_framing_and_share_read_paths() { + let (temporary, path) = fixture(); + let response = as_external(&path); + let calls = Arc::new(AtomicUsize::new(0)); + let expected = Arc::new(Mutex::new(Vec::new())); + let resolver = CountingResolver { + response: Arc::from(response.clone()), + calls: Arc::clone(&calls), + expected: Arc::clone(&expected), + }; + let archive = Archive::open( + MemorySource::new(Arc::<[u8]>::from(std::fs::read(&path).unwrap())), + OpenOptions::default() + .with_access_keys(AccessKeys::new().with_key(private("recipient1"))) + .with_external_resolver(resolver), + ) + .unwrap(); + let mut full = Vec::new(); + archive.copy_to("data", &mut full).unwrap(); + let mut range = Vec::new(); + archive.copy_range_to("data", 1..4, &mut range).unwrap(); + let output = temporary.path().join("external-output"); + extract(&archive, "data", &output).unwrap(); + let mut crypt4gh = Vec::new(); + crypt4gh::export(&archive, "data", vec![public("recipient2")], &mut crypt4gh).unwrap(); + assert_eq!(std::fs::read(output.join("data")).unwrap(), full); + assert_eq!(range, &full[1..4]); + assert!(!crypt4gh.is_empty()); + assert_eq!(calls.load(Ordering::Relaxed), 4); + let (expected_len, policy) = expected.lock().unwrap()[0]; + assert_eq!(expected_len, response.len() as u64); + assert_eq!(policy, OpenLimits::default().max_stored_block_bytes + 4); + + let missing = Archive::open( + MemorySource::new(Arc::<[u8]>::from(std::fs::read(&path).unwrap())), + OpenOptions::default().with_access_keys(AccessKeys::new().with_key(private("recipient1"))), + ) + .unwrap(); + assert!(matches!( + missing.copy_to("data", &mut Vec::new()), + Err(PithosError::ExternalBlockSourceRequired) + )); + let mut corrupt = response.clone(); + corrupt[4] ^= 1; + for response in [ + vec![b'B'; 3], + [b"BLCK".as_slice(), &response, b"x"].concat(), + [b"NOPE".as_slice(), &response[4..]].concat(), + corrupt, + ] { + let resolver = CountingResolver { + response: Arc::from(response), + calls: Arc::new(AtomicUsize::new(0)), + expected: Arc::new(Mutex::new(Vec::new())), + }; + let archive = Archive::open( + MemorySource::new(Arc::<[u8]>::from(std::fs::read(&path).unwrap())), + OpenOptions::default() + .with_access_keys(AccessKeys::new().with_key(private("recipient1"))) + .with_external_resolver(resolver), + ) + .unwrap(); + assert!(archive.copy_to("data", &mut Vec::new()).is_err()); + } +} + +#[test] +fn archive_block_limits_reject_before_local_or_external_acquisition() { + let (_temporary, path) = fixture(); + let bytes = Arc::<[u8]>::from(std::fs::read(&path).unwrap()); + let reads = Arc::new(AtomicUsize::new(0)); + let limits = OpenLimits { + max_stored_block_bytes: 0, + ..OpenLimits::default() + }; + let archive = Archive::open( + CountingSource { + bytes: Arc::clone(&bytes), + reads: Arc::clone(&reads), + }, + OpenOptions::default() + .with_limits(limits) + .with_access_keys(AccessKeys::new().with_key(private("recipient1"))), + ) + .unwrap(); + let before = reads.load(Ordering::Relaxed); + assert!(matches!( + archive.copy_to("data", &mut Vec::new()), + Err(PithosError::LimitExceeded { + field: "stored block", + .. + }) + )); + assert_eq!(reads.load(Ordering::Relaxed), before); + + let response = as_external(&path); + let calls = Arc::new(AtomicUsize::new(0)); + let resolver = CountingResolver { + response: Arc::from(response), + calls: Arc::clone(&calls), + expected: Arc::new(Mutex::new(Vec::new())), + }; + let archive = Archive::open( + MemorySource::new(Arc::<[u8]>::from(std::fs::read(path).unwrap())), + OpenOptions::default() + .with_limits(limits) + .with_access_keys(AccessKeys::new().with_key(private("recipient1"))) + .with_external_resolver(resolver), + ) + .unwrap(); + assert!(matches!( + archive.copy_to("data", &mut Vec::new()), + Err(PithosError::LimitExceeded { + field: "stored block", + .. + }) + )); + assert_eq!(calls.load(Ordering::Relaxed), 0); + + let reads = Arc::new(AtomicUsize::new(0)); + let limits = OpenLimits { + max_decoded_block_bytes: 0, + ..OpenLimits::default() + }; + let archive = Archive::open( + CountingSource { + bytes, + reads: Arc::clone(&reads), + }, + OpenOptions::default() + .with_limits(limits) + .with_access_keys(AccessKeys::new().with_key(private("recipient1"))), + ) + .unwrap(); + let before = reads.load(Ordering::Relaxed); + assert!(matches!( + archive.copy_to("data", &mut Vec::new()), + Err(PithosError::LimitExceeded { + field: "decoded block", + .. + }) + )); + assert_eq!(reads.load(Ordering::Relaxed), before); +} +use crate::adapters::crypt4gh; +use crate::fs::extract; diff --git a/crates/pithos_lib/src/io/pithosreader.rs b/crates/pithos_lib/src/io/pithosreader.rs deleted file mode 100644 index 058687c..0000000 --- a/crates/pithos_lib/src/io/pithosreader.rs +++ /dev/null @@ -1,790 +0,0 @@ -use crate::error::PithosError; -use crate::helpers::archive_path::{validate_existing_candidate, validate_map}; -use crate::helpers::chacha_poly1305::{decrypt_chunk, encrypt_chunk}; -use crate::helpers::crypt4gh::{CRYPT4GH_BLOCK_SIZE, Crypt4GHHeader, HeaderPacket}; -use crate::helpers::x25519_keys::private_key_from_pem_bytes; -use crate::helpers::zstd::decompress_data; -use crate::io::extraction::ExtractionRoot; -use crate::model::deserialization::DeserializationLimits; -use crate::model::structs::{ - BlockDataState, BlockHeader, BlockIndexEntry, BlockLocation, Directory, FileEntry, FileHeader, - FileType, -}; -use crc32fast::hash; -use indexmap::IndexMap; -use std::collections::HashMap; -use std::fs::File; -use std::io::{self, Cursor, Read, Seek, SeekFrom, Write}; -use std::ops::Range; -use std::path::{Path, PathBuf}; -use x25519_dalek::{PublicKey, StaticSecret}; - -pub trait ExternalBlockSource { - fn open(&self, url: &str, max_response_size: u64) -> Result, PithosError>; -} - -#[derive(Clone, Copy, Debug)] -pub struct ReaderLimits { - pub max_directory_bytes: u64, - pub max_parent_directories: u64, - pub max_stored_block_bytes: u64, - pub max_decoded_block_bytes: u64, - pub deserialization: DeserializationLimits, -} - -impl Default for ReaderLimits { - fn default() -> Self { - Self { - max_directory_bytes: 64 * 1024 * 1024, - max_parent_directories: 1024, - max_stored_block_bytes: 64 * 1024 * 1024, - max_decoded_block_bytes: 64 * 1024 * 1024, - deserialization: DeserializationLimits::default(), - } - } -} - -pub struct PithosReaderSimple { - /// Underlying file handle for the Pithos archive - file: File, - /// User's private key - private_key: StaticSecret, - limits: ReaderLimits, - external_block_source: Option>, -} - -impl PithosReaderSimple { - fn open_archive>(pithos_path: P) -> Result { - let mut file = File::open(pithos_path)?; - let header = FileHeader::deserialize(&mut file)?; - if header.version != FileHeader::SUPPORTED_VERSION { - return Err(PithosError::UnsupportedFileVersion { - supported: FileHeader::SUPPORTED_VERSION, - actual: header.version, - }); - } - Ok(file) - } - - fn decode_and_verify_block( - stored_bytes: Vec, - key: &[u8; 32], - expected_hash: &[u8; 32], - block_meta: &BlockIndexEntry, - limits: &ReaderLimits, - ) -> Result, PithosError> { - if block_meta.original_size > limits.max_decoded_block_bytes { - return Err(PithosError::LimitExceeded { - field: "decoded block", - limit: limits.max_decoded_block_bytes, - actual: block_meta.original_size, - }); - } - let mut plaintext = if block_meta.flags.is_encrypted() { - decrypt_chunk(&stored_bytes, key)? - } else { - stored_bytes - }; - if block_meta.flags.get_compression_level() > 0 { - plaintext = decompress_data(&plaintext, block_meta.original_size)?; - } - - let actual_size = plaintext.len() as u64; - if actual_size != block_meta.original_size { - return Err(PithosError::BlockSizeMismatch { - expected: block_meta.original_size, - actual: actual_size, - }); - } - - let actual_hash = *blake3::hash(&plaintext).as_bytes(); - if actual_hash != *expected_hash { - return Err(PithosError::BlockHashMismatch { - expected: *expected_hash, - actual: actual_hash, - }); - } - - Ok(plaintext) - } - - fn checked_stored_size( - meta: &BlockIndexEntry, - limits: &ReaderLimits, - ) -> Result { - if meta.stored_size > limits.max_stored_block_bytes { - return Err(PithosError::LimitExceeded { - field: "stored block", - limit: limits.max_stored_block_bytes, - actual: meta.stored_size, - }); - } - if meta.original_size > limits.max_decoded_block_bytes { - return Err(PithosError::LimitExceeded { - field: "decoded block", - limit: limits.max_decoded_block_bytes, - actual: meta.original_size, - }); - } - usize::try_from(meta.stored_size).map_err(|_| { - PithosError::InvalidDirectoryRange("stored block size does not fit platform".into()) - }) - } - - fn zeroed_buffer(size: usize, field: &'static str) -> Result, PithosError> { - let mut buffer = Vec::new(); - buffer - .try_reserve_exact(size) - .map_err(|_| PithosError::AllocationFailed { - field, - size: size as u64, - })?; - buffer.resize(size, 0); - Ok(buffer) - } - - fn validate_local_block_range(&self, meta: &BlockIndexEntry) -> Result<(), PithosError> { - let end = meta - .offset - .checked_add(4) - .and_then(|value| value.checked_add(meta.stored_size)) - .ok_or_else(|| PithosError::InvalidDirectoryRange("block range overflow".into()))?; - if end > self.file.metadata()?.len() { - return Err(PithosError::InvalidDirectoryRange( - "block range exceeds archive".into(), - )); - } - Ok(()) - } - - fn parse_directory(bytes: &[u8], limits: &ReaderLimits) -> Result { - if bytes.len() < 25 { - return Err(PithosError::DirectoryLengthMismatch { - expected: 25, - actual: bytes.len() as u64, - }); - } - if bytes[..8] != Directory::DIRECTORY_MARKER { - return Err(PithosError::InvalidDirectoryMarker { - expected: Directory::DIRECTORY_MARKER, - actual: bytes[..8].try_into().unwrap(), - }); - } - let expected_len = - u64::from_be_bytes(bytes[bytes.len() - 12..bytes.len() - 4].try_into().unwrap()); - if expected_len != bytes.len() as u64 { - return Err(PithosError::DirectoryLengthMismatch { - expected: bytes.len() as u64, - actual: expected_len, - }); - } - let expected_crc = u32::from_be_bytes(bytes[bytes.len() - 4..].try_into().unwrap()); - let actual_crc = hash(&bytes[..bytes.len() - 4]); - if expected_crc != actual_crc { - return Err(PithosError::DirectoryChecksumMismatch { - expected: actual_crc, - actual: expected_crc, - }); - } - let mut cursor = Cursor::new(bytes); - let directory = Directory::deserialize_with_limits(&mut cursor, &limits.deserialization)?; - if cursor.position() != bytes.len() as u64 { - return Err(PithosError::DirectoryConsumptionMismatch { - expected: bytes.len() as u64, - actual: cursor.position(), - }); - } - Ok(directory) - } - - /// Open a Pithos archive and prepare for reading - #[tracing::instrument(level = "trace", skip(pithos_path, private_key_pem_path))] - pub fn new>( - pithos_path: P, - private_key_pem_path: P, - ) -> Result { - let file = Self::open_archive(&pithos_path)?; - - // Read and parse the PEM-encoded private key - let pem_content = std::fs::read_to_string(private_key_pem_path)?; - let private_key = private_key_from_pem_bytes(pem_content.as_bytes())?; - - Ok(Self { - file, - private_key, - limits: ReaderLimits::default(), - external_block_source: None, - }) - } - - /// Init a simple Pithos reader - #[tracing::instrument(level = "trace", skip(pithos_path, private_key))] - pub fn new_with_key>( - pithos_path: P, - private_key: StaticSecret, - ) -> Result { - let file = Self::open_archive(&pithos_path)?; - - Ok(Self { - file, - private_key, - limits: ReaderLimits::default(), - external_block_source: None, - }) - } - - pub fn with_limits(mut self, limits: ReaderLimits) -> Self { - self.limits = limits; - self - } - - pub fn with_external_block_source(mut self, source: Box) -> Self { - self.external_block_source = Some(source); - self - } - - fn load_stored_block(&mut self, meta: &BlockIndexEntry) -> Result, PithosError> { - let stored_size = Self::checked_stored_size(meta, &self.limits)?; - let mut block_data = Self::zeroed_buffer(stored_size, "stored block")?; - let mut block_header = [0u8; 4]; - - match &meta.location { - BlockLocation::Local => { - self.validate_local_block_range(meta)?; - self.file.seek(SeekFrom::Start(meta.offset))?; - self.file.read_exact(&mut block_header)?; - BlockHeader::deserialize(&mut block_header.as_slice())?; - self.file.read_exact(&mut block_data)?; - } - BlockLocation::External { url } => { - let max_response_size = meta.stored_size.checked_add(5).ok_or_else(|| { - PithosError::ExternalBlockFraming("response size overflow".into()) - })?; - let source = self - .external_block_source - .as_ref() - .ok_or(PithosError::ExternalBlockSourceRequired)?; - let mut response = source.open(url, max_response_size)?; - response.read_exact(&mut block_header).map_err(|error| { - PithosError::ExternalBlockFraming(format!("short block marker: {error}")) - })?; - BlockHeader::deserialize(&mut block_header.as_slice()).map_err(|error| { - PithosError::ExternalBlockFraming(format!("invalid block marker: {error}")) - })?; - response.read_exact(&mut block_data).map_err(|error| { - PithosError::ExternalBlockFraming(format!("short block payload: {error}")) - })?; - let mut extra = [0u8; 1]; - if response.read(&mut extra).map_err(|error| { - PithosError::ExternalBlockFraming(format!("reading block boundary: {error}")) - })? != 0 - { - return Err(PithosError::ExternalBlockFraming( - "response exceeds expected size".into(), - )); - } - } - } - - Ok(block_data) - } - - /// Init a simple Pithos reader - #[tracing::instrument(level = "trace", skip(_pithos_path, _private_key))] - pub fn new_with_keys>( - _pithos_path: P, - _private_key: Vec, - ) -> Result { - Err(PithosError::UnsupportedMultipleReaderKeys) - } - - #[tracing::instrument(level = "trace", skip(self))] - pub fn read_directory(&mut self) -> Result<(Directory, (u64, u64)), PithosError> { - // Read last 12 bytes for crc32 and directory length - let file_len = self.file.metadata()?.len(); - if file_len < 25 { - return Err(PithosError::Other( - "File too small for directory footer".to_string(), - )); - } - self.file.seek(SeekFrom::End(-12))?; - let mut footer = [0u8; 12]; - self.file.read_exact(&mut footer)?; - - let parent_dir_len = u64::from_be_bytes(footer[..8].try_into().map_err(|_| { - PithosError::Conversion("Failed to convert directory length bytes to u64".to_string()) - })?); - if parent_dir_len < 25 { - return Err(PithosError::DirectoryLengthMismatch { - expected: 25, - actual: parent_dir_len, - }); - } - if parent_dir_len > self.limits.max_directory_bytes { - return Err(PithosError::LimitExceeded { - field: "directory", - limit: self.limits.max_directory_bytes, - actual: parent_dir_len, - }); - } - let parent_dir_start = file_len.checked_sub(parent_dir_len).ok_or_else(|| { - PithosError::InvalidDirectoryRange("terminal directory exceeds archive".into()) - })?; - let parent_dir_len_usize = usize::try_from(parent_dir_len).map_err(|_| { - PithosError::InvalidDirectoryRange("directory length does not fit platform".into()) - })?; - - self.file.seek(SeekFrom::Start(parent_dir_start))?; - let mut dir_buf = Vec::new(); - dir_buf - .try_reserve_exact(parent_dir_len_usize) - .map_err(|_| PithosError::LimitExceeded { - field: "directory allocation", - limit: self.limits.max_directory_bytes, - actual: parent_dir_len, - })?; - dir_buf.resize(parent_dir_len_usize, 0); - self.file.read_exact(&mut dir_buf)?; - - // Deserialize full directory - let mut available_file_keys = HashMap::new(); - let mut directory = Self::parse_directory(&dir_buf, &self.limits)?; - available_file_keys.extend( - directory - .decrypt_recipient_with_limits(&self.private_key, &self.limits.deserialization)?, - ); - - // Merge with parent directories - let mut visited = std::collections::HashSet::new(); - let mut depth = 0u64; - let mut child_start = parent_dir_start; - while let Some((start, len)) = directory.parent_directory_offset { - depth = depth - .checked_add(1) - .ok_or_else(|| PithosError::InvalidDirectoryChain("depth overflow".into()))?; - if depth > self.limits.max_parent_directories { - return Err(PithosError::LimitExceeded { - field: "parent directories", - limit: self.limits.max_parent_directories, - actual: depth, - }); - } - if len < 25 || len > self.limits.max_directory_bytes { - return Err(PithosError::InvalidDirectoryRange( - "parent length out of bounds".into(), - )); - } - let end = start.checked_add(len).ok_or_else(|| { - PithosError::InvalidDirectoryRange("parent range overflow".into()) - })?; - if end > child_start || !visited.insert((start, len)) { - return Err(PithosError::InvalidDirectoryChain( - "parent must be backward and nonoverlapping".into(), - )); - } - let len_usize = usize::try_from(len).map_err(|_| { - PithosError::InvalidDirectoryRange("parent length does not fit platform".into()) - })?; - // Read parent directory - self.file.seek(SeekFrom::Start(start))?; - let mut dir_buf = Vec::new(); - dir_buf - .try_reserve_exact(len_usize) - .map_err(|_| PithosError::LimitExceeded { - field: "directory allocation", - limit: self.limits.max_directory_bytes, - actual: len, - })?; - dir_buf.resize(len_usize, 0); - self.file.read_exact(&mut dir_buf)?; - let mut older_directory = Self::parse_directory(&dir_buf, &self.limits)?; - available_file_keys.extend( - older_directory.decrypt_recipient_with_limits( - &self.private_key, - &self.limits.deserialization, - )?, - ); - child_start = start; - - // Merge directories and swap - older_directory.merge(directory)?; - directory = older_directory; - } - - // Remove entries whose encrypted block indexes are unavailable to this reader. - directory - .files - .retain_mut(|id, path, file| match &mut file.block_data { - BlockDataState::Decrypted(_) => true, - BlockDataState::Encrypted(_) => match available_file_keys.get(&id) { - Some(block_key) => match file - .block_data - .decrypt_with_limits(block_key, &self.limits.deserialization) - { - Ok(_) => { - tracing::info!("Successfully decrypted {path}"); - true - } - Err(_) => { - tracing::warn!("Could not decrypt {path}"); - false - } - }, - None => false, - }, - })?; - validate_map(&directory.files)?; - - Ok((directory, (parent_dir_start, parent_dir_len))) - } - - #[tracing::instrument(level = "trace", skip(self, directory))] - pub fn read_file_paths( - &self, - directory: &Directory, - ) -> Result, PithosError> { - Ok(directory - .files - .iter() - .map(|(_, p, f)| (f.file_type, p.to_owned())) - .collect()) - } - - #[tracing::instrument( - level = "trace", - skip(self, inner_path, directory, output_path, ranges) - )] - pub fn read_file( - &mut self, - inner_path: &str, - directory: &Directory, - output_path: Option<&PathBuf>, - ranges: Option>>, - ) -> Result<(), PithosError> { - let file_entry = directory - .files - .get_by_path(inner_path) - .ok_or(PithosError::FileNotFound(inner_path.to_string()))?; - validate_existing_candidate(&directory.files, inner_path, file_entry)?; - - match &file_entry.file_type { - FileType::Data | FileType::Metadata => { - if output_path.is_none() { - let mut output_target: Box = Box::new(io::stdout()); - if let Some(ranges) = ranges { - for range in ranges { - self.read_data_range_to_sink( - range, - file_entry, - &directory.blocks, - &mut output_target, - )?; - } - } else { - self.read_data_to_sink(file_entry, &directory.blocks, output_target)?; - } - return Ok(()); - } - let dest = output_path.unwrap(); - let (root_path, final_path, create_root) = if dest.is_dir() { - (dest.as_path(), inner_path.to_string(), false) - } else { - let parent = dest.parent().unwrap_or_else(|| Path::new(".")); - let name = - dest.file_name() - .and_then(|name| name.to_str()) - .ok_or_else(|| { - PithosError::Conversion("Invalid output file name".into()) - })?; - (parent, name.to_string(), false) - }; - let root = ExtractionRoot::open(root_path, create_root)?; - let pending = root.pending_file(&final_path)?; - let mut output_target: Box = Box::new(pending.writer()?); - if let Some(ranges) = ranges { - for range in ranges { - self.read_data_range_to_sink( - range, - file_entry, - &directory.blocks, - &mut output_target, - )?; - } - } else { - self.read_data_to_sink(file_entry, &directory.blocks, output_target)?; - } - pending.commit()?; - } - FileType::Directory => { - let root_path = output_path - .map(PathBuf::as_path) - .unwrap_or_else(|| Path::new(".")); - ExtractionRoot::open(root_path, true)?.create_dir(inner_path)?; - } - FileType::Symlink => { - let target = file_entry.symlink_target.as_deref().ok_or_else(|| { - PithosError::InvalidSymlinkEntry { - path: inner_path.into(), - reason: "missing target".into(), - } - })?; - let root_path = output_path - .map(PathBuf::as_path) - .unwrap_or_else(|| Path::new(".")); - ExtractionRoot::open(root_path, true)?.create_symlink(inner_path, target)?; - } - } - - Ok(()) - } - - #[tracing::instrument(level = "trace", skip(self, inner_path, directory, output))] - pub fn read_file_to_crypt4gh( - &mut self, - inner_path: &str, - directory: &Directory, - reader_keys: Vec, - output: Option>, - ) -> Result<(), PithosError> { - // Fetch file entry from directory - let file_entry = directory - .get_file_by_path(inner_path) - .ok_or(PithosError::FileNotFound(inner_path.to_string()))?; - - // Validate file type - if ![FileType::Data, FileType::Metadata].contains(&file_entry.file_type) { - return Err(PithosError::InvalidFileType( - "Only data/metadata files can be exported to Crypt4GH".to_string(), - )); - } - - // Generate Crypt4GH header packets from keys - let file_id = directory.files.get_id_by_path(inner_path).ok_or_else(|| { - PithosError::Other(format!("Could not find file id for {inner_path}")) - })?; - let data_key = - directory - .get_file_encryption_key(file_id) - .ok_or(PithosError::FileNotFound(format!( - "Could not extract key for file: {}", - inner_path - )))?; - let packets = HeaderPacket::from_pithos(&self.private_key, reader_keys, &data_key) - .map_err(|e| { - PithosError::Conversion(format!("Conversion to header packet failed: {e})")) - })?; - - // Init output sink - let mut sink = if let Some(sink) = output { - sink - } else { - Box::new(io::stdout()) - }; - - // Write Crypt4GH header - let header = Crypt4GHHeader::new(packets); - let header_bytes: Vec = header.try_into()?; - sink.write_all(&header_bytes)?; - - // Load blocks and write data in 64kb blocks - match &file_entry.block_data { - BlockDataState::Encrypted(_) => { - return Err(PithosError::InvalidBlockDataState( - "Cannot read encrypted block data".to_string(), - )); - } - BlockDataState::Decrypted(blocks) => { - let mut buffer = Vec::with_capacity(65536); // 64kb buffer - for (hash, key) in blocks { - // Fetch block meta from directory - let block_meta = directory - .blocks - .get(hash) - .ok_or(PithosError::BlockHashNotFound(*hash))?; - - let mut block_buf = self.load_stored_block(block_meta)?; - - block_buf = Self::decode_and_verify_block( - block_buf, - key, - hash, - block_meta, - &self.limits, - )?; - - // Write chunk data in 64KiB ChaCha20Poly1305 encrypted blocks - let mut chunk_offset = 0; - while chunk_offset < block_buf.len() { - let remaining_chunk = &block_buf[chunk_offset..]; - let remaining_buffer = CRYPT4GH_BLOCK_SIZE - buffer.len(); - - if remaining_chunk.len() <= remaining_buffer { - // Chunk fits in remaining buffer space - buffer.extend_from_slice(remaining_chunk); - chunk_offset = block_buf.len(); // Consumed the entire chunk - - // Check if buffer is full - if buffer.len() == CRYPT4GH_BLOCK_SIZE { - let encrypted = encrypt_chunk(&buffer, b"", &data_key)?; - sink.write_all(&encrypted)?; - //sink.write_all(&encrypt_chunk(&buffer, b"", &data_key)?)?; - buffer.clear(); - } - } else if buffer.is_empty() && remaining_chunk.len() > CRYPT4GH_BLOCK_SIZE { - // Large chunk that exceeds buffer capacity -> Process in buffer-sized pieces - let chunk_to_process = &remaining_chunk[..CRYPT4GH_BLOCK_SIZE]; - let encrypted = encrypt_chunk(chunk_to_process, b"", &data_key)?; - sink.write_all(&encrypted)?; - //sink.write_all(&encrypt_chunk(&chunk_to_process, b"", &data_key)?)?; - chunk_offset += CRYPT4GH_BLOCK_SIZE; - } else { - // Fill remaining buffer space with part of the chunk - let bytes_to_take = remaining_buffer; - buffer.extend_from_slice(&remaining_chunk[..bytes_to_take]); - chunk_offset += bytes_to_take; - - // Buffer is now full - let encrypted = encrypt_chunk(&buffer, b"", &data_key)?; - sink.write_all(&encrypted)?; - buffer.clear(); - } - } - } - - // Write partial filled buffer if is not empty - if !buffer.is_empty() { - let encrypted = encrypt_chunk(&buffer, b"", &data_key)?; - sink.write_all(&encrypted)?; - //sink.write_all(&encrypt_chunk(&buffer, b"", &data_key)?)?; - buffer.clear() - } - } - } - - Ok(()) - } - - #[tracing::instrument(level = "trace", skip(self, file_entry, block_index, sink))] - fn read_data_to_sink( - &mut self, - file_entry: &FileEntry, - block_index: &IndexMap<[u8; 32], BlockIndexEntry>, - mut sink: Box, - ) -> Result<(), PithosError> { - match &file_entry.block_data { - BlockDataState::Encrypted(_) => { - return Err(PithosError::InvalidBlockDataState( - "Block data is encrypted".to_string(), - )); - } - BlockDataState::Decrypted(blocks) => { - for (hash, key) in blocks { - // Fetch block meta from directory - let block_meta = block_index - .get(hash) - .ok_or(PithosError::BlockHashNotFound(*hash))?; - - let mut block_data = self.load_stored_block(block_meta)?; - - block_data = Self::decode_and_verify_block( - block_data, - key, - hash, - block_meta, - &self.limits, - )?; - - sink.write_all(&block_data)?; - } - } - } - - Ok(()) - } - - #[tracing::instrument(level = "trace", skip(self, byte_range, file_entry, block_index, sink))] - pub fn read_data_range_to_sink( - &mut self, - byte_range: Range, - file_entry: &FileEntry, - block_index: &IndexMap<[u8; 32], BlockIndexEntry>, - sink: &mut Box, - ) -> Result<(), PithosError> { - let mut block_byte_sum: u64 = 0; - - match &file_entry.block_data { - BlockDataState::Encrypted(_) => { - return Err(PithosError::InvalidBlockDataState( - "Block data is still encrypted".to_string(), - )); - } - BlockDataState::Decrypted(blocks) => { - for (hash, key) in blocks { - // Fetch block meta from directory - let block_meta = block_index - .get(hash) - .ok_or(PithosError::BlockHashNotFound(*hash))?; - - Self::checked_stored_size(block_meta, &self.limits)?; - let block_start = block_byte_sum; - let block_end = block_byte_sum - .checked_add(block_meta.original_size) - .ok_or_else(|| { - PithosError::InvalidDirectoryRange("range block size overflow".into()) - })?; - block_byte_sum = block_end; - - // If block ends before start of range; Discard block - if block_end <= byte_range.start { - continue; - } - - // If block starts after end of range; Stop loop - if block_start >= byte_range.end { - break; - } - - let mut block_buf = self.load_stored_block(block_meta)?; - - block_buf = Self::decode_and_verify_block( - block_buf, - key, - hash, - block_meta, - &self.limits, - )?; - - // Calculate the range within this block to write - let write_start = if byte_range.start > block_start { - usize::try_from(byte_range.start - block_start).map_err(|_| { - PithosError::InvalidDirectoryRange( - "range index does not fit platform".into(), - ) - })? - } else { - 0 - }; - let write_end = if byte_range.end < block_end { - usize::try_from(byte_range.end - block_start).map_err(|_| { - PithosError::InvalidDirectoryRange( - "range index does not fit platform".into(), - ) - })? - } else { - block_buf.len() - }; - - // Write only the relevant slice of the block - sink.write_all(&block_buf[write_start..write_end])?; - - // If we've written enough bytes, stop - if block_end >= byte_range.end { - break; - } - } - } - } - - Ok(()) - } -} diff --git a/crates/pithos_lib/src/source.rs b/crates/pithos_lib/src/source.rs new file mode 100644 index 0000000..18fd2eb --- /dev/null +++ b/crates/pithos_lib/src/source.rs @@ -0,0 +1,294 @@ +use std::fs::File; +use std::io; +use std::os::unix::fs::FileExt; +use std::path::PathBuf; +use std::sync::Arc; + +/// Context for acquisition failures at the fixed, offset-based archive boundary. +#[derive(Debug, thiserror::Error)] +pub enum SourceError { + #[error("source {operation} for {path}: {source}")] + PathIo { + operation: &'static str, + path: PathBuf, + #[source] + source: io::Error, + }, + #[error("source {operation} at offset {offset}: {source}")] + Io { + operation: &'static str, + offset: u64, + #[source] + source: io::Error, + }, + #[error("source EOF at offset {offset}: needed {expected} bytes, received {actual}")] + UnexpectedEof { + offset: u64, + expected: usize, + actual: usize, + }, + #[error("source range overflows at offset {offset} for {length} bytes")] + RangeOverflow { offset: u64, length: usize }, + #[error("remote range at offset {offset}: {message}")] + Remote { offset: u64, message: String }, +} + +/// An immutable, positioned byte source for an archive. +#[allow(clippy::len_without_is_empty)] // The fixed public contract is intentionally just length plus positioned exact read. +pub trait ArchiveSource { + fn len(&self) -> Result; + fn read_exact_at(&self, offset: u64, buffer: &mut [u8]) -> Result<(), SourceError>; +} + +/// Linux positioned-file archive source. +pub struct FileSource { + file: File, + length: u64, +} + +impl FileSource { + pub fn open(path: impl AsRef) -> Result { + let path = path.as_ref(); + let file = File::open(path).map_err(|source| SourceError::PathIo { + operation: "open", + path: path.to_path_buf(), + source, + })?; + let length = file + .metadata() + .map_err(|source| SourceError::PathIo { + operation: "metadata", + path: path.to_path_buf(), + source, + })? + .len(); + Ok(Self { file, length }) + } + + pub(crate) fn from_file(file: File) -> Result { + let length = file + .metadata() + .map_err(|source| SourceError::Io { + operation: "metadata", + offset: 0, + source, + })? + .len(); + Ok(Self { file, length }) + } +} + +impl ArchiveSource for FileSource { + fn len(&self) -> Result { + Ok(self.length) + } + + fn read_exact_at(&self, offset: u64, buffer: &mut [u8]) -> Result<(), SourceError> { + let _ = offset + .checked_add(buffer.len() as u64) + .ok_or(SourceError::RangeOverflow { + offset, + length: buffer.len(), + })?; + let mut read = 0; + while read != buffer.len() { + let position = offset + .checked_add(read as u64) + .ok_or(SourceError::RangeOverflow { + offset, + length: buffer.len(), + })?; + let count = self + .file + .read_at(&mut buffer[read..], position) + .map_err(|source| SourceError::Io { + operation: "read", + offset: position, + source, + })?; + if count == 0 { + return Err(SourceError::UnexpectedEof { + offset, + expected: buffer.len(), + actual: read, + }); + } + read += count; + } + Ok(()) + } +} + +#[derive(Clone)] +/// Immutable in-memory archive source. +pub struct MemorySource { + bytes: Arc<[u8]>, +} + +impl MemorySource { + pub fn new(bytes: impl Into>) -> Self { + Self { + bytes: bytes.into(), + } + } +} + +impl ArchiveSource for MemorySource { + fn len(&self) -> Result { + Ok(self.bytes.len() as u64) + } + + fn read_exact_at(&self, offset: u64, buffer: &mut [u8]) -> Result<(), SourceError> { + let end = offset + .checked_add(buffer.len() as u64) + .ok_or(SourceError::RangeOverflow { + offset, + length: buffer.len(), + })?; + let start = usize::try_from(offset).map_err(|_| SourceError::RangeOverflow { + offset, + length: buffer.len(), + })?; + let end = usize::try_from(end).map_err(|_| SourceError::RangeOverflow { + offset, + length: buffer.len(), + })?; + let bytes = self + .bytes + .get(start..end) + .ok_or(SourceError::UnexpectedEof { + offset, + expected: buffer.len(), + actual: self.bytes.len().saturating_sub(start), + })?; + buffer.copy_from_slice(bytes); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::{Arc, Mutex}; + + fn assert_send_sync() {} + + /// Blocking range prototype for source-equivalence tests. It deliberately + /// owns returned bytes and has no transport policy. + struct RemoteRangeSource { + length: u64, + fetch: F, + } + + impl RemoteRangeSource { + fn new(length: u64, fetch: F) -> Self { + Self { length, fetch } + } + } + + impl ArchiveSource for RemoteRangeSource + where + F: Fn(u64, usize) -> Result, SourceError>, + { + fn len(&self) -> Result { + Ok(self.length) + } + + fn read_exact_at(&self, offset: u64, buffer: &mut [u8]) -> Result<(), SourceError> { + let end = + offset + .checked_add(buffer.len() as u64) + .ok_or(SourceError::RangeOverflow { + offset, + length: buffer.len(), + })?; + if end > self.length { + return Err(SourceError::UnexpectedEof { + offset, + expected: buffer.len(), + actual: self.length.saturating_sub(offset) as usize, + }); + } + let response = (self.fetch)(offset, buffer.len())?; + if response.len() != buffer.len() { + return Err(SourceError::UnexpectedEof { + offset, + expected: buffer.len(), + actual: response.len(), + }); + } + buffer.copy_from_slice(&response); + Ok(()) + } + } + + #[test] + fn memory_source_is_exact_and_concurrent() { + assert_send_sync::(); + let source = Arc::new(MemorySource::new(Arc::<[u8]>::from(&b"abcdef"[..]))); + let handles = (0..4).map(|_| { + let source = Arc::clone(&source); + std::thread::spawn(move || { + let mut output = [0; 3]; + source.read_exact_at(2, &mut output).unwrap(); + output + }) + }); + for handle in handles { + assert_eq!(handle.join().unwrap(), *b"cde"); + } + let mut output = [0; 2]; + assert!(matches!( + source.read_exact_at(5, &mut output), + Err(SourceError::UnexpectedEof { offset: 5, .. }) + )); + assert_eq!(source.len().unwrap(), 6); + } + + #[test] + fn remote_range_maps_exact_requests_and_short_responses() { + let requests = Arc::new(Mutex::new(Vec::new())); + let captured = Arc::clone(&requests); + let source = RemoteRangeSource::new(6, move |offset, length| { + captured.lock().unwrap().push((offset, length)); + Ok(b"cde".to_vec()) + }); + let mut output = [0; 3]; + source.read_exact_at(2, &mut output).unwrap(); + assert_eq!(output, *b"cde"); + assert_eq!(*requests.lock().unwrap(), vec![(2, 3)]); + + let short = RemoteRangeSource::new(6, |_offset, _length| Ok(vec![1])); + assert!(matches!( + short.read_exact_at(2, &mut output), + Err(SourceError::UnexpectedEof { actual: 1, .. }) + )); + + let failure = RemoteRangeSource::new(6, |offset, _length| { + Err(SourceError::Remote { + offset, + message: "transport failed".into(), + }) + }); + assert!(matches!( + failure.read_exact_at(2, &mut output), + Err(SourceError::Remote { offset: 2, .. }) + )); + } + + #[test] + fn file_source_reports_contextual_eof() { + let temporary = tempfile::NamedTempFile::new().unwrap(); + std::fs::write(temporary.path(), b"abc").unwrap(); + let source = FileSource::open(temporary.path()).unwrap(); + let mut output = [0; 4]; + assert!(matches!( + source.read_exact_at(0, &mut output), + Err(SourceError::UnexpectedEof { + offset: 0, + actual: 3, + .. + }) + )); + } +} diff --git a/crates/pithos_lib/tests/access.rs b/crates/pithos_lib/tests/access.rs new file mode 100644 index 0000000..be022dd --- /dev/null +++ b/crates/pithos_lib/tests/access.rs @@ -0,0 +1,23 @@ +mod common; + +use common::util::{fixture, open, private_key}; +use pithos_lib::archive::AccessKeys; +use pithos_lib::error::PithosError; + +#[test] +fn no_key_and_wrong_key_preserve_the_same_visible_entry_shape() { + let temporary = tempfile::tempdir().unwrap(); + let path = fixture(&temporary, "recipient1"); + let no_key = open(&path, AccessKeys::new()); + let wrong_key = open(&path, AccessKeys::new().with_key(private_key("recipient2"))); + assert_eq!( + no_key.entries().collect::>(), + wrong_key.entries().collect::>() + ); + for archive in [&no_key, &wrong_key] { + assert!(matches!( + archive.copy_to("data", &mut Vec::new()), + Err(PithosError::ContentUnavailable) + )); + } +} diff --git a/crates/pithos_lib/tests/block_pipeline.rs b/crates/pithos_lib/tests/block_pipeline.rs new file mode 100644 index 0000000..b621ce6 --- /dev/null +++ b/crates/pithos_lib/tests/block_pipeline.rs @@ -0,0 +1,35 @@ +mod common; + +use common::util::{fixture, open, private_key}; +use pithos_lib::archive::AccessKeys; +use std::io::Write; + +struct RecordingSink(Vec); + +impl Write for RecordingSink { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0.extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +#[test] +fn public_archive_rejects_a_bad_block_marker_without_sink_output() { + let temporary = tempfile::tempdir().unwrap(); + let path = fixture(&temporary, "recipient1"); + let mut bytes = std::fs::read(&path).unwrap(); + let marker = bytes + .windows(4) + .position(|window| window == b"BLCK") + .unwrap(); + bytes[marker] ^= 1; + std::fs::write(&path, bytes).unwrap(); + let archive = open(&path, AccessKeys::new().with_key(private_key("recipient1"))); + let mut sink = RecordingSink(Vec::new()); + assert!(archive.copy_to("data", &mut sink).is_err()); + assert!(sink.0.is_empty()); +} diff --git a/crates/pithos_lib/tests/common/append.rs b/crates/pithos_lib/tests/common/append.rs new file mode 100644 index 0000000..07e92e9 --- /dev/null +++ b/crates/pithos_lib/tests/common/append.rs @@ -0,0 +1,177 @@ +use super::{ + archive::open, + keys::{private_key, public_key}, +}; +use pithos_lib::archive::{ + AccessKeys, AppendOptions, Archive, ArchivePath, ArchiveWriter, CdcConfig, EntryMetadata, + ProcessingOptions, WriteOptions, +}; +use pithos_lib::fs::{FsError, append_files, grant_readers}; +use pithos_lib::source::FileSource; +use std::fs::File; +use std::io::Cursor; +use std::path::{Path, PathBuf}; + +#[allow(dead_code)] +pub const SELECTED_ID: u64 = 3; +#[allow(dead_code)] +pub const WITHHELD_ID: u64 = 4; +#[allow(dead_code)] +pub const DIRECTORY_ID: u64 = 2; + +#[allow(dead_code)] +pub struct AppendFixture { + pub archive: PathBuf, +} + +impl AppendFixture { + #[allow(dead_code)] + pub fn open_as(&self, key: &str) -> Archive { + open(&self.archive, AccessKeys::new().with_key(private_key(key))) + } + + #[allow(dead_code)] + pub fn append_source(&self, name: &str, contents: &[u8]) -> PathBuf { + let source = self.archive.parent().unwrap().join(name); + std::fs::write(&source, contents).unwrap(); + source + } +} + +#[allow(dead_code)] +pub fn append_fixture(temporary: &tempfile::TempDir) -> AppendFixture { + let archive = temporary.path().join("append-base.pith"); + let sender = private_key("sender"); + let sender_public = sender.public_key(); + let mut writer = ArchiveWriter::create( + File::create(&archive).unwrap(), + WriteOptions::new(sender, vec![sender_public, public_key("recipient1")]), + ) + .unwrap(); + assert_eq!( + writer + .add_file( + ArchivePath::new("older.txt").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(true, 2).unwrap(), + None, + Cursor::new(b"older base payload"), + ) + .unwrap() + .id, + 0 + ); + assert_eq!( + writer + .add_file( + ArchivePath::new("older-withheld.txt").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(true, 2).unwrap(), + None, + Cursor::new(b"older withheld payload"), + ) + .unwrap() + .id, + 1 + ); + assert_eq!( + writer + .add_directory( + ArchivePath::new("records").unwrap(), + EntryMetadata::new(0, 0, 0o755), + ) + .unwrap() + .id, + DIRECTORY_ID + ); + writer.finish().unwrap(); + let selected = archive.parent().unwrap().join("selected.txt"); + let withheld = archive.parent().unwrap().join("withheld.txt"); + std::fs::write(&selected, b"selected base payload").unwrap(); + std::fs::write(&withheld, b"withheld base payload").unwrap(); + append(&archive, vec![selected, withheld]).unwrap(); + AppendFixture { archive } +} + +#[allow(dead_code)] +pub fn append(archive: &Path, sources: Vec) -> Result<(), FsError> { + append_with_cdc(archive, CdcConfig::default(), sources) +} + +#[allow(dead_code)] +pub fn append_with_cdc( + archive: &Path, + cdc: CdcConfig, + sources: Vec, +) -> Result<(), FsError> { + append_files( + archive, + AppendOptions::new( + private_key("sender"), + vec![public_key("sender"), public_key("recipient1")], + ) + .with_cdc(cdc), + &sources, + ) + .map(|_| ()) +} + +#[allow(dead_code)] +pub fn archive_with_entry(temporary: &tempfile::TempDir, path: &str) -> PathBuf { + let archive = temporary.path().join("hierarchy.pith"); + let sender = private_key("sender"); + let mut writer = ArchiveWriter::create( + File::create(&archive).unwrap(), + WriteOptions::new(sender, vec![public_key("sender"), public_key("recipient1")]), + ) + .unwrap(); + writer + .add_file( + ArchivePath::new(path).unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(true, 2).unwrap(), + None, + Cursor::new(b"existing"), + ) + .unwrap(); + writer.finish().unwrap(); + archive +} + +#[allow(dead_code)] +pub fn archive_with_symlink(temporary: &tempfile::TempDir) -> PathBuf { + let archive = temporary.path().join("symlink.pith"); + let sender = private_key("sender"); + let mut writer = ArchiveWriter::create( + File::create(&archive).unwrap(), + WriteOptions::new(sender, vec![public_key("sender"), public_key("recipient1")]), + ) + .unwrap(); + writer + .add_symlink( + ArchivePath::new("link").unwrap(), + EntryMetadata::new(0, 0, 0o777), + "target", + ) + .unwrap(); + writer.finish().unwrap(); + archive +} + +#[allow(dead_code)] +pub fn grant( + archive: &Path, + sender: &str, + recipients: Vec<&str>, + ids: Vec, +) -> Result<(), FsError> { + grant_readers( + archive, + AppendOptions::new( + private_key(sender), + recipients.into_iter().map(public_key).collect(), + ), + &ids, + ) + .map(|_| ()) +} diff --git a/crates/pithos_lib/tests/common/archive.rs b/crates/pithos_lib/tests/common/archive.rs new file mode 100644 index 0000000..99e4035 --- /dev/null +++ b/crates/pithos_lib/tests/common/archive.rs @@ -0,0 +1,12 @@ +use pithos_lib::archive::{AccessKeys, Archive, OpenOptions}; +use pithos_lib::source::FileSource; +use std::path::Path; + +#[allow(dead_code)] // Integration targets compile this shared module independently. +pub fn open(path: &Path, keys: AccessKeys) -> Archive { + Archive::open( + FileSource::open(path).unwrap(), + OpenOptions::default().with_access_keys(keys), + ) + .unwrap() +} diff --git a/crates/pithos_lib/tests/common/fixtures.rs b/crates/pithos_lib/tests/common/fixtures.rs new file mode 100644 index 0000000..64260e6 --- /dev/null +++ b/crates/pithos_lib/tests/common/fixtures.rs @@ -0,0 +1,29 @@ +use super::keys::{private_key, public_key}; +use pithos_lib::archive::{ + ArchivePath, ArchiveWriter, EntryMetadata, ProcessingOptions, WriteOptions, +}; +use std::fs::File; +use std::path::PathBuf; + +#[allow(dead_code)] // Integration targets compile this shared module independently. +pub fn fixture(directory: &tempfile::TempDir, recipient: &str) -> PathBuf { + let path = directory.path().join("fixture.pith"); + let sender = private_key("sender"); + let recipient = public_key(recipient); + let mut writer = ArchiveWriter::create( + File::create(&path).unwrap(), + WriteOptions::new(sender, vec![recipient]), + ) + .unwrap(); + writer + .add_file( + ArchivePath::new("data").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(true, 0).unwrap(), + Some(29), + std::io::Cursor::new(b"public archive reader fixture"), + ) + .unwrap(); + writer.finish().unwrap(); + path +} diff --git a/crates/pithos_lib/tests/common/keys.rs b/crates/pithos_lib/tests/common/keys.rs new file mode 100644 index 0000000..5ba52eb --- /dev/null +++ b/crates/pithos_lib/tests/common/keys.rs @@ -0,0 +1,17 @@ +use pithos_lib::crypto::{PrivateKey, PublicKey}; + +#[allow(dead_code)] // Integration targets compile this shared module independently. +pub fn private_key(name: &str) -> PrivateKey { + pithos_lib::crypto::parse_private_pem( + &std::fs::read(format!("tests/data/keys/{name}_private.pem")).unwrap(), + ) + .unwrap() +} + +#[allow(dead_code)] // Integration targets compile this shared module independently. +pub fn public_key(name: &str) -> PublicKey { + pithos_lib::crypto::parse_public_pem( + &std::fs::read(format!("tests/data/keys/{name}_public.pem")).unwrap(), + ) + .unwrap() +} diff --git a/crates/pithos_lib/tests/common/mod.rs b/crates/pithos_lib/tests/common/mod.rs index 812d1ed..ada0e4a 100644 --- a/crates/pithos_lib/tests/common/mod.rs +++ b/crates/pithos_lib/tests/common/mod.rs @@ -1 +1,18 @@ -pub mod util; +pub mod append; +pub mod archive; +pub mod fixtures; +pub mod keys; +pub mod ro_crate; +pub mod writer; + +pub mod util { + #[allow(unused_imports)] + // Targets import different subsets through this compatibility facade. + pub use super::archive::open; + #[allow(unused_imports)] + // Targets import different subsets through this compatibility facade. + pub use super::fixtures::fixture; + #[allow(unused_imports)] + // Targets import different subsets through this compatibility facade. + pub use super::keys::{private_key, public_key}; +} diff --git a/crates/pithos_lib/tests/common/ro_crate.rs b/crates/pithos_lib/tests/common/ro_crate.rs new file mode 100644 index 0000000..a61f1dc --- /dev/null +++ b/crates/pithos_lib/tests/common/ro_crate.rs @@ -0,0 +1,136 @@ +use super::keys::{private_key, public_key}; +use pithos_lib::adapters::ro_crate::{LoadedRoCrate, RoCrateError, write_ro_crate}; +use pithos_lib::archive::{ + ArchivePath, ArchiveWriter, CdcConfig, EntryMetadata, ProcessingOptions, WriteOptions, +}; +use std::fs::{self, File}; +use std::path::Path; + +#[allow(dead_code)] +pub fn metadata(has_part: &[&str]) -> String { + let parts = has_part + .iter() + .map(|path| format!(r#"{{"@id":"{path}"}}"#)) + .collect::>() + .join(","); + let entities = has_part + .iter() + .map(|path| format!(r#"{{"@id":"{path}","@type":"File"}}"#)) + .collect::>() + .join(","); + let entity_suffix = if entities.is_empty() { + String::new() + } else { + format!(",{entities}") + }; + format!( + r#"{{"@context":"https://w3id.org/ro/crate/1.2/context","@graph":[{{"@id":"ro-crate-metadata.json","@type":"CreativeWork","conformsTo":{{"@id":"https://w3id.org/ro/crate/1.2"}},"about":{{"@id":"./"}}}},{{"@id":"./","@type":"Dataset","name":"Test Crate","description":"A test RO-Crate","datePublished":"2024-01-01","license":"MIT","hasPart":[{parts}]}}{entity_suffix}]}}"# + ) +} + +#[allow(dead_code)] +pub fn write_raw_zip(path: &Path, entries: &[(&[u8], &[u8], u32)]) { + let mut archive = Vec::new(); + let mut central_directory = Vec::new(); + let mut offsets = Vec::with_capacity(entries.len()); + for (name, content, permissions) in entries { + let offset = archive.len() as u32; + offsets.push(offset); + let checksum = crc32fast::hash(content); + archive.extend_from_slice(&0x0403_4b50u32.to_le_bytes()); + archive.extend_from_slice(&20u16.to_le_bytes()); + archive.extend_from_slice(&0u16.to_le_bytes()); + archive.extend_from_slice(&0u16.to_le_bytes()); + archive.extend_from_slice(&0u16.to_le_bytes()); + archive.extend_from_slice(&0u16.to_le_bytes()); + archive.extend_from_slice(&checksum.to_le_bytes()); + archive.extend_from_slice(&(content.len() as u32).to_le_bytes()); + archive.extend_from_slice(&(content.len() as u32).to_le_bytes()); + archive.extend_from_slice(&(name.len() as u16).to_le_bytes()); + archive.extend_from_slice(&0u16.to_le_bytes()); + archive.extend_from_slice(name); + archive.extend_from_slice(content); + + central_directory.extend_from_slice(&0x0201_4b50u32.to_le_bytes()); + central_directory.extend_from_slice(&20u16.to_le_bytes()); + central_directory.extend_from_slice(&20u16.to_le_bytes()); + central_directory.extend_from_slice(&0u16.to_le_bytes()); + central_directory.extend_from_slice(&0u16.to_le_bytes()); + central_directory.extend_from_slice(&0u16.to_le_bytes()); + central_directory.extend_from_slice(&0u16.to_le_bytes()); + central_directory.extend_from_slice(&checksum.to_le_bytes()); + central_directory.extend_from_slice(&(content.len() as u32).to_le_bytes()); + central_directory.extend_from_slice(&(content.len() as u32).to_le_bytes()); + central_directory.extend_from_slice(&(name.len() as u16).to_le_bytes()); + central_directory.extend_from_slice(&0u16.to_le_bytes()); + central_directory.extend_from_slice(&0u16.to_le_bytes()); + central_directory.extend_from_slice(&0u16.to_le_bytes()); + central_directory.extend_from_slice(&0u16.to_le_bytes()); + central_directory.extend_from_slice(&(*permissions << 16).to_le_bytes()); + central_directory.extend_from_slice(&offset.to_le_bytes()); + central_directory.extend_from_slice(name); + } + let central_offset = archive.len() as u32; + archive.extend_from_slice(¢ral_directory); + archive.extend_from_slice(&0x0605_4b50u32.to_le_bytes()); + archive.extend_from_slice(&0u16.to_le_bytes()); + archive.extend_from_slice(&0u16.to_le_bytes()); + archive.extend_from_slice(&(entries.len() as u16).to_le_bytes()); + archive.extend_from_slice(&(entries.len() as u16).to_le_bytes()); + archive.extend_from_slice(&(central_directory.len() as u32).to_le_bytes()); + archive.extend_from_slice(¢ral_offset.to_le_bytes()); + archive.extend_from_slice(&0u16.to_le_bytes()); + fs::write(path, archive).unwrap(); +} + +#[allow(dead_code)] +pub fn write_loaded(output: &Path, loaded: LoadedRoCrate, cdc: CdcConfig) { + let mut writer = ArchiveWriter::create( + File::create(output).unwrap(), + WriteOptions::new(private_key("sender"), vec![public_key("recipient1")]).with_cdc(cdc), + ) + .unwrap(); + write_ro_crate( + &mut writer, + loaded, + ProcessingOptions::new(false, 0).unwrap(), + ) + .unwrap(); + writer.finish().unwrap(); +} + +#[allow(dead_code)] +pub fn conversion_error(loaded: LoadedRoCrate) -> RoCrateError { + let mut writer = ArchiveWriter::create( + Vec::new(), + WriteOptions::new(private_key("sender"), vec![public_key("recipient1")]), + ) + .unwrap(); + write_ro_crate( + &mut writer, + loaded, + ProcessingOptions::new(false, 0).unwrap(), + ) + .unwrap_err() +} + +#[allow(dead_code)] +pub fn duplicate_conversion_error(loaded: LoadedRoCrate, path: &str) -> RoCrateError { + let mut writer = ArchiveWriter::create( + Vec::new(), + WriteOptions::new(private_key("sender"), vec![public_key("recipient1")]), + ) + .unwrap(); + writer + .add_directory( + ArchivePath::new(path).unwrap(), + EntryMetadata::new(0, 0, 0o755), + ) + .unwrap(); + write_ro_crate( + &mut writer, + loaded, + ProcessingOptions::new(false, 0).unwrap(), + ) + .unwrap_err() +} diff --git a/crates/pithos_lib/tests/common/util.rs b/crates/pithos_lib/tests/common/util.rs deleted file mode 100644 index e32a3ea..0000000 --- a/crates/pithos_lib/tests/common/util.rs +++ /dev/null @@ -1,166 +0,0 @@ -use pithos_lib::error::PithosError; -use pithos_lib::helpers::x25519_keys::{private_key_from_pem_bytes, public_key_from_pem_bytes}; -use pithos_lib::io::pithosreader::PithosReaderSimple; -use pithos_lib::io::pithoswriter::{Content, InputFile, PithosWriter}; -use pithos_lib::model::structs::{Directory, FileType, Reference}; -use std::fs::{File, read, read_to_string}; -use std::io::Write; -use std::path::{Path, PathBuf}; -use tempfile::TempDir; -use x25519_dalek::{PublicKey, StaticSecret}; - -pub type ComparablePithosEntry = (String, FileType, u64, Vec); - -pub fn load_test_keys() -> (StaticSecret, PublicKey, PublicKey) { - // Read sender private key - let sender_pem_content = read_to_string("tests/data/keys/sender_private.pem").unwrap(); - let writer_key = private_key_from_pem_bytes(sender_pem_content.as_bytes()).unwrap(); - - // Read recipient 1 public key - let recipient1_pem_content = read_to_string("tests/data/keys/recipient1_public.pem").unwrap(); - let reader_key_01 = public_key_from_pem_bytes(recipient1_pem_content.as_bytes()).unwrap(); - - // Read recipient 2 public key - let recipient2_pem_content = read_to_string("tests/data/keys/recipient2_public.pem").unwrap(); - let reader_key_02 = public_key_from_pem_bytes(recipient2_pem_content.as_bytes()).unwrap(); - - (writer_key, reader_key_01, reader_key_02) -} - -pub fn minimal_ro_crate_metadata(has_part: &[&str]) -> String { - let parts = has_part - .iter() - .map(|path| format!(r#"{{"@id":"{path}"}}"#)) - .collect::>() - .join(","); - let entities = has_part - .iter() - .map(|path| format!(r#"{{"@id":"{path}","@type":"File"}}"#)) - .collect::>() - .join(","); - - let entity_suffix = if entities.is_empty() { - String::new() - } else { - format!(",{entities}") - }; - format!( - r#"{{"@context":"https://w3id.org/ro/crate/1.2/context","@graph":[{{"@id":"ro-crate-metadata.json","@type":"CreativeWork","conformsTo":{{"@id":"https://w3id.org/ro/crate/1.2"}},"about":{{"@id":"./"}}}},{{"@id":"./","@type":"Dataset","name":"Test Crate","description":"A test RO-Crate","datePublished":"2024-01-01","license":"MIT","hasPart":[{parts}]}}{entity_suffix}]}}"# - ) -} - -pub fn write_zip_entries(path: &Path, entries: &[(&str, &[u8], zip::write::SimpleFileOptions)]) { - let file = File::create(path).unwrap(); - let mut archive = zip::ZipWriter::new(file); - for (name, bytes, options) in entries { - archive.start_file(*name, *options).unwrap(); - archive.write_all(bytes).unwrap(); - } - archive.finish().unwrap(); -} - -pub fn create_pithos_writer( - temp_dir: &TempDir, - cdc: Option<(usize, usize, usize)>, -) -> (PathBuf, StaticSecret, PithosWriter) { - let (sender_key, reader_public_key, _) = load_test_keys(); - let reader_key = private_key_from_pem_bytes( - read_to_string("tests/data/keys/recipient1_private.pem") - .unwrap() - .as_bytes(), - ) - .unwrap(); - let path = temp_dir.path().join("converted.pith"); - let writer = PithosWriter::new( - sender_key, - vec![reader_public_key], - cdc, - Box::new(File::create(&path).unwrap()), - ) - .unwrap(); - (path, reader_key, writer) -} - -pub fn read_pithos_directory( - path: &Path, - reader_key: &StaticSecret, -) -> Result<(Directory, Vec), PithosError> { - let mut reader = PithosReaderSimple::new_with_key(path, reader_key.clone())?; - let (directory, _) = reader.read_directory()?; - let mut entries = directory - .files - .iter() - .map(|(_, path, entry)| { - ( - path.to_string(), - entry.file_type, - entry.file_size, - entry.references.clone(), - ) - }) - .collect::>(); - entries.sort_by(|left, right| left.0.cmp(&right.0)); - Ok((directory, entries)) -} - -pub fn extract_pithos_entry( - path: &Path, - reader_key: &StaticSecret, - inner_path: &str, - temp_dir: &TempDir, -) -> Vec { - let mut reader = PithosReaderSimple::new_with_key(path, reader_key.clone()).unwrap(); - let (directory, _) = reader.read_directory().unwrap(); - let output_path = temp_dir.path().join("extracted-entry"); - reader - .read_file(inner_path, &directory, Some(&output_path), None) - .unwrap(); - read(output_path).unwrap() -} - -pub fn write_dummy_pithos(temp_dir: &TempDir, multifile: bool, metadata: bool) -> PathBuf { - // Dummy file(s) - let mut input_files = vec![InputFile { - file_type: FileType::Data, - inner_path: "t8.shakespeare.txt".to_string(), - data: Content::File("tests/data/t8.shakespeare.sample.txt".to_string()), - metadata: if metadata { - Some(Content::Raw(r#"{"foo":"bar"}"#.to_string())) - } else { - None - }, - encrypt: true, - compression_level: Some(3), - }]; - - if multifile { - input_files.push(InputFile { - file_type: FileType::Data, - inner_path: "SRR33138449.fastq".to_string(), - data: Content::File("tests/data/SRR33138449.sample.fastq".to_string()), - metadata: if metadata { - Some(Content::Raw(r#"{"bar":"baz"}"#.to_string())) - } else { - None - }, - encrypt: true, - compression_level: Some(3), - }) - } - - // Load dummy keys - let (sender_key, r1_key, r2_key) = load_test_keys(); - - // Prepare input for writer - let file_path = temp_dir.path().join("dummy.pith"); - let outfile = File::create(&file_path).unwrap(); - let mut writer = - PithosWriter::new(sender_key, vec![r1_key, r2_key], None, Box::new(outfile)).unwrap(); - - // Process - writer.write_file_header().unwrap(); - writer.process_input_files(input_files).unwrap(); - writer.write_directory().unwrap(); - - file_path -} diff --git a/crates/pithos_lib/tests/common/writer.rs b/crates/pithos_lib/tests/common/writer.rs new file mode 100644 index 0000000..5cfa260 --- /dev/null +++ b/crates/pithos_lib/tests/common/writer.rs @@ -0,0 +1,9 @@ +use pithos_lib::archive::WriteOptions; +use pithos_lib::crypto::PrivateKey; + +#[allow(dead_code)] +pub fn options() -> WriteOptions { + let sender = PrivateKey::generate(); + let recipient = sender.public_key(); + WriteOptions::new(sender, vec![recipient]) +} diff --git a/crates/pithos_lib/tests/data/t8.shakespeare.sample.txt b/crates/pithos_lib/tests/data/t8.shakespeare.sample.txt deleted file mode 100644 index b59d939..0000000 --- a/crates/pithos_lib/tests/data/t8.shakespeare.sample.txt +++ /dev/null @@ -1,2000 +0,0 @@ -This is the 100th Etext file presented by Project Gutenberg, and -is presented in cooperation with World Library, Inc., from their -Library of the Future and Shakespeare CDROMS. Project Gutenberg -often releases Etexts that are NOT placed in the Public Domain!! - -Shakespeare - -*This Etext has certain copyright implications you should read!* - -<> - -*Project Gutenberg is proud to cooperate with The World Library* -in the presentation of The Complete Works of William Shakespeare -for your reading for education and entertainment. HOWEVER, THIS -IS NEITHER SHAREWARE NOR PUBLIC DOMAIN. . .AND UNDER THE LIBRARY -OF THE FUTURE CONDITIONS OF THIS PRESENTATION. . .NO CHARGES MAY -BE MADE FOR *ANY* ACCESS TO THIS MATERIAL. YOU ARE ENCOURAGED!! -TO GIVE IT AWAY TO ANYONE YOU LIKE, BUT NO CHARGES ARE ALLOWED!! - - -**Welcome To The World of Free Plain Vanilla Electronic Texts** - -**Etexts Readable By Both Humans and By Computers, Since 1971** - -*These Etexts Prepared By Hundreds of Volunteers and Donations* - -Information on contacting Project Gutenberg to get Etexts, and -further information is included below. We need your donations. - - -The Complete Works of William Shakespeare - -January, 1994 [Etext #100] - - -The Library of the Future Complete Works of William Shakespeare -Library of the Future is a TradeMark (TM) of World Library Inc. -******This file should be named shaks12.txt or shaks12.zip***** - -Corrected EDITIONS of our etexts get a new NUMBER, shaks13.txt -VERSIONS based on separate sources get new LETTER, shaks10a.txt - -If you would like further information about World Library, Inc. -Please call them at 1-800-443-0238 or email julianc@netcom.com -Please give them our thanks for their Shakespeare cooperation! - - -The official release date of all Project Gutenberg Etexts is at -Midnight, Central Time, of the last day of the stated month. A -preliminary version may often be posted for suggestion, comment -and editing by those who wish to do so. To be sure you have an -up to date first edition [xxxxx10x.xxx] please check file sizes -in the first week of the next month. Since our ftp program has -a bug in it that scrambles the date [tried to fix and failed] a -look at the file size will have to do, but we will try to see a -new copy has at least one byte more or less. - - -Information about Project Gutenberg (one page) - -We produce about two million dollars for each hour we work. The -fifty hours is one conservative estimate for how long it we take -to get any etext selected, entered, proofread, edited, copyright -searched and analyzed, the copyright letters written, etc. This -projected audience is one hundred million readers. If our value -per text is nominally estimated at one dollar, then we produce 2 -million dollars per hour this year we, will have to do four text -files per month: thus upping our productivity from one million. -The Goal of Project Gutenberg is to Give Away One Trillion Etext -Files by the December 31, 2001. [10,000 x 100,000,000=Trillion] -This is ten thousand titles each to one hundred million readers, -which is 10% of the expected number of computer users by the end -of the year 2001. - -We need your donations more than ever! - -All donations should be made to "Project Gutenberg/IBC", and are -tax deductible to the extent allowable by law ("IBC" is Illinois -Benedictine College). (Subscriptions to our paper newsletter go -to IBC, too) - -For these and other matters, please mail to: - -Project Gutenberg -P. O. Box 2782 -Champaign, IL 61825 - -When all other email fails try our Michael S. Hart, Executive Director: -hart@vmd.cso.uiuc.edu (internet) hart@uiucvmd (bitnet) - -We would prefer to send you this information by email -(Internet, Bitnet, Compuserve, ATTMAIL or MCImail). - -****** -If you have an FTP program (or emulator), please -FTP directly to the Project Gutenberg archives: -[Mac users, do NOT point and click. . .type] - -ftp mrcnext.cso.uiuc.edu -login: anonymous -password: your@login -cd etext/etext91 -or cd etext92 -or cd etext93 [for new books] [now also in cd etext/etext93] -or cd etext/articles [get suggest gut for more information] -dir [to see files] -get or mget [to get files. . .set bin for zip files] -GET 0INDEX.GUT -for a list of books -and -GET NEW GUT for general information -and -MGET GUT* for newsletters. - -**Information prepared by the Project Gutenberg legal advisor** - - -***** SMALL PRINT! for COMPLETE SHAKESPEARE ***** - -THIS ELECTRONIC VERSION OF THE COMPLETE WORKS OF WILLIAM -SHAKESPEARE IS COPYRIGHT 1990-1993 BY WORLD LIBRARY, INC., -AND IS PROVIDED BY PROJECT GUTENBERG ETEXT OF -ILLINOIS BENEDICTINE COLLEGE WITH PERMISSION. - -Since unlike many other Project Gutenberg-tm etexts, this etext -is copyright protected, and since the materials and methods you -use will effect the Project's reputation, your right to copy and -distribute it is limited by the copyright and other laws, and by -the conditions of this "Small Print!" statement. - -1. LICENSE - - A) YOU MAY (AND ARE ENCOURAGED) TO DISTRIBUTE ELECTRONIC AND -MACHINE READABLE COPIES OF THIS ETEXT, SO LONG AS SUCH COPIES -(1) ARE FOR YOUR OR OTHERS PERSONAL USE ONLY, AND (2) ARE NOT -DISTRIBUTED OR USED COMMERCIALLY. PROHIBITED COMMERCIAL -DISTRIBUTION INCLUDES BY ANY SERVICE THAT CHARGES FOR DOWNLOAD -TIME OR FOR MEMBERSHIP. - - B) This license is subject to the conditions that you honor -the refund and replacement provisions of this "small print!" -statement; and that you distribute exact copies of this etext, -including this Small Print statement. Such copies can be -compressed or any proprietary form (including any form resulting -from word processing or hypertext software), so long as -*EITHER*: - - (1) The etext, when displayed, is clearly readable, and does - *not* contain characters other than those intended by the - author of the work, although tilde (~), asterisk (*) and - underline (_) characters may be used to convey punctuation - intended by the author, and additional characters may be used - to indicate hypertext links; OR - - (2) The etext is readily convertible by the reader at no - expense into plain ASCII, EBCDIC or equivalent form by the - program that displays the etext (as is the case, for instance, - with most word processors); OR - - (3) You provide or agree to provide on request at no - additional cost, fee or expense, a copy of the etext in plain - ASCII. - -2. LIMITED WARRANTY; DISCLAIMER OF DAMAGES - -This etext may contain a "Defect" in the form of incomplete, -inaccurate or corrupt data, transcription errors, a copyright or -other infringement, a defective or damaged disk, computer virus, -or codes that damage or cannot be read by your equipment. But -for the "Right of Replacement or Refund" described below, the -Project (and any other party you may receive this etext from as -a PROJECT GUTENBERG-tm etext) disclaims all liability to you for -damages, costs and expenses, including legal fees, and YOU HAVE -NO REMEDIES FOR NEGLIGENCE OR UNDER STRICT LIABILITY, OR FOR -BREACH OF WARRANTY OR CONTRACT, INCLUDING BUT NOT LIMITED TO -INDIRECT, CONSEQUENTIAL, PUNITIVE OR INCIDENTAL DAMAGES, EVEN IF -YOU GIVE NOTICE OF THE POSSIBILITY OF SUCH DAMAGES. - -If you discover a Defect in this etext within 90 days of receiv- -ing it, you can receive a refund of the money (if any) you paid -for it by sending an explanatory note within that time to the -person you received it from. If you received it on a physical -medium, you must return it with your note, and such person may -choose to alternatively give you a replacement copy. If you -received it electronically, such person may choose to -alternatively give you a second opportunity to receive it -electronically. - -THIS ETEXT IS OTHERWISE PROVIDED TO YOU "AS-IS". NO OTHER -WARRANTIES OF ANY KIND, EXPRESS OR IMPLIED, ARE MADE TO YOU AS -TO THE ETEXT OR ANY MEDIUM IT MAY BE ON, INCLUDING BUT NOT -LIMITED TO WARRANTIES OF MERCHANTABILITY OR FITNESS FOR A -PARTICULAR PURPOSE. Some states do not allow disclaimers of -implied warranties or the exclusion or limitation of consequen- -tial damages, so the above disclaimers and exclusions may not -apply to you, and you may have other legal rights. - -3. INDEMNITY: You will indemnify and hold the Project, its -directors, officers, members and agents harmless from all lia- -bility, cost and expense, including legal fees, that arise -directly or indirectly from any of the following that you do or -cause: [A] distribution of this etext, [B] alteration, -modification, or addition to the etext, or [C] any Defect. - -4. WHAT IF YOU *WANT* TO SEND MONEY EVEN IF YOU DON'T HAVE TO? -Project Gutenberg is dedicated to increasing the number of -public domain and licensed works that can be freely distributed -in machine readable form. The Project gratefully accepts -contributions in money, time, scanning machines, OCR software, -public domain etexts, royalty free copyright licenses, and -whatever else you can think of. Money should be paid to "Pro- -ject Gutenberg Association / Illinois Benedictine College". - -WRITE TO US! We can be reached at: - Internet: hart@vmd.cso.uiuc.edu - Bitnet: hart@uiucvmd - CompuServe: >internet:hart@.vmd.cso.uiuc.edu - Attmail: internet!vmd.cso.uiuc.edu!Hart - Mail: Prof. Michael Hart - P.O. Box 2782 - Champaign, IL 61825 - -This "Small Print!" by Charles B. Kramer, Attorney -Internet (72600.2026@compuserve.com); TEL: (212-254-5093) -**** SMALL PRINT! FOR __ COMPLETE SHAKESPEARE **** -["Small Print" V.12.08.93] - -<> - - -1609 - -THE SONNETS - -by William Shakespeare - - - - 1 - From fairest creatures we desire increase, - That thereby beauty's rose might never die, - But as the riper should by time decease, - His tender heir might bear his memory: - But thou contracted to thine own bright eyes, - Feed'st thy light's flame with self-substantial fuel, - Making a famine where abundance lies, - Thy self thy foe, to thy sweet self too cruel: - Thou that art now the world's fresh ornament, - And only herald to the gaudy spring, - Within thine own bud buriest thy content, - And tender churl mak'st waste in niggarding: - Pity the world, or else this glutton be, - To eat the world's due, by the grave and thee. - - - 2 - When forty winters shall besiege thy brow, - And dig deep trenches in thy beauty's field, - Thy youth's proud livery so gazed on now, - Will be a tattered weed of small worth held: - Then being asked, where all thy beauty lies, - Where all the treasure of thy lusty days; - To say within thine own deep sunken eyes, - Were an all-eating shame, and thriftless praise. - How much more praise deserved thy beauty's use, - If thou couldst answer 'This fair child of mine - Shall sum my count, and make my old excuse' - Proving his beauty by succession thine. - This were to be new made when thou art old, - And see thy blood warm when thou feel'st it cold. - - - 3 - Look in thy glass and tell the face thou viewest, - Now is the time that face should form another, - Whose fresh repair if now thou not renewest, - Thou dost beguile the world, unbless some mother. - For where is she so fair whose uneared womb - Disdains the tillage of thy husbandry? - Or who is he so fond will be the tomb, - Of his self-love to stop posterity? - Thou art thy mother's glass and she in thee - Calls back the lovely April of her prime, - So thou through windows of thine age shalt see, - Despite of wrinkles this thy golden time. - But if thou live remembered not to be, - Die single and thine image dies with thee. - - - 4 - Unthrifty loveliness why dost thou spend, - Upon thy self thy beauty's legacy? - Nature's bequest gives nothing but doth lend, - And being frank she lends to those are free: - Then beauteous niggard why dost thou abuse, - The bounteous largess given thee to give? - Profitless usurer why dost thou use - So great a sum of sums yet canst not live? - For having traffic with thy self alone, - Thou of thy self thy sweet self dost deceive, - Then how when nature calls thee to be gone, - What acceptable audit canst thou leave? - Thy unused beauty must be tombed with thee, - Which used lives th' executor to be. - - - 5 - Those hours that with gentle work did frame - The lovely gaze where every eye doth dwell - Will play the tyrants to the very same, - And that unfair which fairly doth excel: - For never-resting time leads summer on - To hideous winter and confounds him there, - Sap checked with frost and lusty leaves quite gone, - Beauty o'er-snowed and bareness every where: - Then were not summer's distillation left - A liquid prisoner pent in walls of glass, - Beauty's effect with beauty were bereft, - Nor it nor no remembrance what it was. - But flowers distilled though they with winter meet, - Leese but their show, their substance still lives sweet. - - - 6 - Then let not winter's ragged hand deface, - In thee thy summer ere thou be distilled: - Make sweet some vial; treasure thou some place, - With beauty's treasure ere it be self-killed: - That use is not forbidden usury, - Which happies those that pay the willing loan; - That's for thy self to breed another thee, - Or ten times happier be it ten for one, - Ten times thy self were happier than thou art, - If ten of thine ten times refigured thee: - Then what could death do if thou shouldst depart, - Leaving thee living in posterity? - Be not self-willed for thou art much too fair, - To be death's conquest and make worms thine heir. - - - 7 - Lo in the orient when the gracious light - Lifts up his burning head, each under eye - Doth homage to his new-appearing sight, - Serving with looks his sacred majesty, - And having climbed the steep-up heavenly hill, - Resembling strong youth in his middle age, - Yet mortal looks adore his beauty still, - Attending on his golden pilgrimage: - But when from highmost pitch with weary car, - Like feeble age he reeleth from the day, - The eyes (fore duteous) now converted are - From his low tract and look another way: - So thou, thy self out-going in thy noon: - Unlooked on diest unless thou get a son. - - - 8 - Music to hear, why hear'st thou music sadly? - Sweets with sweets war not, joy delights in joy: - Why lov'st thou that which thou receiv'st not gladly, - Or else receiv'st with pleasure thine annoy? - If the true concord of well-tuned sounds, - By unions married do offend thine ear, - They do but sweetly chide thee, who confounds - In singleness the parts that thou shouldst bear: - Mark how one string sweet husband to another, - Strikes each in each by mutual ordering; - Resembling sire, and child, and happy mother, - Who all in one, one pleasing note do sing: - Whose speechless song being many, seeming one, - Sings this to thee, 'Thou single wilt prove none'. - - - 9 - Is it for fear to wet a widow's eye, - That thou consum'st thy self in single life? - Ah, if thou issueless shalt hap to die, - The world will wail thee like a makeless wife, - The world will be thy widow and still weep, - That thou no form of thee hast left behind, - When every private widow well may keep, - By children's eyes, her husband's shape in mind: - Look what an unthrift in the world doth spend - Shifts but his place, for still the world enjoys it; - But beauty's waste hath in the world an end, - And kept unused the user so destroys it: - No love toward others in that bosom sits - That on himself such murd'rous shame commits. - - - 10 - For shame deny that thou bear'st love to any - Who for thy self art so unprovident. - Grant if thou wilt, thou art beloved of many, - But that thou none lov'st is most evident: - For thou art so possessed with murd'rous hate, - That 'gainst thy self thou stick'st not to conspire, - Seeking that beauteous roof to ruinate - Which to repair should be thy chief desire: - O change thy thought, that I may change my mind, - Shall hate be fairer lodged than gentle love? - Be as thy presence is gracious and kind, - Or to thy self at least kind-hearted prove, - Make thee another self for love of me, - That beauty still may live in thine or thee. - - - 11 - As fast as thou shalt wane so fast thou grow'st, - In one of thine, from that which thou departest, - And that fresh blood which youngly thou bestow'st, - Thou mayst call thine, when thou from youth convertest, - Herein lives wisdom, beauty, and increase, - Without this folly, age, and cold decay, - If all were minded so, the times should cease, - And threescore year would make the world away: - Let those whom nature hath not made for store, - Harsh, featureless, and rude, barrenly perish: - Look whom she best endowed, she gave thee more; - Which bounteous gift thou shouldst in bounty cherish: - She carved thee for her seal, and meant thereby, - Thou shouldst print more, not let that copy die. - - - 12 - When I do count the clock that tells the time, - And see the brave day sunk in hideous night, - When I behold the violet past prime, - And sable curls all silvered o'er with white: - When lofty trees I see barren of leaves, - Which erst from heat did canopy the herd - And summer's green all girded up in sheaves - Borne on the bier with white and bristly beard: - Then of thy beauty do I question make - That thou among the wastes of time must go, - Since sweets and beauties do themselves forsake, - And die as fast as they see others grow, - And nothing 'gainst Time's scythe can make defence - Save breed to brave him, when he takes thee hence. - - - 13 - O that you were your self, but love you are - No longer yours, than you your self here live, - Against this coming end you should prepare, - And your sweet semblance to some other give. - So should that beauty which you hold in lease - Find no determination, then you were - Your self again after your self's decease, - When your sweet issue your sweet form should bear. - Who lets so fair a house fall to decay, - Which husbandry in honour might uphold, - Against the stormy gusts of winter's day - And barren rage of death's eternal cold? - O none but unthrifts, dear my love you know, - You had a father, let your son say so. - - - 14 - Not from the stars do I my judgement pluck, - And yet methinks I have astronomy, - But not to tell of good, or evil luck, - Of plagues, of dearths, or seasons' quality, - Nor can I fortune to brief minutes tell; - Pointing to each his thunder, rain and wind, - Or say with princes if it shall go well - By oft predict that I in heaven find. - But from thine eyes my knowledge I derive, - And constant stars in them I read such art - As truth and beauty shall together thrive - If from thy self, to store thou wouldst convert: - Or else of thee this I prognosticate, - Thy end is truth's and beauty's doom and date. - - - 15 - When I consider every thing that grows - Holds in perfection but a little moment. - That this huge stage presenteth nought but shows - Whereon the stars in secret influence comment. - When I perceive that men as plants increase, - Cheered and checked even by the self-same sky: - Vaunt in their youthful sap, at height decrease, - And wear their brave state out of memory. - Then the conceit of this inconstant stay, - Sets you most rich in youth before my sight, - Where wasteful time debateth with decay - To change your day of youth to sullied night, - And all in war with Time for love of you, - As he takes from you, I engraft you new. - - - 16 - But wherefore do not you a mightier way - Make war upon this bloody tyrant Time? - And fortify your self in your decay - With means more blessed than my barren rhyme? - Now stand you on the top of happy hours, - And many maiden gardens yet unset, - With virtuous wish would bear you living flowers, - Much liker than your painted counterfeit: - So should the lines of life that life repair - Which this (Time's pencil) or my pupil pen - Neither in inward worth nor outward fair - Can make you live your self in eyes of men. - To give away your self, keeps your self still, - And you must live drawn by your own sweet skill. - - - 17 - Who will believe my verse in time to come - If it were filled with your most high deserts? - Though yet heaven knows it is but as a tomb - Which hides your life, and shows not half your parts: - If I could write the beauty of your eyes, - And in fresh numbers number all your graces, - The age to come would say this poet lies, - Such heavenly touches ne'er touched earthly faces. - So should my papers (yellowed with their age) - Be scorned, like old men of less truth than tongue, - And your true rights be termed a poet's rage, - And stretched metre of an antique song. - But were some child of yours alive that time, - You should live twice in it, and in my rhyme. - - - 18 - Shall I compare thee to a summer's day? - Thou art more lovely and more temperate: - Rough winds do shake the darling buds of May, - And summer's lease hath all too short a date: - Sometime too hot the eye of heaven shines, - And often is his gold complexion dimmed, - And every fair from fair sometime declines, - By chance, or nature's changing course untrimmed: - But thy eternal summer shall not fade, - Nor lose possession of that fair thou ow'st, - Nor shall death brag thou wand'rest in his shade, - When in eternal lines to time thou grow'st, - So long as men can breathe or eyes can see, - So long lives this, and this gives life to thee. - - - 19 - Devouring Time blunt thou the lion's paws, - And make the earth devour her own sweet brood, - Pluck the keen teeth from the fierce tiger's jaws, - And burn the long-lived phoenix, in her blood, - Make glad and sorry seasons as thou fleet'st, - And do whate'er thou wilt swift-footed Time - To the wide world and all her fading sweets: - But I forbid thee one most heinous crime, - O carve not with thy hours my love's fair brow, - Nor draw no lines there with thine antique pen, - Him in thy course untainted do allow, - For beauty's pattern to succeeding men. - Yet do thy worst old Time: despite thy wrong, - My love shall in my verse ever live young. - - - 20 - A woman's face with nature's own hand painted, - Hast thou the master mistress of my passion, - A woman's gentle heart but not acquainted - With shifting change as is false women's fashion, - An eye more bright than theirs, less false in rolling: - Gilding the object whereupon it gazeth, - A man in hue all hues in his controlling, - Which steals men's eyes and women's souls amazeth. - And for a woman wert thou first created, - Till nature as she wrought thee fell a-doting, - And by addition me of thee defeated, - By adding one thing to my purpose nothing. - But since she pricked thee out for women's pleasure, - Mine be thy love and thy love's use their treasure. - - - 21 - So is it not with me as with that muse, - Stirred by a painted beauty to his verse, - Who heaven it self for ornament doth use, - And every fair with his fair doth rehearse, - Making a couplement of proud compare - With sun and moon, with earth and sea's rich gems: - With April's first-born flowers and all things rare, - That heaven's air in this huge rondure hems. - O let me true in love but truly write, - And then believe me, my love is as fair, - As any mother's child, though not so bright - As those gold candles fixed in heaven's air: - Let them say more that like of hearsay well, - I will not praise that purpose not to sell. - - - 22 - My glass shall not persuade me I am old, - So long as youth and thou are of one date, - But when in thee time's furrows I behold, - Then look I death my days should expiate. - For all that beauty that doth cover thee, - Is but the seemly raiment of my heart, - Which in thy breast doth live, as thine in me, - How can I then be elder than thou art? - O therefore love be of thyself so wary, - As I not for my self, but for thee will, - Bearing thy heart which I will keep so chary - As tender nurse her babe from faring ill. - Presume not on thy heart when mine is slain, - Thou gav'st me thine not to give back again. - - - 23 - As an unperfect actor on the stage, - Who with his fear is put beside his part, - Or some fierce thing replete with too much rage, - Whose strength's abundance weakens his own heart; - So I for fear of trust, forget to say, - The perfect ceremony of love's rite, - And in mine own love's strength seem to decay, - O'ercharged with burthen of mine own love's might: - O let my looks be then the eloquence, - And dumb presagers of my speaking breast, - Who plead for love, and look for recompense, - More than that tongue that more hath more expressed. - O learn to read what silent love hath writ, - To hear with eyes belongs to love's fine wit. - - - 24 - Mine eye hath played the painter and hath stelled, - Thy beauty's form in table of my heart, - My body is the frame wherein 'tis held, - And perspective it is best painter's art. - For through the painter must you see his skill, - To find where your true image pictured lies, - Which in my bosom's shop is hanging still, - That hath his windows glazed with thine eyes: - Now see what good turns eyes for eyes have done, - Mine eyes have drawn thy shape, and thine for me - Are windows to my breast, where-through the sun - Delights to peep, to gaze therein on thee; - Yet eyes this cunning want to grace their art, - They draw but what they see, know not the heart. - - - 25 - Let those who are in favour with their stars, - Of public honour and proud titles boast, - Whilst I whom fortune of such triumph bars - Unlooked for joy in that I honour most; - Great princes' favourites their fair leaves spread, - But as the marigold at the sun's eye, - And in themselves their pride lies buried, - For at a frown they in their glory die. - The painful warrior famoused for fight, - After a thousand victories once foiled, - Is from the book of honour razed quite, - And all the rest forgot for which he toiled: - Then happy I that love and am beloved - Where I may not remove nor be removed. - - - 26 - Lord of my love, to whom in vassalage - Thy merit hath my duty strongly knit; - To thee I send this written embassage - To witness duty, not to show my wit. - Duty so great, which wit so poor as mine - May make seem bare, in wanting words to show it; - But that I hope some good conceit of thine - In thy soul's thought (all naked) will bestow it: - Till whatsoever star that guides my moving, - Points on me graciously with fair aspect, - And puts apparel on my tattered loving, - To show me worthy of thy sweet respect, - Then may I dare to boast how I do love thee, - Till then, not show my head where thou mayst prove me. - - - 27 - Weary with toil, I haste me to my bed, - The dear respose for limbs with travel tired, - But then begins a journey in my head - To work my mind, when body's work's expired. - For then my thoughts (from far where I abide) - Intend a zealous pilgrimage to thee, - And keep my drooping eyelids open wide, - Looking on darkness which the blind do see. - Save that my soul's imaginary sight - Presents thy shadow to my sightless view, - Which like a jewel (hung in ghastly night) - Makes black night beauteous, and her old face new. - Lo thus by day my limbs, by night my mind, - For thee, and for my self, no quiet find. - - - 28 - How can I then return in happy plight - That am debarred the benefit of rest? - When day's oppression is not eased by night, - But day by night and night by day oppressed. - And each (though enemies to either's reign) - Do in consent shake hands to torture me, - The one by toil, the other to complain - How far I toil, still farther off from thee. - I tell the day to please him thou art bright, - And dost him grace when clouds do blot the heaven: - So flatter I the swart-complexioned night, - When sparkling stars twire not thou gild'st the even. - But day doth daily draw my sorrows longer, - And night doth nightly make grief's length seem stronger - - - 29 - When in disgrace with Fortune and men's eyes, - I all alone beweep my outcast state, - And trouble deaf heaven with my bootless cries, - And look upon my self and curse my fate, - Wishing me like to one more rich in hope, - Featured like him, like him with friends possessed, - Desiring this man's art, and that man's scope, - With what I most enjoy contented least, - Yet in these thoughts my self almost despising, - Haply I think on thee, and then my state, - (Like to the lark at break of day arising - From sullen earth) sings hymns at heaven's gate, - For thy sweet love remembered such wealth brings, - That then I scorn to change my state with kings. - - - 30 - When to the sessions of sweet silent thought, - I summon up remembrance of things past, - I sigh the lack of many a thing I sought, - And with old woes new wail my dear time's waste: - Then can I drown an eye (unused to flow) - For precious friends hid in death's dateless night, - And weep afresh love's long since cancelled woe, - And moan th' expense of many a vanished sight. - Then can I grieve at grievances foregone, - And heavily from woe to woe tell o'er - The sad account of fore-bemoaned moan, - Which I new pay as if not paid before. - But if the while I think on thee (dear friend) - All losses are restored, and sorrows end. - - - 31 - Thy bosom is endeared with all hearts, - Which I by lacking have supposed dead, - And there reigns love and all love's loving parts, - And all those friends which I thought buried. - How many a holy and obsequious tear - Hath dear religious love stol'n from mine eye, - As interest of the dead, which now appear, - But things removed that hidden in thee lie. - Thou art the grave where buried love doth live, - Hung with the trophies of my lovers gone, - Who all their parts of me to thee did give, - That due of many, now is thine alone. - Their images I loved, I view in thee, - And thou (all they) hast all the all of me. - - - 32 - If thou survive my well-contented day, - When that churl death my bones with dust shall cover - And shalt by fortune once more re-survey - These poor rude lines of thy deceased lover: - Compare them with the bett'ring of the time, - And though they be outstripped by every pen, - Reserve them for my love, not for their rhyme, - Exceeded by the height of happier men. - O then vouchsafe me but this loving thought, - 'Had my friend's Muse grown with this growing age, - A dearer birth than this his love had brought - To march in ranks of better equipage: - But since he died and poets better prove, - Theirs for their style I'll read, his for his love'. - - - 33 - Full many a glorious morning have I seen, - Flatter the mountain tops with sovereign eye, - Kissing with golden face the meadows green; - Gilding pale streams with heavenly alchemy: - Anon permit the basest clouds to ride, - With ugly rack on his celestial face, - And from the forlorn world his visage hide - Stealing unseen to west with this disgrace: - Even so my sun one early morn did shine, - With all triumphant splendour on my brow, - But out alack, he was but one hour mine, - The region cloud hath masked him from me now. - Yet him for this, my love no whit disdaineth, - Suns of the world may stain, when heaven's sun staineth. - - - 34 - Why didst thou promise such a beauteous day, - And make me travel forth without my cloak, - To let base clouds o'ertake me in my way, - Hiding thy brav'ry in their rotten smoke? - 'Tis not enough that through the cloud thou break, - To dry the rain on my storm-beaten face, - For no man well of such a salve can speak, - That heals the wound, and cures not the disgrace: - Nor can thy shame give physic to my grief, - Though thou repent, yet I have still the loss, - Th' offender's sorrow lends but weak relief - To him that bears the strong offence's cross. - Ah but those tears are pearl which thy love sheds, - And they are rich, and ransom all ill deeds. - - - 35 - No more be grieved at that which thou hast done, - Roses have thorns, and silver fountains mud, - Clouds and eclipses stain both moon and sun, - And loathsome canker lives in sweetest bud. - All men make faults, and even I in this, - Authorizing thy trespass with compare, - My self corrupting salving thy amiss, - Excusing thy sins more than thy sins are: - For to thy sensual fault I bring in sense, - Thy adverse party is thy advocate, - And 'gainst my self a lawful plea commence: - Such civil war is in my love and hate, - That I an accessary needs must be, - To that sweet thief which sourly robs from me. - - - 36 - Let me confess that we two must be twain, - Although our undivided loves are one: - So shall those blots that do with me remain, - Without thy help, by me be borne alone. - In our two loves there is but one respect, - Though in our lives a separable spite, - Which though it alter not love's sole effect, - Yet doth it steal sweet hours from love's delight. - I may not evermore acknowledge thee, - Lest my bewailed guilt should do thee shame, - Nor thou with public kindness honour me, - Unless thou take that honour from thy name: - But do not so, I love thee in such sort, - As thou being mine, mine is thy good report. - - - 37 - As a decrepit father takes delight, - To see his active child do deeds of youth, - So I, made lame by Fortune's dearest spite - Take all my comfort of thy worth and truth. - For whether beauty, birth, or wealth, or wit, - Or any of these all, or all, or more - Entitled in thy parts, do crowned sit, - I make my love engrafted to this store: - So then I am not lame, poor, nor despised, - Whilst that this shadow doth such substance give, - That I in thy abundance am sufficed, - And by a part of all thy glory live: - Look what is best, that best I wish in thee, - This wish I have, then ten times happy me. - - - 38 - How can my muse want subject to invent - While thou dost breathe that pour'st into my verse, - Thine own sweet argument, too excellent, - For every vulgar paper to rehearse? - O give thy self the thanks if aught in me, - Worthy perusal stand against thy sight, - For who's so dumb that cannot write to thee, - When thou thy self dost give invention light? - Be thou the tenth Muse, ten times more in worth - Than those old nine which rhymers invocate, - And he that calls on thee, let him bring forth - Eternal numbers to outlive long date. - If my slight muse do please these curious days, - The pain be mine, but thine shall be the praise. - - - 39 - O how thy worth with manners may I sing, - When thou art all the better part of me? - What can mine own praise to mine own self bring: - And what is't but mine own when I praise thee? - Even for this, let us divided live, - And our dear love lose name of single one, - That by this separation I may give: - That due to thee which thou deserv'st alone: - O absence what a torment wouldst thou prove, - Were it not thy sour leisure gave sweet leave, - To entertain the time with thoughts of love, - Which time and thoughts so sweetly doth deceive. - And that thou teachest how to make one twain, - By praising him here who doth hence remain. - - - 40 - Take all my loves, my love, yea take them all, - What hast thou then more than thou hadst before? - No love, my love, that thou mayst true love call, - All mine was thine, before thou hadst this more: - Then if for my love, thou my love receivest, - I cannot blame thee, for my love thou usest, - But yet be blamed, if thou thy self deceivest - By wilful taste of what thy self refusest. - I do forgive thy robbery gentle thief - Although thou steal thee all my poverty: - And yet love knows it is a greater grief - To bear greater wrong, than hate's known injury. - Lascivious grace, in whom all ill well shows, - Kill me with spites yet we must not be foes. - - - 41 - Those pretty wrongs that liberty commits, - When I am sometime absent from thy heart, - Thy beauty, and thy years full well befits, - For still temptation follows where thou art. - Gentle thou art, and therefore to be won, - Beauteous thou art, therefore to be assailed. - And when a woman woos, what woman's son, - Will sourly leave her till he have prevailed? - Ay me, but yet thou mightst my seat forbear, - And chide thy beauty, and thy straying youth, - Who lead thee in their riot even there - Where thou art forced to break a twofold truth: - Hers by thy beauty tempting her to thee, - Thine by thy beauty being false to me. - - - 42 - That thou hast her it is not all my grief, - And yet it may be said I loved her dearly, - That she hath thee is of my wailing chief, - A loss in love that touches me more nearly. - Loving offenders thus I will excuse ye, - Thou dost love her, because thou know'st I love her, - And for my sake even so doth she abuse me, - Suff'ring my friend for my sake to approve her. - If I lose thee, my loss is my love's gain, - And losing her, my friend hath found that loss, - Both find each other, and I lose both twain, - And both for my sake lay on me this cross, - But here's the joy, my friend and I are one, - Sweet flattery, then she loves but me alone. - - - 43 - When most I wink then do mine eyes best see, - For all the day they view things unrespected, - But when I sleep, in dreams they look on thee, - And darkly bright, are bright in dark directed. - Then thou whose shadow shadows doth make bright - How would thy shadow's form, form happy show, - To the clear day with thy much clearer light, - When to unseeing eyes thy shade shines so! - How would (I say) mine eyes be blessed made, - By looking on thee in the living day, - When in dead night thy fair imperfect shade, - Through heavy sleep on sightless eyes doth stay! - All days are nights to see till I see thee, - And nights bright days when dreams do show thee me. - - - 44 - If the dull substance of my flesh were thought, - Injurious distance should not stop my way, - For then despite of space I would be brought, - From limits far remote, where thou dost stay, - No matter then although my foot did stand - Upon the farthest earth removed from thee, - For nimble thought can jump both sea and land, - As soon as think the place where he would be. - But ah, thought kills me that I am not thought - To leap large lengths of miles when thou art gone, - But that so much of earth and water wrought, - I must attend, time's leisure with my moan. - Receiving nought by elements so slow, - But heavy tears, badges of either's woe. - - - 45 - The other two, slight air, and purging fire, - Are both with thee, wherever I abide, - The first my thought, the other my desire, - These present-absent with swift motion slide. - For when these quicker elements are gone - In tender embassy of love to thee, - My life being made of four, with two alone, - Sinks down to death, oppressed with melancholy. - Until life's composition be recured, - By those swift messengers returned from thee, - Who even but now come back again assured, - Of thy fair health, recounting it to me. - This told, I joy, but then no longer glad, - I send them back again and straight grow sad. - - - 46 - Mine eye and heart are at a mortal war, - How to divide the conquest of thy sight, - Mine eye, my heart thy picture's sight would bar, - My heart, mine eye the freedom of that right, - My heart doth plead that thou in him dost lie, - (A closet never pierced with crystal eyes) - But the defendant doth that plea deny, - And says in him thy fair appearance lies. - To side this title is impanelled - A quest of thoughts, all tenants to the heart, - And by their verdict is determined - The clear eye's moiety, and the dear heart's part. - As thus, mine eye's due is thy outward part, - And my heart's right, thy inward love of heart. - - - 47 - Betwixt mine eye and heart a league is took, - And each doth good turns now unto the other, - When that mine eye is famished for a look, - Or heart in love with sighs himself doth smother; - With my love's picture then my eye doth feast, - And to the painted banquet bids my heart: - Another time mine eye is my heart's guest, - And in his thoughts of love doth share a part. - So either by thy picture or my love, - Thy self away, art present still with me, - For thou not farther than my thoughts canst move, - And I am still with them, and they with thee. - Or if they sleep, thy picture in my sight - Awakes my heart, to heart's and eye's delight. - - - 48 - How careful was I when I took my way, - Each trifle under truest bars to thrust, - That to my use it might unused stay - From hands of falsehood, in sure wards of trust! - But thou, to whom my jewels trifles are, - Most worthy comfort, now my greatest grief, - Thou best of dearest, and mine only care, - Art left the prey of every vulgar thief. - Thee have I not locked up in any chest, - Save where thou art not, though I feel thou art, - Within the gentle closure of my breast, - From whence at pleasure thou mayst come and part, - And even thence thou wilt be stol'n I fear, - For truth proves thievish for a prize so dear. - - - 49 - Against that time (if ever that time come) - When I shall see thee frown on my defects, - When as thy love hath cast his utmost sum, - Called to that audit by advised respects, - Against that time when thou shalt strangely pass, - And scarcely greet me with that sun thine eye, - When love converted from the thing it was - Shall reasons find of settled gravity; - Against that time do I ensconce me here - Within the knowledge of mine own desert, - And this my hand, against my self uprear, - To guard the lawful reasons on thy part, - To leave poor me, thou hast the strength of laws, - Since why to love, I can allege no cause. - - - 50 - How heavy do I journey on the way, - When what I seek (my weary travel's end) - Doth teach that case and that repose to say - 'Thus far the miles are measured from thy friend.' - The beast that bears me, tired with my woe, - Plods dully on, to bear that weight in me, - As if by some instinct the wretch did know - His rider loved not speed being made from thee: - The bloody spur cannot provoke him on, - That sometimes anger thrusts into his hide, - Which heavily he answers with a groan, - More sharp to me than spurring to his side, - For that same groan doth put this in my mind, - My grief lies onward and my joy behind. - - - 51 - Thus can my love excuse the slow offence, - Of my dull bearer, when from thee I speed, - From where thou art, why should I haste me thence? - Till I return of posting is no need. - O what excuse will my poor beast then find, - When swift extremity can seem but slow? - Then should I spur though mounted on the wind, - In winged speed no motion shall I know, - Then can no horse with my desire keep pace, - Therefore desire (of perfect'st love being made) - Shall neigh (no dull flesh) in his fiery race, - But love, for love, thus shall excuse my jade, - Since from thee going, he went wilful-slow, - Towards thee I'll run, and give him leave to go. - - - 52 - So am I as the rich whose blessed key, - Can bring him to his sweet up-locked treasure, - The which he will not every hour survey, - For blunting the fine point of seldom pleasure. - Therefore are feasts so solemn and so rare, - Since seldom coming in that long year set, - Like stones of worth they thinly placed are, - Or captain jewels in the carcanet. - So is the time that keeps you as my chest - Or as the wardrobe which the robe doth hide, - To make some special instant special-blest, - By new unfolding his imprisoned pride. - Blessed are you whose worthiness gives scope, - Being had to triumph, being lacked to hope. - - - 53 - What is your substance, whereof are you made, - That millions of strange shadows on you tend? - Since every one, hath every one, one shade, - And you but one, can every shadow lend: - Describe Adonis and the counterfeit, - Is poorly imitated after you, - On Helen's cheek all art of beauty set, - And you in Grecian tires are painted new: - Speak of the spring, and foison of the year, - The one doth shadow of your beauty show, - The other as your bounty doth appear, - And you in every blessed shape we know. - In all external grace you have some part, - But you like none, none you for constant heart. - - - 54 - O how much more doth beauty beauteous seem, - By that sweet ornament which truth doth give! - The rose looks fair, but fairer we it deem - For that sweet odour, which doth in it live: - The canker blooms have full as deep a dye, - As the perfumed tincture of the roses, - Hang on such thorns, and play as wantonly, - When summer's breath their masked buds discloses: - But for their virtue only is their show, - They live unwooed, and unrespected fade, - Die to themselves. Sweet roses do not so, - Of their sweet deaths, are sweetest odours made: - And so of you, beauteous and lovely youth, - When that shall vade, by verse distills your truth. - - - 55 - Not marble, nor the gilded monuments - Of princes shall outlive this powerful rhyme, - But you shall shine more bright in these contents - Than unswept stone, besmeared with sluttish time. - When wasteful war shall statues overturn, - And broils root out the work of masonry, - Nor Mars his sword, nor war's quick fire shall burn: - The living record of your memory. - 'Gainst death, and all-oblivious enmity - Shall you pace forth, your praise shall still find room, - Even in the eyes of all posterity - That wear this world out to the ending doom. - So till the judgment that your self arise, - You live in this, and dwell in lovers' eyes. - - - 56 - Sweet love renew thy force, be it not said - Thy edge should blunter be than appetite, - Which but to-day by feeding is allayed, - To-morrow sharpened in his former might. - So love be thou, although to-day thou fill - Thy hungry eyes, even till they wink with fulness, - To-morrow see again, and do not kill - The spirit of love, with a perpetual dulness: - Let this sad interim like the ocean be - Which parts the shore, where two contracted new, - Come daily to the banks, that when they see: - Return of love, more blest may be the view. - Or call it winter, which being full of care, - Makes summer's welcome, thrice more wished, more rare. - - - 57 - Being your slave what should I do but tend, - Upon the hours, and times of your desire? - I have no precious time at all to spend; - Nor services to do till you require. - Nor dare I chide the world-without-end hour, - Whilst I (my sovereign) watch the clock for you, - Nor think the bitterness of absence sour, - When you have bid your servant once adieu. - Nor dare I question with my jealous thought, - Where you may be, or your affairs suppose, - But like a sad slave stay and think of nought - Save where you are, how happy you make those. - So true a fool is love, that in your will, - (Though you do any thing) he thinks no ill. - - - 58 - That god forbid, that made me first your slave, - I should in thought control your times of pleasure, - Or at your hand th' account of hours to crave, - Being your vassal bound to stay your leisure. - O let me suffer (being at your beck) - Th' imprisoned absence of your liberty, - And patience tame to sufferance bide each check, - Without accusing you of injury. - Be where you list, your charter is so strong, - That you your self may privilage your time - To what you will, to you it doth belong, - Your self to pardon of self-doing crime. - I am to wait, though waiting so be hell, - Not blame your pleasure be it ill or well. - - - 59 - If there be nothing new, but that which is, - Hath been before, how are our brains beguiled, - Which labouring for invention bear amis - The second burthen of a former child! - O that record could with a backward look, - Even of five hundred courses of the sun, - Show me your image in some antique book, - Since mind at first in character was done. - That I might see what the old world could say, - To this composed wonder of your frame, - Whether we are mended, or whether better they, - Or whether revolution be the same. - O sure I am the wits of former days, - To subjects worse have given admiring praise. - - - 60 - Like as the waves make towards the pebbled shore, - So do our minutes hasten to their end, - Each changing place with that which goes before, - In sequent toil all forwards do contend. - Nativity once in the main of light, - Crawls to maturity, wherewith being crowned, - Crooked eclipses 'gainst his glory fight, - And Time that gave, doth now his gift confound. - Time doth transfix the flourish set on youth, - And delves the parallels in beauty's brow, - Feeds on the rarities of nature's truth, - And nothing stands but for his scythe to mow. - And yet to times in hope, my verse shall stand - Praising thy worth, despite his cruel hand. - - - 61 - Is it thy will, thy image should keep open - My heavy eyelids to the weary night? - Dost thou desire my slumbers should be broken, - While shadows like to thee do mock my sight? - Is it thy spirit that thou send'st from thee - So far from home into my deeds to pry, - To find out shames and idle hours in me, - The scope and tenure of thy jealousy? - O no, thy love though much, is not so great, - It is my love that keeps mine eye awake, - Mine own true love that doth my rest defeat, - To play the watchman ever for thy sake. - For thee watch I, whilst thou dost wake elsewhere, - From me far off, with others all too near. - - - 62 - Sin of self-love possesseth all mine eye, - And all my soul, and all my every part; - And for this sin there is no remedy, - It is so grounded inward in my heart. - Methinks no face so gracious is as mine, - No shape so true, no truth of such account, - And for my self mine own worth do define, - As I all other in all worths surmount. - But when my glass shows me my self indeed - beated and chopt with tanned antiquity, - Mine own self-love quite contrary I read: - Self, so self-loving were iniquity. - 'Tis thee (my self) that for my self I praise, - Painting my age with beauty of thy days. - - - 63 - Against my love shall be as I am now - With Time's injurious hand crushed and o'erworn, - When hours have drained his blood and filled his brow - With lines and wrinkles, when his youthful morn - Hath travelled on to age's steepy night, - And all those beauties whereof now he's king - Are vanishing, or vanished out of sight, - Stealing away the treasure of his spring: - For such a time do I now fortify - Against confounding age's cruel knife, - That he shall never cut from memory - My sweet love's beauty, though my lover's life. - His beauty shall in these black lines be seen, - And they shall live, and he in them still green. - - - 64 - When I have seen by Time's fell hand defaced - The rich-proud cost of outworn buried age, - When sometime lofty towers I see down-rased, - And brass eternal slave to mortal rage. - When I have seen the hungry ocean gain - Advantage on the kingdom of the shore, - And the firm soil win of the watery main, - Increasing store with loss, and loss with store. - When I have seen such interchange of State, - Or state it self confounded, to decay, - Ruin hath taught me thus to ruminate - That Time will come and take my love away. - This thought is as a death which cannot choose - But weep to have, that which it fears to lose. - - - 65 - Since brass, nor stone, nor earth, nor boundless sea, - But sad mortality o'ersways their power, - How with this rage shall beauty hold a plea, - Whose action is no stronger than a flower? - O how shall summer's honey breath hold out, - Against the wrackful siege of batt'ring days, - When rocks impregnable are not so stout, - Nor gates of steel so strong but time decays? - O fearful meditation, where alack, - Shall Time's best jewel from Time's chest lie hid? - Or what strong hand can hold his swift foot back, - Or who his spoil of beauty can forbid? - O none, unless this miracle have might, - That in black ink my love may still shine bright. - - - 66 - Tired with all these for restful death I cry, - As to behold desert a beggar born, - And needy nothing trimmed in jollity, - And purest faith unhappily forsworn, - And gilded honour shamefully misplaced, - And maiden virtue rudely strumpeted, - And right perfection wrongfully disgraced, - And strength by limping sway disabled - And art made tongue-tied by authority, - And folly (doctor-like) controlling skill, - And simple truth miscalled simplicity, - And captive good attending captain ill. - Tired with all these, from these would I be gone, - Save that to die, I leave my love alone. - - - 67 - Ah wherefore with infection should he live, - And with his presence grace impiety, - That sin by him advantage should achieve, - And lace it self with his society? - Why should false painting imitate his cheek, - And steal dead seeming of his living hue? - Why should poor beauty indirectly seek, - Roses of shadow, since his rose is true? - Why should he live, now nature bankrupt is, - Beggared of blood to blush through lively veins, - For she hath no exchequer now but his, - And proud of many, lives upon his gains? - O him she stores, to show what wealth she had, - In days long since, before these last so bad. - - - 68 - Thus is his cheek the map of days outworn, - When beauty lived and died as flowers do now, - Before these bastard signs of fair were born, - Or durst inhabit on a living brow: - Before the golden tresses of the dead, - The right of sepulchres, were shorn away, - To live a second life on second head, - Ere beauty's dead fleece made another gay: - In him those holy antique hours are seen, - Without all ornament, it self and true, - Making no summer of another's green, - Robbing no old to dress his beauty new, - And him as for a map doth Nature store, - To show false Art what beauty was of yore. - - - 69 - Those parts of thee that the world's eye doth view, - Want nothing that the thought of hearts can mend: - All tongues (the voice of souls) give thee that due, - Uttering bare truth, even so as foes commend. - Thy outward thus with outward praise is crowned, - But those same tongues that give thee so thine own, - In other accents do this praise confound - By seeing farther than the eye hath shown. - They look into the beauty of thy mind, - And that in guess they measure by thy deeds, - Then churls their thoughts (although their eyes were kind) - To thy fair flower add the rank smell of weeds: - But why thy odour matcheth not thy show, - The soil is this, that thou dost common grow. - - - 70 - That thou art blamed shall not be thy defect, - For slander's mark was ever yet the fair, - The ornament of beauty is suspect, - A crow that flies in heaven's sweetest air. - So thou be good, slander doth but approve, - Thy worth the greater being wooed of time, - For canker vice the sweetest buds doth love, - And thou present'st a pure unstained prime. - Thou hast passed by the ambush of young days, - Either not assailed, or victor being charged, - Yet this thy praise cannot be so thy praise, - To tie up envy, evermore enlarged, - If some suspect of ill masked not thy show, - Then thou alone kingdoms of hearts shouldst owe. - - - 71 - No longer mourn for me when I am dead, - Than you shall hear the surly sullen bell - Give warning to the world that I am fled - From this vile world with vilest worms to dwell: - Nay if you read this line, remember not, - The hand that writ it, for I love you so, - That I in your sweet thoughts would be forgot, - If thinking on me then should make you woe. - O if (I say) you look upon this verse, - When I (perhaps) compounded am with clay, - Do not so much as my poor name rehearse; - But let your love even with my life decay. - Lest the wise world should look into your moan, - And mock you with me after I am gone. - - - 72 - O lest the world should task you to recite, - What merit lived in me that you should love - After my death (dear love) forget me quite, - For you in me can nothing worthy prove. - Unless you would devise some virtuous lie, - To do more for me than mine own desert, - And hang more praise upon deceased I, - Than niggard truth would willingly impart: - O lest your true love may seem false in this, - That you for love speak well of me untrue, - My name be buried where my body is, - And live no more to shame nor me, nor you. - For I am shamed by that which I bring forth, - And so should you, to love things nothing worth. - - - 73 - That time of year thou mayst in me behold, - When yellow leaves, or none, or few do hang - Upon those boughs which shake against the cold, - Bare ruined choirs, where late the sweet birds sang. - In me thou seest the twilight of such day, - As after sunset fadeth in the west, - Which by and by black night doth take away, - Death's second self that seals up all in rest. - In me thou seest the glowing of such fire, - That on the ashes of his youth doth lie, - As the death-bed, whereon it must expire, - Consumed with that which it was nourished by. - This thou perceiv'st, which makes thy love more strong, - To love that well, which thou must leave ere long. - - - 74 - But be contented when that fell arrest, - Without all bail shall carry me away, - My life hath in this line some interest, - Which for memorial still with thee shall stay. - When thou reviewest this, thou dost review, - The very part was consecrate to thee, - The earth can have but earth, which is his due, - My spirit is thine the better part of me, - So then thou hast but lost the dregs of life, - The prey of worms, my body being dead, - The coward conquest of a wretch's knife, - Too base of thee to be remembered, - The worth of that, is that which it contains, - And that is this, and this with thee remains. - - - 75 - So are you to my thoughts as food to life, - Or as sweet-seasoned showers are to the ground; - And for the peace of you I hold such strife - As 'twixt a miser and his wealth is found. - Now proud as an enjoyer, and anon - Doubting the filching age will steal his treasure, - Now counting best to be with you alone, - Then bettered that the world may see my pleasure, - Sometime all full with feasting on your sight, - And by and by clean starved for a look, - Possessing or pursuing no delight - Save what is had, or must from you be took. - Thus do I pine and surfeit day by day, - Or gluttoning on all, or all away. - - - 76 - Why is my verse so barren of new pride? - So far from variation or quick change? - Why with the time do I not glance aside - To new-found methods, and to compounds strange? - Why write I still all one, ever the same, - And keep invention in a noted weed, - That every word doth almost tell my name, - Showing their birth, and where they did proceed? - O know sweet love I always write of you, - And you and love are still my argument: - So all my best is dressing old words new, - Spending again what is already spent: - For as the sun is daily new and old, - So is my love still telling what is told. - - - 77 - Thy glass will show thee how thy beauties wear, - Thy dial how thy precious minutes waste, - These vacant leaves thy mind's imprint will bear, - And of this book, this learning mayst thou taste. - The wrinkles which thy glass will truly show, - Of mouthed graves will give thee memory, - Thou by thy dial's shady stealth mayst know, - Time's thievish progress to eternity. - Look what thy memory cannot contain, - Commit to these waste blanks, and thou shalt find - Those children nursed, delivered from thy brain, - To take a new acquaintance of thy mind. - These offices, so oft as thou wilt look, - Shall profit thee, and much enrich thy book. - - - 78 - So oft have I invoked thee for my muse, - And found such fair assistance in my verse, - As every alien pen hath got my use, - And under thee their poesy disperse. - Thine eyes, that taught the dumb on high to sing, - And heavy ignorance aloft to fly, - Have added feathers to the learned's wing, - And given grace a double majesty. - Yet be most proud of that which I compile, - Whose influence is thine, and born of thee, - In others' works thou dost but mend the style, - And arts with thy sweet graces graced be. - But thou art all my art, and dost advance - As high as learning, my rude ignorance. - - - 79 - Whilst I alone did call upon thy aid, - My verse alone had all thy gentle grace, - But now my gracious numbers are decayed, - And my sick muse doth give an other place. - I grant (sweet love) thy lovely argument - Deserves the travail of a worthier pen, - Yet what of thee thy poet doth invent, - He robs thee of, and pays it thee again, - He lends thee virtue, and he stole that word, - From thy behaviour, beauty doth he give - And found it in thy cheek: he can afford - No praise to thee, but what in thee doth live. - Then thank him not for that which he doth say, - Since what he owes thee, thou thy self dost pay. - - - 80 - O how I faint when I of you do write, - Knowing a better spirit doth use your name, - And in the praise thereof spends all his might, - To make me tongue-tied speaking of your fame. - But since your worth (wide as the ocean is) - The humble as the proudest sail doth bear, - My saucy bark (inferior far to his) - On your broad main doth wilfully appear. - Your shallowest help will hold me up afloat, - Whilst he upon your soundless deep doth ride, - Or (being wrecked) I am a worthless boat, - He of tall building, and of goodly pride. - Then if he thrive and I be cast away, - The worst was this, my love was my decay. - - - 81 - Or I shall live your epitaph to make, - Or you survive when I in earth am rotten, - From hence your memory death cannot take, - Although in me each part will be forgotten. - Your name from hence immortal life shall have, - Though I (once gone) to all the world must die, - The earth can yield me but a common grave, - When you entombed in men's eyes shall lie, - Your monument shall be my gentle verse, - Which eyes not yet created shall o'er-read, - And tongues to be, your being shall rehearse, - When all the breathers of this world are dead, - You still shall live (such virtue hath my pen) - Where breath most breathes, even in the mouths of men. - - - 82 - I grant thou wert not married to my muse, - And therefore mayst without attaint o'erlook - The dedicated words which writers use - Of their fair subject, blessing every book. - Thou art as fair in knowledge as in hue, - Finding thy worth a limit past my praise, - And therefore art enforced to seek anew, - Some fresher stamp of the time-bettering days. - And do so love, yet when they have devised, - What strained touches rhetoric can lend, - Thou truly fair, wert truly sympathized, - In true plain words, by thy true-telling friend. - And their gross painting might be better used, - Where cheeks need blood, in thee it is abused. - - - 83 - I never saw that you did painting need, - And therefore to your fair no painting set, - I found (or thought I found) you did exceed, - That barren tender of a poet's debt: - And therefore have I slept in your report, - That you your self being extant well might show, - How far a modern quill doth come too short, - Speaking of worth, what worth in you doth grow. - This silence for my sin you did impute, - Which shall be most my glory being dumb, - For I impair not beauty being mute, - When others would give life, and bring a tomb. - There lives more life in one of your fair eyes, - Than both your poets can in praise devise. - - - 84 - Who is it that says most, which can say more, - Than this rich praise, that you alone, are you? - In whose confine immured is the store, - Which should example where your equal grew. - Lean penury within that pen doth dwell, - That to his subject lends not some small glory, - But he that writes of you, if he can tell, - That you are you, so dignifies his story. - Let him but copy what in you is writ, - Not making worse what nature made so clear, - And such a counterpart shall fame his wit, - Making his style admired every where. - You to your beauteous blessings add a curse, - Being fond on praise, which makes your praises worse. - - - 85 - My tongue-tied muse in manners holds her still, - While comments of your praise richly compiled, - Reserve their character with golden quill, - And precious phrase by all the Muses filed. - I think good thoughts, whilst other write good words, - And like unlettered clerk still cry Amen, - To every hymn that able spirit affords, - In polished form of well refined pen. - Hearing you praised, I say 'tis so, 'tis true, - And to the most of praise add something more, - But that is in my thought, whose love to you - (Though words come hindmost) holds his rank before, - Then others, for the breath of words respect, - Me for my dumb thoughts, speaking in effect. - - - 86 - Was it the proud full sail of his great verse, - Bound for the prize of (all too precious) you, - That did my ripe thoughts in my brain inhearse, - Making their tomb the womb wherein they grew? - Was it his spirit, by spirits taught to write, - Above a mortal pitch, that struck me dead? - No, neither he, nor his compeers by night - Giving him aid, my verse astonished. - He nor that affable familiar ghost - Which nightly gulls him with intelligence, - As victors of my silence cannot boast, - I was not sick of any fear from thence. - But when your countenance filled up his line, - Then lacked I matter, that enfeebled mine. - - - 87 - Farewell! thou art too dear for my possessing, - And like enough thou know'st thy estimate, - The charter of thy worth gives thee releasing: - My bonds in thee are all determinate. - For how do I hold thee but by thy granting, - And for that riches where is my deserving? - The cause of this fair gift in me is wanting, - And so my patent back again is swerving. - Thy self thou gav'st, thy own worth then not knowing, - Or me to whom thou gav'st it, else mistaking, - So thy great gift upon misprision growing, - Comes home again, on better judgement making. - Thus have I had thee as a dream doth flatter, - In sleep a king, but waking no such matter. - - - 88 - When thou shalt be disposed to set me light, - And place my merit in the eye of scorn, - Upon thy side, against my self I'll fight, - And prove thee virtuous, though thou art forsworn: - With mine own weakness being best acquainted, - Upon thy part I can set down a story - Of faults concealed, wherein I am attainted: - That thou in losing me, shalt win much glory: - And I by this will be a gainer too, - For bending all my loving thoughts on thee, - The injuries that to my self I do, - Doing thee vantage, double-vantage me. - Such is my love, to thee I so belong, - That for thy right, my self will bear all wrong. - - - 89 - Say that thou didst forsake me for some fault, - And I will comment upon that offence, - Speak of my lameness, and I straight will halt: - Against thy reasons making no defence. - Thou canst not (love) disgrace me half so ill, - To set a form upon desired change, - As I'll my self disgrace, knowing thy will, - I will acquaintance strangle and look strange: - Be absent from thy walks and in my tongue, - Thy sweet beloved name no more shall dwell, - Lest I (too much profane) should do it wronk: - And haply of our old acquaintance tell. - For thee, against my self I'll vow debate, - For I must ne'er love him whom thou dost hate. - - - 90 - Then hate me when thou wilt, if ever, now, - Now while the world is bent my deeds to cross, - join with the spite of fortune, make me bow, - And do not drop in for an after-loss: - Ah do not, when my heart hath 'scaped this sorrow, - Come in the rearward of a conquered woe, - Give not a windy night a rainy morrow, - To linger out a purposed overthrow. - If thou wilt leave me, do not leave me last, - When other petty griefs have done their spite, - But in the onset come, so shall I taste - At first the very worst of fortune's might. - And other strains of woe, which now seem woe, - Compared with loss of thee, will not seem so. - - - 91 - Some glory in their birth, some in their skill, - Some in their wealth, some in their body's force, - Some in their garments though new-fangled ill: - Some in their hawks and hounds, some in their horse. - And every humour hath his adjunct pleasure, - Wherein it finds a joy above the rest, - But these particulars are not my measure, - All these I better in one general best. - Thy love is better than high birth to me, - Richer than wealth, prouder than garments' costs, - Of more delight than hawks and horses be: - And having thee, of all men's pride I boast. - Wretched in this alone, that thou mayst take, - All this away, and me most wretchcd make. - - - 92 - But do thy worst to steal thy self away, - For term of life thou art assured mine, - And life no longer than thy love will stay, - For it depends upon that love of thine. - Then need I not to fear the worst of wrongs, - When in the least of them my life hath end, - I see, a better state to me belongs - Than that, which on thy humour doth depend. - Thou canst not vex me with inconstant mind, - Since that my life on thy revolt doth lie, - O what a happy title do I find, - Happy to have thy love, happy to die! - But what's so blessed-fair that fears no blot? - Thou mayst be false, and yet I know it not. - - - 93 - So shall I live, supposing thou art true, - Like a deceived husband, so love's face, - May still seem love to me, though altered new: - Thy looks with me, thy heart in other place. - For there can live no hatred in thine eye, - Therefore in that I cannot know thy change, - In many's looks, the false heart's history - Is writ in moods and frowns and wrinkles strange. - But heaven in thy creation did decree, - That in thy face sweet love should ever dwell, - Whate'er thy thoughts, or thy heart's workings be, - Thy looks should nothing thence, but sweetness tell. - How like Eve's apple doth thy beauty grow, - If thy sweet virtue answer not thy show. - - - 94 - They that have power to hurt, and will do none, - That do not do the thing, they most do show, - Who moving others, are themselves as stone, - Unmoved, cold, and to temptation slow: - They rightly do inherit heaven's graces, - And husband nature's riches from expense, - Tibey are the lords and owners of their faces, - Others, but stewards of their excellence: - The summer's flower is to the summer sweet, - Though to it self, it only live and die, - But if that flower with base infection meet, - The basest weed outbraves his dignity: - For sweetest things turn sourest by their deeds, - Lilies that fester, smell far worse than weeds. - - - 95 - How sweet and lovely dost thou make the shame, - Which like a canker in the fragrant rose, - Doth spot the beauty of thy budding name! - O in what sweets dost thou thy sins enclose! - That tongue that tells the story of thy days, - (Making lascivious comments on thy sport) - Cannot dispraise, but in a kind of praise, - Naming thy name, blesses an ill report. - O what a mansion have those vices got, - Which for their habitation chose out thee, - Where beauty's veil doth cover every blot, - And all things turns to fair, that eyes can see! - Take heed (dear heart) of this large privilege, - The hardest knife ill-used doth lose his edge. - - - 96 - Some say thy fault is youth, some wantonness, - Some say thy grace is youth and gentle sport, - Both grace and faults are loved of more and less: - Thou mak'st faults graces, that to thee resort: - As on the finger of a throned queen, - The basest jewel will be well esteemed: - So are those errors that in thee are seen, - To truths translated, and for true things deemed. - How many lambs might the stern wolf betray, - If like a lamb he could his looks translate! - How many gazers mightst thou lead away, - if thou wouldst use the strength of all thy state! - But do not so, I love thee in such sort, - As thou being mine, mine is thy good report. - - - 97 - How like a winter hath my absence been - From thee, the pleasure of the fleeting year! - What freezings have I felt, what dark days seen! - What old December's bareness everywhere! - And yet this time removed was summer's time, - The teeming autumn big with rich increase, - Bearing the wanton burden of the prime, - Like widowed wombs after their lords' decease: - Yet this abundant issue seemed to me - But hope of orphans, and unfathered fruit, - For summer and his pleasures wait on thee, - And thou away, the very birds are mute. - Or if they sing, 'tis with so dull a cheer, - That leaves look pale, dreading the winter's near. - - - 98 - From you have I been absent in the spring, - When proud-pied April (dressed in all his trim) - Hath put a spirit of youth in every thing: - That heavy Saturn laughed and leaped with him. - Yet nor the lays of birds, nor the sweet smell - Of different flowers in odour and in hue, - Could make me any summer's story tell: - Or from their proud lap pluck them where they grew: - Nor did I wonder at the lily's white, - Nor praise the deep vermilion in the rose, - They were but sweet, but figures of delight: - Drawn after you, you pattern of all those. - Yet seemed it winter still, and you away, - As with your shadow I with these did play. - - - 99 - The forward violet thus did I chide, - Sweet thief, whence didst thou steal thy sweet that smells, - If not from my love's breath? The purple pride - Which on thy soft check for complexion dwells, - In my love's veins thou hast too grossly dyed. - The lily I condemned for thy hand, - And buds of marjoram had stol'n thy hair, - The roses fearfully on thorns did stand, - One blushing shame, another white despair: - A third nor red, nor white, had stol'n of both, - And to his robbery had annexed thy breath, - But for his theft in pride of all his growth - A vengeful canker eat him up to death. - More flowers I noted, yet I none could see, - But sweet, or colour it had stol'n from thee. - - - 100 - Where art thou Muse that thou forget'st so long, - To speak of that which gives thee all thy might? - Spend'st thou thy fury on some worthless song, - Darkening thy power to lend base subjects light? - Return forgetful Muse, and straight redeem, - In gentle numbers time so idly spent, - Sing to the ear that doth thy lays esteem, - And gives thy pen both skill and argument. - Rise resty Muse, my love's sweet face survey, - If time have any wrinkle graven there, - If any, be a satire to decay, - And make time's spoils despised everywhere. - Give my love fame faster than Time wastes life, - So thou prevent'st his scythe, and crooked knife. - - - 101 - O truant Muse what shall be thy amends, - For thy neglect of truth in beauty dyed? - Both truth and beauty on my love depends: - So dost thou too, and therein dignified: - Make answer Muse, wilt thou not haply say, - 'Truth needs no colour with his colour fixed, - Beauty no pencil, beauty's truth to lay: - But best is best, if never intermixed'? - Because he needs no praise, wilt thou be dumb? - Excuse not silence so, for't lies in thee, - To make him much outlive a gilded tomb: - And to be praised of ages yet to be. - Then do thy office Muse, I teach thee how, - To make him seem long hence, as he shows now. - - - 102 - My love is strengthened though more weak in seeming, - I love not less, though less the show appear, - That love is merchandized, whose rich esteeming, - The owner's tongue doth publish every where. - Our love was new, and then but in the spring, - When I was wont to greet it with my lays, - As Philomel in summer's front doth sing, - And stops her pipe in growth of riper days: - Not that the summer is less pleasant now - Than when her mournful hymns did hush the night, - But that wild music burthens every bough, - And sweets grown common lose their dear delight. - Therefore like her, I sometime hold my tongue: - Because I would not dull you with my song. - - - 103 - Alack what poverty my muse brings forth, - That having such a scope to show her pride, - The argument all bare is of more worth - Than when it hath my added praise beside. - O blame me not if I no more can write! - Look in your glass and there appears a face, - That over-goes my blunt invention quite, - Dulling my lines, and doing me disgrace. - Were it not sinful then striving to mend, - To mar the subject that before was well? - For to no other pass my verses tend, - Than of your graces and your gifts to tell. diff --git a/crates/pithos_lib/tests/format_validation.rs b/crates/pithos_lib/tests/format_validation.rs new file mode 100644 index 0000000..69ac880 --- /dev/null +++ b/crates/pithos_lib/tests/format_validation.rs @@ -0,0 +1,160 @@ +use pithos_lib::archive::{ + AccessKeys, Archive, ArchivePath, ArchiveWriter, EntryMetadata, EntryReference, OpenLimits, + OpenOptions, ProcessingOptions, WriteOptions, +}; +use pithos_lib::crypto::PrivateKey; +use pithos_lib::error::PithosError; +use pithos_lib::source::MemorySource; + +#[test] +fn public_open_rejects_invalid_header() { + assert!(matches!( + Archive::open(MemorySource::new(&b"NOTPIT"[..]), OpenOptions::default()), + Err(PithosError::Deserialization(_)) + )); +} + +#[test] +fn public_open_rejects_a_footer_claiming_an_impossible_directory() { + let mut bytes = b"PITH\x80\x02".to_vec(); + bytes.extend_from_slice(&u64::MAX.to_be_bytes()); + bytes.extend_from_slice(&0u32.to_be_bytes()); + assert!(matches!( + Archive::open(MemorySource::new(bytes), OpenOptions::default()), + Err(PithosError::LimitExceeded { + field: "directory", + .. + }) + )); +} + +fn archive_with_entries(entries: Vec<(&str, EntryMetadata)>) -> (Vec, PrivateKey) { + let sender = PrivateKey::generate(); + let recipient = sender.duplicate(); + let mut writer = ArchiveWriter::create( + Vec::new(), + WriteOptions::new(sender, vec![recipient.public_key()]), + ) + .unwrap(); + for (index, (path, metadata)) in entries.into_iter().enumerate() { + let content = [u8::try_from(index).unwrap()]; + writer + .add_file( + ArchivePath::new(path).unwrap(), + metadata, + ProcessingOptions::new(false, 0).unwrap(), + Some(content.len() as u64), + std::io::Cursor::new(content), + ) + .unwrap(); + } + (writer.finish().unwrap(), recipient) +} + +#[test] +fn open_limits_reject_oversized_entry_descriptor_reference_and_relationship_counts() { + let cases = [ + ( + OpenLimits { + max_entries: 1, + ..OpenLimits::default() + }, + archive_with_entries(vec![ + ("first", EntryMetadata::new(0, 0, 0o644)), + ("second", EntryMetadata::new(0, 0, 0o644)), + ]), + ), + ( + OpenLimits { + max_descriptors: 1, + ..OpenLimits::default() + }, + archive_with_entries(vec![ + ("first", EntryMetadata::new(0, 0, 0o644)), + ("second", EntryMetadata::new(0, 0, 0o644)), + ]), + ), + ( + OpenLimits { + max_references: 0, + ..OpenLimits::default() + }, + archive_with_entries(vec![ + ("first", EntryMetadata::new(0, 0, 0o644)), + ( + "second", + EntryMetadata::new(0, 0, 0o644).with_references(vec![EntryReference { + target_file_id: 0, + relationship: 0, + }]), + ), + ]), + ), + ( + OpenLimits { + max_relationships: 0, + ..OpenLimits::default() + }, + archive_with_entries(vec![ + ("first", EntryMetadata::new(0, 0, 0o644)), + ( + "second", + EntryMetadata::new(0, 0, 0o644).with_references(vec![EntryReference { + target_file_id: 0, + relationship: 0, + }]), + ), + ]), + ), + ]; + + for (limits, (bytes, recipient)) in cases { + let result = Archive::open( + MemorySource::new(bytes), + OpenOptions::default() + .with_limits(limits) + .with_access_keys(AccessKeys::new().with_key(recipient)), + ); + match result { + Err(PithosError::LimitExceeded { .. } | PithosError::Deserialization(_)) => {} + Err(error) => panic!("unexpected error: {error}"), + Ok(_) => panic!("oversized directory unexpectedly opened"), + } + } +} + +#[test] +fn adversarial_wire_path_order_stays_within_open_limits() { + let sender = PrivateKey::generate(); + let recipient = sender.duplicate(); + let mut writer = ArchiveWriter::create( + Vec::new(), + WriteOptions::new(sender, vec![recipient.public_key()]), + ) + .unwrap(); + for index in (0..100).rev() { + writer + .add_directory( + ArchivePath::new(format!("root/{index:03}")).unwrap(), + EntryMetadata::new(0, 0, 0o755), + ) + .unwrap(); + } + writer + .add_directory( + ArchivePath::new("root").unwrap(), + EntryMetadata::new(0, 0, 0o755), + ) + .unwrap(); + let archive = Archive::open( + MemorySource::new(writer.finish().unwrap()), + OpenOptions::default() + .with_limits(OpenLimits { + max_entries: 101, + ..OpenLimits::default() + }) + .with_access_keys(AccessKeys::new().with_key(recipient)), + ) + .unwrap(); + assert_eq!(archive.entries().len(), 101); +} diff --git a/crates/pithos_lib/tests/reader.rs b/crates/pithos_lib/tests/reader.rs index 774eab2..103ca1a 100644 --- a/crates/pithos_lib/tests/reader.rs +++ b/crates/pithos_lib/tests/reader.rs @@ -1,1504 +1,150 @@ -pub mod common; +mod common; -use crate::common::util::{create_pithos_writer, load_test_keys, write_dummy_pithos}; -use pithos_lib::error::PithosError; -use pithos_lib::helpers::chacha_poly1305::decrypt_chunk; -use pithos_lib::helpers::crypt4gh::{ - CRYPT4GH_ENCRYPTED_BLOCK_SIZE, Crypt4GHError, Crypt4GHHeader, Packet, PacketData, -}; -use pithos_lib::helpers::directory::DirectoryBuilder; -use pithos_lib::helpers::file_entry_map::Key; -use pithos_lib::helpers::ro_crate::{RoCrateSource, read_ro_crate_directory, read_ro_crate_zip}; -use pithos_lib::helpers::x25519_keys::{private_key_from_pem_bytes, public_key_from_pem_bytes}; -use pithos_lib::io::pithosreader::{ExternalBlockSource, PithosReaderSimple, ReaderLimits}; -use pithos_lib::io::pithoswriter::{Content, InputFile, PithosWriter}; -use pithos_lib::model::deserialization::DeserializationError; -use pithos_lib::model::structs::{ - BlockDataState, BlockLocation, FileEntry, FileHeader, FileType, Reference, +use common::util::{fixture, open, private_key, public_key}; +use pithos_lib::adapters::crypt4gh::{self, Crypt4GHError}; +use pithos_lib::archive::{ + AccessKeys, ArchivePath, ArchiveWriter, EntryKind, EntryMetadata, ProcessingOptions, + WriteOptions, }; -use rocraters::ro_crate::graph_vector::GraphVector; -use rocraters::ro_crate::read::CrateReadError; -use rocraters::ro_crate::schema::RoCrateSchemaVersion; -use std::fs::{File, OpenOptions, read, read_to_string, write}; -use std::io::{Cursor, Read, Seek, SeekFrom, Write}; -use std::path::{Path, PathBuf}; -use std::sync::{Arc, Mutex}; -use tempfile::TempDir; -use x25519_dalek::StaticSecret; - -fn reader_key() -> StaticSecret { - private_key_from_pem_bytes( - std::fs::read("tests/data/keys/recipient1_private.pem") - .unwrap() - .as_slice(), - ) - .unwrap() -} - -fn empty_entry(file_type: FileType, target: Option<&str>) -> FileEntry { - FileEntry { - file_type, - block_data: BlockDataState::Decrypted(vec![]), - created: 0, - modified: 0, - file_size: 0, - permissions: 0o644, - references: vec![], - symlink_target: target.map(str::to_owned), - } -} - -fn caller_directory( - archive: &Path, - entry_path: &str, - file_type: FileType, - target: Option<&str>, -) -> (PithosReaderSimple, pithos_lib::model::structs::Directory) { - let mut reader = PithosReaderSimple::new_with_key(archive, reader_key()).unwrap(); - let (mut directory, _) = reader.read_directory().unwrap(); - directory - .files - .insert(Key::new(9000, entry_path), empty_entry(file_type, target)) - .unwrap(); - (reader, directory) -} - -#[test] -fn test_reader_rejects_invalid_header_magic_during_construction() { - let temp_dir = TempDir::new().unwrap(); - let archive = write_dummy_pithos(&temp_dir, false, false); - let mut bytes = read(&archive).unwrap(); - bytes[0] = b'X'; - write(&archive, bytes).unwrap(); - - let error = match PithosReaderSimple::new_with_key(&archive, reader_key()) { - Ok(_) => panic!("reader construction must reject an invalid header magic"), - Err(error) => error, - }; - assert!(matches!( - error, - PithosError::Deserialization(DeserializationError::InvalidMarker(_)) - )); -} - -#[test] -fn test_reader_rejects_invalid_header_version_during_construction() { - let temp_dir = TempDir::new().unwrap(); - let archive = write_dummy_pithos(&temp_dir, false, false); - let mut bytes = read(&archive).unwrap(); - bytes[4..6].copy_from_slice(&[0x80, 0x04]); - write(&archive, bytes).unwrap(); - - let error = match PithosReaderSimple::new_with_key(&archive, reader_key()) { - Ok(_) => panic!("reader construction must reject an unsupported header version"), - Err(error) => error, - }; - let message = error.to_string(); - assert!( - message.contains("0x0100"), - "missing supported version: {message}" - ); - assert!( - message.contains("0x0200"), - "missing actual version: {message}" - ); -} - -fn write_zip_entry(path: &Path, name: &str, content: &[u8]) { - let file = File::create(path).unwrap(); - let mut archive = zip::ZipWriter::new(file); - archive - .start_file(name, zip::write::SimpleFileOptions::default()) - .unwrap(); - archive.write_all(content).unwrap(); - archive.finish().unwrap(); -} - -fn append_empty_directory(path: &Path) { - let (writer_key, _, reader_key) = load_test_keys(); - let mut writer = PithosWriter::new_from_file(writer_key, vec![reader_key], None, path).unwrap(); - writer.write_directory().unwrap(); - drop(writer); -} - -fn complete_test_directory(parent: Option<(u64, u64)>) -> Vec { - let mut directory = pithos_lib::model::structs::Directory { - identifier: *b"PITHOSDR", - parent_directory_offset: parent, - files: pithos_lib::helpers::file_entry_map::FileEntryMap::new(), - blocks: indexmap::IndexMap::new(), - relations: vec![], - encryption: indexmap::IndexMap::new(), - dir_len: 0, - crc32: 0, - }; - let mut bytes = Vec::new(); - directory.serialize(&mut bytes).unwrap(); - directory.dir_len = bytes.len() as u64; - bytes.clear(); - directory.serialize(&mut bytes).unwrap(); - let crc_start = bytes.len() - 4; - let crc = crc32fast::hash(&bytes[..crc_start]); - bytes[crc_start..].copy_from_slice(&crc.to_be_bytes()); - bytes -} - -fn directory_bounds(bytes: &[u8]) -> (usize, usize) { - let length = - u64::from_be_bytes(bytes[bytes.len() - 12..bytes.len() - 4].try_into().unwrap()) as usize; - (bytes.len() - length, length) -} - -fn write_identity_integrity_archive( - temp_dir: &TempDir, -) -> ( - PathBuf, - StaticSecret, - pithos_lib::model::structs::Directory, - [u8; 32], -) { - let (archive, key, mut writer) = create_pithos_writer(temp_dir, None); - writer.write_file_header().unwrap(); - writer - .process_input(InputFile { - file_type: FileType::Data, - inner_path: "integrity.txt".into(), - data: Content::Raw("deterministic integrity payload".into()), - metadata: None, - encrypt: false, - compression_level: Some(0), - }) - .unwrap(); - writer.write_directory().unwrap(); - drop(writer); - - let mut reader = PithosReaderSimple::new_with_key(&archive, key.clone()).unwrap(); - let (directory, _) = reader.read_directory().unwrap(); - let file_entry = directory.get_file_by_path("integrity.txt").unwrap(); - let hash = match &file_entry.block_data { - BlockDataState::Decrypted(blocks) => blocks[0].0, - BlockDataState::Encrypted(_) => panic!("identity archive has encrypted block data"), - }; - (archive, key, directory, hash) -} - -fn mutate_block_payload( - archive: &Path, - directory: &pithos_lib::model::structs::Directory, - hash: [u8; 32], -) { - let block = directory.blocks.get(&hash).unwrap(); - let mut bytes = read(archive).unwrap(); - let offset = block.offset as usize; - assert_eq!(&bytes[offset..offset + 4], b"BLCK"); - bytes[offset + 4] ^= 1; - write(archive, bytes).unwrap(); -} - -fn external_directory( - mut directory: pithos_lib::model::structs::Directory, - hash: [u8; 32], -) -> pithos_lib::model::structs::Directory { - let block = directory.blocks.get_mut(&hash).unwrap(); - block.location = BlockLocation::External { - url: "https://invalid.invalid/external-block".into(), - }; - block.offset = u64::MAX; - directory -} - -struct RecordingExternalSource { - response: Vec, - requested_url: Arc>>, - requested_max: Arc>>, -} - -struct SharedWriter(Arc>>); - -impl Write for SharedWriter { - fn write(&mut self, bytes: &[u8]) -> std::io::Result { - self.0.lock().unwrap().extend_from_slice(bytes); - Ok(bytes.len()) - } - - fn flush(&mut self) -> std::io::Result<()> { - Ok(()) - } -} - -impl ExternalBlockSource for RecordingExternalSource { - fn open(&self, url: &str, max_response_size: u64) -> Result, PithosError> { - *self.requested_url.lock().unwrap() = Some(url.to_owned()); - *self.requested_max.lock().unwrap() = Some(max_response_size); - Ok(Box::new(Cursor::new(self.response.clone()))) - } -} - -fn external_block_bytes( - archive: &Path, - directory: &pithos_lib::model::structs::Directory, - hash: [u8; 32], -) -> Vec { - let block = directory.blocks.get(&hash).unwrap(); - let bytes = read(archive).unwrap(); - let start = block.offset as usize; - let end = start + 4 + block.stored_size as usize; - bytes[start..end].to_vec() -} - -#[test] -fn test_external_blocks_use_injected_source_for_all_paths() { - let temp_dir = TempDir::new().unwrap(); - let (archive, key, directory, hash) = write_identity_integrity_archive(&temp_dir); - let valid_response = external_block_bytes(&archive, &directory, hash); - let directory = external_directory(directory, hash); - let requested_url = Arc::new(Mutex::new(None)); - let requested_max = Arc::new(Mutex::new(None)); - let source = RecordingExternalSource { - response: valid_response.clone(), - requested_url: requested_url.clone(), - requested_max: requested_max.clone(), - }; - let mut reader = PithosReaderSimple::new_with_key(&archive, key.clone()) - .unwrap() - .with_external_block_source(Box::new(source)); - let output = temp_dir.path().join("external-full-success.txt"); - reader - .read_file("integrity.txt", &directory, Some(&output), None) - .unwrap(); - assert_eq!(read(&output).unwrap(), b"deterministic integrity payload"); - assert_eq!( - requested_url.lock().unwrap().as_deref(), - Some("https://invalid.invalid/external-block") - ); - assert_eq!( - *requested_max.lock().unwrap(), - Some(4 + directory.blocks.get(&hash).unwrap().stored_size + 1) - ); - - let source = RecordingExternalSource { - response: valid_response.clone(), - requested_url: Arc::new(Mutex::new(None)), - requested_max: Arc::new(Mutex::new(None)), - }; - let mut reader = PithosReaderSimple::new_with_key(&archive, key.clone()) - .unwrap() - .with_external_block_source(Box::new(source)); - let range_bytes = Arc::new(Mutex::new(Vec::new())); - let mut range_output: Box = Box::new(SharedWriter(range_bytes.clone())); - let file_entry = directory.get_file_by_path("integrity.txt").unwrap(); - reader - .read_data_range_to_sink(2..10, file_entry, &directory.blocks, &mut range_output) - .unwrap(); - assert_eq!(*range_bytes.lock().unwrap(), b"terminis"); - - let recipient = public_key_from_pem_bytes( - read_to_string("tests/data/keys/recipient2_public.pem") - .unwrap() - .as_bytes(), - ) - .unwrap(); - let source = RecordingExternalSource { - response: valid_response, - requested_url: Arc::new(Mutex::new(None)), - requested_max: Arc::new(Mutex::new(None)), - }; - let mut reader = PithosReaderSimple::new_with_key(&archive, key) - .unwrap() - .with_external_block_source(Box::new(source)); - let crypt4gh_output = Arc::new(Mutex::new(Vec::new())); - reader - .read_file_to_crypt4gh( - "integrity.txt", - &directory, - vec![recipient], - Some(Box::new(SharedWriter(crypt4gh_output.clone()))), - ) - .unwrap(); - assert!(!crypt4gh_output.lock().unwrap().is_empty()); -} - -#[test] -fn test_external_blocks_reject_invalid_framing_without_plaintext() { - let cases = [ - ( - b"BLCK".to_vec(), - "external block framing error: short block payload", - ), - ( - [b"BLCK".as_slice(), b"deterministic integrity payload", b"x"].concat(), - "external block framing error: response exceeds expected size", - ), - ( - [b"NOPE".as_slice(), b"deterministic integrity payload"].concat(), - "external block framing error: invalid block marker", - ), - ]; - for (response, expected_error) in cases { - let temp_dir = TempDir::new().unwrap(); - let (archive, key, directory, hash) = write_identity_integrity_archive(&temp_dir); - let directory = external_directory(directory, hash); - let source = RecordingExternalSource { - response, - requested_url: Arc::new(Mutex::new(None)), - requested_max: Arc::new(Mutex::new(None)), - }; - let output = temp_dir.path().join("invalid-external.txt"); - let mut reader = PithosReaderSimple::new_with_key(&archive, key) - .unwrap() - .with_external_block_source(Box::new(source)); - let result = reader.read_file("integrity.txt", &directory, Some(&output), None); - assert!(result.unwrap_err().to_string().contains(expected_error)); - assert!(!output.exists()); - } -} - -#[test] -fn test_external_blocks_fail_closed_without_source() { - let temp_dir = TempDir::new().unwrap(); - let (archive, key, directory, hash) = write_identity_integrity_archive(&temp_dir); - let directory = external_directory(directory, hash); - - let full_output = temp_dir.path().join("external-full.txt"); - let mut reader = PithosReaderSimple::new_with_key(&archive, key.clone()).unwrap(); - let full_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { - reader.read_file("integrity.txt", &directory, Some(&full_output), None) - })) - .map_err(|panic| { - let message = panic - .downcast_ref::() - .cloned() - .or_else(|| panic.downcast_ref::<&str>().map(ToString::to_string)) - .unwrap_or_else(|| "unknown panic".into()); - PithosError::Other(message) - }) - .and_then(|result| result); - assert_eq!( - full_result.unwrap_err().to_string(), - "external block source required" - ); - assert!(!full_output.exists()); - - let range_output = temp_dir.path().join("external-range.txt"); - let mut reader = PithosReaderSimple::new_with_key(&archive, key.clone()).unwrap(); - let range_result = reader.read_file( - "integrity.txt", - &directory, - Some(&range_output), - #[allow(clippy::single_range_in_vec_init)] - Some(vec![0..1]), - ); - assert_eq!( - range_result.unwrap_err().to_string(), - "external block source required" - ); - assert!(!range_output.exists()); - - let recipient = public_key_from_pem_bytes( - read_to_string("tests/data/keys/recipient2_public.pem") - .unwrap() - .as_bytes(), - ) - .unwrap(); - let mut reader = PithosReaderSimple::new_with_key(&archive, key).unwrap(); - let crypt4gh_result = reader.read_file_to_crypt4gh( - "integrity.txt", - &directory, - vec![recipient], - Some(Box::new(Vec::::new())), - ); - assert_eq!( - crypt4gh_result.unwrap_err().to_string(), - "external block source required" - ); -} - -#[test] -fn test_block_integrity_valid_identity_read() { - let temp_dir = TempDir::new().unwrap(); - let (archive, key, directory, _) = write_identity_integrity_archive(&temp_dir); - let output = temp_dir.path().join("valid.txt"); - let mut reader = PithosReaderSimple::new_with_key(&archive, key).unwrap(); - - reader - .read_file("integrity.txt", &directory, Some(&output), None) - .unwrap(); - assert_eq!(read(&output).unwrap(), b"deterministic integrity payload"); -} +use pithos_lib::error::PithosError; +use pithos_lib::fs::extract; #[test] -fn test_block_integrity_full_read_rejects_corruption_without_commit() { - let temp_dir = TempDir::new().unwrap(); - let (archive, key, directory, hash) = write_identity_integrity_archive(&temp_dir); - mutate_block_payload(&archive, &directory, hash); - let output = temp_dir.path().join("corrupt-full.txt"); - let mut reader = PithosReaderSimple::new_with_key(&archive, key).unwrap(); - +fn public_reader_lists_copies_ranges_extracts_and_exports() { + let temporary = tempfile::tempdir().unwrap(); + let path = fixture(&temporary, "recipient1"); + let archive = open(&path, AccessKeys::new().with_key(private_key("recipient1"))); + let entry = archive.entry("data").unwrap().unwrap(); assert!(matches!( - reader.read_file("integrity.txt", &directory, Some(&output), None), - Err(PithosError::BlockHashMismatch { .. }) + entry.kind, + EntryKind::File { + available: true, + .. + } )); - assert!(!output.exists()); -} + assert_eq!(entry.permissions, 0o644); -#[test] -fn test_block_integrity_range_read_rejects_corruption_without_commit() { - let temp_dir = TempDir::new().unwrap(); - let (archive, key, directory, hash) = write_identity_integrity_archive(&temp_dir); - mutate_block_payload(&archive, &directory, hash); - let output = temp_dir.path().join("corrupt-range.txt"); - let mut reader = PithosReaderSimple::new_with_key(&archive, key).unwrap(); + let mut full = Vec::new(); + archive.copy_to("data", &mut full).unwrap(); + assert_eq!(full, b"public archive reader fixture"); + let mut range = Vec::new(); + archive.copy_range_to("data", 7..14, &mut range).unwrap(); + assert_eq!(range, b"archive"); - assert!(matches!( - reader.read_file( - "integrity.txt", - &directory, - Some(&output), - #[allow(clippy::single_range_in_vec_init)] - Some(vec![0..1]), - ), - Err(PithosError::BlockHashMismatch { .. }) - )); - assert!(!output.exists()); -} + let output = temporary.path().join("out"); + extract(&archive, "data", &output).unwrap(); + assert_eq!(std::fs::read(output.join("data")).unwrap(), full); -#[test] -fn test_block_integrity_crypt4gh_rejects_corruption_before_block_output() { - let temp_dir = TempDir::new().unwrap(); - let (archive, key, directory, hash) = write_identity_integrity_archive(&temp_dir); - mutate_block_payload(&archive, &directory, hash); - let recipient = public_key_from_pem_bytes( - read_to_string("tests/data/keys/recipient2_public.pem") - .unwrap() - .as_bytes(), + let mut exported = Vec::new(); + crypt4gh::export( + &archive, + "data", + vec![public_key("recipient2")], + &mut exported, ) .unwrap(); - let output = temp_dir.path().join("corrupt.crypt4gh"); - let mut reader = PithosReaderSimple::new_with_key(&archive, key).unwrap(); - let sink = Box::new(File::create(&output).unwrap()); - - assert!(matches!( - reader.read_file_to_crypt4gh("integrity.txt", &directory, vec![recipient], Some(sink)), - Err(PithosError::BlockHashMismatch { .. }) - )); - let output_bytes = read(&output).unwrap(); - let header = Crypt4GHHeader::try_from(output_bytes.as_slice()).unwrap(); - let header_len = 16 - + header - .header_packets - .iter() - .map(|packet| packet.length as usize) - .sum::(); - assert_eq!(output_bytes.len(), header_len); -} - -#[test] -fn test_block_integrity_size_mismatch_precedes_output() { - let temp_dir = TempDir::new().unwrap(); - let (archive, key, mut directory, hash) = write_identity_integrity_archive(&temp_dir); - directory.blocks.get_mut(&hash).unwrap().original_size += 1; - let output = temp_dir.path().join("wrong-size.txt"); - let mut reader = PithosReaderSimple::new_with_key(&archive, key).unwrap(); - - assert!(matches!( - reader.read_file("integrity.txt", &directory, Some(&output), None), - Err(PithosError::BlockSizeMismatch { .. }) - )); - assert!(!output.exists()); -} - -#[test] -fn test_directory_integrity_valid_terminal_archive() { - let temp_dir = TempDir::new().unwrap(); - let archive = write_dummy_pithos(&temp_dir, false, false); - let mut reader = PithosReaderSimple::new_with_key(&archive, reader_key()).unwrap(); - - assert!(reader.read_directory().is_ok()); -} - -#[test] -fn test_reader_reads_directory_built_without_manual_length_update() { - let temp_dir = TempDir::new().unwrap(); - let directory = DirectoryBuilder::new() - .set_relations(vec![]) - .build() - .unwrap(); - let archive = temp_dir.path().join("builder-directory.pith"); - let mut bytes = FileHeader::default().serialize_to_bytes().unwrap(); - directory.serialize(&mut bytes).unwrap(); - write(&archive, bytes).unwrap(); - - let mut reader = PithosReaderSimple::new_with_key(&archive, reader_key()).unwrap(); - let (read_directory, _) = reader.read_directory().unwrap(); - - assert_eq!(read_directory, directory); + assert!(exported.starts_with(b"crypt4gh")); + assert!(exported.len() > full.len()); } #[test] -fn test_directory_integrity_terminal_marker_mutation() { - let temp_dir = TempDir::new().unwrap(); - let archive = write_dummy_pithos(&temp_dir, false, false); - let mut bytes = read(&archive).unwrap(); - let (start, _) = directory_bounds(&bytes); - bytes[start] ^= 1; - write(&archive, bytes).unwrap(); - - let mut reader = PithosReaderSimple::new_with_key(&archive, reader_key()).unwrap(); +fn unavailable_content_is_visible_and_invalid_ranges_fail_before_copy() { + let temporary = tempfile::tempdir().unwrap(); + let path = fixture(&temporary, "recipient1"); + let archive = open(&path, AccessKeys::new()); assert!(matches!( - reader.read_directory(), - Err(PithosError::InvalidDirectoryMarker { .. }) + archive.entries().next().unwrap().kind, + EntryKind::File { + available: false, + .. + } )); -} - -#[test] -fn test_directory_integrity_terminal_crc_mutation() { - let temp_dir = TempDir::new().unwrap(); - let archive = write_dummy_pithos(&temp_dir, false, false); - let mut bytes = read(&archive).unwrap(); - *bytes.last_mut().unwrap() ^= 1; - write(&archive, bytes).unwrap(); - - let mut reader = PithosReaderSimple::new_with_key(&archive, reader_key()).unwrap(); assert!(matches!( - reader.read_directory(), - Err(PithosError::DirectoryChecksumMismatch { .. }) + archive.copy_to("data", &mut Vec::new()), + Err(PithosError::ContentUnavailable) )); -} - -#[test] -fn test_directory_integrity_rejects_extra_unconsumed_byte() { - let temp_dir = TempDir::new().unwrap(); - let archive = write_dummy_pithos(&temp_dir, false, false); - let mut bytes = read(&archive).unwrap(); - let (start, length) = directory_bounds(&bytes); - let mut directory = bytes.split_off(start); - directory.insert(directory.len() - 12, 0); - let new_length = (length + 1) as u64; - let footer_start = directory.len() - 12; - directory[footer_start..footer_start + 8].copy_from_slice(&new_length.to_be_bytes()); - let crc_start = directory.len() - 4; - let crc = crc32fast::hash(&directory[..crc_start]); - directory[crc_start..].copy_from_slice(&crc.to_be_bytes()); - bytes.extend_from_slice(&directory); - write(&archive, bytes).unwrap(); - let mut reader = PithosReaderSimple::new_with_key(&archive, reader_key()).unwrap(); + let archive = open(&path, AccessKeys::new().with_key(private_key("recipient1"))); assert!(matches!( - reader.read_directory(), - Err(PithosError::DirectoryConsumptionMismatch { .. }) + archive.copy_range_to("data", 99..100, &mut Vec::new()), + Err(PithosError::InvalidReadRange { .. }) )); } #[test] -fn test_directory_integrity_valid_append_chain() { - let temp_dir = TempDir::new().unwrap(); - let archive = write_dummy_pithos(&temp_dir, false, false); - append_empty_directory(&archive); - - let mut reader = PithosReaderSimple::new_with_key(&archive, reader_key()).unwrap(); - assert!(reader.read_directory().is_ok()); -} - -#[test] -fn test_robust_terminal_footer_max_length_returns_error() { - let temp_dir = TempDir::new().unwrap(); - let archive = write_dummy_pithos(&temp_dir, false, false); - let mut bytes = read(&archive).unwrap(); - let footer_start = bytes.len() - 12; - bytes[footer_start..footer_start + 8].copy_from_slice(&u64::MAX.to_be_bytes()); - write(&archive, bytes).unwrap(); - - let result = PithosReaderSimple::new_with_key(&archive, reader_key()) - .unwrap() - .read_directory(); - assert!(matches!( - result, - Err(PithosError::LimitExceeded { - field: "directory", - .. - }) - )); -} - -#[test] -fn test_robust_overlapping_parent_is_rejected_as_invalid_chain() { - let temp_dir = TempDir::new().unwrap(); - let archive = write_dummy_pithos(&temp_dir, false, false); - append_empty_directory(&archive); - let bytes = read(&archive).unwrap(); - let (final_start, final_length) = directory_bounds(&bytes); - let mut final_directory = pithos_lib::model::structs::Directory::deserialize( - &mut std::io::Cursor::new(&bytes[final_start..]), - ) - .unwrap(); - final_directory.parent_directory_offset = Some(((final_start + 1) as u64, final_length as u64)); - final_directory.dir_len = 0; - final_directory.crc32 = 0; - let mut replacement = Vec::new(); - final_directory.serialize(&mut replacement).unwrap(); - final_directory.dir_len = replacement.len() as u64; - replacement.clear(); - final_directory.serialize(&mut replacement).unwrap(); - let crc = crc32fast::hash(&replacement[..replacement.len() - 4]); - let crc_start = replacement.len() - 4; - replacement[crc_start..].copy_from_slice(&crc.to_be_bytes()); - let mut archive_bytes = bytes[..final_start].to_vec(); - archive_bytes.extend_from_slice(&replacement); - write(&archive, archive_bytes).unwrap(); - - let mut reader = PithosReaderSimple::new_with_key(&archive, reader_key()).unwrap(); - let error = reader.read_directory().unwrap_err(); - assert!(error.to_string().contains("chain")); -} - -#[test] -fn test_robust_parent_boundary_uses_immediate_child() { - let temp_dir = TempDir::new().unwrap(); - let middle = complete_test_directory(None); - let oldest = complete_test_directory(Some((33, middle.len() as u64))); - let terminal = complete_test_directory(Some((6, oldest.len() as u64))); - assert_eq!(oldest.len(), 27); - assert_eq!(middle.len(), 25); - let archive = temp_dir.path().join("three-directories.pithos"); - let mut bytes = FileHeader::default().serialize_to_bytes().unwrap(); - bytes.extend_from_slice(&oldest); - bytes.extend_from_slice(&middle); - bytes.extend_from_slice(&terminal); - write(&archive, bytes).unwrap(); - - let mut reader = PithosReaderSimple::new_with_key(&archive, reader_key()).unwrap(); - let error = reader.read_directory().unwrap_err(); +fn crypt4gh_export_rejects_zero_recipients_before_sink_output() { + let temporary = tempfile::tempdir().unwrap(); + let path = fixture(&temporary, "recipient1"); + let archive = open(&path, AccessKeys::new().with_key(private_key("recipient1"))); + let mut exported = Vec::new(); + let error = crypt4gh::export(&archive, "data", Vec::new(), &mut exported).unwrap_err(); assert!( - matches!(error, PithosError::InvalidDirectoryChain(_)), - "unexpected chain fixture error: {error:?}" + (&error as &(dyn std::error::Error + 'static)) + .downcast_ref::() + .is_some() ); + assert!(exported.is_empty()); } #[test] -fn test_robust_oversized_stored_block_returns_error_without_commit() { - let temp_dir = TempDir::new().unwrap(); - let (archive, key, mut directory, _) = write_identity_integrity_archive(&temp_dir); - let hash = match directory - .get_file_by_path("integrity.txt") - .unwrap() - .block_data - .clone() - { - BlockDataState::Decrypted(blocks) => blocks[0].0, - BlockDataState::Encrypted(_) => unreachable!(), +fn public_reader_accepts_empty_eof_ranges_and_rejects_reversed_ranges() { + let temporary = tempfile::tempdir().unwrap(); + let path = fixture(&temporary, "recipient1"); + let archive = open(&path, AccessKeys::new().with_key(private_key("recipient1"))); + let size = match archive.entry("data").unwrap().unwrap().kind { + EntryKind::File { size, .. } => size, + _ => panic!("fixture data entry must be a file"), }; - directory.blocks.get_mut(&hash).unwrap().stored_size = u64::MAX; - let output = temp_dir.path().join("oversized.txt"); - let result = PithosReaderSimple::new_with_key(&archive, key) - .unwrap() - .read_file("integrity.txt", &directory, Some(&output), None); - assert!(matches!( - result, - Err(PithosError::LimitExceeded { - field: "stored block", - .. - }) - )); - assert!(!output.exists()); -} - -#[test] -fn test_robust_zero_crypt4gh_packet_length_returns_error() { - let mut bytes = Vec::from(b"crypt4gh".as_slice()); - bytes.extend_from_slice(&1u32.to_le_bytes()); - bytes.extend_from_slice(&1u32.to_le_bytes()); - bytes.extend_from_slice(&0u32.to_le_bytes()); - bytes.extend_from_slice(&[0u8; 68]); - let result = Crypt4GHHeader::try_from(bytes.as_slice()); - assert!(matches!(result, Err(Crypt4GHError::FromBytesError(_)))); -} - -#[test] -fn test_robust_skipped_range_enforces_decoded_block_limit() { - let temp_dir = TempDir::new().unwrap(); - let (archive, key, directory, hash) = write_identity_integrity_archive(&temp_dir); - let file_entry = directory.get_file_by_path("integrity.txt").unwrap(); - let block_end = directory.blocks.get(&hash).unwrap().original_size; - let limits = ReaderLimits { - max_decoded_block_bytes: 0, - ..ReaderLimits::default() - }; - let mut reader = PithosReaderSimple::new_with_key(&archive, key) - .unwrap() - .with_limits(limits); - let mut sink: Box = Box::new(Vec::::new()); - assert!(matches!( - reader.read_data_range_to_sink( - block_end..block_end + 1, - file_entry, - &directory.blocks, - &mut sink, - ), - Err(PithosError::LimitExceeded { - field: "decoded block", - .. - }) - )); -} - -#[test] -fn test_robust_skipped_range_enforces_stored_block_limit() { - let temp_dir = TempDir::new().unwrap(); - let (archive, key, directory, hash) = write_identity_integrity_archive(&temp_dir); - let file_entry = directory.get_file_by_path("integrity.txt").unwrap(); - let block_end = directory.blocks.get(&hash).unwrap().original_size; - let limits = ReaderLimits { - max_stored_block_bytes: 0, - ..ReaderLimits::default() - }; - let mut reader = PithosReaderSimple::new_with_key(&archive, key) - .unwrap() - .with_limits(limits); - let mut sink: Box = Box::new(Vec::::new()); + let mut output = Vec::new(); + archive + .copy_range_to("data", size..size, &mut output) + .unwrap(); + assert!(output.is_empty()); assert!(matches!( - reader.read_data_range_to_sink( - block_end..block_end + 1, - file_entry, - &directory.blocks, - &mut sink, + archive.copy_range_to( + "data", + std::ops::Range { start: 2, end: 1 }, + &mut Vec::new() ), - Err(PithosError::LimitExceeded { - field: "stored block", - .. - }) - )); -} - -#[test] -fn test_robust_configured_directory_limit_returns_limit_error() { - let temp_dir = TempDir::new().unwrap(); - let archive = write_dummy_pithos(&temp_dir, false, false); - let limits = ReaderLimits { - max_directory_bytes: 24, - ..ReaderLimits::default() - }; - let mut reader = PithosReaderSimple::new_with_key(&archive, reader_key()) - .unwrap() - .with_limits(limits); - assert!(matches!( - reader.read_directory(), - Err(PithosError::LimitExceeded { - field: "directory", - .. - }) - )); -} - -#[test] -fn test_robust_configured_stored_block_limit_returns_limit_error() { - let temp_dir = TempDir::new().unwrap(); - let (archive, key, directory, _) = write_identity_integrity_archive(&temp_dir); - let limits = ReaderLimits { - max_stored_block_bytes: 0, - ..ReaderLimits::default() - }; - let mut reader = PithosReaderSimple::new_with_key(&archive, key) - .unwrap() - .with_limits(limits); - assert!(matches!( - reader.read_file("integrity.txt", &directory, None, None), - Err(PithosError::LimitExceeded { - field: "stored block", - .. - }) - )); -} - -#[test] -fn test_robust_configured_decoded_block_limit_returns_limit_error() { - let temp_dir = TempDir::new().unwrap(); - let (archive, key, directory, _) = write_identity_integrity_archive(&temp_dir); - let limits = ReaderLimits { - max_decoded_block_bytes: 0, - ..ReaderLimits::default() - }; - let mut reader = PithosReaderSimple::new_with_key(&archive, key) - .unwrap() - .with_limits(limits); - assert!(matches!( - reader.read_file("integrity.txt", &directory, None, None), - Err(PithosError::LimitExceeded { - field: "decoded block", - .. - }) - )); -} - -#[test] -fn test_robust_configured_parent_depth_returns_limit_error() { - let temp_dir = TempDir::new().unwrap(); - let archive = write_dummy_pithos(&temp_dir, false, false); - append_empty_directory(&archive); - let limits = ReaderLimits { - max_parent_directories: 0, - ..ReaderLimits::default() - }; - let mut reader = PithosReaderSimple::new_with_key(&archive, reader_key()) - .unwrap() - .with_limits(limits); - assert!(matches!( - reader.read_directory(), - Err(PithosError::LimitExceeded { - field: "parent directories", - .. - }) - )); -} - -#[test] -fn test_directory_integrity_parent_embedded_length_mismatch() { - let temp_dir = TempDir::new().unwrap(); - let archive = write_dummy_pithos(&temp_dir, false, false); - append_empty_directory(&archive); - let mut bytes = read(&archive).unwrap(); - let (final_start, final_length) = directory_bounds(&bytes); - let final_directory = &bytes[final_start..final_start + final_length]; - let mut cursor = std::io::Cursor::new(final_directory); - let directory = pithos_lib::model::structs::Directory::deserialize(&mut cursor).unwrap(); - let (parent_start, parent_length) = directory.parent_directory_offset.unwrap(); - let parent_start = parent_start as usize; - let parent_length = parent_length as usize; - let parent = &mut bytes[parent_start..parent_start + parent_length]; - parent[parent_length - 12..parent_length - 4] - .copy_from_slice(&((parent_length + 1) as u64).to_be_bytes()); - let crc = crc32fast::hash(&parent[..parent_length - 4]); - parent[parent_length - 4..].copy_from_slice(&crc.to_be_bytes()); - write(&archive, bytes).unwrap(); - - let mut reader = PithosReaderSimple::new_with_key(&archive, reader_key()).unwrap(); - assert!(matches!( - reader.read_directory(), - Err(PithosError::DirectoryLengthMismatch { .. }) + Err(PithosError::InvalidReadRange { .. }) )); } #[test] -fn test_reader_single_file() { - let temp_dir = TempDir::new().unwrap(); - let pithos_file = write_dummy_pithos(&temp_dir, false, false); - let key_pem = PathBuf::from("tests/data/keys/recipient1_private.pem".to_string()); - - let mut reader = PithosReaderSimple::new(pithos_file, key_pem).unwrap(); - let (directory, _) = reader.read_directory().unwrap(); - - let inner_paths = reader.read_file_paths(&directory).unwrap(); - - assert_eq!(inner_paths.len(), 1); - assert_eq!(inner_paths[0].0, FileType::Data); - assert_eq!(inner_paths[0].1, "t8.shakespeare.txt"); -} - -#[test] -fn test_safe_extraction_rejects_caller_constructed_unsafe_paths() { - let temp_dir = TempDir::new().unwrap(); - let pithos_file = write_dummy_pithos(&temp_dir, false, false); - let key = private_key_from_pem_bytes( - std::fs::read("tests/data/keys/recipient1_private.pem") - .unwrap() - .as_slice(), +fn extraction_is_no_clobber_no_follow_and_staged() { + let temporary = tempfile::tempdir().unwrap(); + let path = temporary.path().join("archive.pith"); + let sender = private_key("sender"); + let mut writer = ArchiveWriter::create( + std::fs::File::create(&path).unwrap(), + WriteOptions::new(sender, vec![public_key("recipient1")]), ) .unwrap(); - let mut reader = PithosReaderSimple::new_with_key(&pithos_file, key).unwrap(); - let (mut directory, _) = reader.read_directory().unwrap(); - let entry = FileEntry { - file_type: FileType::Data, - block_data: BlockDataState::Decrypted(vec![]), - created: 0, - modified: 0, - file_size: 0, - permissions: 0o644, - references: vec![], - symlink_target: None, - }; - directory - .files - .insert(Key::new(999, "../outside"), entry.clone()) - .unwrap(); - directory - .files - .insert(Key::new(1000, "/absolute"), entry) - .unwrap(); - let output = temp_dir.path().join("output"); - std::fs::create_dir(&output).unwrap(); - assert!( - reader - .read_file("../outside", &directory, Some(&output), None) - .is_err() - ); - assert!( - reader - .read_file("/absolute", &directory, Some(&output), None) - .is_err() - ); - assert!(!temp_dir.path().join("outside").exists()); -} - -#[test] -fn test_safe_extraction_rejects_preexisting_parent_symlink() { - let temp = TempDir::new().unwrap(); - let archive = write_dummy_pithos(&temp, false, false); - let outside = temp.path().join("outside"); - std::fs::create_dir(&outside).unwrap(); - let output = temp.path().join("output"); - std::fs::create_dir(&output).unwrap(); - std::os::unix::fs::symlink(&outside, output.join("nested")).unwrap(); - let (mut reader, directory) = caller_directory(&archive, "nested/file", FileType::Data, None); - assert!( - reader - .read_file("nested/file", &directory, Some(&output), None) - .is_err() - ); - assert!(!outside.join("file").exists()); -} - -#[test] -fn test_safe_extraction_rejects_existing_final_symlink() { - let temp = TempDir::new().unwrap(); - let archive = write_dummy_pithos(&temp, false, false); - let outside = temp.path().join("outside"); - std::fs::write(&outside, b"unchanged").unwrap(); - let output = temp.path().join("output"); - std::fs::create_dir(&output).unwrap(); - std::os::unix::fs::symlink(&outside, output.join("file")).unwrap(); - let (mut reader, directory) = caller_directory(&archive, "file", FileType::Data, None); - assert!( - reader - .read_file("file", &directory, Some(&output), None) - .is_err() - ); - assert_eq!(std::fs::read(&outside).unwrap(), b"unchanged"); -} - -#[test] -fn test_safe_extraction_rejects_existing_final_regular_file() { - let temp = TempDir::new().unwrap(); - let archive = write_dummy_pithos(&temp, false, false); - let output = temp.path().join("output"); - std::fs::create_dir(&output).unwrap(); - std::fs::write(output.join("file"), b"unchanged").unwrap(); - let (mut reader, directory) = caller_directory(&archive, "file", FileType::Data, None); - assert!( - reader - .read_file("file", &directory, Some(&output), None) - .is_err() - ); - assert_eq!(std::fs::read(output.join("file")).unwrap(), b"unchanged"); -} - -#[test] -fn test_safe_extraction_rejects_existing_final_directory_for_data() { - let temp = TempDir::new().unwrap(); - let archive = write_dummy_pithos(&temp, false, false); - let output = temp.path().join("output"); - std::fs::create_dir_all(output.join("file")).unwrap(); - let (mut reader, directory) = caller_directory(&archive, "file", FileType::Data, None); - assert!( - reader - .read_file("file", &directory, Some(&output), None) - .is_err() - ); - assert!(output.join("file").is_dir()); -} - -#[test] -fn test_safe_extraction_cleans_pending_file_after_read_failure() { - let temp = TempDir::new().unwrap(); - let archive = write_dummy_pithos(&temp, false, false); - let output = temp.path().join("output"); - std::fs::create_dir(&output).unwrap(); - let (mut reader, mut directory) = caller_directory(&archive, "failed", FileType::Data, None); - for (_, path, entry) in directory.files.iter_mut() { - if path == "failed" { - entry.block_data = BlockDataState::Decrypted(vec![([9; 32], [0; 32])]); - } - } - assert!( - reader - .read_file("failed", &directory, Some(&output), None) - .is_err() - ); - assert!(!output.join("failed").exists()); - assert!(!std::fs::read_dir(&output).unwrap().any(|item| { - item.unwrap() - .file_name() - .to_string_lossy() - .starts_with(".pithos-tmp-") - })); -} - -#[test] -fn test_safe_extraction_extracts_safe_nested_regular_file() { - let temp = TempDir::new().unwrap(); - let (archive, key, mut writer) = create_pithos_writer(&temp, None); - writer.write_file_header().unwrap(); - writer - .process_input(InputFile { - file_type: FileType::Data, - inner_path: "nested/file".into(), - data: Content::Raw("nested payload".into()), - metadata: None, - encrypt: false, - compression_level: Some(0), - }) - .unwrap(); - writer.write_directory().unwrap(); - let mut reader = PithosReaderSimple::new_with_key(&archive, key).unwrap(); - let (directory, _) = reader.read_directory().unwrap(); - let output = temp.path().join("output"); - std::fs::create_dir(&output).unwrap(); - reader - .read_file("nested/file", &directory, Some(&output), None) - .unwrap(); - assert_eq!( - std::fs::read(output.join("nested/file")).unwrap(), - b"nested payload" - ); -} - -#[test] -fn test_safe_extraction_creates_contained_and_dangling_symlinks() { - let temp = TempDir::new().unwrap(); - let archive = write_dummy_pithos(&temp, false, false); - let output = temp.path().join("output"); - std::fs::create_dir_all(output.join("nested")).unwrap(); - let (mut reader, mut directory) = caller_directory( - &archive, - "nested/link", - FileType::Symlink, - Some("../target"), - ); - directory - .files - .insert( - Key::new(9001, "dangling"), - empty_entry(FileType::Symlink, Some("missing/target")), - ) - .unwrap(); - reader - .read_file("nested/link", &directory, Some(&output), None) - .unwrap(); - reader - .read_file("dangling", &directory, Some(&output), None) - .unwrap(); - assert_eq!( - std::fs::read_link(output.join("nested/link")).unwrap(), - PathBuf::from("../target") - ); - assert_eq!( - std::fs::read_link(output.join("dangling")).unwrap(), - PathBuf::from("missing/target") - ); -} - -#[test] -fn test_safe_extraction_rejects_caller_constructed_ancestor_conflict() { - let temp = TempDir::new().unwrap(); - let archive = write_dummy_pithos(&temp, false, false); - let output = temp.path().join("output"); - std::fs::create_dir(&output).unwrap(); - let (mut reader, mut directory) = caller_directory(&archive, "a", FileType::Data, None); - directory - .files - .insert(Key::new(9001, "a/child"), empty_entry(FileType::Data, None)) - .unwrap(); - assert!( - reader - .read_file("a", &directory, Some(&output), None) - .is_err() - ); - assert!(!output.join("a").exists()); -} - -#[test] -fn test_reader_targeted_validation_rejects_reverse_caller_ancestor_conflict() { - let temp = TempDir::new().unwrap(); - let archive = write_dummy_pithos(&temp, false, false); - let output = temp.path().join("output"); - std::fs::create_dir(&output).unwrap(); - let (mut reader, mut directory) = caller_directory(&archive, "a/child", FileType::Data, None); - directory - .files - .insert(Key::new(9001, "a"), empty_entry(FileType::Data, None)) - .unwrap(); - assert!( - reader - .read_file("a/child", &directory, Some(&output), None) - .is_err() - ); - assert!(!output.join("a/child").exists()); -} - -#[test] -fn test_reader_reads_unrelated_entries_without_full_map_revalidation() { - let temp = TempDir::new().unwrap(); - let archive = write_dummy_pithos(&temp, false, false); - let output = temp.path().join("output"); - std::fs::create_dir(&output).unwrap(); - let (mut reader, mut directory) = caller_directory(&archive, "safe", FileType::Data, None); - directory - .files - .insert( - Key::new(9001, "conflict"), - empty_entry(FileType::Data, None), - ) - .unwrap(); - directory - .files - .insert( - Key::new(9002, "conflict/child"), - empty_entry(FileType::Data, None), - ) - .unwrap(); - - reader - .read_file("safe", &directory, Some(&output), None) - .unwrap(); - assert!(output.join("safe").exists()); -} - -#[test] -fn test_reader_hides_files_without_keys() { - let temp_dir = TempDir::new().unwrap(); - let pithos_file = write_dummy_pithos(&temp_dir, false, false); - let unauthorized_key = StaticSecret::from([0x42; 32]); - - let mut reader = PithosReaderSimple::new_with_key(pithos_file, unauthorized_key).unwrap(); - let (directory, _) = reader.read_directory().unwrap(); - - assert!(directory.files.is_empty()); - assert!(reader.read_file_paths(&directory).unwrap().is_empty()); -} - -#[test] -fn test_reader_file_ranges() { - let temp_dir = TempDir::new().unwrap(); - let pithos_file = write_dummy_pithos(&temp_dir, false, false); - let key_pem = PathBuf::from("tests/data/keys/recipient1_private.pem".to_string()); - - let mut reader = PithosReaderSimple::new(pithos_file, key_pem).unwrap(); - let (directory, _) = reader.read_directory().unwrap(); - - let temp_dir = TempDir::new().unwrap(); - let outfile_single = temp_dir.path().join("range.txt"); - let outfile_multi = temp_dir.path().join("ranges.txt"); - if directory.get_file_by_path("t8.shakespeare.txt").is_some() { - reader - .read_file( - "t8.shakespeare.txt", - &directory, - Some(&outfile_single), - #[allow(clippy::single_range_in_vec_init)] - Some(vec![261..272]), - ) - .unwrap(); - assert_eq!("Shakespeare", read_to_string(outfile_single).unwrap()); - - reader - .read_file( - "t8.shakespeare.txt", - &directory, - Some(&outfile_multi), - Some(vec![261..272, 1434..1441]), + for (path, content) in [ + ("data", b"payload".as_slice()), + ("nested/data", b"nested payload"), + ] { + writer + .add_file( + ArchivePath::new(path).unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(true, 0).unwrap(), + Some(content.len() as u64), + std::io::Cursor::new(content), ) .unwrap(); - assert_eq!("Shakespeare Etexts", read_to_string(outfile_multi).unwrap()); } -} - -#[test] -fn test_reader_multiple_files() { - let temp_dir = TempDir::new().unwrap(); - let pithos_file = write_dummy_pithos(&temp_dir, true, true); - let key_pem = PathBuf::from("tests/data/keys/recipient1_private.pem".to_string()); - - let mut reader = PithosReaderSimple::new(pithos_file, key_pem).unwrap(); - let (directory, _) = reader.read_directory().unwrap(); - let inner_paths = reader.read_file_paths(&directory).unwrap(); - - assert_eq!(inner_paths.len(), 4); - - assert_eq!(inner_paths[0].0, FileType::Metadata); - assert_eq!(inner_paths[0].1, "t8.shakespeare.txt.meta"); - assert_eq!(inner_paths[1].0, FileType::Data); - assert_eq!(inner_paths[1].1, "t8.shakespeare.txt"); - - assert_eq!(inner_paths[2].0, FileType::Metadata); - assert_eq!(inner_paths[2].1, "SRR33138449.fastq.meta"); - assert_eq!(inner_paths[3].0, FileType::Data); - assert_eq!(inner_paths[3].1, "SRR33138449.fastq"); - - let file_entry = directory.get_file_by_path("t8.shakespeare.txt").unwrap(); - let reference = Reference { - target_file_id: 0, - relationship: 0, - }; - assert!(file_entry.references.contains(&reference)); - - let file_entry = directory.get_file_by_path("SRR33138449.fastq").unwrap(); - let reference = Reference { - target_file_id: 2, - relationship: 0, - }; - assert!(file_entry.references.contains(&reference)); -} - -#[test] -fn test_rocrate_read_directory_1_2() { - let loaded = read_ro_crate_directory("tests/data/dummy_dir").unwrap(); - assert_eq!( - loaded.source, - RoCrateSource::Directory(PathBuf::from("tests/data/dummy_dir")) - ); - - let mut expected_data_entity_ids = vec![ - "conclusions.txt", - "dummy_results.txt", - "dataset/", - "literature/", - ]; - let mut data_entity_ids = loaded - .ro_crate - .graph - .iter() - .filter_map(|entity| match entity { - GraphVector::DataEntity(entity) => Some(entity.id.as_str()), - _ => None, - }) - .collect::>(); - data_entity_ids.sort_unstable(); - expected_data_entity_ids.sort_unstable(); - assert_eq!(data_entity_ids, expected_data_entity_ids); - - let mut contextual_entity_ids = loaded - .ro_crate - .graph - .iter() - .filter_map(|entity| match entity { - GraphVector::ContextualEntity(entity) => Some(entity.id.as_str()), - _ => None, - }) - .collect::>(); - contextual_entity_ids.sort_unstable(); - let mut expected_contextual_entity_ids = vec![ - "mailto:josiah.carberry@example.com", - "https://orcid.org/0000-0002-1825-0097", - ]; - expected_contextual_entity_ids.sort_unstable(); - assert_eq!(contextual_entity_ids, expected_contextual_entity_ids); - - assert_eq!( - loaded - .ro_crate - .graph - .iter() - .filter(|entity| matches!(entity, GraphVector::MetadataDescriptor(_))) - .count(), - 1 - ); - assert_eq!( - loaded - .ro_crate - .graph - .iter() - .filter(|entity| matches!(entity, GraphVector::RootDataEntity(_))) - .count(), - 1 - ); - assert_eq!( - loaded.ro_crate.get_rocrate_version(), - Some(RoCrateSchemaVersion::V1_2) - ); -} - -#[test] -fn test_rocrate_read_zip_1_1() { - let loaded = read_ro_crate_zip("tests/data/ro-crate.zip").unwrap(); - - assert_eq!( - loaded.source, - RoCrateSource::Zip(PathBuf::from("tests/data/ro-crate.zip")) - ); - assert!(!loaded.ro_crate.graph.is_empty()); -} - -#[test] -fn test_rocrate_directory_requires_metadata() { - let temp_dir = TempDir::new().unwrap(); - - let error = read_ro_crate_directory(temp_dir.path()).unwrap_err(); - - assert!(matches!(error, PithosError::MissingRoCrateMetadata(_))); -} - -#[test] -fn test_rocrate_rejects_incomplete_root() { - let temp_dir = TempDir::new().unwrap(); - let metadata_path = temp_dir.path().join("ro-crate-metadata.json"); - let metadata = r#"{ - "@context": "https://w3id.org/ro/crate/1.2/context", - "@graph": [ - { - "@id": "ro-crate-metadata.json", - "@type": "CreativeWork", - "conformsTo": {"@id": "https://w3id.org/ro/crate/1.2"}, - "about": {"@id": "./"} - }, - { - "@id": "./", - "@type": "Dataset" - } - ] - }"#; - std::fs::write(metadata_path, metadata).unwrap(); - - let error = read_ro_crate_directory(temp_dir.path()).unwrap_err(); - - assert!(matches!( - error, - PithosError::RoCrate(CrateReadError::JsonError(_)) - )); -} - -#[test] -fn test_rocrate_zip_returns_error_for_invalid_archive() { - let temp_dir = TempDir::new().unwrap(); - let zip_path = temp_dir.path().join("invalid.zip"); - std::fs::write(&zip_path, b"not a ZIP archive").unwrap(); - - let error = read_ro_crate_zip(&zip_path).unwrap_err(); - - assert!(matches!(error, PithosError::Zip(_))); -} - -#[test] -fn test_rocrate_zip_requires_root_metadata() { - let temp_dir = TempDir::new().unwrap(); - let zip_path = temp_dir.path().join("without-metadata.zip"); - write_zip_entry(&zip_path, "data.txt", b"data"); - - let error = read_ro_crate_zip(&zip_path).unwrap_err(); - - assert!(matches!(error, PithosError::MissingRoCrateMetadata(_))); -} - -#[test] -fn test_rocrate_zip_rejects_metadata_below_wrapper_directory() { - let temp_dir = TempDir::new().unwrap(); - let zip_path = temp_dir.path().join("wrapped.zip"); - write_zip_entry( - &zip_path, - "wrapper/ro-crate-metadata.json", - br#"{"@context":"https://w3id.org/ro/crate/1.1/context","@graph":[]}"#, - ); - - let error = read_ro_crate_zip(&zip_path).unwrap_err(); - - assert!(matches!(error, PithosError::MissingRoCrateMetadata(_))); -} - -#[test] -fn test_read_to_crypt4gh() { - // Prepare Pithos file - let temp_dir = TempDir::new().unwrap(); - let pithos_file = write_dummy_pithos(&temp_dir, false, false); - let reader_key_pem = PathBuf::from("tests/data/keys/recipient1_private.pem".to_string()); - - // Read recipient public key - let recipient_pem_content = read_to_string("tests/data/keys/recipient2_public.pem").unwrap(); - let recipient_key = public_key_from_pem_bytes(recipient_pem_content.as_bytes()).unwrap(); - - // Export file in Crypt4GH format - let mut reader = PithosReaderSimple::new(pithos_file, reader_key_pem).unwrap(); - let (directory, _) = reader.read_directory().unwrap(); - - let crypt4gh_output_path = temp_dir.path().join("t8.shakespeare.crypt4gh"); - let crypt4gh_output = Box::new( - OpenOptions::new() - .create(true) - .truncate(true) - .write(true) - .open(&crypt4gh_output_path) - .unwrap(), - ); - reader - .read_file_to_crypt4gh( - "t8.shakespeare.txt", - &directory, - vec![recipient_key], - Some(crypt4gh_output), - ) - .unwrap(); - - // Read recipient private key - let reader_pem_content = read_to_string("tests/data/keys/recipient2_private.pem").unwrap(); - let reader_key = private_key_from_pem_bytes(reader_pem_content.as_bytes()).unwrap(); - - // Read Crypt4GH file and deserialize/decrypt header - let mut buffer = Vec::new(); - let mut input = File::open(&crypt4gh_output_path).unwrap(); - input.read_to_end(&mut buffer).unwrap(); - - let mut header = Crypt4GHHeader::try_from(buffer.as_slice()).unwrap(); - let block_start: u64 = 16 // Magic bytes + version + packet count - + header - .header_packets - .iter() - .map(|hp| hp.length as u64) - .sum::(); - let mut data_key = None; - 'outer: for header_packet in header.header_packets.iter_mut() { - header_packet.decrypt(&reader_key).unwrap(); - if let PacketData::Decrypted(packets) = &header_packet.packet_data { - for packet in packets { - if let Packet::Encryption(encryption_packet) = packet { - data_key = Some(*encryption_packet.get_encryption_key()); - break 'outer; - } - } - } - } - let data_key = data_key.unwrap(); - - // Decrypt blocks and write to file - input.seek(SeekFrom::Start(block_start)).unwrap(); - - let raw_output_file = temp_dir.path().join("t8.shakespeare.txt"); - let mut output = OpenOptions::new() - .create(true) - .truncate(true) - .write(true) - .open(&raw_output_file) - .unwrap(); - - let mut buffer = [0u8; CRYPT4GH_ENCRYPTED_BLOCK_SIZE]; - let mut buffer_pos = 0; - loop { - // Read data into the remaining space in buffer - let bytes_read = input.read(&mut buffer[buffer_pos..]).unwrap(); - if bytes_read == 0 { - // End of file - process any remaining data in buffer - if buffer_pos > 0 { - let raw = decrypt_chunk(&buffer[..buffer_pos], &data_key).unwrap(); - output.write_all(raw.as_slice()).unwrap(); - } - break; - } - buffer_pos += bytes_read; - - // If buffer is full, process it and reset - if buffer_pos == CRYPT4GH_ENCRYPTED_BLOCK_SIZE { - let raw = decrypt_chunk(&buffer, &data_key).unwrap(); - output.write_all(raw.as_slice()).unwrap(); - buffer_pos = 0; - } - } - - let original_content = read_to_string("tests/data/t8.shakespeare.sample.txt").unwrap(); - let decrypted_content = read_to_string(&raw_output_file).unwrap(); - assert_eq!(original_content, decrypted_content); + writer.finish().unwrap(); + let archive = open(&path, AccessKeys::new().with_key(private_key("recipient1"))); + let root = temporary.path().join("output"); + std::fs::create_dir(&root).unwrap(); + std::fs::write(root.join("data"), b"unchanged").unwrap(); + assert!(extract(&archive, "data", &root).is_err()); + assert_eq!(std::fs::read(root.join("data")).unwrap(), b"unchanged"); + std::fs::remove_file(root.join("data")).unwrap(); + let outside = temporary.path().join("outside"); + std::fs::create_dir(&outside).unwrap(); + std::os::unix::fs::symlink(&outside, root.join("nested")).unwrap(); + assert!(extract(&archive, "nested/data", &root).is_err()); + assert!(!outside.join("data").exists()); } diff --git a/crates/pithos_lib/tests/robustness.rs b/crates/pithos_lib/tests/robustness.rs deleted file mode 100644 index 5649ad6..0000000 --- a/crates/pithos_lib/tests/robustness.rs +++ /dev/null @@ -1,58 +0,0 @@ -use pithos_lib::error::PithosError; -use pithos_lib::helpers::directory::DirectoryBuilder; -use pithos_lib::helpers::file_entry_map::FileEntryMap; -use pithos_lib::io::pithosreader::PithosReaderSimple; -use pithos_lib::io::pithoswriter::Content; -use pithos_lib::model::structs::{BlockDataState, FileEntry, FileType, Reference}; -use std::error::Error; -use x25519_dalek::StaticSecret; - -#[test] -fn new_with_keys_returns_error_instead_of_panicking() { - let result = PithosReaderSimple::new_with_keys( - "/nonexistent-pithos-robustness-test", - Vec::::new(), - ); - - assert!(matches!( - result, - Err(PithosError::UnsupportedMultipleReaderKeys) - )); -} - -#[test] -fn reference_content_returns_error_instead_of_panicking() { - let reference = Reference { - target_file_id: 0, - relationship: 0, - }; - - let result = FileEntry::new_from_content(FileType::Data, &Content::Reference(reference)); - - assert!(matches!( - result, - Err(PithosError::UnsupportedReferenceContent) - )); -} - -#[test] -fn exhausted_file_ids_return_error_instead_of_panicking() -> Result<(), Box> { - let mut directory = DirectoryBuilder::new() - .files(FileEntryMap::new_with_max(u64::MAX)) - .build()?; - let file_entry = FileEntry { - file_type: FileType::Data, - block_data: BlockDataState::Decrypted(Vec::new()), - created: 0, - modified: 0, - file_size: 0, - permissions: 0o644, - references: Vec::new(), - symlink_target: None, - }; - - let result = directory.add_file("exhausted", &file_entry); - - assert!(matches!(result, Err(PithosError::FileIdExhausted))); - Ok(()) -} diff --git a/crates/pithos_lib/tests/source_equivalence.rs b/crates/pithos_lib/tests/source_equivalence.rs new file mode 100644 index 0000000..c5b6a46 --- /dev/null +++ b/crates/pithos_lib/tests/source_equivalence.rs @@ -0,0 +1,226 @@ +use pithos_lib::archive::{ + AccessKeys, Archive, ArchivePath, ArchiveWriter, EntryKind, EntryMetadata, OpenOptions, + ProcessingOptions, WriteOptions, +}; +use pithos_lib::crypto::PrivateKey; +use pithos_lib::error::PithosError; +use pithos_lib::source::{ArchiveSource, FileSource, MemorySource, SourceError}; +use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, +}; + +fn private_key() -> PrivateKey { + pithos_lib::crypto::parse_private_pem( + &std::fs::read("tests/data/keys/recipient1_private.pem").unwrap(), + ) + .unwrap() +} + +fn options() -> OpenOptions { + OpenOptions::default().with_access_keys(AccessKeys::new().with_key(private_key())) +} + +#[test] +fn file_source_open_errors_include_the_requested_path() { + let temporary = tempfile::tempdir().unwrap(); + let missing = temporary.path().join("distinct-missing-archive.pith"); + + let error = match FileSource::open(&missing) { + Ok(_) => panic!("missing source unexpectedly opened"), + Err(error) => error, + }; + + assert!(error.to_string().contains(missing.to_str().unwrap())); + assert!(std::error::Error::source(&error).is_some()); +} + +fn fixture(temporary: &tempfile::TempDir) -> std::path::PathBuf { + let sender = pithos_lib::crypto::parse_private_pem( + &std::fs::read("tests/data/keys/sender_private.pem").unwrap(), + ) + .unwrap(); + let recipient = pithos_lib::crypto::parse_public_pem( + &std::fs::read("tests/data/keys/recipient1_public.pem").unwrap(), + ) + .unwrap(); + let path = temporary.path().join("fixture.pith"); + let mut writer = ArchiveWriter::create( + std::fs::File::create(&path).unwrap(), + WriteOptions::new(sender, vec![recipient]), + ) + .unwrap(); + writer + .add_file( + ArchivePath::new("data").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(true, 0).unwrap(), + Some(23), + std::io::Cursor::new(b"verified source fixture"), + ) + .unwrap(); + writer.finish().unwrap(); + path +} + +#[test] +fn file_and_memory_sources_produce_the_same_validated_index() { + let temporary = tempfile::tempdir().unwrap(); + let path = fixture(&temporary); + let file = Archive::open(FileSource::open(&path).unwrap(), options()).unwrap(); + let memory = Archive::open( + MemorySource::new(Arc::<[u8]>::from(std::fs::read(path).unwrap())), + options(), + ) + .unwrap(); + assert_eq!( + file.entries().collect::>(), + memory.entries().collect::>() + ); +} + +struct Fragmented(Arc<[u8]>); + +impl ArchiveSource for Fragmented { + fn len(&self) -> Result { + Ok(self.0.len() as u64) + } + + fn read_exact_at(&self, offset: u64, output: &mut [u8]) -> Result<(), SourceError> { + let start = usize::try_from(offset).map_err(|_| SourceError::RangeOverflow { + offset, + length: output.len(), + })?; + let end = start + .checked_add(output.len()) + .ok_or(SourceError::RangeOverflow { + offset, + length: output.len(), + })?; + let input = self.0.get(start..end).ok_or(SourceError::UnexpectedEof { + offset, + expected: output.len(), + actual: self.0.len().saturating_sub(start), + })?; + for (target, source) in output.iter_mut().zip(input) { + *target = *source; + } + Ok(()) + } +} + +struct CountingSource { + bytes: Arc<[u8]>, + reads: Arc, +} + +struct InconsistentLength(Arc<[u8]>); + +impl ArchiveSource for InconsistentLength { + fn len(&self) -> Result { + Ok(self.0.len() as u64 + 1) + } + + fn read_exact_at(&self, offset: u64, output: &mut [u8]) -> Result<(), SourceError> { + Fragmented(Arc::clone(&self.0)).read_exact_at(offset, output) + } +} + +struct FailingSource; + +impl ArchiveSource for FailingSource { + fn len(&self) -> Result { + Ok(64) + } + + fn read_exact_at(&self, offset: u64, _output: &mut [u8]) -> Result<(), SourceError> { + Err(SourceError::Remote { + offset, + message: "injected transport failure".into(), + }) + } +} + +impl ArchiveSource for CountingSource { + fn len(&self) -> Result { + Ok(self.bytes.len() as u64) + } + + fn read_exact_at(&self, offset: u64, output: &mut [u8]) -> Result<(), SourceError> { + self.reads.fetch_add(1, Ordering::Relaxed); + Fragmented(Arc::clone(&self.bytes)).read_exact_at(offset, output) + } +} + +#[test] +fn fragmented_source_is_equivalent_and_source_errors_keep_context() { + let temporary = tempfile::tempdir().unwrap(); + let path = fixture(&temporary); + let bytes = Arc::<[u8]>::from(std::fs::read(path).unwrap()); + let archive = Archive::open(Fragmented(Arc::clone(&bytes)), options()).unwrap(); + assert_eq!(archive.entries().count(), 1); + + let error = match Archive::open(MemorySource::new(&bytes[..4]), options()) { + Ok(_) => panic!("truncated source must fail"), + Err(error) => error, + }; + assert!(matches!( + error, + PithosError::Source(SourceError::UnexpectedEof { offset: 0, .. }) + )); +} + +#[test] +fn unavailable_entries_remain_visible_and_invalid_ranges_do_not_read() { + let temporary = tempfile::tempdir().unwrap(); + let path = fixture(&temporary); + let bytes = Arc::<[u8]>::from(std::fs::read(path).unwrap()); + let archive = Archive::open( + CountingSource { + bytes: Arc::clone(&bytes), + reads: Arc::new(AtomicUsize::new(0)), + }, + OpenOptions::default(), + ) + .unwrap(); + let entry = archive.entries().next().unwrap(); + assert!(matches!( + entry.kind, + EntryKind::File { + available: false, + .. + } + )); + + let reads = Arc::new(AtomicUsize::new(0)); + let archive = Archive::open( + CountingSource { + bytes, + reads: Arc::clone(&reads), + }, + options(), + ) + .unwrap(); + let before = reads.load(Ordering::Relaxed); + assert!( + archive + .copy_range_to("data", 99..100, &mut Vec::new()) + .is_err() + ); + assert_eq!(reads.load(Ordering::Relaxed), before); +} + +#[test] +fn short_inconsistent_and_failing_sources_keep_their_acquisition_context() { + let temporary = tempfile::tempdir().unwrap(); + let path = fixture(&temporary); + let bytes = Arc::<[u8]>::from(std::fs::read(path).unwrap()); + assert!(matches!( + Archive::open(InconsistentLength(bytes), options()), + Err(PithosError::Source(SourceError::UnexpectedEof { .. })) + )); + assert!(matches!( + Archive::open(FailingSource, options()), + Err(PithosError::Source(SourceError::Remote { offset: 0, .. })) + )); +} From a14dc14d21fb55baaf012483872a66952f9c3835 Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Tue, 11 Aug 2026 23:00:28 +0200 Subject: [PATCH 007/166] refactor: replace writer with streaming lifecycle --- .../pithos_lib/src/archive/path_validation.rs | 490 ++++ crates/pithos_lib/src/archive/writer.rs | 1994 +++++++++++++ crates/pithos_lib/src/fs/ingest.rs | 779 +++++ crates/pithos_lib/src/io/pithoswriter.rs | 729 ----- crates/pithos_lib/src/io/util.rs | 179 -- .../tests/data/SRR33138449.sample.fastq | 2500 ----------------- crates/pithos_lib/tests/fs_ingest.rs | 191 ++ crates/pithos_lib/tests/streaming_memory.rs | 104 + crates/pithos_lib/tests/writer.rs | 1546 ---------- .../pithos_lib/tests/writer_configuration.rs | 94 + crates/pithos_lib/tests/writer_failures.rs | 429 +++ crates/pithos_lib/tests/writer_processing.rs | 120 + 12 files changed, 4201 insertions(+), 4954 deletions(-) create mode 100644 crates/pithos_lib/src/archive/path_validation.rs create mode 100644 crates/pithos_lib/src/archive/writer.rs create mode 100644 crates/pithos_lib/src/fs/ingest.rs delete mode 100644 crates/pithos_lib/src/io/pithoswriter.rs delete mode 100644 crates/pithos_lib/src/io/util.rs delete mode 100644 crates/pithos_lib/tests/data/SRR33138449.sample.fastq create mode 100644 crates/pithos_lib/tests/fs_ingest.rs create mode 100644 crates/pithos_lib/tests/streaming_memory.rs delete mode 100644 crates/pithos_lib/tests/writer.rs create mode 100644 crates/pithos_lib/tests/writer_configuration.rs create mode 100644 crates/pithos_lib/tests/writer_failures.rs create mode 100644 crates/pithos_lib/tests/writer_processing.rs diff --git a/crates/pithos_lib/src/archive/path_validation.rs b/crates/pithos_lib/src/archive/path_validation.rs new file mode 100644 index 0000000..3cf248e --- /dev/null +++ b/crates/pithos_lib/src/archive/path_validation.rs @@ -0,0 +1,490 @@ +use crate::error::PithosError; +use crate::format::entries::WireEntries; +use crate::format::wire::{BlockDataState, FileEntry, FileType}; + +fn invalid_path(path: &str, reason: impl Into) -> PithosError { + PithosError::InvalidArchivePath { + path: path.to_string(), + reason: reason.into(), + } +} + +pub(crate) fn validate_entry_path(path: &str) -> Result<(), PithosError> { + if path.is_empty() { + return Err(invalid_path(path, "path is empty")); + } + if path.contains('\0') { + return Err(invalid_path(path, "NUL is not allowed")); + } + if path.contains('\\') { + return Err(invalid_path(path, "backslash is not allowed")); + } + if path.starts_with('/') || path.ends_with('/') { + return Err(invalid_path(path, "path must not start or end with /")); + } + if path.as_bytes().get(1) == Some(&b':') { + return Err(invalid_path(path, "drive forms are not allowed")); + } + for component in path.split('/') { + if component.is_empty() { + return Err(invalid_path(path, "empty path components are not allowed")); + } + if component == "." || component == ".." { + return Err(invalid_path(path, "dot components are not allowed")); + } + } + Ok(()) +} + +pub(crate) fn validate_symlink_target(path: &str, target: &str) -> Result<(), PithosError> { + validate_entry_path(path).map_err(|error| PithosError::InvalidSymlinkTarget { + path: path.to_string(), + target: target.to_string(), + reason: error.to_string(), + })?; + if target.is_empty() { + return Err(PithosError::InvalidSymlinkTarget { + path: path.to_string(), + target: target.to_string(), + reason: "target is empty".into(), + }); + } + if target.contains('\0') || target.contains('\\') { + return Err(PithosError::InvalidSymlinkTarget { + path: path.to_string(), + target: target.to_string(), + reason: "invalid separator or NUL".into(), + }); + } + if target.starts_with('/') || target.as_bytes().get(1) == Some(&b':') { + return Err(PithosError::InvalidSymlinkTarget { + path: path.to_string(), + target: target.to_string(), + reason: "absolute or drive target".into(), + }); + } + let mut depth = path.split('/').count() - 1; + for component in target.split('/') { + if component.is_empty() || component == "." { + return Err(PithosError::InvalidSymlinkTarget { + path: path.to_string(), + target: target.to_string(), + reason: "empty or dot component".into(), + }); + } + if component == ".." { + if depth == 0 { + return Err(PithosError::InvalidSymlinkTarget { + path: path.to_string(), + target: target.to_string(), + reason: "target escapes archive root".into(), + }); + } + depth -= 1; + } else { + depth += 1; + } + } + Ok(()) +} + +pub(crate) fn validate_entry(path: &str, entry: &FileEntry) -> Result<(), PithosError> { + validate_entry_path(path)?; + match (&entry.file_type, &entry.symlink_target, &entry.block_data) { + (FileType::Symlink, Some(target), BlockDataState::Decrypted(blocks)) + if blocks.is_empty() => + { + validate_symlink_target(path, target) + } + (FileType::Symlink, None, _) => Err(PithosError::InvalidSymlinkEntry { + path: path.into(), + reason: "missing target".into(), + }), + (FileType::Symlink, Some(_), BlockDataState::Decrypted(blocks)) if !blocks.is_empty() => { + Err(PithosError::InvalidSymlinkEntry { + path: path.into(), + reason: "symlink has block references".into(), + }) + } + (FileType::Symlink, Some(_), BlockDataState::Encrypted(_)) => { + Err(PithosError::InvalidSymlinkEntry { + path: path.into(), + reason: "encrypted symlink block data".into(), + }) + } + (_, Some(_), _) => Err(PithosError::InvalidSymlinkEntry { + path: path.into(), + reason: "non-symlink has a target".into(), + }), + _ => Ok(()), + } +} + +fn validate_candidate_hierarchy( + map: &WireEntries, + path: &str, + entry: &FileEntry, +) -> Result<(), PithosError> { + for (index, _) in path.match_indices('/') { + let ancestor = &path[..index]; + if map + .get_by_path(ancestor) + .is_some_and(|existing| existing.file_type != FileType::Directory) + { + return Err(PithosError::InvalidArchivePath { + path: path.into(), + reason: format!("file entry {ancestor} is an ancestor"), + }); + } + } + + if entry.file_type != FileType::Directory + && let Some(successor) = map.first_path_after(path) + && successor.starts_with(path) + && successor.as_bytes().get(path.len()) == Some(&b'/') + { + return Err(PithosError::InvalidArchivePath { + path: path.into(), + reason: format!("entry is an ancestor of {successor}"), + }); + } + + Ok(()) +} + +#[cfg(test)] +pub(crate) fn validate_existing_candidate( + map: &WireEntries, + path: &str, + entry: &FileEntry, +) -> Result<(), PithosError> { + validate_entry(path, entry)?; + validate_candidate_hierarchy(map, path, entry) +} + +pub(crate) fn validate_new_candidate( + map: &WireEntries, + path: &str, + entry: &FileEntry, +) -> Result<(), PithosError> { + validate_entry(path, entry)?; + if map.get_by_path(path).is_some() { + return Err(PithosError::PathOccupied(format!( + "File path already occupied: {path}" + ))); + } + validate_candidate_hierarchy(map, path, entry) +} + +pub(crate) fn validate_wire_map(map: &WireEntries) -> Result<(), PithosError> { + for (_, path, entry) in map.iter() { + validate_entry(path, entry)?; + } + + validate_wire_hierarchy(map) +} + +pub(crate) fn validate_wire_hierarchy(map: &WireEntries) -> Result<(), PithosError> { + let mut entries = map.iter_ordered(); + let Some((mut previous_path, mut previous_entry)) = entries.next() else { + return Ok(()); + }; + + for (path, entry) in entries { + if previous_entry.file_type != FileType::Directory + && path.starts_with(previous_path) + && path.as_bytes().get(previous_path.len()) == Some(&b'/') + { + return Err(PithosError::InvalidArchivePath { + path: previous_path.into(), + reason: format!("entry is an ancestor of {path}"), + }); + } + + previous_path = path; + previous_entry = entry; + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn entry(file_type: FileType, target: Option<&str>, blocks: BlockDataState) -> FileEntry { + FileEntry { + file_type, + block_data: blocks, + created: 0, + modified: 0, + file_size: 0, + permissions: 0o644, + references: vec![], + symlink_target: target.map(str::to_owned), + } + } + + #[test] + fn archive_path_valid_entry_corpus() { + for path in ["file", "nested/file", "ユニコード/file"] { + assert!(validate_entry_path(path).is_ok()); + } + } + #[test] + fn archive_path_invalid_entry_corpus() { + for path in [ + "", + "/file", + "file/", + "a//b", + "a/./b", + "a/../b", + "a\\b", + "C:", + "C:/x", + "\\\\server\\x", + "a\0b", + ] { + assert!(validate_entry_path(path).is_err(), "{path:?}"); + } + } + #[test] + fn archive_path_symlink_target_corpus() { + for target in ["target", "nested/target", "../target", "../dangling"] { + assert!(validate_symlink_target("nested/link", target).is_ok()); + } + for target in [ + "", + ".", + "target/.", + "target//child", + "target/", + "a\0b", + "a\\b", + "/absolute", + "C:", + "C:/target", + "\\\\server\\target", + "../target", + "../../target", + ] { + assert!( + validate_symlink_target("link", target).is_err(), + "{target:?}" + ); + } + assert!(validate_symlink_target("", "target").is_err()); + assert!(validate_symlink_target("bad//link", "target").is_err()); + assert!(validate_symlink_target("bad/../link", "target").is_err()); + } + + #[test] + fn archive_path_entry_invariants_corpus() { + assert!( + validate_entry( + "link", + &entry( + FileType::Symlink, + None, + BlockDataState::Decrypted(vec![].into()) + ) + ) + .is_err() + ); + assert!( + validate_entry( + "file", + &entry( + FileType::Data, + Some("target"), + BlockDataState::Decrypted(vec![].into()) + ) + ) + .is_err() + ); + assert!( + validate_entry( + "link", + &entry( + FileType::Symlink, + Some("target"), + BlockDataState::Decrypted(vec![([0; 32], [0; 32])].into()) + ) + ) + .is_err() + ); + assert!( + validate_entry( + "link", + &entry( + FileType::Symlink, + Some("target"), + BlockDataState::Encrypted(vec![]) + ) + ) + .is_err() + ); + } + + #[test] + fn archive_path_candidate_and_map_conflicts_are_order_independent() { + let file = entry( + FileType::Data, + None, + BlockDataState::Decrypted(vec![].into()), + ); + let link = entry( + FileType::Symlink, + Some("target"), + BlockDataState::Decrypted(vec![].into()), + ); + let directory = entry( + FileType::Directory, + None, + BlockDataState::Decrypted(vec![].into()), + ); + + for (ancestor_path, ancestor) in [("a", file.clone()), ("a", link.clone())] { + for order in [0, 1] { + let mut map = WireEntries::new(); + if order == 0 { + map.insert(0, ancestor_path, ancestor.clone()).unwrap(); + map.insert(1, "a/child", file.clone()).unwrap(); + } else { + map.insert(0, "a/child", file.clone()).unwrap(); + map.insert(1, ancestor_path, ancestor.clone()).unwrap(); + } + assert!(validate_wire_map(&map).is_err()); + } + } + + for order in [0, 1] { + let mut map = WireEntries::new(); + if order == 0 { + map.insert(0, "a/child", file.clone()).unwrap(); + map.insert(1, "a", file.clone()).unwrap(); + } else { + map.insert(0, "a", file.clone()).unwrap(); + map.insert(1, "a/child", file.clone()).unwrap(); + } + assert!(validate_wire_map(&map).is_err()); + } + + for order in [0, 1] { + let mut map = WireEntries::new(); + if order == 0 { + map.insert(0, "a", directory.clone()).unwrap(); + map.insert(1, "a/child", file.clone()).unwrap(); + } else { + map.insert(0, "a/child", file.clone()).unwrap(); + map.insert(1, "a", directory.clone()).unwrap(); + } + assert!(validate_wire_map(&map).is_ok()); + } + } + + #[test] + fn archive_path_candidate_validation_handles_component_boundaries_and_depth() { + let file = entry( + FileType::Data, + None, + BlockDataState::Decrypted(vec![].into()), + ); + let directory = entry( + FileType::Directory, + None, + BlockDataState::Decrypted(vec![].into()), + ); + + let mut map = WireEntries::new(); + map.insert(0, "a", file.clone()).unwrap(); + assert!(validate_new_candidate(&map, "a/child", &file).is_err()); + + let mut map = WireEntries::new(); + map.insert(0, "a/child", file.clone()).unwrap(); + assert!(validate_new_candidate(&map, "a", &file).is_err()); + + let mut map = WireEntries::new(); + map.insert(0, "a", directory.clone()).unwrap(); + assert!(validate_new_candidate(&map, "a/child", &file).is_ok()); + + let mut map = WireEntries::new(); + for (id, path) in [(0, "ab"), (1, "a!"), (2, "a.b"), (3, "a/child")] { + map.insert(id, path, file.clone()).unwrap(); + } + assert!(validate_new_candidate(&map, "a", &file).is_err()); + assert!(validate_new_candidate(&map, "a!x", &file).is_ok()); + assert!(validate_new_candidate(&map, "a.bx", &file).is_ok()); + assert!(validate_new_candidate(&map, "abx", &file).is_ok()); + + let mut map = WireEntries::new(); + map.insert(0, "ユニコード", file.clone()).unwrap(); + assert!(validate_new_candidate(&map, "ユニコード/子", &file).is_err()); + + let deep = (0..64) + .map(|part| format!("part{part}")) + .collect::>(); + let ancestor = deep.join("/"); + let descendant = format!("{ancestor}/leaf"); + let mut map = WireEntries::new(); + map.insert(0, descendant, file.clone()).unwrap(); + assert!(validate_new_candidate(&map, &ancestor, &file).is_err()); + } + + #[test] + fn archive_path_existing_and_new_candidate_exact_path_semantics() { + let file = entry( + FileType::Data, + None, + BlockDataState::Decrypted(vec![].into()), + ); + let mut map = WireEntries::new(); + map.insert(0, "occupied", file.clone()).unwrap(); + + assert!(validate_existing_candidate(&map, "occupied", &file).is_ok()); + assert!(matches!( + validate_new_candidate(&map, "occupied", &file), + Err(PithosError::PathOccupied(message)) if message == "File path already occupied: occupied" + )); + } + + #[test] + fn archive_path_component_order_keeps_descendants_adjacent() { + let file = entry( + FileType::Data, + None, + BlockDataState::Decrypted(vec![].into()), + ); + let mut map = WireEntries::new(); + for (id, path) in [(0, "a"), (1, "a!"), (2, "a/child")] { + map.insert(id, path, file.clone()).unwrap(); + } + + assert_eq!( + map.iter_ordered().map(|(path, _)| path).collect::>(), + ["a", "a/child", "a!"] + ); + assert!(validate_wire_map(&map).is_err()); + } + + #[test] + fn descending_wire_paths_keep_component_order_without_vector_insertion() { + let file = entry( + FileType::Data, + None, + BlockDataState::Decrypted(vec![].into()), + ); + let mut map = WireEntries::new(); + for id in (0..10_000u64).rev() { + map.insert(id, format!("entry-{id:05}"), file.clone()) + .unwrap(); + } + + assert_eq!(map.first_path_after("entry-04999"), Some("entry-05000")); + assert_eq!( + map.iter_ordered().next().map(|(path, _)| path), + Some("entry-00000") + ); + } +} diff --git a/crates/pithos_lib/src/archive/writer.rs b/crates/pithos_lib/src/archive/writer.rs new file mode 100644 index 0000000..6d255b0 --- /dev/null +++ b/crates/pithos_lib/src/archive/writer.rs @@ -0,0 +1,1994 @@ +//! Streaming archive construction with a consuming publication boundary. + +use crate::archive::{AppendSnapshot, ArchivePath, FileId, Span, segment_from_wire}; +use crate::archive::{validate_new_candidate, validate_symlink_target}; +use crate::block; +use crate::crypto::{FileKey, PrivateKey, PublicKey}; +use crate::error::PithosError; +use crate::format::codec; +use crate::format::entries::WireEntries; +use crate::format::wire::{ + BlockDataState, BlockHeader, BlockIndexEntry, BlockLocation, EncryptionSection, FileEntry, + FileHeader, FileType, ProcessingFlags, RecipientData, Reference, +}; +use fastcdc::v2020::{Normalization, StreamCDC}; +use indexmap::IndexMap; +use std::collections::{HashMap, HashSet}; +use std::io::{self, Read, Write}; +use thiserror::Error; +use x25519_dalek::{PublicKey as LegacyPublicKey, StaticSecret}; +use zeroize::Zeroizing; + +/// Validated content-defined chunking parameters. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct CdcConfig { + min_size: usize, + avg_size: usize, + max_size: usize, +} + +impl CdcConfig { + /// The characterized current FastCDC defaults, always using level-one normalization. + pub const DEFAULT: Self = Self { + min_size: fastcdc::v2020::MINIMUM_MAX, + avg_size: fastcdc::v2020::AVERAGE_MAX, + max_size: fastcdc::v2020::MAXIMUM_MAX, + }; + + pub fn new(min_size: usize, avg_size: usize, max_size: usize) -> Result { + if !(fastcdc::v2020::MINIMUM_MIN..=fastcdc::v2020::MINIMUM_MAX).contains(&min_size) + || !(fastcdc::v2020::AVERAGE_MIN..=fastcdc::v2020::AVERAGE_MAX).contains(&avg_size) + || !(fastcdc::v2020::MAXIMUM_MIN..=fastcdc::v2020::MAXIMUM_MAX).contains(&max_size) + || min_size > avg_size + || avg_size > max_size + { + return Err(PithosError::InvalidCdcConfig { + min_size, + avg_size, + max_size, + }); + } + Ok(Self { + min_size, + avg_size, + max_size, + }) + } + + pub fn min_size(self) -> usize { + self.min_size + } + pub fn avg_size(self) -> usize { + self.avg_size + } + pub fn max_size(self) -> usize { + self.max_size + } +} + +impl Default for CdcConfig { + fn default() -> Self { + Self::DEFAULT + } +} + +/// Per-block processing requested for a content entry. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ProcessingOptions { + encrypted: bool, + compression_level: u8, +} + +impl ProcessingOptions { + pub(crate) const fn append_default() -> Self { + Self { + encrypted: true, + compression_level: 2, + } + } + + pub fn new(encrypted: bool, compression_level: u8) -> Result { + if compression_level > 7 { + return Err(PithosError::ReservedProcessingBits(compression_level)); + } + Ok(Self { + encrypted, + compression_level, + }) + } + + pub fn encrypted(self) -> bool { + self.encrypted + } + pub fn compression_level(self) -> u8 { + self.compression_level + } + + fn flags(self) -> ProcessingFlags { + ProcessingFlags::new(self.encrypted, Some(self.compression_level)) + } +} + +impl Default for ProcessingOptions { + fn default() -> Self { + Self { + encrypted: true, + compression_level: 3, + } + } +} + +/// Caller-supplied, host-independent metadata for an archive entry. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct EntryMetadata { + pub created: u64, + pub modified: u64, + pub permissions: u32, + pub references: Vec, +} + +impl EntryMetadata { + pub fn new(created: u64, modified: u64, permissions: u32) -> Self { + Self { + created, + modified, + permissions, + references: Vec::new(), + } + } + + pub fn with_references(mut self, references: Vec) -> Self { + self.references = references; + self + } +} + +/// A validated reference to a previously staged entry. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct EntryReference { + pub target_file_id: u64, + pub relationship: u64, +} + +/// The identifier assigned after an entry delta has been committed. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct WrittenEntry { + pub id: u64, +} + +/// Creation options. A writer always has one sender and at least one recipient. +pub struct WriteOptions { + sender: PrivateKey, + recipients: Vec, + cdc: CdcConfig, +} + +impl WriteOptions { + pub fn new(sender: PrivateKey, recipients: Vec) -> Self { + Self { + sender, + recipients, + cdc: CdcConfig::default(), + } + } + + pub fn with_cdc(mut self, cdc: CdcConfig) -> Self { + self.cdc = cdc; + self + } + + /// Check all creation metadata before taking ownership of an output sink. + pub fn validate(&self) -> Result<(), PithosError> { + if self.recipients.is_empty() { + return Err(PithosError::WriterRequiresRecipient); + } + let mut recipients = HashSet::with_capacity(self.recipients.len()); + if self + .recipients + .iter() + .any(|recipient| !recipients.insert(recipient)) + { + return Err(PithosError::DuplicateRecipientKey); + } + Ok(()) + } +} + +/// A creation error that returns the non-published sink to its owner. +pub struct CreateError { + error: PithosError, + sink: W, +} + +impl CreateError { + pub fn error(&self) -> &PithosError { + &self.error + } + pub fn into_incomplete(self) -> W { + self.sink + } + pub fn into_parts(self) -> (PithosError, W) { + (self.error, self.sink) + } +} + +impl std::fmt::Display for CreateError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.error.fmt(formatter) + } +} + +impl std::fmt::Debug for CreateError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("CreateError") + .field("error", &self.error) + .field("sink", &"[REDACTED]") + .finish() + } +} + +impl std::error::Error for CreateError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(&self.error) + } +} + +/// A finalization error that returns the incomplete sink. It never denotes publication. +pub struct FinishError { + error: PithosError, + sink: W, +} + +impl FinishError { + pub fn error(&self) -> &PithosError { + &self.error + } + pub fn into_incomplete(self) -> W { + self.sink + } + pub fn into_parts(self) -> (PithosError, W) { + (self.error, self.sink) + } +} + +impl std::fmt::Display for FinishError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.error.fmt(formatter) + } +} + +impl std::fmt::Debug for FinishError { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("FinishError") + .field("error", &self.error) + .field("sink", &"[REDACTED]") + .finish() + } +} + +impl std::error::Error for FinishError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(&self.error) + } +} + +#[derive(Debug, Error)] +pub enum WriterError { + #[error("writer is poisoned; recover only with into_incomplete")] + Poisoned, + #[error("{0}")] + Pithos( + #[from] + #[source] + PithosError, + ), +} + +/// A failed recovery attempt retains an open writer, which can only be dropped +/// rather than finalized. `into_incomplete` is intentionally poisoned-only. +pub struct IncompleteWriter(ArchiveWriter); + +impl std::fmt::Debug for IncompleteWriter { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("IncompleteWriter(..)") + } +} + +trait WriterRuntime { + fn file_key(&mut self) -> Result; + fn block_nonce(&mut self) -> Result<[u8; 12], PithosError>; + fn block_list_nonce(&mut self) -> Result<[u8; 12], PithosError>; + fn recipient_list_nonce(&mut self) -> Result<[u8; 12], PithosError>; + fn encode_block( + &mut self, + plaintext: &[u8], + flags: ProcessingFlags, + nonce: [u8; 12], + ) -> Result; + fn seal_block_list( + &mut self, + block_data: &mut BlockDataState, + file_key: &FileKey, + nonce: [u8; 12], + ) -> Result<(), PithosError>; + fn seal_recipient_list( + &mut self, + recipient_data: &mut RecipientData, + shared_key: crate::crypto::SharedSecret, + nonce: [u8; 12], + ) -> Result<(), PithosError>; +} + +struct ProductionRuntime; + +impl WriterRuntime for ProductionRuntime { + fn file_key(&mut self) -> Result { + Ok(FileKey::from_bytes(StaticSecret::random().to_bytes())) + } + + fn block_nonce(&mut self) -> Result<[u8; 12], PithosError> { + Ok(crate::crypto::random_nonce()) + } + + fn block_list_nonce(&mut self) -> Result<[u8; 12], PithosError> { + Ok(crate::crypto::random_nonce()) + } + + fn recipient_list_nonce(&mut self) -> Result<[u8; 12], PithosError> { + Ok(crate::crypto::random_nonce()) + } + + fn encode_block( + &mut self, + plaintext: &[u8], + flags: ProcessingFlags, + nonce: [u8; 12], + ) -> Result { + block::encode(plaintext, flags, nonce) + } + + fn seal_block_list( + &mut self, + block_data: &mut BlockDataState, + file_key: &FileKey, + nonce: [u8; 12], + ) -> Result<(), PithosError> { + block_data.encrypt_with_nonce(file_key, nonce) + } + + fn seal_recipient_list( + &mut self, + recipient_data: &mut RecipientData, + shared_key: crate::crypto::SharedSecret, + nonce: [u8; 12], + ) -> Result<(), PithosError> { + recipient_data.encrypt_with_secret_and_nonce(shared_key, nonce) + } +} + +struct CountingSink { + sink: W, + offset: u64, +} + +/// One fully prepared archive entry publication. It owns all data that may allocate +/// or fail before the live directory is touched. +struct EntryDelta { + id: u64, + path: ArchivePath, + entry: FileEntry, + descriptors: IndexMap<[u8; 32], BlockIndexEntry>, + recipient_access: Option<(u64, FileKey)>, +} + +trait Reservable { + fn reserve_capacity( + &mut self, + additional: usize, + field: &'static str, + ) -> Result<(), PithosError>; +} + +impl Reservable for Vec { + fn reserve_capacity( + &mut self, + additional: usize, + field: &'static str, + ) -> Result<(), PithosError> { + Vec::try_reserve(self, additional).map_err(|_| allocation_failed(field, additional)) + } +} + +impl Reservable for Zeroizing> { + fn reserve_capacity( + &mut self, + additional: usize, + field: &'static str, + ) -> Result<(), PithosError> { + self.try_reserve(additional) + .map_err(|_| allocation_failed(field, additional)) + } +} + +impl Reservable for HashSet +where + T: Eq + std::hash::Hash, + S: std::hash::BuildHasher, +{ + fn reserve_capacity( + &mut self, + additional: usize, + field: &'static str, + ) -> Result<(), PithosError> { + HashSet::try_reserve(self, additional).map_err(|_| allocation_failed(field, additional)) + } +} + +impl Reservable for HashMap +where + K: Eq + std::hash::Hash, + S: std::hash::BuildHasher, +{ + fn reserve_capacity( + &mut self, + additional: usize, + field: &'static str, + ) -> Result<(), PithosError> { + HashMap::try_reserve(self, additional).map_err(|_| allocation_failed(field, additional)) + } +} + +impl Reservable for IndexMap +where + S: std::hash::BuildHasher, +{ + fn reserve_capacity( + &mut self, + additional: usize, + field: &'static str, + ) -> Result<(), PithosError> { + IndexMap::try_reserve(self, additional).map_err(|_| allocation_failed(field, additional)) + } +} + +fn reserve( + collection: &mut impl Reservable, + additional: usize, + field: &'static str, +) -> Result<(), PithosError> { + collection.reserve_capacity(additional, field) +} + +fn allocation_failed(field: &'static str, size: usize) -> PithosError { + PithosError::AllocationFailed { + field, + size: u64::try_from(size).unwrap_or(u64::MAX), + } +} + +fn append_block_reference( + entry: &mut FileEntry, + hash: [u8; 32], + key: &crate::crypto::BlockKey, +) -> Result<(), PithosError> { + let BlockDataState::Decrypted(references) = &mut entry.block_data else { + return Err(PithosError::InvalidBlockDataState( + "block data already/still encrypted".into(), + )); + }; + reserve(references, 1, "entry block references")?; + references.push((hash, *key.expose_for_protocol())); + Ok(()) +} + +impl CountingSink { + fn into_inner(self) -> W { + self.sink + } +} + +impl Write for CountingSink { + fn write(&mut self, bytes: &[u8]) -> io::Result { + let accepted = self.sink.write(bytes)?; + if accepted > bytes.len() { + return Err(io::Error::other("sink accepted more bytes than requested")); + } + let accepted = u64::try_from(accepted) + .map_err(|_| io::Error::other("archive output offset does not fit u64"))?; + self.offset = self + .offset + .checked_add(accepted) + .ok_or_else(|| io::Error::other("archive output offset overflow"))?; + usize::try_from(accepted) + .map_err(|_| io::Error::other("sink accepted byte count does not fit usize")) + } + + fn flush(&mut self) -> io::Result<()> { + self.sink.flush() + } +} + +/// A streaming archive writer. Dropping it leaves an intentionally incomplete sink. +pub struct ArchiveWriter { + sender: StaticSecret, + cdc: CdcConfig, + sink: CountingSink, + directory: crate::format::wire::Directory, + poisoned: bool, + runtime: Box, + append_snapshot: Option, + append_next_id: Option>, + planned_ids: Option>, + granted_access_ids: Option>, +} + +#[cfg(test)] +#[derive(Debug, Eq, PartialEq)] +struct MetadataSnapshot { + files: usize, + descriptors: usize, + next_id: Option, + recipient_records: usize, +} + +impl ArchiveWriter { + /// Writes exactly one current-format header before returning an open writer. + pub fn create(sink: W, options: WriteOptions) -> Result> { + if let Err(error) = options.validate() { + return Err(CreateError { error, sink }); + } + let sender = options.sender.into_dalek_static_secret(); + let recipients = options + .recipients + .into_iter() + .map(PublicKey::into_dalek_public_key) + .collect::>(); + let encryption = IndexMap::from_iter([( + LegacyPublicKey::from(&sender).to_bytes(), + EncryptionSection::new(&recipients), + )]); + let directory = crate::format::wire::Directory::new(None, WireEntries::new(), encryption); + let mut sink = CountingSink { sink, offset: 0 }; + if let Err(error) = codec::encode_header(&FileHeader::default(), &mut sink) { + return Err(CreateError { + error: error.into(), + sink: sink.into_inner(), + }); + } + Ok(Self { + sender, + cdc: options.cdc, + sink, + directory, + poisoned: false, + runtime: Box::new(ProductionRuntime), + append_snapshot: None, + append_next_id: None, + planned_ids: None, + granted_access_ids: None, + }) + } + + /// Seeds a child directory for the direct appender. + /// It intentionally writes no header and retains ancestor state only for + /// validation and block reuse, never for child-directory serialization. + pub(crate) fn append( + sink: W, + sender: PrivateKey, + recipients: Vec, + cdc: CdcConfig, + snapshot: AppendSnapshot, + ) -> Result { + WriteOptions::new(sender.duplicate(), recipients.clone()).validate()?; + let sender = sender.into_dalek_static_secret(); + let recipients = recipients + .into_iter() + .map(PublicKey::into_dalek_public_key) + .collect::>(); + let parent = snapshot.terminal_directory(); + let maximum_id = snapshot.maximum_id(); + let files = WireEntries::with_maximum_id(maximum_id.map_or(0, |id| id.0)); + let encryption = IndexMap::from_iter([( + LegacyPublicKey::from(&sender).to_bytes(), + EncryptionSection::new(&recipients), + )]); + let directory = crate::format::wire::Directory::new( + Some((parent.start(), parent.len())), + files, + encryption, + ); + Ok(Self { + sender, + cdc, + sink: CountingSink { + sink, + offset: snapshot.archive_len(), + }, + directory, + poisoned: false, + runtime: Box::new(ProductionRuntime), + append_next_id: Some(maximum_id.map(|id| id.0.checked_add(1)).unwrap_or(Some(0))), + append_snapshot: Some(snapshot), + planned_ids: None, + granted_access_ids: None, + }) + } + + #[cfg(test)] + fn with_test_runtime( + sink: W, + options: WriteOptions, + runtime: Box, + initial_offset: u64, + ) -> Result> { + let mut writer = Self::create(sink, options)?; + writer.runtime = runtime; + writer.sink.offset = initial_offset; + Ok(writer) + } + + pub fn add_file( + &mut self, + path: ArchivePath, + metadata: EntryMetadata, + processing: ProcessingOptions, + expected_size: Option, + content: R, + ) -> Result { + self.add_content( + FileType::Data, + path, + metadata, + processing, + expected_size, + content, + ) + } + + pub(crate) fn add_file_planned( + &mut self, + expected_id: u64, + path: ArchivePath, + metadata: EntryMetadata, + processing: ProcessingOptions, + expected_size: Option, + content: R, + ) -> Result { + self.assert_planned_id(expected_id)?; + self.add_file(path, metadata, processing, expected_size, content) + } + + pub(crate) fn prepare_planned_ids(&mut self, ids: &[u64]) -> Result<(), WriterError> { + let Some(Some(mut next)) = self.append_next_id else { + return Err(PithosError::PlannedIdsRequireAppendWriter.into()); + }; + let mut planned = HashSet::with_capacity(ids.len()); + for (index, id) in ids.iter().enumerate() { + if *id != next || !planned.insert(*id) { + return Err(PithosError::DuplicateFileId(format!( + "planned file id {id} does not match append allocation" + )) + .into()); + } + if index + 1 < ids.len() { + next = next.checked_add(1).ok_or(PithosError::FileIdExhausted)?; + } + } + self.planned_ids = Some(planned); + Ok(()) + } + + pub fn add_metadata( + &mut self, + path: ArchivePath, + metadata: EntryMetadata, + processing: ProcessingOptions, + expected_size: Option, + content: R, + ) -> Result { + self.add_content( + FileType::Metadata, + path, + metadata, + processing, + expected_size, + content, + ) + } + + pub fn add_directory( + &mut self, + path: ArchivePath, + metadata: EntryMetadata, + ) -> Result { + self.ensure_open()?; + let delta = self.stage_entry(FileType::Directory, path, metadata, 0, None)?; + self.prepare_delta(&delta)?; + Ok(self.commit_delta(delta)) + } + + pub(crate) fn add_directory_planned( + &mut self, + expected_id: u64, + path: ArchivePath, + metadata: EntryMetadata, + ) -> Result { + self.assert_planned_id(expected_id)?; + self.add_directory(path, metadata) + } + + pub fn add_symlink( + &mut self, + path: ArchivePath, + metadata: EntryMetadata, + target: impl Into, + ) -> Result { + self.ensure_open()?; + let target = target.into(); + validate_symlink_target(path.as_str(), &target)?; + let delta = self.stage_entry(FileType::Symlink, path, metadata, 0, Some(target))?; + self.prepare_delta(&delta)?; + Ok(self.commit_delta(delta)) + } + + pub(crate) fn add_symlink_planned( + &mut self, + expected_id: u64, + path: ArchivePath, + metadata: EntryMetadata, + target: impl Into, + ) -> Result { + self.assert_planned_id(expected_id)?; + self.add_symlink(path, metadata, target) + } + + /// Adds recovered ancestor content keys to every recipient of an otherwise empty child. + /// The snapshot retains the sole zeroizing key owner; plaintext is only borrowed while the + /// transient recipient records are assembled for sealing during `finish`. + pub(crate) fn grant_file_keys(&mut self, ids: &[FileId]) -> Result<(), PithosError> { + self.ensure_open().map_err(|error| match error { + WriterError::Pithos(error) => error, + WriterError::Poisoned => PithosError::WriterPoisoned, + })?; + if ids.is_empty() { + return Err(PithosError::GrantRequiresFileId); + } + if !self.directory.files.is_empty() || !self.directory.blocks.is_empty() { + return Err(PithosError::GrantChildContainsContent); + } + let snapshot = self + .append_snapshot + .as_ref() + .ok_or(PithosError::GrantRequiresAppendSnapshot)?; + let mut granted_ids = HashSet::with_capacity(ids.len()); + for id in ids { + if !granted_ids.insert(id.0) { + return Err(PithosError::DuplicateRecipientFileId); + } + snapshot.with_file_key(*id, |_| ())?; + } + for section in self.directory.encryption.values_mut() { + for recipient in section.recipients.values_mut() { + let RecipientData::Decrypted(records) = &mut recipient.recipient_data else { + return Err(PithosError::WriterUnsealedRecipientList); + }; + reserve(records, ids.len(), "recipient access records")?; + } + } + let mut recipients = self + .directory + .encryption + .values_mut() + .flat_map(|section| section.recipients.values_mut()) + .collect::>(); + for id in ids { + snapshot.with_file_key(*id, |key| { + let record_key = Zeroizing::new(*key.expose_for_protocol()); + for recipient in &mut recipients { + let RecipientData::Decrypted(records) = &mut recipient.recipient_data else { + unreachable!("recipient lists were validated before grant publication"); + }; + records.push((id.0, *record_key)); + } + })?; + } + self.granted_access_ids = Some(granted_ids); + Ok(()) + } + + fn add_content( + &mut self, + file_type: FileType, + path: ArchivePath, + metadata: EntryMetadata, + processing: ProcessingOptions, + expected_size: Option, + content: R, + ) -> Result { + self.ensure_open()?; + let mut delta = self.stage_entry(file_type, path, metadata, 0, None)?; + let mut stream = StreamCDC::with_level( + content, + self.cdc.min_size, + self.cdc.avg_size, + self.cdc.max_size, + Normalization::Level1, + ); + let mut size = 0u64; + let mut block_references = HashMap::new(); + for chunk in &mut stream { + let chunk = match chunk { + Ok(chunk) => chunk, + Err(error) => return self.poison(error.into()), + }; + let chunk_data = Zeroizing::new(chunk.data); + size = match size.checked_add(chunk.length as u64) { + Some(size) => size, + None => return self.poison(PithosError::WriterSizeOverflow), + }; + let nonce = match self.runtime.block_nonce() { + Ok(nonce) => nonce, + Err(error) => return self.poison(error), + }; + let encoded = match self + .runtime + .encode_block(&chunk_data, processing.flags(), nonce) + { + Ok(encoded) => encoded, + Err(error) => return self.poison(error), + }; + let hash = encoded.hash; + if let Err(error) = reserve(&mut block_references, 1, "block references") { + return self.poison(error); + } + if block_references + .insert(hash, *encoded.key.expose_for_protocol()) + .is_some_and(|existing| existing != *encoded.key.expose_for_protocol()) + { + return self.poison(PithosError::DuplicateBlockReference); + } + if let Err(error) = append_block_reference(&mut delta.entry, hash, &encoded.key) { + return self.poison(error); + } + if let Some(existing) = self.directory.blocks.get(&hash) { + if existing.original_size != chunk.length as u64 { + return self.poison(PithosError::BlockIndexConflict { + hash, + existing_original_size: existing.original_size, + new_original_size: chunk.length as u64, + }); + } + continue; + } + if let Some(existing) = self + .append_snapshot + .as_ref() + .and_then(|snapshot| snapshot.descriptor(crate::archive::types::BlockHash(hash))) + { + if existing.original_size != chunk.length as u64 { + return self.poison(PithosError::BlockIndexConflict { + hash, + existing_original_size: existing.original_size, + new_original_size: chunk.length as u64, + }); + } + continue; + } + if let Some(existing) = delta.descriptors.get(&hash) { + if existing.original_size != chunk.length as u64 { + return self.poison(PithosError::BlockIndexConflict { + hash, + existing_original_size: existing.original_size, + new_original_size: chunk.length as u64, + }); + } + continue; + } + let descriptor = BlockIndexEntry { + offset: self.sink.offset, + stored_size: match u64::try_from(encoded.stored.len()) { + Ok(size) => size, + Err(_) => return self.poison(PithosError::WriterSizeOverflow), + }, + original_size: chunk.length as u64, + flags: encoded.flags, + location: BlockLocation::Local, + }; + if let Err(error) = reserve(&mut delta.descriptors, 1, "new block descriptors") { + return self.poison(error); + } + if let Err(error) = self.write_block(&encoded.stored) { + return self.poison(error); + } + delta.descriptors.insert(hash, descriptor); + } + if expected_size.is_some_and(|expected| expected != size) { + return self.poison(PithosError::WriterExpectedSizeMismatch { + expected: expected_size.unwrap(), + actual: size, + }); + } + delta.entry.file_size = size; + if let Err(error) = self.validate_unsealed_content(&delta) { + return self.poison(error); + } + let file_key = match self.runtime.file_key() { + Ok(file_key) => file_key, + Err(error) => return self.poison(error), + }; + let nonce = match self.runtime.block_list_nonce() { + Ok(nonce) => nonce, + Err(error) => return self.poison(error), + }; + if let Err(error) = + self.runtime + .seal_block_list(&mut delta.entry.block_data, &file_key, nonce) + { + return self.poison(error); + } + delta.recipient_access = Some((delta.id, file_key)); + // Block bytes may already be orphaned on failure, so preparation failures + // poison while every live metadata collection remains unchanged. + if let Err(error) = self.prepare_delta(&delta) { + return self.poison(error); + } + Ok(self.commit_delta(delta)) + } + + fn entry( + &self, + file_type: FileType, + metadata: EntryMetadata, + size: u64, + target: Option, + ) -> FileEntry { + FileEntry { + file_type, + block_data: BlockDataState::Decrypted(Zeroizing::new(Vec::new())), + created: metadata.created, + modified: metadata.modified, + file_size: size, + permissions: metadata.permissions, + references: metadata + .references + .into_iter() + .map(|reference| Reference { + target_file_id: reference.target_file_id, + relationship: reference.relationship, + }) + .collect(), + symlink_target: target, + } + } + + fn validate_candidate(&self, path: &ArchivePath, entry: &FileEntry) -> Result<(), PithosError> { + if let Some(snapshot) = &self.append_snapshot { + snapshot.ensure_path_available(path)?; + snapshot.ensure_candidate_hierarchy(path, entry.file_type == FileType::Directory)?; + } + validate_new_candidate(&self.directory.files, path.as_str(), entry)?; + for reference in &entry.references { + let child_has_relationship = self + .directory + .relations + .iter() + .any(|(id, _)| *id == reference.relationship); + let ancestor_has_relationship = self + .append_snapshot + .as_ref() + .is_some_and(|snapshot| snapshot.has_relationship(reference.relationship)); + if !child_has_relationship && !ancestor_has_relationship { + return Err(PithosError::UnknownRelationshipId(reference.relationship)); + } + let child_has_target = self + .directory + .get_file_by_id(reference.target_file_id) + .is_some(); + let ancestor_has_target = self + .append_snapshot + .as_ref() + .is_some_and(|snapshot| snapshot.entry(FileId(reference.target_file_id)).is_some()); + let planned_has_target = self + .planned_ids + .as_ref() + .is_some_and(|ids| ids.contains(&reference.target_file_id)); + if !child_has_target && !ancestor_has_target && !planned_has_target { + return Err(PithosError::MissingReferenceTarget( + reference.target_file_id, + )); + } + } + Ok(()) + } + + fn assert_planned_id(&self, expected_id: u64) -> Result<(), WriterError> { + let Some(next_id) = self.append_next_id else { + return Err(PithosError::PlannedIdsRequireAppendWriter.into()); + }; + match next_id { + Some(actual) if actual == expected_id => Ok(()), + Some(actual) => Err(PithosError::DuplicateFileId(format!( + "planned file id {expected_id} does not match next append id {actual}" + )) + .into()), + None => Err(PithosError::FileIdExhausted.into()), + } + } + + fn stage_entry( + &self, + file_type: FileType, + path: ArchivePath, + metadata: EntryMetadata, + size: u64, + target: Option, + ) -> Result { + let entry = self.entry(file_type, metadata, size, target); + self.validate_candidate(&path, &entry)?; + let id = match self.append_next_id { + Some(Some(id)) => id, + Some(None) => return Err(PithosError::FileIdExhausted.into()), + None => self.directory.next_free_file_index()?, + }; + if let Some(snapshot) = &self.append_snapshot { + snapshot.ensure_id_available(FileId(id))?; + } + Ok(EntryDelta { + id, + path, + entry, + descriptors: IndexMap::new(), + recipient_access: None, + }) + } + + /// Validate every insertion and reserve every live allocation before publication. + fn prepare_delta(&mut self, delta: &EntryDelta) -> Result<(), PithosError> { + self.validate_candidate(&delta.path, &delta.entry)?; + self.directory + .files + .prevalidate_insert(delta.id, delta.path.as_str())?; + for (hash, descriptor) in &delta.descriptors { + if let Some(existing) = self.directory.blocks.get(hash) { + return Err(PithosError::BlockIndexConflict { + hash: *hash, + existing_original_size: existing.original_size, + new_original_size: descriptor.original_size, + }); + } + } + self.directory.files.reserve_one()?; + reserve( + &mut self.directory.blocks, + delta.descriptors.len(), + "archive block descriptors", + )?; + if delta.recipient_access.is_some() { + for section in self.directory.encryption.values_mut() { + for recipient in section.recipients.values_mut() { + if let RecipientData::Decrypted(records) = &mut recipient.recipient_data { + reserve(records, 1, "recipient access records")?; + } + } + } + } + Ok(()) + } + + /// Verify the just-streamed plaintext block list before its keys are sealed. + fn validate_unsealed_content(&self, delta: &EntryDelta) -> Result<(), PithosError> { + let BlockDataState::Decrypted(references) = &delta.entry.block_data else { + return Err(PithosError::WriterUnsealedBlockList); + }; + let mut keys = HashMap::with_capacity(references.len()); + let actual = references.iter().try_fold(0u64, |total, (hash, key)| { + if keys + .insert(*hash, *key) + .is_some_and(|existing| existing != *key) + { + return Err(PithosError::DuplicateBlockReference); + } + let descriptor_size = delta + .descriptors + .get(hash) + .map(|descriptor| descriptor.original_size) + .or_else(|| { + self.directory + .blocks + .get(hash) + .map(|descriptor| descriptor.original_size) + }) + .or_else(|| { + self.append_snapshot + .as_ref() + .and_then(|snapshot| { + snapshot.descriptor(crate::archive::types::BlockHash(*hash)) + }) + .map(|descriptor| descriptor.original_size) + }) + .ok_or(PithosError::MissingBlockDescriptor)?; + total + .checked_add(descriptor_size) + .ok_or(PithosError::AccessibleFileSizeMismatch { + expected: delta.entry.file_size, + actual: u64::MAX, + }) + })?; + if actual != delta.entry.file_size { + return Err(PithosError::AccessibleFileSizeMismatch { + expected: delta.entry.file_size, + actual, + }); + } + Ok(()) + } + + /// All capacity and collision checks happen in `prepare_delta`, making this + /// publication allocation-free and infallible. + fn commit_delta(&mut self, delta: EntryDelta) -> WrittenEntry { + let id = delta.id; + for (hash, descriptor) in delta.descriptors { + debug_assert!(!self.directory.blocks.contains_key(&hash)); + self.directory.blocks.insert(hash, descriptor); + } + self.directory + .files + .insert_prepared(id, delta.path.as_str(), delta.entry); + if let Some((id, access)) = delta.recipient_access { + for section in self.directory.encryption.values_mut() { + for recipient in section.recipients.values_mut() { + if let RecipientData::Decrypted(records) = &mut recipient.recipient_data { + records.push((id, *access.expose_for_protocol())); + } + } + } + } + if let Some(next_id) = &mut self.append_next_id { + *next_id = id.checked_add(1); + } + WrittenEntry { id } + } + + fn write_block(&mut self, bytes: &[u8]) -> Result<(), PithosError> { + codec::encode_block_marker(&BlockHeader::default(), &mut self.sink)?; + self.sink.write_all(bytes)?; + Ok(()) + } + + fn ensure_open(&self) -> Result<(), WriterError> { + if self.poisoned { + Err(WriterError::Poisoned) + } else { + Ok(()) + } + } + + fn poison(&mut self, error: PithosError) -> Result { + self.poisoned = true; + Err(error.into()) + } + + #[cfg(test)] + fn metadata_snapshot(&self) -> MetadataSnapshot { + MetadataSnapshot { + files: self.directory.files.len(), + descriptors: self.directory.blocks.len(), + next_id: self.directory.next_free_file_index().ok(), + recipient_records: self + .directory + .encryption + .values() + .flat_map(|section| section.recipients.values()) + .map(|recipient| match &recipient.recipient_data { + RecipientData::Decrypted(records) => records.len(), + RecipientData::Encrypted(_) => 0, + }) + .sum(), + } + } + + /// Recover the sink only after a poisoned operation. An open writer must be + /// finalized or dropped; it cannot be relabeled as an incomplete recovery. + pub fn into_incomplete(self) -> Result>> { + if self.poisoned { + Ok(self.sink.into_inner()) + } else { + Err(Box::new(IncompleteWriter(self))) + } + } + + /// Validate, write the one terminal directory, flush, and return the completed sink. + pub fn finish(mut self) -> Result> { + let result = (|| -> Result<(), PithosError> { + if self.poisoned { + return Err(PithosError::WriterPoisoned); + } + self.validate_entry_state()?; + self.validate_required_access_records()?; + self.seal_recipient_lists()?; + self.validate_publishable()?; + codec::update_directory_len(&mut self.directory)?; + codec::update_directory_crc(&mut self.directory)?; + if let Some(snapshot) = &self.append_snapshot { + let span = Span::new(self.sink.offset, self.directory.dir_len)?; + let child = + segment_from_wire(&self.directory, span, Some(snapshot.terminal_directory()))?; + snapshot.validate_prospective_child(child)?; + } + codec::encode_directory(&self.directory, &mut self.sink)?; + self.sink.flush()?; + Ok(()) + })(); + match result { + Ok(()) => Ok(self.sink.into_inner()), + Err(error) => Err(FinishError { + error, + sink: self.sink.into_inner(), + }), + } + } + + fn validate_publishable(&self) -> Result<(), PithosError> { + self.validate_entry_state()?; + for section in self.directory.encryption.values() { + for recipient in section.recipients.values() { + if matches!(recipient.recipient_data, RecipientData::Decrypted(_)) { + return Err(PithosError::WriterUnsealedRecipientList); + } + } + } + Ok(()) + } + + fn seal_recipient_lists(&mut self) -> Result<(), PithosError> { + let sender = LegacyPublicKey::from(&self.sender).to_bytes(); + let Some(section) = self.directory.encryption.get_mut(&sender) else { + return Ok(()); + }; + for (recipient_key, recipient) in &mut section.recipients { + let recipient_key = LegacyPublicKey::from(*recipient_key); + let shared_key = + crate::crypto::derive_shared(self.sender.as_bytes(), recipient_key.as_bytes())?; + let nonce = self.runtime.recipient_list_nonce()?; + self.runtime + .seal_recipient_list(&mut recipient.recipient_data, shared_key, nonce)?; + } + Ok(()) + } + + fn validate_entry_state(&self) -> Result<(), PithosError> { + for (_, _, entry) in self.directory.files.iter() { + match entry.file_type { + FileType::Data | FileType::Metadata + if !matches!(entry.block_data, BlockDataState::Encrypted(_)) => + { + return Err(PithosError::WriterUnsealedBlockList); + } + FileType::Directory | FileType::Symlink => match &entry.block_data { + BlockDataState::Decrypted(entries) if entries.is_empty() => {} + _ => return Err(PithosError::WriterNoContentHasBlockMaterial), + }, + _ => {} + } + } + self.directory + .validate_references_and_accessible_blocks_with( + |id| { + self.directory.get_file_by_id(id).is_some() + || self + .append_snapshot + .as_ref() + .is_some_and(|snapshot| snapshot.entry(FileId(id)).is_some()) + }, + |relationship| { + self.append_snapshot + .as_ref() + .is_some_and(|snapshot| snapshot.has_relationship(relationship)) + }, + |hash| { + self.directory + .blocks + .get(&hash) + .map(|descriptor| descriptor.original_size) + .or_else(|| { + self.append_snapshot + .as_ref() + .and_then(|snapshot| { + snapshot.descriptor(crate::archive::types::BlockHash(hash)) + }) + .map(|descriptor| descriptor.original_size) + }) + }, + )?; + Ok(()) + } + + fn validate_required_access_records(&self) -> Result<(), PithosError> { + let mut content_ids = self + .directory + .files + .iter() + .filter_map(|(id, _, entry)| { + matches!(entry.file_type, FileType::Data | FileType::Metadata).then_some(id) + }) + .collect::>(); + if let Some(granted_ids) = &self.granted_access_ids { + content_ids.extend(granted_ids); + } + for section in self.directory.encryption.values() { + for recipient in section.recipients.values() { + let RecipientData::Decrypted(records) = &recipient.recipient_data else { + return Err(PithosError::WriterUnsealedRecipientList); + }; + let ids = records.iter().map(|(id, _)| *id).collect::>(); + if ids.len() != records.len() || ids != content_ids { + return Err(PithosError::WriterUnsealedRecipientList); + } + } + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::archive::{AccessKeys, Archive, OpenOptions}; + use crate::crypto::PrivateKey; + use crate::format::limits::DeserializationLimits; + use crate::source::MemorySource; + use std::io::Cursor; + use std::sync::Arc; + + #[derive(Clone, Copy, Eq, PartialEq)] + enum RuntimeOperation { + FileKey, + BlockNonce, + BlockEncoding, + BlockListNonce, + BlockListSealing, + RecipientListNonce, + RecipientListSealing, + } + + impl RuntimeOperation { + const fn index(self) -> usize { + match self { + Self::FileKey => 0, + Self::BlockNonce => 1, + Self::BlockEncoding => 2, + Self::BlockListNonce => 3, + Self::BlockListSealing => 4, + Self::RecipientListNonce => 5, + Self::RecipientListSealing => 6, + } + } + } + + struct TestRuntime { + file_key: [u8; 32], + failure: Option<(RuntimeOperation, usize)>, + calls: [usize; 7], + } + + impl TestRuntime { + fn new(failure: Option<(RuntimeOperation, usize)>) -> Self { + Self { + file_key: [8; 32], + failure, + calls: [0; 7], + } + } + + fn operation(&mut self, operation: RuntimeOperation) -> Result { + let index = operation.index(); + let call = self.calls[index]; + self.calls[index] += 1; + if self.failure == Some((operation, call)) { + return Err(PithosError::Io(io::Error::other( + "writer runtime failure injected for testing", + ))); + } + Ok(call) + } + + fn nonce(&mut self, operation: RuntimeOperation) -> Result<[u8; 12], PithosError> { + let call = self.operation(operation)?; + let mut nonce = [operation.index() as u8; 12]; + nonce[11] = u8::try_from(call).unwrap(); + Ok(nonce) + } + } + + impl WriterRuntime for TestRuntime { + fn file_key(&mut self) -> Result { + self.operation(RuntimeOperation::FileKey)?; + Ok(FileKey::from_bytes(self.file_key)) + } + + fn block_nonce(&mut self) -> Result<[u8; 12], PithosError> { + self.nonce(RuntimeOperation::BlockNonce) + } + + fn block_list_nonce(&mut self) -> Result<[u8; 12], PithosError> { + self.nonce(RuntimeOperation::BlockListNonce) + } + + fn recipient_list_nonce(&mut self) -> Result<[u8; 12], PithosError> { + self.nonce(RuntimeOperation::RecipientListNonce) + } + + fn encode_block( + &mut self, + plaintext: &[u8], + flags: ProcessingFlags, + nonce: [u8; 12], + ) -> Result { + self.operation(RuntimeOperation::BlockEncoding)?; + block::encode(plaintext, flags, nonce) + } + + fn seal_block_list( + &mut self, + block_data: &mut BlockDataState, + file_key: &FileKey, + nonce: [u8; 12], + ) -> Result<(), PithosError> { + self.operation(RuntimeOperation::BlockListSealing)?; + block_data.encrypt_with_nonce(file_key, nonce) + } + + fn seal_recipient_list( + &mut self, + recipient_data: &mut RecipientData, + shared_key: crate::crypto::SharedSecret, + nonce: [u8; 12], + ) -> Result<(), PithosError> { + self.operation(RuntimeOperation::RecipientListSealing)?; + recipient_data.encrypt_with_secret_and_nonce(shared_key, nonce) + } + } + + fn options() -> WriteOptions { + let sender = PrivateKey::generate(); + WriteOptions::new(sender.duplicate(), vec![sender.public_key()]) + } + + fn deterministic_options() -> WriteOptions { + let sender = PrivateKey::from_dalek_static_secret(&StaticSecret::from([3; 32])); + WriteOptions::new(sender.duplicate(), vec![sender.public_key()]) + } + + fn content_writer(failure: Option<(RuntimeOperation, usize)>) -> ArchiveWriter> { + ArchiveWriter::with_test_runtime( + Vec::new(), + deterministic_options().with_cdc(CdcConfig::new(64, 256, 1024).unwrap()), + Box::new(TestRuntime::new(failure)), + 0, + ) + .unwrap() + } + + fn add_test_content(writer: &mut ArchiveWriter>) -> Result { + let mut state = 1u32; + let content = (0..4096) + .map(|_| { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + (state >> 24) as u8 + }) + .collect::>(); + writer.add_file( + ArchivePath::new("data").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(false, 0).unwrap(), + None, + Cursor::new(content), + ) + } + + #[test] + fn injected_transform_failure_poison_preserves_live_directory() { + let mut writer = ArchiveWriter::with_test_runtime( + Vec::new(), + options(), + Box::new(TestRuntime::new(Some((RuntimeOperation::BlockEncoding, 0)))), + 0, + ) + .unwrap(); + assert!( + writer + .add_file( + ArchivePath::new("data").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::default(), + None, + Cursor::new(b"content"), + ) + .is_err() + ); + assert_eq!(writer.metadata_snapshot().files, 0); + assert_eq!(writer.metadata_snapshot().descriptors, 0); + assert!(writer.into_incomplete().is_ok()); + } + + #[test] + fn selected_content_runtime_failures_poison_without_publishing_metadata() { + for failure in [ + (RuntimeOperation::BlockNonce, 0), + (RuntimeOperation::BlockEncoding, 1), + (RuntimeOperation::FileKey, 0), + (RuntimeOperation::BlockListNonce, 0), + (RuntimeOperation::BlockListSealing, 0), + ] { + let mut writer = content_writer(Some(failure)); + let before = writer.metadata_snapshot(); + assert!(add_test_content(&mut writer).is_err()); + assert_eq!(writer.metadata_snapshot(), before); + assert!(writer.poisoned); + assert!(matches!( + writer.add_directory( + ArchivePath::new("later").unwrap(), + EntryMetadata::new(0, 0, 0o755), + ), + Err(WriterError::Poisoned) + )); + assert!(writer.into_incomplete().is_ok()); + } + } + + #[test] + fn recipient_runtime_failures_return_an_incomplete_sink() { + for failure in [ + (RuntimeOperation::RecipientListNonce, 0), + (RuntimeOperation::RecipientListSealing, 0), + ] { + let mut writer = content_writer(Some(failure)); + add_test_content(&mut writer).unwrap(); + let error = writer.finish().unwrap_err(); + assert!(matches!(error.error(), PithosError::Io(_))); + let sink = error.into_incomplete(); + assert_eq!(&sink[..4], b"PITH"); + assert!(!sink.windows(8).any(|window| window == b"PITHOSDR")); + } + } + + #[test] + fn deterministic_runtime_produces_identical_complete_archives() { + // This is the executable deterministic-writer vector; it replaces controlled-vectors.toml. + fn write() -> Vec { + let mut writer = content_writer(None); + add_test_content(&mut writer).unwrap(); + writer.finish().unwrap() + } + + let archive = write(); + assert_eq!( + blake3::hash(&archive).to_hex().as_str(), + "4831e3667b0a3565c46338516f1a1bfe8c1060b437f1fbba96cff6df2cbe72d1" + ); + assert_eq!(&archive[..6], b"PITH\x80\x02"); + assert_eq!( + archive + .windows(8) + .filter(|window| *window == b"PITHOSDR") + .count(), + 1 + ); + assert_eq!(archive, write()); + } + + #[test] + fn invalid_transient_states_cannot_be_published() { + let mut writer = content_writer(None); + add_test_content(&mut writer).unwrap(); + writer + .directory + .files + .try_for_each_mut(|_, entry| { + entry.block_data = BlockDataState::Decrypted(Zeroizing::new(Vec::new())); + Ok::<_, ()>(()) + }) + .unwrap(); + assert!(matches!( + writer.validate_publishable(), + Err(PithosError::WriterUnsealedBlockList) + )); + + let mut writer = content_writer(None); + writer + .add_directory( + ArchivePath::new("directory").unwrap(), + EntryMetadata::new(0, 0, 0o755), + ) + .unwrap(); + writer + .directory + .files + .try_for_each_mut(|_, entry| { + entry.block_data = BlockDataState::Encrypted(vec![1]); + Ok::<_, ()>(()) + }) + .unwrap(); + assert!(matches!( + writer.validate_publishable(), + Err(PithosError::WriterNoContentHasBlockMaterial) + )); + + let mut writer = content_writer(None); + add_test_content(&mut writer).unwrap(); + let recipient = writer + .directory + .encryption + .values_mut() + .next() + .unwrap() + .recipients + .values_mut() + .next() + .unwrap(); + recipient.recipient_data = RecipientData::Decrypted(Zeroizing::new(Vec::new())); + assert!(matches!( + writer.validate_publishable(), + Err(PithosError::WriterUnsealedRecipientList) + )); + } + + #[test] + fn failed_content_delta_preserves_all_live_metadata_and_next_id() { + let mut writer = ArchiveWriter::create(Vec::new(), options()).unwrap(); + let before = writer.metadata_snapshot(); + assert!(matches!( + writer.add_file( + ArchivePath::new("data").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(false, 0).unwrap(), + Some(99), + Cursor::new(b"actual"), + ), + Err(WriterError::Pithos( + PithosError::WriterExpectedSizeMismatch { .. } + )) + )); + assert_eq!(writer.metadata_snapshot(), before); + assert!(writer.poisoned); + } + + #[test] + fn directory_and_symlink_validation_failures_preserve_live_metadata() { + let mut writer = ArchiveWriter::create(Vec::new(), options()).unwrap(); + writer + .add_directory( + ArchivePath::new("parent").unwrap(), + EntryMetadata::new(0, 0, 0o755), + ) + .unwrap(); + let before = writer.metadata_snapshot(); + assert!( + writer + .add_directory( + ArchivePath::new("parent").unwrap(), + EntryMetadata::new(0, 0, 0o755), + ) + .is_err() + ); + assert_eq!(writer.metadata_snapshot(), before); + assert!( + writer + .add_symlink( + ArchivePath::new("parent/link").unwrap(), + EntryMetadata::new(0, 0, 0o777), + "../../outside", + ) + .is_err() + ); + assert_eq!(writer.metadata_snapshot(), before); + assert!(!writer.poisoned); + } + + #[test] + fn repeated_compatible_descriptor_keeps_the_earliest_descriptor() { + let mut writer = ArchiveWriter::create(Vec::new(), options()).unwrap(); + for path in ["first", "second"] { + writer + .add_file( + ArchivePath::new(path).unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(false, 0).unwrap(), + Some(4), + Cursor::new(b"same"), + ) + .unwrap(); + } + assert_eq!(writer.directory.blocks.len(), 1); + let descriptor = writer.directory.blocks.values().next().unwrap(); + assert_eq!(descriptor.offset, 6); + assert_eq!(descriptor.original_size, 4); + } + + #[test] + fn compression_profiles_round_trip_with_current_flags_and_empty_content() { + let mut state = 0x243f_6a88_u32; + let incompressible = (0..16 * 1024) + .map(|index| { + state = state + .wrapping_mul(1_664_525) + .wrapping_add(1_013_904_223) + .rotate_left((index % 31) as u32); + (state ^ index as u32).to_le_bytes()[index % 4] + }) + .collect::>(); + + for payload in [vec![b'A'; 16 * 1024], incompressible] { + for compression in 0..=7 { + for encrypted in [false, true] { + let recipient = PrivateKey::generate(); + let mut writer = ArchiveWriter::create( + Vec::new(), + WriteOptions::new(PrivateKey::generate(), vec![recipient.public_key()]), + ) + .unwrap(); + writer + .add_file( + ArchivePath::new("data").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(encrypted, compression).unwrap(), + Some(payload.len() as u64), + Cursor::new(&payload), + ) + .unwrap(); + for descriptor in writer.directory.blocks.values() { + assert_eq!(descriptor.flags.is_encrypted(), encrypted); + let level = descriptor.flags.get_compression_level(); + assert!(level == 0 || level == compression); + } + let bytes = writer.finish().unwrap(); + let archive = Archive::open( + MemorySource::new(bytes), + OpenOptions::default() + .with_access_keys(AccessKeys::new().with_key(recipient)), + ) + .unwrap(); + let mut copied = Vec::new(); + archive.copy_to("data", &mut copied).unwrap(); + assert_eq!(copied, payload); + } + } + } + + let recipient = PrivateKey::generate(); + let mut writer = ArchiveWriter::create( + Vec::new(), + WriteOptions::new(PrivateKey::generate(), vec![recipient.public_key()]), + ) + .unwrap(); + writer + .add_file( + ArchivePath::new("empty").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(false, 0).unwrap(), + Some(0), + Cursor::new([]), + ) + .unwrap(); + assert!(writer.directory.blocks.is_empty()); + let bytes = writer.finish().unwrap(); + let archive = Archive::open( + MemorySource::new(bytes), + OpenOptions::default().with_access_keys(AccessKeys::new().with_key(recipient)), + ) + .unwrap(); + let mut copied = Vec::new(); + archive.copy_to("empty", &mut copied).unwrap(); + assert!(copied.is_empty()); + } + + #[test] + fn counter_overflow_is_not_saturated() { + let mut writer = ArchiveWriter::with_test_runtime( + Vec::new(), + options(), + Box::new(TestRuntime::new(None)), + u64::MAX, + ) + .unwrap(); + assert!( + writer + .add_file( + ArchivePath::new("data").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(false, 0).unwrap(), + None, + Cursor::new(b"content"), + ) + .is_err() + ); + assert!(writer.poisoned); + assert_eq!(writer.directory.files.iter().count(), 0); + assert!(writer.into_incomplete().is_ok()); + } + + #[test] + fn file_id_exhaustion_does_not_mutate_or_poison_before_streaming() { + let mut writer = ArchiveWriter::create(Vec::new(), options()).unwrap(); + writer.directory.files = WireEntries::with_maximum_id(u64::MAX); + assert!(matches!( + writer.add_directory( + ArchivePath::new("data").unwrap(), + EntryMetadata::new(0, 0, 0o755), + ), + Err(WriterError::Pithos(PithosError::FileIdExhausted)) + )); + assert!(!writer.poisoned); + assert_eq!(writer.directory.files.iter().count(), 0); + } + + #[test] + fn grant_child_is_metadata_only_and_seals_selected_recovered_keys() { + let sender = PrivateKey::generate(); + let recipient = PrivateKey::generate(); + let mut writer = ArchiveWriter::create( + Vec::new(), + WriteOptions::new(sender.duplicate(), vec![sender.public_key()]), + ) + .unwrap(); + for path in ["first", "second"] { + writer + .add_file( + ArchivePath::new(path).unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(true, 0).unwrap(), + None, + Cursor::new(path.as_bytes()), + ) + .unwrap(); + } + let prefix = writer.finish().unwrap(); + let snapshot = Archive::open( + MemorySource::new(Arc::<[u8]>::from(prefix.clone())), + OpenOptions::default().with_access_keys(AccessKeys::new().with_key(sender.duplicate())), + ) + .unwrap() + .into_append_snapshot(); + let parent = snapshot.terminal_directory(); + let mut child = ArchiveWriter::append( + Vec::new(), + sender, + vec![recipient.public_key()], + CdcConfig::default(), + snapshot, + ) + .unwrap(); + child.grant_file_keys(&[FileId(0), FileId(1)]).unwrap(); + assert!(child.directory.files.is_empty()); + assert!(child.directory.blocks.is_empty()); + assert_eq!( + child.directory.parent_directory_offset, + Some((parent.start(), parent.len())) + ); + let transient_recipient_data = &child + .directory + .encryption + .values() + .next() + .unwrap() + .recipients + .values() + .next() + .unwrap() + .recipient_data; + assert!(matches!( + transient_recipient_data, + RecipientData::Decrypted(records) if records.iter().map(|(id, _)| *id).eq([0, 1]) + )); + + let bytes = child.finish().unwrap(); + let directory = + codec::decode_directory(&mut Cursor::new(&bytes), &DeserializationLimits::default()) + .unwrap(); + assert_eq!(directory.files.len(), 0); + assert_eq!(directory.blocks.len(), 0); + assert_eq!(directory.encryption.len(), 1); + let sealed_recipient_data = &directory + .encryption + .values() + .next() + .unwrap() + .recipients + .values() + .next() + .unwrap() + .recipient_data; + assert!(matches!(sealed_recipient_data, RecipientData::Encrypted(_))); + } + + #[test] + fn planned_append_ids_reject_mismatches_before_content_io_and_allow_forward_references() { + struct PanicRead; + impl Read for PanicRead { + fn read(&mut self, _: &mut [u8]) -> io::Result { + panic!("planned ID mismatch read content") + } + } + + let sender = PrivateKey::generate(); + let recipient = sender.public_key(); + let mut parent = ArchiveWriter::create( + Vec::new(), + WriteOptions::new(sender.duplicate(), vec![recipient]), + ) + .unwrap(); + parent + .add_file( + ArchivePath::new("ancestor").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(false, 0).unwrap(), + Some(0), + Cursor::new([]), + ) + .unwrap(); + let snapshot = Archive::open( + MemorySource::new(Arc::<[u8]>::from(parent.finish().unwrap())), + OpenOptions::default().with_access_keys(AccessKeys::new().with_key(sender.duplicate())), + ) + .unwrap() + .into_append_snapshot(); + let mut child = ArchiveWriter::append( + Vec::new(), + sender, + vec![recipient], + CdcConfig::default(), + snapshot, + ) + .unwrap(); + child.prepare_planned_ids(&[1, 2]).unwrap(); + assert!(matches!( + child.add_file_planned( + 2, + ArchivePath::new("never-read").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(false, 0).unwrap(), + None, + PanicRead, + ), + Err(WriterError::Pithos(PithosError::DuplicateFileId(_))) + )); + child + .add_file_planned( + 1, + ArchivePath::new("first").unwrap(), + EntryMetadata::new(0, 0, 0o644).with_references(vec![EntryReference { + target_file_id: 2, + relationship: 0, + }]), + ProcessingOptions::new(false, 0).unwrap(), + Some(0), + Cursor::new([]), + ) + .unwrap(); + child + .add_file_planned( + 2, + ArchivePath::new("second").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(false, 0).unwrap(), + Some(0), + Cursor::new([]), + ) + .unwrap(); + assert!(child.finish().is_ok()); + } + + #[test] + fn planned_append_ids_allow_u64_max_only_as_the_final_id() { + let mut writer = content_writer(None); + writer.append_next_id = Some(Some(u64::MAX)); + + writer.prepare_planned_ids(&[u64::MAX]).unwrap(); + assert!(matches!( + writer.prepare_planned_ids(&[u64::MAX, 0]), + Err(WriterError::Pithos(PithosError::FileIdExhausted)) + )); + } +} diff --git a/crates/pithos_lib/src/fs/ingest.rs b/crates/pithos_lib/src/fs/ingest.rs new file mode 100644 index 0000000..cd1832f --- /dev/null +++ b/crates/pithos_lib/src/fs/ingest.rs @@ -0,0 +1,779 @@ +use crate::archive::validate_symlink_target; +use crate::archive::{ + AppendSnapshot, ArchivePath, ArchiveWriter, EntryMetadata, ProcessingOptions, WriterError, +}; +use crate::error::PithosError; +use crate::fs::FsError; +use cap_std::fs::{ + Dir, FileTypeExt as CapFileTypeExt, MetadataExt as CapMetadataExt, + PermissionsExt as CapPermissionsExt, +}; +use cap_std::time::{SystemClock, SystemTime as CapSystemTime}; +use rustix::fs::{Mode, OFlags}; +use std::collections::BTreeMap; +use std::fs::{self, File}; +use std::os::unix::fs::{FileTypeExt, MetadataExt, PermissionsExt}; +use std::path::{Component, Path, PathBuf}; +use std::time::SystemTime; + +fn archive_path(path: &Path) -> Result { + let display_path = path.to_str().ok_or_else(|| FsError::InvalidUtf8Path { + path: path.to_path_buf(), + })?; + let mut components = Vec::new(); + for component in path.components() { + let Component::Normal(component) = component else { + return Err(FsError::Core { + operation: "validate archive path", + path: path.to_path_buf(), + source: PithosError::InvalidArchivePath { + path: display_path.to_owned(), + reason: "archive paths must contain only normal components".into(), + }, + }); + }; + components.push(component.to_str().ok_or_else(|| FsError::InvalidUtf8Path { + path: path.to_path_buf(), + })?); + } + ArchivePath::new(components.join("/")).map_err(|source| FsError::Core { + operation: "validate archive path", + path: path.to_path_buf(), + source, + }) +} + +fn symlink_target(path: PathBuf) -> Result { + path.to_str() + .map(str::to_owned) + .ok_or(FsError::InvalidUtf8Path { path }) +} + +fn metadata(metadata: &fs::Metadata) -> EntryMetadata { + let timestamp = |value: Result| { + value + .ok() + .and_then(|time| time.duration_since(SystemTime::UNIX_EPOCH).ok()) + .map_or(0, |duration| duration.as_secs()) + }; + EntryMetadata::new( + timestamp(metadata.created()), + timestamp(metadata.modified()), + metadata.permissions().mode() & 0o7777, + ) +} + +fn cap_metadata(metadata: &cap_std::fs::Metadata) -> EntryMetadata { + let timestamp = |value: Result| { + value + .ok() + .and_then(|time| time.duration_since(SystemClock::UNIX_EPOCH).ok()) + .map_or(0, |duration| duration.as_secs()) + }; + EntryMetadata::new( + timestamp(metadata.created()), + timestamp(metadata.modified()), + metadata.permissions().mode() & 0o7777, + ) +} + +fn open_regular_file_no_follow(path: &Path) -> std::io::Result { + rustix::fs::open( + path, + OFlags::RDONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::empty(), + ) + .map_err(std::io::Error::from) + .map(File::from) +} + +fn same_file_identity(left: &fs::Metadata, right: &fs::Metadata) -> bool { + left.file_type().is_file() + && right.file_type().is_file() + && left.dev() == right.dev() + && left.ino() == right.ino() +} + +fn open_directory_no_follow(path: &Path) -> std::io::Result { + rustix::fs::open( + path, + OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::empty(), + ) + .map_err(std::io::Error::from) + .map(|fd| Dir::from_std_file(File::from(fd))) +} + +fn open_at_no_follow(parent: &Dir, name: &Path, directory: bool) -> std::io::Result { + let flags = OFlags::RDONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC; + let flags = if directory { + flags | OFlags::DIRECTORY + } else { + flags + }; + rustix::fs::openat(parent, name, flags, Mode::empty()) + .map_err(std::io::Error::from) + .map(File::from) +} + +fn same_cap_file_identity(left: &cap_std::fs::Metadata, right: &fs::Metadata) -> bool { + left.dev() == right.dev() && left.ino() == right.ino() +} + +enum ManifestKind { + File { expected_size: u64 }, + Directory, + Symlink { target: String }, +} + +struct ManifestEntry { + source: Option, + source_key: Option, + path: ArchivePath, + metadata: EntryMetadata, + kind: ManifestKind, +} + +/// A complete, validated filesystem-to-archive mapping. +/// +/// Synthetic directory parents use deterministic zero timestamps and mode 0755. +pub struct InputManifest { + entries: Vec, + sources: BTreeMap, +} + +impl InputManifest { + /// Materialize the already validated manifest with typed writer operations. + /// A filesystem race after preflight is still reported to the caller. + pub fn ingest( + &self, + writer: &mut ArchiveWriter, + processing: ProcessingOptions, + ) -> Result<(), WriterError> { + for entry in &self.entries { + match &entry.kind { + ManifestKind::File { expected_size } => { + let source = self.source(entry)?; + writer.add_file( + entry.path.clone(), + entry.metadata.clone(), + processing, + Some(*expected_size), + source.try_clone().map_err(PithosError::Io)?, + )?; + } + ManifestKind::Directory => { + writer.add_directory(entry.path.clone(), entry.metadata.clone())?; + } + ManifestKind::Symlink { target } => { + writer.add_symlink( + entry.path.clone(), + entry.metadata.clone(), + target.clone(), + )?; + } + } + } + Ok(()) + } + + /// Executes a preassigned append plan. The ID assertion occurs before a source is cloned + /// or read, so a stale plan cannot consume filesystem content. + pub(crate) fn ingest_planned( + &self, + ids: &[u64], + writer: &mut ArchiveWriter, + processing: ProcessingOptions, + ) -> Result<(), WriterError> { + if ids.len() != self.entries.len() { + return Err(PithosError::AppendPlanLengthMismatch.into()); + } + writer.prepare_planned_ids(ids)?; + for (entry, id) in self.entries.iter().zip(ids) { + match &entry.kind { + ManifestKind::File { expected_size } => { + let source = self.source(entry)?; + writer.add_file_planned( + *id, + entry.path.clone(), + entry.metadata.clone(), + processing, + Some(*expected_size), + source.try_clone().map_err(PithosError::Io)?, + )?; + } + ManifestKind::Directory => { + writer.add_directory_planned( + *id, + entry.path.clone(), + entry.metadata.clone(), + )?; + } + ManifestKind::Symlink { target } => { + writer.add_symlink_planned( + *id, + entry.path.clone(), + entry.metadata.clone(), + target.clone(), + )?; + } + } + } + Ok(()) + } + + fn source(&self, entry: &ManifestEntry) -> Result<&File, WriterError> { + entry + .source_key + .and_then(|key| self.sources.get(&key)) + .ok_or_else(|| PithosError::MissingManifestSource.into()) + } + + pub(crate) fn source_with_identity(&self, device: u64, inode: u64) -> Option<&Path> { + self.entries.iter().find_map(|entry| match &entry.kind { + ManifestKind::File { .. } => { + let metadata = self.source(entry).ok()?.metadata().ok()?; + (metadata.dev() == device && metadata.ino() == inode) + .then_some(entry.source.as_deref()) + .flatten() + } + ManifestKind::Directory | ManifestKind::Symlink { .. } => None, + }) + } + + /// Check every planned entry against the complete validated ancestor before streaming. + pub(crate) fn validate_append( + &self, + snapshot: &AppendSnapshot, + ) -> Result, PithosError> { + let entry_count = + u64::try_from(self.entries.len()).map_err(|_| PithosError::FileIdExhausted)?; + if snapshot + .maximum_id() + .is_some_and(|id| id.0.checked_add(entry_count).is_none()) + { + return Err(PithosError::FileIdExhausted); + } + snapshot.validate_child_relationships(&DEFAULT_RELATIONSHIPS)?; + let mut next_id = snapshot + .maximum_id() + .map_or(Some(0), |id| id.0.checked_add(1)); + let mut ids = Vec::with_capacity(self.entries.len()); + for entry in &self.entries { + snapshot.ensure_path_available(&entry.path)?; + snapshot.ensure_candidate_hierarchy( + &entry.path, + matches!(&entry.kind, ManifestKind::Directory), + )?; + let id = next_id.ok_or(PithosError::FileIdExhausted)?; + ids.push(id); + next_id = id.checked_add(1); + } + Ok(ids) + } +} + +const DEFAULT_RELATIONSHIPS: [(u64, &str); 10] = [ + (0, "Describes"), + (1, "Annotates"), + (2, "Derived_From"), + (3, "Source_Of"), + (4, "Previous_Version"), + (5, "Next_Version"), + (6, "Part_of"), + (7, "Contains"), + (8, "Input_To"), + (9, "Output_From"), +]; + +fn manifest_entry( + source: PathBuf, + path: ArchivePath, + sources: &mut BTreeMap, + next_source_key: &mut u64, +) -> Result { + let host = fs::symlink_metadata(&source).map_err(|source_error| FsError::Host { + operation: "inspect input", + path: source.clone(), + source: source_error, + })?; + let (metadata, kind, source_key) = if host.file_type().is_dir() { + (metadata(&host), ManifestKind::Directory, None) + } else if host.file_type().is_symlink() { + let target = + symlink_target( + fs::read_link(&source).map_err(|source_error| FsError::Host { + operation: "read symlink", + path: source.clone(), + source: source_error, + })?, + )?; + validate_symlink_target(path.as_str(), &target).map_err(|source_error| FsError::Core { + operation: "validate symlink", + path: source.clone(), + source: source_error, + })?; + (metadata(&host), ManifestKind::Symlink { target }, None) + } else if host.file_type().is_file() { + let file = open_regular_file_no_follow(&source).map_err(|source_error| FsError::Host { + operation: "open input", + path: source.clone(), + source: source_error, + })?; + let opened = file.metadata().map_err(|source_error| FsError::Host { + operation: "inspect input", + path: source.clone(), + source: source_error, + })?; + if !same_file_identity(&host, &opened) { + return Err(FsError::UnsupportedEntry { + path: source, + kind: "entry changed while opening", + }); + } + let expected_size = opened.len(); + let source_key = *next_source_key; + *next_source_key = source_key.checked_add(1).ok_or_else(|| FsError::Core { + operation: "allocate source identity", + path: source.clone(), + source: PithosError::FileIdExhausted, + })?; + sources.insert(source_key, file); + ( + metadata(&opened), + ManifestKind::File { expected_size }, + Some(source_key), + ) + } else { + let kind = if host.file_type().is_fifo() { + "FIFO" + } else if host.file_type().is_socket() { + "socket" + } else if host.file_type().is_block_device() { + "block device" + } else if host.file_type().is_char_device() { + "character device" + } else { + "special entry" + }; + return Err(FsError::UnsupportedEntry { path: source, kind }); + }; + Ok(ManifestEntry { + source: Some(source), + source_key, + path, + metadata, + kind, + }) +} + +fn collect_directory_manifest( + directory: &Dir, + source_root: &Path, + prefix: &Path, + entries: &mut BTreeMap, + sources: &mut BTreeMap, + next_source_key: &mut u64, +) -> Result<(), FsError> { + let mut pending = vec![( + (directory.try_clone()).map_err(|source| FsError::Host { + operation: "retain input directory", + path: source_root.join(prefix), + source, + })?, + prefix.to_path_buf(), + )]; + while let Some((directory, prefix)) = pending.pop() { + let directory_path = source_root.join(&prefix); + let mut children = directory + .read_dir(".") + .map_err(|source| FsError::Host { + operation: "read input directory", + path: directory_path.clone(), + source, + })? + .collect::, _>>() + .map_err(|source| FsError::Host { + operation: "read input directory entry", + path: directory_path, + source, + })?; + children.sort_by_key(|entry| entry.file_name()); + let mut child_directories = Vec::new(); + for child in children { + let name = child.file_name(); + let relative = prefix.join(&name); + let source = source_root.join(&relative); + let path = archive_path(&relative)?; + let host = directory + .symlink_metadata(&name) + .map_err(|source_error| FsError::Host { + operation: "inspect input", + path: source.clone(), + source: source_error, + })?; + let entry = if host.file_type().is_dir() { + let opened = open_at_no_follow(&directory, Path::new(&name), true).map_err( + |source_error| FsError::Host { + operation: "open input directory", + path: source.clone(), + source: source_error, + }, + )?; + let opened_metadata = opened.metadata().map_err(|source_error| FsError::Host { + operation: "inspect opened input directory", + path: source.clone(), + source: source_error, + })?; + if !opened_metadata.file_type().is_dir() + || !same_cap_file_identity(&host, &opened_metadata) + { + return Err(FsError::UnsupportedEntry { + path: source, + kind: "entry changed while opening", + }); + } + child_directories.push((Dir::from_std_file(opened), relative)); + ManifestEntry { + source: Some(source), + source_key: None, + path, + metadata: metadata(&opened_metadata), + kind: ManifestKind::Directory, + } + } else if host.file_type().is_file() { + let file = open_at_no_follow(&directory, Path::new(&name), false).map_err( + |source_error| FsError::Host { + operation: "open input", + path: source.clone(), + source: source_error, + }, + )?; + let opened = file.metadata().map_err(|source_error| FsError::Host { + operation: "inspect opened input", + path: source.clone(), + source: source_error, + })?; + if !opened.file_type().is_file() || !same_cap_file_identity(&host, &opened) { + return Err(FsError::UnsupportedEntry { + path: source, + kind: "entry changed while opening", + }); + } + let source_key = *next_source_key; + *next_source_key = source_key.checked_add(1).ok_or_else(|| FsError::Core { + operation: "allocate source identity", + path: source.clone(), + source: PithosError::FileIdExhausted, + })?; + sources.insert(source_key, file); + ManifestEntry { + source: Some(source), + source_key: Some(source_key), + path, + metadata: metadata(&opened), + kind: ManifestKind::File { + expected_size: opened.len(), + }, + } + } else if host.file_type().is_symlink() { + let target = symlink_target(directory.read_link_contents(&name).map_err( + |source_error| FsError::Host { + operation: "read symlink", + path: source.clone(), + source: source_error, + }, + )?)?; + let current = + directory + .symlink_metadata(&name) + .map_err(|source_error| FsError::Host { + operation: "inspect read symlink", + path: source.clone(), + source: source_error, + })?; + if !current.file_type().is_symlink() + || current.dev() != host.dev() + || current.ino() != host.ino() + { + return Err(FsError::UnsupportedEntry { + path: source, + kind: "entry changed while reading symlink", + }); + } + validate_symlink_target(path.as_str(), &target).map_err(|source_error| { + FsError::Core { + operation: "validate symlink", + path: source.clone(), + source: source_error, + } + })?; + ManifestEntry { + source: Some(source), + source_key: None, + path, + metadata: cap_metadata(&host), + kind: ManifestKind::Symlink { target }, + } + } else { + let kind = if host.file_type().is_fifo() { + "FIFO" + } else if host.file_type().is_socket() { + "socket" + } else if host.file_type().is_block_device() { + "block device" + } else if host.file_type().is_char_device() { + "character device" + } else { + "special entry" + }; + return Err(FsError::UnsupportedEntry { path: source, kind }); + }; + insert_entry(entries, entry)?; + } + pending.extend(child_directories.into_iter().rev()); + } + Ok(()) +} + +fn insert_entry( + entries: &mut BTreeMap, + entry: ManifestEntry, +) -> Result<(), FsError> { + let path = entry.path.clone(); + if let Some(existing) = entries.get(&path) { + if matches!(&existing.kind, ManifestKind::Directory) + != matches!(&entry.kind, ManifestKind::Directory) + { + return Err(FsError::Core { + operation: "validate input hierarchy", + path: PathBuf::from(path.as_str()), + source: PithosError::InvalidArchivePath { + path: path.as_str().to_owned(), + reason: "directory conflicts with a non-directory entry".into(), + }, + }); + } + return Err(FsError::Core { + operation: "validate input hierarchy", + path: PathBuf::from(path.as_str()), + source: PithosError::PathOccupied(path.as_str().to_owned()), + }); + } + entries.insert(path, entry); + Ok(()) +} + +fn add_synthetic_parents( + entries: &mut BTreeMap, +) -> Result<(), FsError> { + let paths = entries.keys().cloned().collect::>(); + for path in paths { + let components = path.as_str().split('/').collect::>(); + for depth in 1..components.len() { + let parent = ArchivePath::new(components[..depth].join("/")).map_err(|source| { + FsError::Core { + operation: "construct synthetic parent", + path: PathBuf::from(path.as_str()), + source, + } + })?; + match entries.get(&parent) { + Some(ManifestEntry { + kind: ManifestKind::Directory, + .. + }) => {} + Some(_) => { + return Err(FsError::Core { + operation: "validate input hierarchy", + path: PathBuf::from(path.as_str()), + source: PithosError::InvalidArchivePath { + path: path.as_str().to_owned(), + reason: format!( + "non-directory entry {} is an ancestor", + parent.as_str() + ), + }, + }); + } + None => { + entries.insert( + parent.clone(), + ManifestEntry { + source: None, + source_key: None, + path: parent, + metadata: EntryMetadata::new(0, 0, 0o755), + kind: ManifestKind::Directory, + }, + ); + } + } + } + } + Ok(()) +} + +fn validate_hierarchy(entries: &BTreeMap) -> Result<(), FsError> { + let mut ordered = entries.iter(); + let Some((mut previous_path, mut previous_entry)) = ordered.next() else { + return Ok(()); + }; + for (path, entry) in ordered { + if !matches!(&previous_entry.kind, ManifestKind::Directory) + && previous_path.is_ancestor_of(path) + { + return Err(FsError::Core { + operation: "validate input hierarchy", + path: PathBuf::from(previous_path.as_str()), + source: PithosError::InvalidArchivePath { + path: previous_path.as_str().to_owned(), + reason: "a non-directory entry has descendants".into(), + }, + }); + } + previous_path = path; + previous_entry = entry; + } + Ok(()) +} + +/// Build and validate the complete create-side filesystem manifest without +/// opening an output sink. +pub fn build_input_manifest(inputs: &[PathBuf]) -> Result { + let mut manifest = BTreeMap::new(); + let mut sources = BTreeMap::new(); + let mut next_source_key = 0; + for source in inputs { + let root = fs::symlink_metadata(source).map_err(|source_error| FsError::Host { + operation: "inspect input", + path: source.clone(), + source: source_error, + })?; + // The root directory is not an archive entry, but its metadata must still + // be readable before output creation. + let _ = metadata(&root); + if root.file_type().is_dir() { + let directory = + open_directory_no_follow(source).map_err(|source_error| FsError::Host { + operation: "open input directory", + path: source.clone(), + source: source_error, + })?; + let opened = directory + .symlink_metadata(".") + .map_err(|source_error| FsError::Host { + operation: "inspect opened input directory", + path: source.clone(), + source: source_error, + })?; + if !opened.file_type().is_dir() + || root.dev() != opened.dev() + || root.ino() != opened.ino() + { + return Err(FsError::UnsupportedEntry { + path: source.clone(), + kind: "entry changed while opening", + }); + } + collect_directory_manifest( + &directory, + source, + Path::new(""), + &mut manifest, + &mut sources, + &mut next_source_key, + )?; + } else { + let name = source.file_name().ok_or_else(|| FsError::InvalidUtf8Path { + path: source.clone(), + })?; + let path = archive_path(Path::new(name))?; + insert_entry( + &mut manifest, + manifest_entry(source.clone(), path, &mut sources, &mut next_source_key)?, + )?; + } + } + add_synthetic_parents(&mut manifest)?; + validate_hierarchy(&manifest)?; + Ok(InputManifest { + entries: manifest.into_values().collect(), + sources, + }) +} + +/// Ingest one filesystem entry using its validated archive path. +pub fn ingest_path( + writer: &mut ArchiveWriter, + source: &Path, + path: ArchivePath, + processing: ProcessingOptions, +) -> Result<(), FsError> { + let mut sources = BTreeMap::new(); + let entry = manifest_entry(source.to_path_buf(), path, &mut sources, &mut 0)?; + InputManifest { + entries: vec![entry], + sources, + } + .ingest(writer, processing) + .map_err(|source| FsError::Writer { + operation: "ingest input", + archive_path: "input".to_owned(), + source, + }) +} + +/// Ingest a directory recursively after validating its complete manifest. +pub fn ingest_directory( + writer: &mut ArchiveWriter, + source: &Path, + processing: ProcessingOptions, +) -> Result<(), FsError> { + let archive_path = source.display().to_string(); + build_input_manifest(&[source.to_path_buf()])? + .ingest(writer, processing) + .map_err(|source| FsError::Writer { + operation: "ingest directory", + archive_path, + source, + }) +} + +/// Validate the complete create-side filesystem manifest without opening an +/// output sink. Prefer [`build_input_manifest`] when the manifest will be used. +pub fn validate_input_manifest(inputs: &[PathBuf]) -> Result<(), FsError> { + build_input_manifest(inputs).map(|_| ()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn synthetic_parents_are_deterministic() { + let path = ArchivePath::new("nested/child/file").unwrap(); + let mut entries = BTreeMap::new(); + entries.insert( + path.clone(), + ManifestEntry { + source: Some(PathBuf::from("file")), + source_key: None, + path, + metadata: EntryMetadata::new(1, 2, 0o644), + kind: ManifestKind::File { expected_size: 0 }, + }, + ); + + add_synthetic_parents(&mut entries).unwrap(); + + let paths = entries.keys().map(|path| path.as_str()).collect::>(); + assert_eq!(paths, ["nested", "nested/child", "nested/child/file"]); + for path in ["nested", "nested/child"] { + let entry = entries.get(&ArchivePath::new(path).unwrap()).unwrap(); + assert!(entry.source.is_none()); + assert!(matches!(&entry.kind, ManifestKind::Directory)); + assert_eq!(entry.metadata, EntryMetadata::new(0, 0, 0o755)); + } + } +} diff --git a/crates/pithos_lib/src/io/pithoswriter.rs b/crates/pithos_lib/src/io/pithoswriter.rs deleted file mode 100644 index 2e9ac4a..0000000 --- a/crates/pithos_lib/src/io/pithoswriter.rs +++ /dev/null @@ -1,729 +0,0 @@ -use crate::error::PithosError; -use crate::helpers::archive_path::{validate_map, validate_new_candidate}; -use crate::helpers::directory::DirectoryBuilder; -use crate::helpers::file_entry_map::{FileEntryMap, Key}; -use crate::helpers::hash::{Hasher, Hashes}; -use crate::helpers::ro_crate::{ - LoadedRoCrate, RO_CRATE_METADATA_FILE, RoCrateSource, ZipEntryDescriptor, ZipEntryKind, - inspect_ro_crate_zip_manifest, -}; -use crate::helpers::zstd::{ZstdError, map_to_zstd_level}; -use crate::io::pithosreader::{PithosReaderSimple, ReaderLimits}; -use crate::io::util::{create_stream_cdc, extract_filename}; -use crate::model::serialization::SerializationError; -use crate::model::structs::{ - BlockHeader, BlockIndexEntry, BlockLocation, Directory, EncryptionSection, FileEntry, FileType, - ProcessingFlags, Reference, -}; -use crate::{ - helpers::{ - chacha_poly1305::encrypt_chunk, - zstd::{compress_data, probe_compression_ratio}, - }, - model::structs::FileHeader, -}; -use fastcdc::v2020::ChunkData; -use indexmap::IndexMap; -use std::fs; -use std::fs::{File, symlink_metadata}; -use std::io::{Cursor, Read, Write}; -use std::path::{Path, PathBuf}; -use x25519_dalek::{PublicKey, StaticSecret}; -use zip::ZipArchive; - -#[derive(Debug)] -pub enum Content { - File(String), // Path to file with content - Raw(String), // Raw string content - Reference(Reference), // Reference to already existing file entry (Data -> copies BlockIndex; Metadata -> ) -} - -#[derive(Debug)] -pub struct InputFile { - pub file_type: FileType, - pub data: Content, - pub metadata: Option, - pub inner_path: String, // Internal path - pub encrypt: bool, - pub compression_level: Option, -} - -impl TryFrom<&PathBuf> for InputFile { - type Error = PithosError; - - fn try_from(file_path: &PathBuf) -> Result { - let metadata = symlink_metadata(file_path)?; - let file_type = FileType::try_from(&metadata)?; - let file_path_str = file_path - .to_str() - .ok_or(PithosError::Conversion( - "Invalid UTF-8 in file path".to_string(), - ))? - .to_string(); - let inner_path = match &file_type { - FileType::Directory => file_path_str.as_str(), - _ => extract_filename(&file_path_str).ok_or(PithosError::Conversion( - "Input file is missing file name.".to_string(), - ))?, - }; - - Ok(InputFile { - file_type, - inner_path: inner_path.to_string(), - data: if file_type == FileType::Data - || file_type == FileType::Directory - || file_type == FileType::Symlink - { - Content::File(file_path_str) - } else { - Content::Raw("".to_string()) - }, - metadata: None, - encrypt: true, - compression_level: Some(2), - }) - } -} - -fn collect_directory_entries(directory: &Path) -> Result, PithosError> { - if !directory.is_dir() { - return Err(PithosError::Conversion(format!( - "Input path is not a directory: {}", - directory.display() - ))); - } - - let mut entries = Vec::new(); - for entry in walkdir::WalkDir::new(directory) { - let entry = entry?; - let relative_path = entry.path().strip_prefix(directory)?; - - if relative_path.as_os_str().is_empty() { - continue; - } - - let inner_path = relative_path - .components() - .map(|component| { - component - .as_os_str() - .to_str() - .ok_or(PithosError::Conversion( - "Invalid UTF-8 in file path".to_string(), - )) - }) - .collect::, _>>()? - .join("/"); - let file_type = FileType::try_from(&symlink_metadata(entry.path())?)?; - let file_path = entry - .path() - .to_str() - .ok_or(PithosError::Conversion( - "Invalid UTF-8 in file path".to_string(), - ))? - .to_string(); - - entries.push(InputFile { - file_type, - inner_path, - data: Content::File(file_path), - metadata: None, - encrypt: true, - compression_level: Some(2), - }); - } - - entries.sort_by(|a, b| { - a.file_type - .cmp(&b.file_type) - .then_with(|| { - a.inner_path - .to_lowercase() - .cmp(&b.inner_path.to_lowercase()) - }) - .then_with(|| a.inner_path.cmp(&b.inner_path)) - }); - - Ok(entries) -} - -fn file_entry_from_ro_crate_zip_descriptor(descriptor: &ZipEntryDescriptor) -> FileEntry { - let file_type = match descriptor.kind { - ZipEntryKind::Directory => FileType::Directory, - ZipEntryKind::File => FileType::Data, - ZipEntryKind::Symlink => FileType::Symlink, - }; - - FileEntry::new_from_archive( - file_type, - descriptor.uncompressed_size, - descriptor.timestamp, - descriptor.timestamp, - descriptor.permissions, - None, - ) -} - -pub struct PithosWriter { - // Input - writer_key: StaticSecret, //TODO: Multiple sender keys for individual EncryptionSections - cdc: Option<(usize, usize, usize)>, - sink: Box, - - // Processing - directory: Directory, // Single or merged from multiple - ancestor_blocks: IndexMap<[u8; 32], BlockIndexEntry>, - written_bytes: u64, -} - -impl PithosWriter { - #[tracing::instrument(level = "trace", skip(writer_key, reader_keys, cdc, sink))] - pub fn new( - writer_key: StaticSecret, - reader_keys: Vec, - cdc: Option<(usize, usize, usize)>, - sink: Box, - ) -> Result { - // Init encryption section - let encryption_sections = IndexMap::from_iter([( - PublicKey::from(&writer_key).to_bytes(), - EncryptionSection::new(&reader_keys), - )]); - - Ok(PithosWriter { - writer_key, - cdc, - sink, - directory: DirectoryBuilder::new() - .encryption(encryption_sections) - .build()?, - ancestor_blocks: IndexMap::new(), - written_bytes: 0, - }) - } - - #[tracing::instrument(level = "trace", skip(writer_key, reader_keys, cdc, pithos_file))] - pub fn new_from_file>( - writer_key: StaticSecret, - reader_keys: Vec, - cdc: Option<(usize, usize, usize)>, - pithos_file: P, - ) -> Result { - Self::new_from_file_with_limits( - writer_key, - reader_keys, - cdc, - pithos_file, - ReaderLimits::default(), - ) - } - - #[tracing::instrument( - level = "trace", - skip(writer_key, reader_keys, cdc, pithos_file, limits) - )] - pub fn new_from_file_with_limits>( - writer_key: StaticSecret, - reader_keys: Vec, - cdc: Option<(usize, usize, usize)>, - pithos_file: P, - limits: ReaderLimits, - ) -> Result { - // Read existing directories - let mut reader = - PithosReaderSimple::new_with_key(&pithos_file, writer_key.clone())?.with_limits(limits); - let (directory, offset) = reader.read_directory()?; - - // Open Pithos file in append write mode - let file = fs::OpenOptions::new() - //.read(true) //? - .append(true) - .open(pithos_file)?; - let written_bytes = file.metadata()?.len(); - let sink = Box::new(file); - - Ok(PithosWriter { - directory: DirectoryBuilder::new() - .parent_directory_offset(Some(offset)) - .files(FileEntryMap::new_with_max( - directory.files.get_current_max_id(), - )) - .encryption(IndexMap::from_iter([( - PublicKey::from(&writer_key).to_bytes(), - EncryptionSection::new(&reader_keys), - )])) - .build()?, - ancestor_blocks: directory.blocks.clone(), - written_bytes, - writer_key, - cdc, - sink, - }) - } - - pub fn get_directory_mut(&mut self) -> &mut Directory { - &mut self.directory - } - - /* ----- Processing ---------- */ - /// Processes a single data chunk (block) for storage. - /// - /// # Arguments - /// * `chunk` - Mutable reference to the `ChunkData` containing the block's raw data. - /// * `processing_flags` - Reference to `ProcessingFlags` specifying compression and encryption options. - /// - /// # Returns - /// Returns a tuple containing: - /// * `BlockIndexEntry` - Metadata about the processed block for indexing. - /// * `[u8; 32]` - The SHAKE256 hash of the block data. - /// - /// # Errors - /// Returns `PithosWriterError` if any step fails. - #[tracing::instrument(level = "trace", skip(self, chunk, processing_flags))] - pub fn process_block( - &mut self, - chunk: &mut ChunkData, - processing_flags: &ProcessingFlags, - ) -> Result<(BlockIndexEntry, Hashes, bool), PithosError> { - // Calculate block hashes - let mut hasher = Hasher::new(); - hasher.update(&chunk.data); - let hashes = hasher.finalize(); - - // Check if block already exists in directory - if let Some(entry) = self.directory.block_hash_exists(&hashes.blake3) { - // Return already existing block entry - return Ok((entry, hashes, true)); - } - if let Some(entry) = self.ancestor_blocks.get(hashes.blake3.as_bytes()) { - return Ok((entry.clone(), hashes, true)); - } - - // Init BlockIndexEntry - let mut block_index_entry = BlockIndexEntry { - offset: self.written_bytes, - stored_size: chunk.data.len() as u64, - original_size: chunk.length as u64, - flags: *processing_flags, - location: BlockLocation::Local, //TODO: Remote files - }; - - // Compression - let compression_level = map_to_zstd_level(processing_flags); - if compression_level > 0 - && probe_compression_ratio(&chunk.data, Some(compression_level))? < 0.85 - { - chunk.data = compress_data(chunk.data.as_slice(), Some(compression_level))?; - } else { - // No compression, as the input is likely to have high entropy - block_index_entry.flags.set_compression_level(0); - } - - // Encryption - if processing_flags.is_encrypted() { - chunk.data = encrypt_chunk(chunk.data.as_slice(), b"", &hashes.shake256)?; - } - - // Update stored size to processed block length - block_index_entry.stored_size = chunk.data.len() as u64; - - Ok((block_index_entry, hashes, false)) - } - - /// Processes an entire file and adds it to the directory index. - /// - /// This function splits the file content into chunks using FastCDC, processes each chunk, - /// writes the processed blocks to the sink, and updates the file entry and directory index - /// accordingly. After all chunks are processed, the block index for the file is encrypted and - /// the file entry is added to the directory. A reference to the processed file is returned. - /// - /// # Arguments - /// * `file_entry` - Mutable reference to a `FileEntry` representing the file's metadata and block index. - /// * `processing_flags` - Reference to `ProcessingFlags` specifying compression and encryption options. - /// * `content` - Reader representing the file's content stream. - /// - /// # Returns - /// Returns a `Reference` struct describing the relationship and ID of the processed file. - /// - /// # Errors - /// Returns `PithosWriterError` if any step fails. - #[tracing::instrument(level = "trace", skip(self, file_entry, processing_flags, content))] - pub fn process_file_entry( - &mut self, - entry_path: &str, - file_entry: &mut FileEntry, - processing_flags: &ProcessingFlags, - content: R, - ) -> Result { - validate_new_candidate(&self.directory.files, entry_path, file_entry)?; - self.process_file_entry_prevalidated(entry_path, file_entry, processing_flags, content) - } - - fn process_file_entry_prevalidated( - &mut self, - entry_path: &str, - file_entry: &mut FileEntry, - processing_flags: &ProcessingFlags, - content: R, - ) -> Result { - // Directory or Symlink FileEntry are just added to Pithos directory - let file_entry_key = Key::new( - self.directory.next_free_file_index()?, - entry_path.to_string(), - ); - - if [FileType::Directory, FileType::Symlink].contains(&file_entry.file_type) { - self.directory - .add_file_to_index(&file_entry_key, file_entry)?; - - return Reference::try_from((&file_entry_key, file_entry)); - } - - // Split content in chunks - let fastcdc_stream = create_stream_cdc(content, self.cdc); - - // Iterate over CDC blocks - for result in fastcdc_stream { - // Process chunk - let mut chunk = result?; - let (block_entry, hashes, deduplicated) = - self.process_block(&mut chunk, processing_flags)?; - - // Add block to file entry - file_entry.add_block_data((*hashes.blake3.as_bytes(), hashes.shake256))?; - if !deduplicated { - // If block is no duplicate write block into sink and add to directory index - self.write_block(&chunk.data)?; - self.directory - .add_block_to_index(*hashes.blake3.as_bytes(), block_entry)?; - } - } - - // Create random key and encrypt file block index - let enc_key = StaticSecret::random().to_bytes(); - file_entry.block_data.encrypt(enc_key)?; - - // Add file entry to directory - self.directory - .add_file_to_index(&file_entry_key, file_entry)?; - self.directory - .add_file_to_all_recipients((file_entry_key.id(), enc_key)); - - // Return reference according to FileType - Reference::try_from((&file_entry_key, file_entry)) - } - - #[tracing::instrument(level = "trace", skip(self, input))] - pub fn process_input(&mut self, input: InputFile) -> Result { - let data_check = FileEntry::new_from_content(input.file_type, &input.data)?; - validate_new_candidate(&self.directory.files, &input.inner_path, &data_check)?; - if let Some(metadata) = &input.metadata - && !matches!(metadata, Content::Reference(_)) - { - let metadata_check = FileEntry::new_from_content(FileType::Metadata, metadata)?; - validate_new_candidate( - &self.directory.files, - &format!("{}.meta", input.inner_path), - &metadata_check, - )?; - } - // Create FileEntry with its ProcessingFlags from data file input - let mut data_file = data_check; - let processing_flags = ProcessingFlags::new(input.encrypt, input.compression_level); - - // First process metadata to add reference - if let Some(metadata) = input.metadata { - let meta_file_path = &format!("{}.meta", input.inner_path); - - let reference = match &metadata { - Content::File(disk_path) => { - let mut meta_file = FileEntry::new_from_content(FileType::Metadata, &metadata)?; - let handle = File::open(disk_path)?; - self.process_file_entry_prevalidated( - meta_file_path, - &mut meta_file, - &processing_flags, - handle, - )? - } - Content::Raw(raw_content) => { - let mut meta_file = FileEntry::new_from_content(FileType::Metadata, &metadata)?; - let handle = Cursor::new(raw_content.clone().into_bytes()); - self.process_file_entry_prevalidated( - meta_file_path, - &mut meta_file, - &processing_flags, - handle, - )? - } - Content::Reference(reference) => reference.clone(), - }; - - data_file.references.push(reference); - } - - // Process data FileEntry - let data_reference = match input.data { - Content::File(disk_path) - if [FileType::Data, FileType::Metadata].contains(&input.file_type) => - { - let handle = File::open(disk_path)?; - self.process_file_entry_prevalidated( - &input.inner_path, - &mut data_file, - &processing_flags, - handle, - )? - } - Content::File(_) => self.process_file_entry_prevalidated( - &input.inner_path, - &mut data_file, - &processing_flags, - Cursor::new(Vec::::new()), - )?, - Content::Raw(raw_content) => { - let handle = Cursor::new(raw_content.into_bytes()); - self.process_file_entry_prevalidated( - &input.inner_path, - &mut data_file, - &processing_flags, - handle, - )? - } - Content::Reference(reference) => { - // Clone content (block_data) into new FileEntry - let ref_fe = self - .directory - .get_file_by_id(reference.target_file_id) - .ok_or(PithosError::FileNotFound(format!( - "Could not find reference. FileEntry with id {} does not exist.", - reference.target_file_id - )))?; - data_file.block_data = ref_fe.block_data.clone(); - - // Fetch encryption key of referenced file - // Add file entry to directory and make it available for all recipients - if let Some(enc_key) = self - .directory - .get_file_encryption_key(reference.target_file_id) - { - let file_entry_key = - Key::new(self.directory.next_free_file_index()?, &input.inner_path); - self.directory.add_file(&input.inner_path, &data_file)?; - self.directory - .add_file_to_all_recipients((file_entry_key.id(), enc_key)); - - Reference::try_from((&file_entry_key, &mut data_file))? - } else { - return Err(PithosError::FileNotFound(format!( - "Could not extract key for file: {}", - input.inner_path - ))); - } - } - }; - - Ok(data_reference) - } - - #[tracing::instrument(level = "trace", skip(self, files))] - pub fn process_input_files(&mut self, files: Vec) -> Result<(), PithosError> { - for file in files { - tracing::info!("Processing [{:?}] {}", file.file_type, file.inner_path); - match file.file_type { - FileType::Directory => self.process_directory(file.inner_path)?, - _ => { - self.process_input(file)?; - } - } - } - Ok(()) - } - - #[tracing::instrument(level = "trace", skip(self, directory))] - pub fn process_directory>(&mut self, directory: P) -> Result<(), PithosError> { - for entry in collect_directory_entries(directory.as_ref())? { - tracing::info!("Processing [{:?}] {}", entry.file_type, entry.inner_path); - self.process_input(entry)?; - } - - Ok(()) - } - - #[tracing::instrument(level = "trace", skip(self, directories))] - pub fn process_directories>( - &mut self, - directories: Vec

, - ) -> Result<(), PithosError> { - for directory in directories { - self.process_directory(directory)? - } - - Ok(()) - } - - #[tracing::instrument(level = "trace", skip(self, block, compression_level))] - pub fn compress_block( - &self, - mut block: Vec, - compression_level: u8, - ) -> Result, ZstdError> { - // Compress chunk - let chunk_sample = if block.len() > 8192 { - &block[..8192] - } else { - &block - }; - - if probe_compression_ratio(chunk_sample, Some(compression_level as i32))? < 0.85 { - block = compress_data(&block, Some(compression_level as i32))?; - } - Ok(block) - } - - #[tracing::instrument(level = "trace", skip(self))] - pub fn write_file_header(&mut self) -> Result<(), SerializationError> { - FileHeader::default().serialize(&mut self.sink)?; - self.written_bytes = self.written_bytes.saturating_add(6); - Ok(()) - } - - #[tracing::instrument(level = "trace", skip(self, block))] - pub fn write_block(&mut self, block: &[u8]) -> Result<(), SerializationError> { - // Write BlockHeader - BlockHeader::default().serialize(&mut self.sink)?; - self.written_bytes = self.written_bytes.saturating_add(4); // Add BlockHeader length - - // Write block data - self.sink.write_all(block)?; - self.written_bytes = self.written_bytes.saturating_add(block.len() as u64); - - Ok(()) - } - - #[tracing::instrument(level = "trace", skip(self))] - pub fn write_directory(&mut self) -> Result<(), PithosError> { - validate_map(&self.directory.files)?; - // Encrypt recipients of writer section - self.directory.encrypt_recipients(&self.writer_key)?; - - // Update len and crc32 - self.directory.update_len()?; - self.directory.update_crc32()?; - - // Write directory - self.directory.serialize(&mut self.sink)?; - - Ok(()) - } - - fn process_ro_crate_directory(&mut self, directory: &Path) -> Result<(), PithosError> { - let mut entries = collect_directory_entries(directory)?; - let metadata_index = entries - .iter() - .position(|entry| entry.inner_path == RO_CRATE_METADATA_FILE) - .ok_or_else(|| PithosError::MissingRoCrateMetadata(directory.to_path_buf()))?; - - if entries[metadata_index].file_type != FileType::Data { - return Err(PithosError::InvalidRoCrateSource { - path: directory.join(RO_CRATE_METADATA_FILE), - expected: "regular metadata file", - }); - } - - let mut metadata = entries.remove(metadata_index); - metadata.file_type = FileType::Metadata; - let metadata_reference = self.process_input(metadata)?; - - for mut entry in entries { - if entry.file_type == FileType::Data { - entry.metadata = Some(Content::Reference(metadata_reference.clone())); - } - self.process_input(entry)?; - } - - Ok(()) - } - - fn process_ro_crate_zip(&mut self, path: &Path) -> Result<(), PithosError> { - let manifest = inspect_ro_crate_zip_manifest(path)?; - let metadata = manifest.metadata.clone(); - let metadata_index = metadata - .archive_index - .ok_or_else(|| PithosError::MissingRoCrateMetadata(path.to_path_buf()))?; - let mut archive = ZipArchive::new(File::open(path)?)?; - let processing_flags = ProcessingFlags::new(true, Some(2)); - - let mut metadata_file_entry = file_entry_from_ro_crate_zip_descriptor(&metadata); - metadata_file_entry.file_type = FileType::Metadata; - let metadata_reference = { - let member = archive.by_index(metadata_index)?; - self.process_file_entry( - &metadata.inner_path, - &mut metadata_file_entry, - &processing_flags, - member, - )? - }; - - for descriptor in manifest.entries { - let mut file_entry = file_entry_from_ro_crate_zip_descriptor(&descriptor); - if file_entry.file_type == FileType::Data { - file_entry.references.push(metadata_reference.clone()); - } - - match descriptor.kind { - ZipEntryKind::Directory => { - self.process_file_entry( - &descriptor.inner_path, - &mut file_entry, - &processing_flags, - Cursor::new(Vec::::new()), - )?; - } - ZipEntryKind::File => { - let index = descriptor.archive_index.ok_or_else(|| { - PithosError::UnsupportedZipEntry(descriptor.inner_path.clone()) - })?; - { - let member = archive.by_index(index)?; - self.process_file_entry( - &descriptor.inner_path, - &mut file_entry, - &processing_flags, - member, - )?; - } - } - ZipEntryKind::Symlink => { - let index = descriptor.archive_index.ok_or_else(|| { - PithosError::UnsupportedZipEntry(descriptor.inner_path.clone()) - })?; - let target = { - let mut member = archive.by_index(index)?; - let mut target = String::new(); - member.read_to_string(&mut target)?; - target - }; - file_entry.symlink_target = Some(target); - self.process_file_entry( - &descriptor.inner_path, - &mut file_entry, - &processing_flags, - Cursor::new(Vec::::new()), - )?; - } - } - } - - Ok(()) - } - - #[tracing::instrument(level = "trace", skip(self, loaded))] - pub fn process_ro_crate(&mut self, loaded: &LoadedRoCrate) -> Result<(), PithosError> { - match &loaded.source { - RoCrateSource::Directory(path) => self.process_ro_crate_directory(path), - RoCrateSource::Zip(path) => self.process_ro_crate_zip(path), - } - } -} diff --git a/crates/pithos_lib/src/io/util.rs b/crates/pithos_lib/src/io/util.rs deleted file mode 100644 index 1194c73..0000000 --- a/crates/pithos_lib/src/io/util.rs +++ /dev/null @@ -1,179 +0,0 @@ -use crate::error::PithosError; -use crate::helpers::archive_path::{validate_entry_path, validate_symlink_target}; -use crate::io::extraction::ExtractionRoot; -use fastcdc::v2020::{Normalization, StreamCDC}; -use std::env::current_dir; -use std::io::Read; -use std::os::fd::AsRawFd; -use std::path::{Path, PathBuf}; -use std::time::SystemTime; - -#[tracing::instrument(level = "trace", skip(path))] -pub fn extract_filename(path: &str) -> Option<&str> { - Path::new(path).file_name()?.to_str() -} - -#[tracing::instrument(level = "trace")] -pub fn current_timestamp() -> Result { - Ok(SystemTime::now() - .duration_since(SystemTime::UNIX_EPOCH)? - .as_secs()) -} - -#[tracing::instrument(level = "trace", skip(file))] -pub fn get_symlink_target(file: &std::fs::File) -> Result { - let fd = file.as_raw_fd(); - let proc_path = format!("/proc/self/fd/{fd}"); - Ok(std::fs::read_link(proc_path)?.to_string_lossy().to_string()) -} - -#[tracing::instrument(level = "trace", skip(path, base_dir))] -pub fn create_dir(path: &str, base_dir: Option<&PathBuf>) -> Result<(), PithosError> { - validate_entry_path(path)?; - let root = base_dir.cloned().unwrap_or(current_dir()?); - ExtractionRoot::open(&root, true)?.create_dir(path) -} - -#[tracing::instrument(level = "trace", skip(path, target, base_dir))] -pub fn create_symlink( - path: &str, - target: &str, - base_dir: Option<&PathBuf>, -) -> Result<(), PithosError> { - validate_entry_path(path)?; - validate_symlink_target(path, target)?; - let root = base_dir.cloned().unwrap_or(current_dir()?); - ExtractionRoot::open(&root, true)?.create_symlink(path, target) -} - -#[tracing::instrument(level = "trace", skip(content, cdc))] -pub fn create_stream_cdc(content: R, cdc: Option<(usize, usize, usize)>) -> StreamCDC { - match cdc { - Some((min, avg, max)) => { - StreamCDC::with_level(content, min, avg, max, Normalization::Level1) - } - None => StreamCDC::with_level( - content, - fastcdc::v2020::MINIMUM_MAX, - fastcdc::v2020::AVERAGE_MAX, - fastcdc::v2020::MAXIMUM_MAX, - Normalization::Level1, - ), - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::fs::{self, File}; - use std::io::Cursor; - use std::time::UNIX_EPOCH; - use tempfile::TempDir; - - #[test] - fn extract_filename_handles_paths_and_roots() { - assert_eq!(extract_filename("file.txt"), Some("file.txt")); - assert_eq!(extract_filename("nested/file.txt"), Some("file.txt")); - assert_eq!(extract_filename("/"), None); - } - - #[test] - fn current_timestamp_is_current() { - let before = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_secs(); - let timestamp = current_timestamp().unwrap(); - let after = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_secs(); - - assert!((before..=after).contains(×tamp)); - } - - #[test] - fn get_symlink_target_reports_the_open_file() { - let temp_dir = TempDir::new().unwrap(); - let target = temp_dir.path().join("target.txt"); - let link = temp_dir.path().join("link.txt"); - fs::write(&target, b"content").unwrap(); - std::os::unix::fs::symlink(&target, &link).unwrap(); - - let file = File::open(&link).unwrap(); - assert_eq!(get_symlink_target(&file).unwrap(), target.to_string_lossy()); - } - - #[test] - fn create_dir_creates_nested_paths_with_a_base_directory() { - let temp_dir = TempDir::new().unwrap(); - let base_dir = temp_dir.path().join("base"); - create_dir("nested/path", Some(&base_dir)).unwrap(); - assert!(base_dir.join("nested/path").is_dir()); - } - - #[test] - fn create_symlink_preserves_relative_targets_at_the_requested_path() { - let temp_dir = TempDir::new().unwrap(); - let base_dir = temp_dir.path().join("output"); - fs::create_dir(&base_dir).unwrap(); - - create_dir("nested", Some(&base_dir)).unwrap(); - create_symlink("nested/link", "../target.txt", Some(&base_dir)).unwrap(); - - let link = base_dir.join("nested/link"); - assert!( - fs::symlink_metadata(&link) - .unwrap() - .file_type() - .is_symlink() - ); - assert_eq!(fs::read_link(link).unwrap(), PathBuf::from("../target.txt")); - } - - #[test] - fn create_symlink_rejects_absolute_targets() { - let temp_dir = TempDir::new().unwrap(); - let base_dir = temp_dir.path().join("output"); - let target = temp_dir.path().join("outside-target"); - fs::create_dir(&base_dir).unwrap(); - - assert!(create_symlink("link", target.to_str().unwrap(), Some(&base_dir)).is_err()); - assert!(fs::symlink_metadata(temp_dir.path().join("outside-target")).is_err()); - } - - #[test] - fn create_stream_cdc_respects_custom_limits_and_preserves_content() { - let content = (0..4096) - .map(|index| (index % 251) as u8) - .collect::>(); - let chunks = create_stream_cdc(Cursor::new(content.clone()), Some((64, 256, 1024))) - .collect::, _>>() - .unwrap(); - - let mut reconstructed = Vec::new(); - let mut next_offset = 0; - for chunk in chunks { - assert_eq!(chunk.offset, next_offset); - assert_eq!(chunk.length, chunk.data.len()); - assert!(chunk.length <= 1024); - next_offset += chunk.length as u64; - reconstructed.extend_from_slice(&chunk.data); - } - - assert_eq!(reconstructed, content); - } - - #[test] - fn create_stream_cdc_defaults_preserve_content() { - let content = b"small stream".to_vec(); - let reconstructed = create_stream_cdc(Cursor::new(content.clone()), None) - .collect::, _>>() - .unwrap() - .into_iter() - .flat_map(|chunk| chunk.data) - .collect::>(); - - assert_eq!(reconstructed, content); - } -} diff --git a/crates/pithos_lib/tests/data/SRR33138449.sample.fastq b/crates/pithos_lib/tests/data/SRR33138449.sample.fastq deleted file mode 100644 index 9856756..0000000 --- a/crates/pithos_lib/tests/data/SRR33138449.sample.fastq +++ /dev/null @@ -1,2500 +0,0 @@ -@SRR33138449.1 1 length=250 -AAGTGAGGCTACCCTATGCCTGAACACCATAAGGTTAGCCTCTTACTCGTTGGAAATCAACACAGGGGGCTGGGAATGCCGCAGAAATATAGATTACTTTCTTTAATAGTGATTTGTTGCACGCTTTTATTTTTCACCTGGATGATAAGAGATTCACTGTGTGAATTGCATATTAAACAGGGGAGTTATGAGCTGGCGGCATTTTTAGCCTGCAATTTGAAAGAGTAAGAGTCTTTGGCGGGAAGTTATT -+SRR33138449.1 1 length=250 -BCDCCFFFCCFFGGGGGGGGGGHHHHHGHHHHHHHHHHHHHHHHHHHHGHGHHHHHHIHHHHGHHHGGGGGGGGHHHHHHGGGGGHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHGGGGGHHHHHHHHHHHHHHHHHHHHHHGHHHHHHHHHHHHHHHHHHGHHHHHHHHHHHHGGGGGHGHHHHHHHFGGEGGGGGGHGEFFGGGGGGGGGGGGGGGFGFGFGGGGGFFFFAFF@FFFFFFF -@SRR33138449.1 1 length=251 -GCTCCGAACCATTCCTGTTCCCGATTAAAAACTTATCTAATAGCCAAATCCATTCGTATTTAAAAAAACATTCATTAAAGAAATAAATTTGGGTGCAATGAGAATGCGCAACGCCGTAAGAAAGGCGGGAATAACTTCCCGCCAAAGACTCTTACTCTTTCAAATTGCAGGCTAAAAATGCCGCCAGCTCATAACTCCCCTGTTTAATATGCAATTCACACAGTGAATCTCTTATCATCCAGGTGAAAAAA -+SRR33138449.1 1 length=251 -?AABB@@ABBFFGGGGGFGGGGGGGGGHHHHGHHHHHGHHHHHHHHHGHHHGHHHHHHHHHHBDGHGGGHHHHHHHHHHHHHHGHHFHHHHHGGGGHGHGHHGHGHGGGGGGGGGGFGGHHHHHFECCCCGFHHFHHHGBDGGGGHHHFHHEGHFFGHHGGFHH:G:GHFHGGBB;FGFFEG;BA-@EFFBFFF9FF9EFFEBFFFF/BFBBF/;BFFFFED9/BFFF///BF/F/9BFE9/B///;BBF- -@SRR33138449.2 2 length=251 -AGCCAAATCCATTCGTATTTAAAAAAACATTCATTAAAGAAATAAATTTGGGTGCAATGAGAATGCGCAACGCCGTAAGAAAGGCGGGAATAACTTCCCGCCAAAGACTCTTACTCTTTCAAATTGCAGGCTAAAAATGCCGCCAGCTCATAACTCCCCTGTTTAATATGCAATTCACACAGTGAATCTCTTATCATCCAGGTGAAAAATAAAAGCGTGCAACAAATCACTATTAAAGAAAGTAATCTATA -+SRR33138449.2 2 length=251 -BBBCCFFFFFFFGGGGGGGGGGHHHGGGHHHHHHHHHHHHHHHHHHHHHHHGGGGHHHHHHHHHHHGGGGGGGGGGGGHHHHHFGGGGGGHHHHHHHHHGGGGGGHHHHHHHGHHHHHHHHHHHHFHFBGEHHFGGHHHHHFGG?CGHHGHHHHHGHHGHGHHHGHHHHFHGGGGGGGGGGGFGFGGGGGGGGGGGGGGBFFFFGGGGGFFFFFBFDFFFEFF?EFFFFFFFFFFFFFFFFFFFFFFFFF0 -@SRR33138449.2 2 length=251 -GTACAAAGTGAGGCTACCCTATGCCTGAACACCATAAGGTTAGCCTCTTACTCGTTGGAAATCAACACAGGGGGCTGGGAATGCCGCAGAAATATAGATTACTTTCTTTAATAGTGATTTGTTGCACGCTTTTATTTTTCACCTGGATGATAAGAGATTCACTGTGTGAATTGCATATTAAACAGGGGAGTTATGAGCTGGCGGCATTTTTAGCCTGCAATTTGAAAGAGTAAGAGTCTTTGGCGGGAAGT -+SRR33138449.2 2 length=251 -ABBBBFFFFFFFGFGGGGGGGGHHHHFFHHHHCHHHHHGHGHFHHHHHHHHHHCHGHHHHHHHHHHHGHHGGGFGGGGDGFGHHHGGGGCGHHHHHHGHGHHHHHHHHHHGHHHHHHGHHHHHHHHHFGGGGHHHHHHHHHHHGHHFHGHHHHHGFHHHGHHEHGBFFBDGHFFGHDGHGD1GGFEG?E@DHHHHBGHHG/EGGGFGGGGGEFBFGGGFFFFF09CFFFFGFFFF//9:BFFBFBFD-9./ -@SRR33138449.3 3 length=251 -GTACAAAGTGAGGCTACCCTATGCCTGAACACCATAAGGTTAGCCTCTTACTCGTTGGAAATCAACACAGGGGGCTGGGAATGCCGCAGAAATATAGATTACTTTCTTTAATAGTGATTTGTTGCACGCTTTTATTTTTCACCTGGATGATAAGAGATTCACTGTGTGAATTGCATATTAAACAGGGGAGTTATGAGCTGGCGGCATTTTTAGCCTGCAATTTGAAAGAGTAAGAGTCTTTGGCGGGAAGT -+SRR33138449.3 3 length=251 -AAABBFFFBBFB4FGGGGGEGGBHFHHFHHHHGHHHHFHFFGFHHHHHHHHHHGHGGHHGHHHGFHFGHCGFGGCGGGGHBHFHHGGECGFHHHEHHGHHHHGHHHHHHHDHFHGHHHHHHHHHGFHFGGGGHHGHFGHHHHHFHEBBGHFFHHGHEHHHHHHHHHA>D1>DC1A1A0E0AAEFCGHFF10AF0FGGCGBHFCBFEGC/ECGEGHBDFG/A>///B1B0@B@FE>EC/BGEBHHHEHFH22FFHG>FGF2F0>BB1BF>FG2DGB11G/10/?/CG?0FGDHHB1<@DCGG1FGAFGFGF11AF0CG0EDAEGHHCGGHFGHHHFHGH0FEC/BBEHHHHHHBHFFHGFHHFHHEGFGEF/FFGHHGHHHHHH2DGFHGHGHGHGHHDFHHFEEGGCHHHFEGCGHHHHF1DGGGEGGGHHFFGDFFHGEHHHHHHGHHFGGHHFHHGHHHHH0G1GHDF1DDE1>FGHHGFHHHGHHHFHHHHGGGFEHHHFHHGFHHHBBGGGCHGHHFGFFGFFGFGGGEFGGG? -@SRR33138449.5 5 length=251 -GACTCTTACTCTTTCAAATTGCAGGCTAAAAATGCCGCCAGCTCATAACTCCCCTGTTTAATATGCAATTCACACAGTGAATCTCTTATCATCCAGGTGAAAAATAAAAGCGTGCAACAAATCACTATTAAAGAAAGTAATCTATATTTCTGCGGCATTCCCATCCCCCTGTGTTGATTTCCAACGAGTAAGAGGCTAACCTTATGGCGTTCAGGCATAGGGTAGCCTCACTTTGGACTGACCCCAACAAG -+SRR33138449.5 5 length=251 -1>AA11B3DFF3AGGB3B33A1ABA00BF1A10BBGE?AAE/1BG112G1DAAECF11FBEFHBEBE1GGB1DEG/BEG1D21FGGGFFHFFD1FFC11BFGF001B1GBGE?E//FGA/0BFFCG2GBFF1FBFFHHHFH2DFG2B2>DBBGC/B/>1>@G11@0FAA?G3EFFGFEEEEFCFGHFBBFHGGFEGHFF??EFGHHHHH3FDGHFGFFG3BG2GCGFD/EC?>ABBFFFFFFFGGGDGGGGGGAEGGGGHHHHHHHHH5G5GGGHHHFHFGFFGGHGHHHHHEEGHFGGEFD1GEFGHGHGBDAGGFCCFGGFG3GHGEGEE@EE/EGG0CGHHHCEGG@GFDFHFHFFHHHHHHFHHGGGHHGGHHHHHHHHHHHHGGGGGHHHHHHHFHHBHHEGFHFHHHHFHFHHHFHHHHHHGHGHHFFFHHHHHHHHGHHGGEBGGGFFFFGGGGG@G;BBFFFFFFFFFFFFBFF -@SRR33138449.14 14 length=251 -CTCTTTGACTGCTCCCGAACATCCCTGCGTTTGGTACAACACACTGTAAGCACAGCGGTATCCTTTGCGAGGAGAAAGGTGTCGTTGATAGAGCATTATCAACAGTAAAAGCAGTTTATTCATGAAGGCTCCGAACCATTCCTGTTCCCGATTAAAAACTTATCTAATAGCCAAATCCATTCGTATTTAAAAAAACATTCATTAAAGAAATAAATTTGGGTGCAATGAGAATGCGCAACGCCGTAAGAAAG -+SRR33138449.14 14 length=251 -AAAAAFD5BDFFGGFAE?AEEGFHHGGHGA?EDG2AEGHGHA2EHFEGFHHHFGGEFCFEGEHFHH5GE0@F/??FCBF3@BGFGC2ED@GFDGCFHFDDGFHF2FGHEGF0G3BFHBDHHFFH3?32FGFEGGGCCHHGHGFAABBBBBBBFFGGGGGGGGGGGHGHGHHHGHGHHHHHHGHFHGHGEEGGHHHHHFHGCEG0EGHHHHHGHFFFGHEHHHHHHHHHHHHGGHHHHGGHFHHGHGHHHHHHHHHHHHGHHHGGGDGGHHHHEHFHHHHHGGGGGHHHHGHHGHHHHFHHFFBGHHGHFGHGHHGHFFGGHHHHCCGEHHFFGHHFEFFGHGGFBFFGG0.AEDGFGGEGEFGBF.CB--BACFFDDFFEBBF?D---@.B/ -@SRR33138449.18 18 length=250 -TGCCTGAACACCATAAGGTTAGCCTCTTACTCGTTGGAAATCAACACAGGGGGCTGGGAATGCCGCAGAAATATAGATTACTTTCTTTAATAGTGATTTGTTGCACGCTTTTATTTTTCACCTGGATGATAAGAGATTCACTGTGTGAATTGCATATTAAACAGGGGAGTTATGAGCTGGCGGCATTTTTAGCCTGCAATTTGAAAGAGTAAGAGTCTTTGGCGGGAAGTTATTCCCGCCTTTCTTACGG -+SRR33138449.18 18 length=250 -CCCCCFFFFFFCGGGGGGGGGGHGHHHHHHHHGHGHHHHHHHHHHHGHHHGGGGGGGGGHHHHHGGGGGHHHHHHHHHHHHHHHHHHHHHHHHGHHHHHHHHHHHHGG/FGHHHHHHHHHHGHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHGGGGGHHHHHHHHHHGGGGGGHHHHHGHHHHHHHGGGGGGGGGGGGGGGGGGGGGGGGGGFFFFFFFFFFFFFFFFFFFFFFFF? -@SRR33138449.18 18 length=251 -GCGCAACGCCGTAAGAAAGGCGGGAATAACTTCCCGCCAAAGACTCTTACTCTTTCAAATTGCAGGCTAAAAATGCCGCCAGCTCATAACTCCCCTGTTTAATATGCAATTCACACAGTGAATCTCTTATCATCCAGGTGAAAAATAAAAGCGTGCAACAAATCACTATTAAAGAAAGTAATCTATATTTCTGCGGCATTCCCAGCCCCCTGTGTTGATTTCCAACGAGTAAGAGGCTAACCTTATGGTGT -+SRR33138449.18 18 length=251 -ABBBBBBBBBBBGGGGGGGCGGGGCGHGHHHHFHHDCEFFHH3CFHHFHGHHHHHHHHHHHFHHBFDEHHHHHFBHGEEG/@FFHBHEHHEFHEGGGFHHBHHHH2>FHFHGHHHGDFGFGHHEGBGHHGGDG?G0CFCGCGHGHFHHGGHFC-C?CGHHHHHFFFHHGHHFFFF00;B/CB9FFB9;FF0;CBFA?--@BBB/FFEFFFCAAFBDFGFF5BGBGBGFBAGCEEECEHHFH5FBFGFGFFFGFEGGD?1FF1A333D5DFFHEHEHHEFHGHHCGHEGEECEB@3FDCFG/E?EGD33B??GHGFFG/GDGE/BC1?2FHFF21FHFBD1FGG1FFG1F/0F1B@11GFGF1B22FG2G1EG1B>F1FFFE011GE@E/EA1B1B1F1>DF//0/1BF22B2DGGGHBDGFB2C/0///@FBG1GHHHHHHGHHHGHHGHFHHDGHHFBD1FFGG?EEHGHHDFHHHHFGCC??BDFGHCFBGHHHEHHHHHHHHHHFFFGGH1AAA@11A11A111AEACB33DAG1AG1EAEEABF11B1FD21DFFEHHEGFB0BF1FGGHHCEGFFDB1G1>ECCGB1FF@DGHD@FGHFD1B2BD/>E?/BDEGDD11FFB>EEC/<@@?B/<-CCHHHF0D<0C/<;-:.;/:000;;00:F00 -@SRR33138449.26 26 length=251 -GAAATATAGATTACTTTCTTTAATAGTGATTTGTTGCACGCTTTTATTTTTCACCTGGATGATAAGAGATTCACTGTGTGAATTGCATATTAAACAGGGGAGTTATGAGCTGGCGGCATTTTTAGCCTGCAATTTGAAAGAGTAAGAGTCTTTGGCGGGAAGTTATTCCCGCCTTTCTTACGGCGTTGCGCATTCTCATTGCACCCAAATTTATTTCTTTAATGAATGTTTTTTTAAATACGAATGGATTT -+SRR33138449.26 26 length=251 -ABBB@FFFFBDFGBGGFBGGGGFHHF6FGFFHFHHHHHHG?EFFHDGHHHHGFGFHHH2GGFGHHHHG2FEFHFHHHHAF5FGDFEHHHHHHBFHHHGFC?EEHBGH4FBGGHCEEG/EH4G4?3FFHHGHHHHHHHFFHFF3BDFHH3F>2@2FF@@@->AGDDGH00DDGGGDHHHHFB.;@CFG?ECG-AABFGGG0BBBFFD?E?FFFFFFFFFFFFBFFFFFFFFFF9@FA/BBB/A9.AFF/;FF -@SRR33138449.26 26 length=251 -GCTCCGAACCATTCCTGTTCCCGATTAAAAACTTATCTAATAGCCAAATCCATTCGCATTTAAAAAAACATTCATTAAAGAAATAAATTTGGGTGCAATGAGAATGCGCAACGCCGTAAGAAAGGCGGGAATAACTTCCCGCCAAAGACTCTTACTCTTTCAAATTGCAGGCTAAAAATGCCGCCAGCCCATAACTCCCCTGTTTAATATGCAATTCACACAGTGAATCTCTCATCATCCAGGTGAAAAAT -+SRR33138449.26 26 length=251 -BBBB?ADA@DCFCGGGDAGCGFGFEEGBDGFFFHFFFFHHFFGGHFFHHFCGHHFH1BEEHHHDFBEEEGHHHFHFHFHHGHEFHHFBGFF4EGGGHBFFF33@DG3EE?@EEG?//CAFHFHF0?BC--CBGD1F>10<.A1A1>DADD@1A3BBGBBFDDC?E0EDGDFECF1DAGGHH22BBA00BB1BAG22ABF1DDF1AAEEEFH1BGFGDF1@1BGFEFFHHFF1EEFEHH2EGH1B111>E>>/EEG//?/C10FBC@-@0FGGGF1>F.<-<-/>1>1@33D3>1AF31BF1B3331FB3113AAF3D22B1EFC1FFE221DG0DF2FF2B1//AE/DA12D22BD2F11AGA////>CHHGFF1?10<-<---<>>A?DADDFFDGDGCBDFGG0AA?C1GFHFFFFHF2FBD22BABF0B1FADGHHAFGHHH21D1//EGGFHHHGHHG1FF0F1@122@F20>>FFHFHGBGFFFBE/EEGE/EEEEG0FBGB0GGC//AFHHGH1>DB.-<GFHHCCCCCH1GHDDFFCC@@CCCC-AABFFFFFFFGGGGGGGGGGHGGFEEGHHHHHHHHHGGGGGGHHHHHGHHGGHHHHHHHHHHHHHHHHHHHHHHHHHEHGGG?EGHHHHHFHHGGGGGHHHHHHGGGGGGGGGGGGGHHHHHHHHHHHGGHHHHHGFHHHHHHHHHHHHHHHHHHGGGACFGGGGGEEGGGGGGGEFGFFGBFGGGGGCCFGG.CGBEFFAFFFFFFFFFFFFFFFFF;FFFFFF/BFFFFFFFFEFFEFBFFFFFEBF -@SRR33138449.34 34 length=251 -GCCTGTAATCTCTGCTTAAAAGCACAGAATCTAAGATCCCTGCCATTTGGCGGGGATTTTTTTATTGGATAAACCGCTTTATATGTAGAGAAGAACAGTCCCAGGAACAATCCAGATCCGGCACCGGAATGTCGCAATCACGACAGACCGCACCATCGCACCCGGAACAATCACAATTATTATTCTTCTTTTTACGTATTAACTCGTTCAGGTGTTGCTTCTGCCTGATTGTCTGTGCCGCAAGTCGGCAA -+SRR33138449.34 34 length=251 ->1AAAFFFFFF3F3DFGGFFG11BGCHCBGGFHFGGHFHFHBGF1GGGDFGGG?//EGHHHGGHHF1BDBGHFFG/>EEF1>FEGHHBGBDGFFFGH0111BBFFFCACGF1FFGDDGF0<@/<DHHCC.0CGCGCC-:.A-=A-9C09BE.?A?@@---/90///9;////9///9//:BFBFFF/;-;/9//99BBA--//9;/BAEFFFFFBFFFF-:B/B///9//9-----9/-9>@9 -@SRR33138449.35 35 length=251 -GAATTGCATATTAAACAGGGGAGTTATGAGCTGGCGGCATTTTTAGCCTGCAATTTGAAAGAGTAAGAGTCTTTGGCGGGAAGTTATTCCCGCCTTTCTTACGGCGTTGCGCATTCTCATTGCACCCAAATTTATTTCTTTAATGAATGTTTTTTTAAATACGAATGGATTTGGCTATTAGATAAGTTTTTAATCGGGAACAGGAATGGTTCGGAGCCTTCATGAATAAACTGCTTTTACTGTTGATAATG -+SRR33138449.35 35 length=251 -AAABCFFFFFFFGGGGGGGGGGFGHHHHHHGHHHGGGGGGHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHGHHGGGGGGHHHHHHHHGGGGGHHHHHGGGGGGGGGGGGGGHHHHHHHHHHHGGHHHHHHHHHHHHHHHHHHHHHFHHHGGGHGHHHHGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGFFFFFFFFFFFFFFFFFFFFFFFFFFFFHFFFFFFFFFFFHFHFHHFFFFFFFF: -@SRR33138449.35 35 length=251 -CTCTTTGACTGCTCCCGAACATCCCTGCGTTTGGTACAACACACTGTAAGCACAGCGGTATCCTTTGCGAGGAGAAAGGTGTCGTTGATAGAGCATTATCAACAGTAAAAGCAGTTTATTCATGAAGGCTCCGAACCATTCCTGTTCCCGATTAAAAACTTATCTAATAGCCAAATCCATTCGTATTTAAAAAAACATTCATTAAAGAAATAAATTTGGGTGCAATGAGAATGCGCAACGCCGTAAGAAAA -+SRR33138449.35 35 length=251 -CCCCCFFFFFFFGGGGGGGGGGHHHHHHGEGGGGHHHHHHHGGGHHHHHHHHHHHFGGGGGHHHGHHHGEGGCEEHHHHHHHHGGGHGHHHHGHHHHFHHHHHGHHHHHHHHHHGHHHHHHHHHGHHHHHHGGGDGGHHHHHHHHHHHHGC?GGHHHHGGHHGHHHHHHGGHHHHHHHHFFHGHHHHHHHGHHFGGHHFFGGGGGBFGFFGGGGGGGGBADGAFGGGGEBFGGGBDFFFAFC;AFFFFFF/ -@SRR33138449.36 36 length=180 -ATTAAAAACTTATCTAATAGCCAAATCCATTCGTATTTAAAAAAACATTCATTAAAGAAATAAATTTGGGTGCAATGAGAATGCGCAACGCCGTAAGAAAGGCGGGAATAACTTCCCGCCAAAGACTCTTACTCTTTCAAATTGCAGGCTAAAAATGCCGCCAGCTCATAACTCCCCTGT -+SRR33138449.36 36 length=180 -CCCCCFFFCFFFGGGGGGGGGGHHHHHHHHHHHHHHHHHHHHHGGGHHHHHHHHHHHHHHHHHHHHHHHGEEGHHHHHHHHHGHGGGGGGGGGFFGHHHHHGHGGGCDHHHHHHHHHGGGGGHHHHGHHHHHHHHHHHHHHFHHHHHHGHHHHHHFHHHGGGGGGHHHHHHGFFGGGGGG -@SRR33138449.36 36 length=180 -ACAGGGGAGTTATGAGCTGGCGGCATTTTTAGCCTGCAATTTGAAAGAGTAAGAGTCTTTGGCGGGAAGTTATTCCCGCCTTTCTTACGGCGTTGCGCATTCTCATTGCACCCAAATTTATTTCTTTAATGAATGTTTTTTTAAATACGAATGGATTTGGCTATTAGATAAGTTTTTAAT -+SRR33138449.36 36 length=180 -DDDCDDDDDCFFGGGGGGGGGGGGGGGHHHHHHHHHGHHHHHHHHHHHGGHGHHHFHGHHHHGGGG?EE@GGHHHHHGGGGGHHHHGHGEGGBDCGGGGGGGHFGFHGGHHHGFHHHHHHGHHHHHHGGFHHHHGGHHHGGGGHHHHHHECEHHHFHHFGGGGGGGBFGGFG0FFGGGGG -@SRR33138449.37 37 length=249 -GGGGAGTTATGAGCTGGCGGCATTTTTAGCCTGCAATTTGAAAGAGTAAGAGTCTTTGGCGGGAAGTTATTCCCGCCTTTCTTACGGCGTTGCGCATTCTCATTGCACCCAAATTTATTTCTTTAATGAATGTTTTTTTAAATACGAATGGATTTGGCTATTAGATAAGTTTTTAATCGGGAACAGGAATGGTTCGGAGCCTTCATGAATAAACTGCTCTTACTGTTGATAATGCTCTATCAACGACAC -+SRR33138449.37 37 length=249 -ABAA3DAFFFFFGD5FFEEGGGGHHHHHGHHHGHHF5FHGHFGFHCDEGGHHHHFFHHGFAEG/>ECB4GFBDG1EEEEHBHHFDCEG@DGCDG?@GG2FGFHHFHBGGD/CCFHFGGGHDFHHGDFHBHFBFFGHGGGGGHH0:C:CGHC0GHHHGHHCGH0F0FFBFGGFGG.0F0;EG@?A/BEGFFBFBF/-9;@BEFF/FF/BBBBFBFFF/B/;BBFBFFF/9B/FFFFFFFBFFFFFDABBB -@SRR33138449.37 37 length=251 -CGACAGACCGCACCATCGCACCCGGAACAATCACAATTATTATTCTTCTTTTTACGTATTACCTCGTTCAGGTGTTGCTTCTGCCTGATTGTCTGTGCCGCAAGTCGGCAATCTGCAAAACGCTGACGTATCTCGCCCCAGCCTGCAATAACACCTTTCTGTTGAATAATCTCTTTGACTGCTCCCGAACATCCCTGCGTTTGGTACAACACACTGTAAGCACAGCGGTATCCTTTGCGAGGAGAATGGTG -+SRR33138449.37 37 length=251 -11>11@FFADDDGCFG1FGAG00EE?ECFFGFHHFHGEGHGGHFHHFBGFAGFHHF0FGHHHBGFEGFGCHGBB@EHFGHEDFB1GECBFHHHHEEHGHEE///>EEE/EEGHHHFD1GHGC/?EGHCE/FHHHH/A<>CCHFFHHHHGG1GHHGGGHHHGHGGACGGHA>ABBBDBFBFFGGGGGEGCEEHCFCBEHFFHFHFHHGHHGHFFFHFHFGFGHFHHHHHEHGFHFG@FBEHGFHHGHHGFHBHHEFGFGFEGHGGGFGBEEGGCEHHHFHHFHHFCDDGDGD/GHHGHHEGGGGGHGFGHF1FBHHHHDGHHFHFGHH>FHHHHFBGDDFHGFHHHFHFGG?CDGGFGEFFGA:C-ECA0CBFFEEGABFBBFFGFFFFFDCA.AFFFBFF/@@---.9F/B9/B/BDFD -@SRR33138449.39 39 length=250 -GTTATGAGCTGGCGGCATTTTTAGCCTGCAATTTGAAAGAGTAAGAGTCTTTGGCGGGAAGTTATTCCCGCCTTTCTTACGGCGTTGCGCATTCTCATTGCACCCAAATTTATTTCTTTAATGAATGTTTTTTTAAATACGAATGGATTTGGCTATTAGATAAGTTTTTAATCGGGAACAGGAATGGTTCGGAGCCTTCATGAATAAACTGCTTTTACTGTTGATAATGCTCTATCAACGACACCTTTCT -+SRR33138449.39 39 length=250 -BBBBBFFBFBCFFFGCGCGGGGHGHHHHHHHHHHHHHCFGHHFHHHGGHGHHHHHGGGEGGFHHHFGHHGGGGGGHHHHHGGGGGGGGGGGGGHHHFHGHHHHGFGEHHHHHHHHHHHHHBGFGHHHHHHHGGGHCGGHFHGHHHHHHHHHHFFFHHGGGHFHHHHHHGGFFGEA@@?EGEGFGGGGGGGGGDFFFFFFFFFFFFFFFBFFEFFFFE/FFFFFFFFFFFFFFFFFFFFFD.@ADFEFFFF -@SRR33138449.39 39 length=250 -GAACGATGGTTACAAATCTCTGGACGAAGGTCAGAAAGTGTCCTTCACCATCGAAAGCGGTGCTAAAGGCCCGGCAGCTGGTAACGTAACCAGCCTGTAATCTCTGCTTAAAAGCACAGAATCTAAGATCCCTGCCATTTGGCGGGGATTTTTTTATTGGATAAACCGCTTTATATGTCGAGAAGAACAGTCCCTGGAACAATCCAGATCCCGCACCGGACTGTCGAGATAACGCCAGACCCCACCATCG -+SRR33138449.39 39 length=250 -AAA1>11>ADFFGFFFGBFGGGFHHGCEGG11BGG11CDGGCF2DBG1D1BF/0FEE0//AAAG22AGECE/>?EECFBGHG1B1A00G/FGAAFBFB>DGGB1F1BGHEFH1EGBGFEGCFDHFFFBFGGCCFEFFF1GB1CCGGG-?@@@2FGBFGGGGFH1BDFHAEE/<@/AAFFFBFCGGGGC0ECBAF0BE1BEA?GGHGGFHCBFHFDBDF0GG1GGFDGFCGFHHFGDEFBHGEGHAAECFEGG?0FGHHHHHHFHFEEE>EFHFHGHB1>BDGFHFGHHBFB=1FHHHHGFHHGHHGHHHHGFGGGHFFHHGBHHHHFEFFHGCCGHHGGGFGFFBGFGGGGFFC -@SRR33138449.43 43 length=251 -GCGGTATCCTTTGCGAGGAGAAAGGTGTCGTTGATAGAGCATTATCAACAGTAAAAGCAGTTTATTCATGAAGGCTCCGAACCATTCCTGTTCCCGATTAAAAACTTATCTAATAGCCAAATCCATTCGTATTTAAAAAAACATTCATTAAAGAAATAAATTTGGGTGCAATGAGAATGCGCAACGCCGTAAGAAAGGCGGGCATAAATTTCCGCCAAAGACTCTTACGCTTTCAAATTACAGCCTAAAAA -+SRR33138449.43 43 length=251 -A?A?AAAFFF5DDDAE?ECE4CFGCB3BFEF2FFGFD5G5GFHHHGFHHHCGFFHGBBCG5A5FEF5FGFGHHGH3GFGGGGG1GHEF4G3@4F3FFEGGHGHHGEBFG3GHE44?FGHH0DGE34BG4?B?CFGHHGF?DGHGB>FGGHHHDBCDGC?-:A@.:.CGFGAAB?--9.;BF///9:..99-;/9.9B/;B/.....///;/;9/://../9/BB -@SRR33138449.44 44 length=250 -GAGTAAGAGTCTTTGGCGGGAAGTTATTCCCGCCTTTCTTACGGCGTTGCGCATTCTCATTGCACCCAAATTTATTTCTTTAATGAATGTTTTTTTAAATACGAATGGATTTGGCTATTAGATAAGTTTTTAATCGGGAACAGGAATGGTTCGGAGCCTTCATGAATAAACTGCTTTTACTGTTGATAATGCTCTATCAACGACACCTTTCTCCTCGCAAAGGATACCGCTGTGCTTACAGTGTGTTGTA -+SRR33138449.44 44 length=250 -A?AA1DFFFFFDFGGGG?000ECAF2FGBGFEGGCEDGHH2GA/EAE/>/EEFEEDHHHFHHEEDGGEGAGHHEFHHHFHH2FGFGFGHDGHHGGCEB1DFHA/FGGFFFGFEBFF1FG1@1GFDD?GGHE/FDGGGG-?F00.CFCGG=DDC?A@CE/CGFG0CHFBFCGHEFFBFB;9FG;CBBFG0FFGFFFGG0CBFB.=9>EFFFFFFFFF?;-9=FFFFFFBA@@@FFFFFFFFFFFBFFF?99 -@SRR33138449.44 44 length=251 -ATCCAGATCCGGCACCGGAATGTCGCAATCACGACAGACCGCACCATCGCACCCGGAACAATCACAATTATTATTCTTCTTTTTACGTATTACCTCGTTCAGGTGTTGCTTCTGCCTGATTGTCTGTGCCGCAATTCGGCAATCTGCAAAACGCTGACGTATCTCGCCCCAGCCTGCAATAACACCTTTCTGTTGAATAATCTCTTTGACTGCTCCCGAACATCCCTGCGTTTGGAACAACACACTGTAAG -+SRR33138449.44 44 length=251 -1A1>1D1DFFC>EEGGGGAAEG12AC00BBBGG//AFCG/AEAEEAEHCEGGG/>///>C>DF1G11FGBGG2GBGHH2GH1GFHHGCBBFGBHHFG/FFG2F1BGAAGHHBG1DGHH0FG1F>GGHFFFGGCC/?FFCAC-FGFFFB11<<-AFFFFBFFGGGGGGGGGFHHHHHHHHHFHHHHHHHHGGGGGGHHHGHGHFGGGGGHGHHHHHGHGHHHHHGGHHFGHGGEEFFGHHHHHGGGG>EGHHHFBBGGGCEG/GHHGHHHHHHHHHHHHHGHHGHHHGHFGHFHHHHHHHHHGHGHGGGGGGFHHGHHGFGHFDEGGGGHHHHGHHHHHHHHGHGHGHFHHHGHGHGHFHHFGFFGGGGGGGGGFGGFGFFBBFFGGGFFFFFFFFFFFFF -@SRR33138449.45 45 length=250 -GAGTAAGAGTCTTTGGCGGGAAGTTATTCCCGCCTTTCTTACGGCGTTGCGCATTCTCATTGCACCCAAATTTATTTCTTTAATGAATGTTTTTTTAAATACGAATGGATTTGGCTATTAGATAAGTTTTTAATCGGGAACAGGAATGGTTCGGAGCCTTCATGAATAAACTGCTTTTACTGTTGATAATGCTCTATCAACGACCCCTTTCTCCTCGCAAAGGATACCGCTGTGCTTACAGTGTGTTGTA -+SRR33138449.45 45 length=250 -AAAA?FFFFFFFBGGGGGGG22BGFDGHHHHG?AFGHHGF55FECGEE@EG?/EEHHFH4DHFFFFGGEFGHFHHHHFHHHBDFF4FHGHHFHGEG/G1GDFHGEAGHFFGGGHFGHHHEGHHHHGFBDBEDGGFHHB<0GFHHFHHCGEBHGHFCFDGGHFFFFGFFGGGFGFGFBFFFFFFDA?DFFBFGGGGGB0;ACA@=ADFCFDFBFB/ -@SRR33138449.52 52 length=250 -GTATTACCTCGTTCAGGTGTTGCTTCTGCCTGATTGTCTGTGCCGCAAGTCGGCAATCTGCAAAACGCTGACGTATCTCGCCCCAGCCTGCAATAACACCTTTCTGTTGAATAATCTCTTTGACTGCTCCCGAACATCCCTGCGTTTGGTACAACACACTGTAAGCACAGCGGTATCCTTTGCGAGGAGAAAGGTGTCGTTGATAGAGCATTATCAACAGTAAAAGCAGTTTATTCATGAAGGCTCCGAA -+SRR33138449.52 52 length=250 -AAABCFFFFFCCGGGGGFGGEGHHHHHHHHHHHHHHHHHGGHHHGGGGGHHGGGGGHGHHHHHHHHGGGGGGGFHHHHHHGGGGGGHGHHHHHHHGHHHGGHHHHHHHHHHHFHHHHHHHGHHHHHHHHHGGGGGGHHHHHGHGGGGGGHHHHHHHGHGHHHGGBFHHHHHGGGGGHHHHHHHGGGGCGGGGGGGGFGFGGGGGGGGGGGGGGGGGGGGFGGFFFFFFFBFFFFFFFFFFFFFFFFFDFF -@SRR33138449.52 52 length=251 -TTCTTACGGCGTTGCGCATTCTCATTGCACCCAAATTTATTTCTTTAATGAATGTTTTTTTAAATACGAATGGATTTGGCTATTAGATAAGTTTTTAATCGGGAACAGGAATGGTTCGGAGCCTTCATGAATAAACTGCTTTTACTGTTGATAATGCTCTATCAACGACACCTTTCTCCTCGCAAAGGATACCGCTGTGCTTACAGTGTGTTGTACCAAACGCAGGGATGTTCGGGAGCAGTCAAAGAGAT -+SRR33138449.52 52 length=251 -BBBBBFF?ADDDE?2EFCEGGGGHGGFFFHFGCG2FFEGGGHHHHHHEHHEFHGFBGHGGCEBFGFBEFGHHHHHHFHFHGDGHH4DGFFFHGHHGABGD/AE/EFCEEFHHEFHHGGCD@@FGHGFHHFGHHHBGFHBGGFHHHHGGHHHHHHHGGHBGFDGBGGGF@CCEHHHEHHHHHDGC?@:GGBCF0:-;AEFFFFGGFF0C;B9BE/C0CFGEED-BC-9EFAFBFFAB;9../9BFFFBFBB/ -@SRR33138449.53 53 length=250 -GCGTTGCGCATTCTCATTGCACCCAAATTTATTTCTTTAATGAATGTTTTTTTAAATACGAATGGATTTGGCTATTAGATAAGTTTTTAATCGGGAACAGGAATGGTTCGGAGCCTTCATGAATAAACTGCTTTTACTGTTGATAATGCTCTATCAACGACACCTTTCTCCTCGCAAAGGATACCGCTGTGCTTACAGTGTGTTGTACCAAACGCAGGGATGTTCGGGAGCAGTCAAAGAGATTATTCAA -+SRR33138449.53 53 length=250 -CCCCCCCCBCBBGGGGGGGGGGHGGHHHHFHHFHHHHHGHHGFHHHHHHHGGGHHHHHHHGHHHHHHHHHHGHHHHHHHHHGGDFGHGGHHHGFEEEGHHHAFHHHHHHGCFEGGHHHHHHHHHHHGHHHHHHHHGHHHHHHHHHHHGFHHHHHHHHHGEFGGGHHHHHHHHHGCCCDBHFHGHFGGGGGHHHHHHHGHHHHHHAFFBFFFGGGGGGGGGGGGGGGFFFFFFFFFFFFFFFFFFFFFFFF -@SRR33138449.53 53 length=250 -GCTTTATATGTAGAGAAGAACAGTCCCAGGAACAATCCAGATCCGGCACCGGAATGTCGCAATCACGACAGACCGCACCATCGCACCCGGAACAATCACAATTATTATTCTTCTTTTTACGTATTACCTCGTTCAGGTGTTGCTTCTGCCTGATTGTCTGTGCCGCAAGTCGGCAATCTGCAAAACGCTGACGTATCTCGCCCCAGCCTGCAATAACACCTTTCTGTTGAATAATCTCTTTGACTGCTCC -+SRR33138449.53 53 length=250 -A1>AAFFFFFF3ED1GFCC1FFFEGHHFHG0FEBCGFFEHHHFHG?FCEAAAEAFFHHGHE/EGGFEEE?E0>/@EEEEFGHGGGGCGE/1>DBDFFGGGGGGFFGGGGGFGH1EEE0011A0EAEHHFFGHFHFFG?E//GAAFGHFHFE@FCGGGEFHEHHBGHGFFGHHBFGHHHFFHHGHHHHHFFEGBFHFFFHHGGGGG?HGFHFHH?CGCCE/FBDGGEGHF?D?GGBFHHFCACCEGAHFGGGFB@-C-:EC./CFCGCGCEC/EFB000FG0BBBFFFBFBFG0099AFB0B0;09CFFGGFFFF/9EF@F@-AFFFFFFFFFFFB- -@SRR33138449.55 55 length=251 -TTCTCATTGCACCCAAATTTATTTCTTTAATGAATGTTTTTTTAAATACGAATGGATTTGGCTATTAGATAAGTTTTTAATCGGGAACAGGAATGGTTCGGAGCCTTCATGAATAAACTGCTTTTACTGTTGATAATGCTCTATCAACGACACCTTTCTCCTCGCAAAGGATACCGCTGTGCTTACAGTGTGTTGTACCAAACGCAGGGATGTTCGGGAGCAGTCAAAGAGATGATTCAACAGAAAGTTGT -+SRR33138449.55 55 length=251 -AAAA1@BD31BD1E1FGGFDFF3FBF3DF3DGFBFGBFFGCEEC1D2FF//BBA1GHHHGBGGHGHHFFFHHH2BF2AAFG2/?E/>FGF/00FEG1F1E//>>FB11BFBGDFHBDGBDBDGFGGHFG2FDFHE2FGDGHBFFDB>F/?@@CGH<<1GFGDGA@CF-GF01<11.>@-A/=G0GBFD0=0=::.CG0;CH.CE@-A@GFBA/009-9A-.CFFFGB0F/--B/9/BB////9BFB/9/;/ -@SRR33138449.55 55 length=251 -GTCCCAGGAACAATCGAGAGCCGGCACCGGAATGTCGCAGTCACGACAGACCCCACCATCGCACCCGAAACAATCACAATTATTATTCTTCTTTTTACGTATTACCTCGTTCAGGTGTTGCTTCTGCCTGATTGTCTGTGCCGCAAGCCGGCAATCTGCAAAACGCTGACTTATCTCGCCCCAGCCTGCAATAACACCTTTCGGTTGAATAATCTCTTTGACCGCTCCCGCACTTCCCTGCGTTTGGGACA -+SRR33138449.55 55 length=251 -1>A?AFFAAB1111F1A110000AA0A0EG///F12/A///B11//AE/0B>///BBG1FE/>//E>/>/EF/1>>FH>11B22BFD222D2FGH1?1/01BFDBD11FFFHDH11@11@10?1?F111111?A----.<-AFFFFBFFGFGFGGGFGGHHHHGHHHHHGEFGHHGHGGGGFFHHHHHFGGGGGGAAGFFHHHGEGHHHHEHG5FGFHFGFEGHEHFHHGG?EFEFF0FGGFGGE?FEH?4GHFHHHFFHGHHHGHF3BDGFGHHHFBHHHHGGFHF2FGF1FEG?FGCFFHHFHHGFBDBDEGGGHGHHGHG0DHHHFHHHB0CGHGHHHHEHFC:/BFBFFBFFAGGF0FGGFFBBB;F/FEFFFFFEFD.A?FFF -@SRR33138449.57 57 length=249 -CGAATGGATTTGGCTATTAGATAAGTTTTTAATCGGGAACAGGAATGGTTCGGAGCCTTCATGAATAAACTGCTTTTACTGTTGATAATGCTCTATCAACGACACCTTTCTCCTCGCAAAGGATACCGCTGTGCTTACAGTGTGTTGTACCAAACGCAGGGATGTTCGGGAGCAGTCAAAGAGATTATTCAACAGAAAGGTGTTATTGCAGGCTGGGGCGAGATACGTCAGCGTTTTGCAGATTGCCGA -+SRR33138449.57 57 length=249 -ABBBAFFFFFFFGGGGGGGGGGFFHHHHGGHHHHGDEGGGHHHHHHHHHHHGGDFGGHHHHHHHGHHHHHHHHHGHHHHHGHHHHHHHHHHHHHHHHHGHGGGGGGHHHGHGHHHGGGFGFGGHHHHGGGGGHHHHHHHHHHHHHHHHHHHHHGHGGGGGGGGGFHHGGGGGGHHHHHGFFHHHHHHGHHHHHHGHCHHHGHHGGGGGGGGGGGGDGFGFFFFFFFFFFFFFFAACFFFFFFFFFFFFF -@SRR33138449.57 57 length=251 -GTCCTTCACCATCGAAAGCGGTGCTAAAGGCCCGGCAGCTGGTAACGTAACCAGCCTGTAATCTCTGCTTAAAAGCACAGAATCTAAGATCCCTGCCATTTGGCGGGGATTTTTTTATTGGATAAACCGCTTTATATGTAGAGAAGAACAGTCCCAGGAACAATCCAGATCCGGCACCTGAATGTCGCCATCACGCCAGACCTCACCATCGCACCCGGAACGATCACCATTATTATTCTTCTTTTTACGTA -+SRR33138449.57 57 length=251 -A3ABBFFFFFFBGGGG?FGEGAEAEFFFHHEFGGG?EGGHHHHBFFFFG11FGEEFHH3G5EGHGHEFHGHGBFFFHE2F@3GHFHFHHGHGFFFHHC@FFBG3/EC?//GGGHGG/FFFF11FFHGHGC@DAHH0=0=00DDB////CBCC0CC00;./;CB0B9CF0;C0AB-:B.F/;BF0C;9..;.;99....-9.A/9BEFF/B9A?--;=----..;.././//9:B////9///9B;BE/9B/ -@SRR33138449.58 58 length=86 -CGAATGGATTTGGCTATTAGATAAGTTTTTAATCGGGAACAGGAATGGTTCGGAGCCTTCATGAATAAACTGCTTTTACTGTTGAT -+SRR33138449.58 58 length=86 -BBBCCFFFFFFFGGGGGGGGGGHHHGHHHGHHHHEGGGCGHHHHHHHHGHHEFEGGGHHHHHHHHHHHGGHHGHHHHGDGHGHHHH -@SRR33138449.58 58 length=86 -ATCAACAGTAAAAGCAGTTTATTCATGAAGGCTCCGAACCATTCCTGTTCCCGATTAAAAACTTATCTAATAGCCAAATCCATTCG -+SRR33138449.58 58 length=86 -BBBBBFFFFFFFGGGGGGGGGGHDHGHFHHHHHHGGGGGGHFFHHGHFHHHHGGGGGFFHHGHHHHFHHGHHHFHHEFGGHHHHFH -@SRR33138449.59 59 length=251 -GGATTTGGCTATTAGATAAGTTTTTAATCGGGAACAGGAATGGTTCGGAGCCTTCATGAATAAACTGCTTTTACTGTTGATAATGCTCTATCAACGACACCTTTCTCCTCGCAAAGGATACCGCTGTGCTTACAGTGTGTTGTACCAAACGCAGGGATGTTCGGGAGCAGTCAAAGAGATTATTCAACAGAAAGGTGTTATTGCAGGCTGGGGCGAGATACGTCAGCGTTTTGCAGATTGCCGACTTGCGG -+SRR33138449.59 59 length=251 -ABBBAFFFFFFFGGFGFGFGGFHFFHHDFFGGGGGHFGAGFFHEGHGG2EAAEGFHHHHGHHFHDHHHHHHHFGHHFHHFHFFFHHHHFHHFGHHGEGCEFHFHHHFHHHGGGGGHHHHHHGEGGGGHHGHHHHHBFFFBGHHFGGDHHHFGGGGGGHFGHHGGGGGGEHGHGFFHHGHGFHHHG0;GFHHHHHHGHEGEHFGCEEEFFHHHHHHFFHHHGHGHHHFF2?FEHFHHHHFHGHGHFHHHEGHFGFGGGHHGEHHFCCC@GFGHFGG/CCGCHFHD00DFGFHFCC.:.C;GCGBF-;-9AAA?/B9:/0BE.9B?/BFFB/B/BB9FFFFFFFFB/BF?BFFFFEB/// -@SRR33138449.60 60 length=251 -GTAAAATGGTTCAACGCTGACAAAGGCTTCGGCTTCATCACTCCTGACGATGGCTCTAAAGACGTGTTCGTACACTTCTCTGCTATCCAGAACGATGGTTACAAATCTCTGGACGAAGGTCAGAAAGTGTCCTTCACCATCGAAAGCGGTGCTAAAGGCCCGGCAGCTGGTAACGTAACCAGCCTGTAATCTCTGCTTAAAAGCACAGAATCTAAGATCCCTGCCATTTGGCGGGGATTTTTTTATTGGAT -+SRR33138449.60 60 length=251 -BABCCFFFFFFFGGGGGGGGGGHHHHHHHHGGGGGHHHHHHHHHHHHHGGGGGGHHHHHHHHHEFGHHHGGHHGHHHHHHHHHHGHHHHHHHHGGHGGHHHHHHHHHHHHHHHHGGGGGHHHHHHHHHHHHHHHHHHHHHHGGHGGGGGEGGHHHHHHGHGGGGGGGHHHHHGHGGHHGHHHHGHHHHHHGHHHHFGFGGFGGGGGGGGFGGGGEFGGGFGGGGGGGGGGGGFF-D>AFFFFFFEFFFFF0 -@SRR33138449.60 60 length=251 -GGTTCGGAGCCTTCATGAATAAACTGCTTTTACTGTTGATAATGCTCTATCAACGACACCTTTCTCCTCGCAAAGGATACCGCTGTGCTTACAGTGTGTTGTACCAAACGCAGGGATGTTCGGGAGCAGTCAAAGAGATTATTCAACAGAAAGGTGTTATTGCAGGCTGGGGCGAGATACGTCAGCGTTTTGCAGATTGCCGACTTGCGGCACAGACAATCAGGCAGAAGCAACACCTGAACGAGGTAATA -+SRR33138449.60 60 length=251 -A3AAADBAB@AFGFGGGGGGGGHFHHHHGHHHHGHHHHGGHHHHHHHHHHGFFHGEEGGGFHHGHGHHHGGGGGHGHHGHFECGGGHHHHHGHHHGFHHFHHHHFHFGHGGGGGFGHGFFBF>A@DFHHHHHBGHFFGFFHHHHHHGFHHHFHH@>FHF@G3GFGBGG1GHHFHHFEH/EE/E@?CGEG4F?C/D@D/0FG<@DHFFFHHFDGHG1?GCFGBGH/>>@GG11=FB0.01>133111>111110000A00B10AD222B10A/A0122FGEDH11DFF11AF0100A11112B12BGH1AD1F0FF1D2BF>//EE/B0BDF021>2<2B?F><0////<1>2>F2>1?1110CG000??C.0<..>00<110>00ACFF1F3B1B31113BB3FGHBBDGFGB2FGH22B2D11A202BE//EAFFHD1EHG1EFGG?EHBCB11F//>//B1DE2G11BBB10B/F1@2B1>FE@E/<B////D11>0<<0<.-.<<--.0=.0;.C0:C..;.0000;00;9..99.;--99----99--;///9B-A-/////-9:://;/B;------9; -@SRR33138449.63 63 length=251 -GCTAAAGGCCCGGCAGCTGGTAACGTAACCAGCCTGTAATCTCTGCTTAAAAGCACAGAATCTAAGATCCCTGCCATTTGGCGGGGATTTTTTTATTGGATAAACCGCTTTATATGTAGAGAAGAACAGTCCCAGGAACAATCCAGATCCGGCACCGGAATGTCGCAATCACGACAGACCGCACCATCGCACCCGGAACAATCACAATTATTATTCTTCTTTTTACGTATTACCTCGTTCAGGTGTTGCTT -+SRR33138449.63 63 length=251 -CCCDDFFFDEDDGGGGGGGGGGHHGHHHGHHHHGHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHGGGGCEHHHHGGHHHFHFFGHHHGGGGGGHHHHFGHHGHHHGHHHHHGFHHHGGEHHHHGEHHHHEFHHGGGGGGGGGGGFGGGDGGGGGGGGGGGFGFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFFFFFFFFFFFFFFBFFBAFFF -@SRR33138449.63 63 length=251 -GTTGATAATGCTCTATCAACGACACCTTTCTCCTCGCAAAGGATACCGCTGTGCTTACAGTGTGTTGTACCAAACGCAGGGATGTTCGGGAGCAGTCAAAGAGATTATTCAACAGAAAGGTGTTATTGCAGGCTGGGGCGAGATACGTCAGCGTTTTGCAGATTGCCGACTTGCGGCACAGACAATCAGGCAGAAGCAACACCTGAACGAGGTAATACGTAAAAAGAAGAATAATAATTGTGATTGTTCCG -+SRR33138449.63 63 length=251 -BCCCBFFBFFFFGGGGGGGGGGGGGGHHHHGHFHHGGGGGHHHHHHHGGGCGHHHHHHHHGGFGHHHFHHEGGEHGFGGGGGHGHHHGCFGFGHGGHHHGHHHHHHHHHHHHHHHGHHHHDGF?HHHHGGFFGGHGCDGGGGGGHHFGHEDHFADGGHHHHHHFFHGGCFGGEG?FACABFFGGGGFGGGBEGGFFFFEFFEFFFFFFFDFFAFFFFFE9FFEFFFFFFFFFFFFFFFFFBFFFBFFFF/9 -@SRR33138449.64 64 length=250 -CTTCTCTGCTATCCAGAACGATGGTTACAAATCTCTGGACGAAGGTCAGAAAGTGTCCTTCACCATCGAAAGCGGTGCTAAAGGCCCGGCAGCTGGTAACGTAACCAGCCTGTAATCTCTGCTTAAAAGCACAGAATCTAAGATCCCTGCCATTTGGCGGGGATTTTTTTATTGGATAAACCGCTTTATATGTAGAGAAGAACAGTCCCAGGAACAATCCAGATCCGGCACCGGAATGTCGCAATCACGA -+SRR33138449.64 64 length=250 -CCCCCFFFFFFFGGGGGGGGGGGGHHHHHHHHHHHHHHHHGGGGGFGGHHGHHHHHHHHHHHHHGHHGFHGDHGGGGGHHHHHHHHGGGGGGGHHHHHHHGGGHGEHHHGHHHHHHHHHHHHHHHHHHHHHHHHHHHGHGFHHHGHHGHHGHHHHHHHGGGG?DHHHHGGGGGGGGGGGGBFCGGGGGGGGGFFBFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF;FFFFFA-BFFFE?EFFFFFFFF -@SRR33138449.64 64 length=251 -ATCAACGACACCTTTCTCCTCGCAAAGGATACCGCTGTGCTTACAGTGTGTTGTACCAAACGCAGGGATGTTCGGGAGCAGTCAAAGAGATTATTCAACAGAAAGGTGTTATTGCAGGCTGGGGCGAGATACGTCAGCGTTTTGCAGATTGCCGACTTGCGGCACAGACAATCAGGCAGAAGCAACACCTGAACGAGGTAATACGTAAAAAGAAGAATAATAATTGTGATTGTTCCGGGTGCGATGGTGCG -+SRR33138449.64 64 length=251 -AA3AAF@ABBBBGCGGFDFBGGGGGGGCCHHGGGGAF2FHHHHFHHAGADFGEGFFHHFFHGGGGGGFGGGFFEGGGGGGHHHHHFHFHHHHBGHH4FHHHHEFFH3BB?GFHHHFHFGHHG?DFGGGGGGHGHHGGEDGADGHHGFHHFE0GD-@DGHHCGG?-FFGGGFBBFGFFE.9BCFGFGGGFFBBFF.ADAAFFFBFFB/.A/;9EFFF//BFFFBB//9;9F///;:/9@@;@.999.A99B; -@SRR33138449.65 65 length=251 -CCTTTCTCCTCGCAAAGGATACCGCTGTGCTTACAGTGTGTTGTACCAAACGCAGGGATGTTCGGGAGCAGTCAAAGAGATTATTCAACAGAAAGGTGTTATTGCAGGCTGGGGCGAGATACGTCAGCGTTTTGCAGATTGCCGACTTGCGGCACAGACAATCAGGCAGAAGCAACACCTGAACGAGGTAATACGTAAAAAGAAGAATAATAATTGTGATTGTTCCGGGTGCGATGGTGCGGTCTGTCGTG -+SRR33138449.65 65 length=251 -CCCCCFFFFFFCGGGGGGGGGGHGGGGGHHHHHHHHHHHHHHHHHHHHHHHGGGGGGGGGHHHGGGGGGHHHHHHHHHHHHHHHHHHHHHHHHHHHFGHGHHHHHHHHGHHGGGGGGGGGHHGHHGGHGDGGGHHHHHHHHGHGGGGGHHGGGGGHHHHHHHHHHHGGGGGGGGGGGGGGGGGGGBGG:FFGFFFFFFFFFFFFFEFFFEFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFAFFFFFFD -@SRR33138449.65 65 length=251 -CTCCTGACGATGGCTCTAAAGACGTGTTCGTACACTTCTCTGCTATCCAGAACGATGGTTACAAATCTCTGGACGAAGGTCAGAAAGTGTCCTTCACCATCGAAAGCGGTGCTAAAGGCCCGGCAGCTGGTAACGTAACCAGCCTGTAATCTCTGCTTAAAAGCACAGAATCTAAGATCCCTGCCATTTGGCGGGGCTTTTTTTATTGGATAAACCGCTTTATATGTAGAGAAGAACAGTCCCAGGAACAA -+SRR33138449.65 65 length=251 -AAA@BFFFBABBGGGGGGGFGGHGGCHFHFFFHGHHHHHHHHHGBGHHGHGHHGCHGGDGHHHHGHHHHHHHHHEGFGGHFHHHHHHFHFHHHHFHHHHHGGFHGGFGEECFHB43F3GHECGGGGGFEHGEHHGGHHGHGHHGGHHHHHGHHGHHHHHGEFGFHHEHEFGBG0GHFHHGHFGGHHHHGEBF?CCF-;EFFF;DEFBFFFFB;;;/@@9..;BF///9//9//;BBB/99/:BFFE.:AFB -@SRR33138449.66 66 length=251 -CCTTTCTCCTCGCAAAGGATACCGCTGTGCTTACAGTGTGTTGTACCAAACGCAGGGATGTTCGGGAGCAGTCAAAGAGATTATTCAACAGAAAGGTGTTATTGCAGGCTGGGGCGAGATACGTCAGCGTTTTGCAGATTGCCGACTTGCGGCACAGACAATCAGGCAGAAGCAACACCTGAACGAGGTAATACGTAAAAAGAAGAATAATAATTGTGATTGTTCCGGGTGCGATGGTGCGGTCTGTCGTG -+SRR33138449.66 66 length=251 -AAAABFFFDFFAEGGGGGBGGGGEGGGAGFHGHHHHHHHHHFGFDEGDGGEG?CEGGFCFHH5EFGCECHGFGFGHHHHHHHFHHHGDFHHHFHGE3BBEHHHFFFFHGFGAEED@BDCGGHGHGFFGGEDCFHHGGBGFH1<1-://;B-9------;--/;///9///:///99// -@SRR33138449.67 67 length=249 -GAACAATCCAGATCCGGCACCGGAATGTCGCAATCACGACAGACCGCACCATCGCACCCGGAACAATCACAATTATTATTCTTCTTTTTACGTATTACCTCGTTCAGGTGTTGCTTCTGCCTGATTGTCTGTGCCGCAAGTCGGCAATCTGCAAAACGCTGACGTATCTCGCCCCAGCCTGCAATAACACCTTTCTGTTGAATAATCTCTTTGACTGCTCCCGAACATCCCTGCGTTTGGTACAACACA -+SRR33138449.67 67 length=249 -CCCCCFFFFFFFGGGGGGGGGGGGGGHHHHHGGGHHHGGGGGHHHGGGGGHHHGGGGGGGGGGGHHHHGHHGHHHHFHHHHHHHHHHHHHHGHHGHHHHHHGGGHHHGHHFFHHHHHHHHHHHFHHHHHHHHHHHGGGGCHHGGGGGHHHHHGHGHHGGGGCHGGHHHHHHGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGFFFFFFFFFFFFFFFFEFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF -@SRR33138449.67 67 length=251 -GGATACCGCTGTGCTTACAGTGTGTTGTACCAAACGCAGGGATGTTCGGGAGCAGTCAAAGAGATTATTCAACAGAAAGGTGTTATTGCAGGCTGGGGCGAGATACGTCAGCGTTTTGCAGATTGCCGACTTGCGGCACAGACAATCAGGCAGAAGCAACACCTGAACGAGGTAATACGTAAAAAGAAGAATAATAATTGTGATTGTTCCGGGTGCGATGGTGCGGTCTGTCGTGATTGCGACATTCCGGT -+SRR33138449.67 67 length=251 ->A?AAFFBBBBBGGGGGGCFGGGCFHGHFHHHHHCGG?E?FCHGHHHGGGCEDEFHHHDGGHHFHHHHHHGHHHGEHFHHFEGGHHGHHHHHGHGGEGGGGGCFFFEHEFGGGDDGGGGFGHHEHGHGGGGCHHG?DCGHFHHHHFFBHHHHFGBGBGCCAEGHGHGEF.CG;FBBFGGBFFGGGDEFGGFBBBBFGFFFBBBFFFFFFFFFFDD?;AEFFFFFFFDFDFFBEEDB/BBFD-9DDFFFF;- -@SRR33138449.68 68 length=251 -GGATACCGCTGTGCTTACAGTGTGTTGTACCAAACGCAGGGATGTTCGGGAGCAGTCAAAGAGATTATTCAACAGAAAGGTGTTATTGCAGGCTGGGGCGAGATACGTCAGCGTTTTGCAGATTGCCGACTTGCGGCACAGACAATCAGGCAGAAGCAACACCTGAACGAGGTAATACGTAAAAAGAAGAATAATAATTGTGATTGTTCCGGGTGCGATGGTGCGGTCTGTCGTGATTGCGACATTCCGGT -+SRR33138449.68 68 length=251 -A@AAAFFB@BDAGGGGFGGGGGGHHHHHHHFHHFHGG222EAGFHHHGGGGGGHBGHGHFHHHHEFGHHHFHFHHFGFHHFHBEGHHHGHHGGHGDCE?EGGGGHHGHHGGGGGGGGAGHHFFGHHHGDGGDHE-DGGCCGHEHCGHHFFGH?CF/FHBGAEDHHFGHGGGGGGGGGGGBFE?BBAAFF/BFGFFFFFFFFFEFFFFFFFAB-;@?BFFF.//9@-9@DFFFFADFFFFFDBB??FFBF@9 -@SRR33138449.68 68 length=251 -CCATTAAAGCAGTGTAGTAAGGCAAGTCCCTTCAAGAGTTATCGTTGATACCCCTCGTAGTGCACATTCCTTTAACGCTTCAAAATCTGTAAAGCACGCCATATCGCCGAAAGGCACACTTAATTATTAAAGGTAATACACTATGTCCGGTAAAATGACTGGTATCGTAAAATGGTTCAACGCTGACAACGGCTTCGGCTTCATCACTCCTGACGATGGCTCTAAAGTCGTTTTCGTACACTTGTCTGCTA -+SRR33138449.68 68 length=251 ->11>A3BD1FFFFGGFBF33B11BAAGFFFFHBDHHCAGFHFHCFGFHHHFG1FFGEGFAFCFHHHFHGFB2FD2FA?EFGFGHFFGHHHG2EGFCE@?FAFC2DF/EEEE/>ECF0FGCHGFFF2FHHHEGHB1FDB>1FFGHFHHHC/>//@2>1DFBBF?DDGCFGFA=GFHB1<C- -@SRR33138449.70 70 length=251 -CCTGTAATCTCTGCTTAAAAGCACAGAATCTAAGATCCCTGCCATTTGGCGGGGATTTTTTTATTGGATAAACCGCTTTATATGTAGAGAAGAACAGTCCCAGGAACAATCCAGATCCGGCACCGGAATGTCACAATCACGACAGACCGCACCATCGCACCCGGAACAATCACAATTATTATTCTTCTTTTTACGTATTACCTCGTTCAGGTGTTGCTTCTGCCTGATTGTCTGTGCCGCAAGTCGGCAAT -+SRR33138449.70 70 length=251 -BCCCCFFFFFFFGGGGGGGGGGHHHHHHHHHHHHHHHHHHHHHHHHHHHHGGGG@GHHHHGGHHHGHHHHHHHFGGGGGHHHHGHHHHHHHHHHHHGGFHHHHGEGGHHDFHHHHHHGHGGGGGGGGGGHHH1FGHHFHGGGGGGHHHGGGGGHHHGGGGGGGGGGGGGGGGGGFGGFFFFFFFFFFFFFHFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFHFFFFFFFFFFFFFFEFFFFFF0 -@SRR33138449.70 70 length=248 -GTGCTTACAGTGTGTTGTACCAAACGCAGGGATGTTCGGGAGCAGTCAAAGAGATTATTCAACAGAAAGGTGTTATTGCAGGCTGGGGCGAGATACGTCAGCGTTTTGCAGATTGCCGACTTGCGGCACAGACAATCAGGCAGAAGCAACACCTGAACGAGGTAATACGTAAAAAGAAGAATAATAATTGTGATTGTTCCGGGTGCGATGGTGCGGTCTGTCGTGATTGCGACATTCCGGTGCCGGAT -+SRR33138449.70 70 length=248 -BBCCCFFFFFFFGGGGGGGGGGHHHGGGGGGDHGHHHGGGGGGHHHHGFHHHHHHHHHHHHHHHHHGHHHBFBEFGFGHHHHGEHGGGGGGGGGHHGHHGGHG/EGGHHHHGHHHHHGCCFDHHGGGGCGHHHHHBGFGFFHGGH0GGHGHEFHHFHFHGGGEHHFFFHFGFGGFDEGGGFGGGFEFFFF;FGGGGFFGGGGF9BDB.9DDBBFFA;CFF9F:ADBBBB0.@A>.FF0BDBFA/CB;9 -@SRR33138449.71 71 length=248 -GTGCTTACAGTGTGTTGTACCAAACGCAGGGATGTTCGGGAGCAGTCAAAGAGATTATTCAACAGAAAGGTGTTATTGCAGGCTGGGGCGAGATACGTCAGCGTTTTGCAGATTGCCGACTTGCGGCACAGACAATCAGGCAGAAGCAACACCTGAACGAGGTAATACGTAAAAAGAAGAATAATAATTGTGATTGTTCCGGGTGCGATGGTGCGGTCTGTCGTGATTGCGACATTCCGGTGCCGGAT -+SRR33138449.71 71 length=248 -CCCCCFFFFFFFGGGGGGGGGGHHHGGGGGGDFGHHHGGGGGGHHHHHGHHHHHHHHHHHHGHHHHHHHHGEFFHHHGHHHHGHHGGGGGGGGGHHGGHGGHGFGGGHHHHHHHHHHGGGGGHHGGGGGHHGHHHHHGHFHHGGHHHHHHHGGHHHHHGGGGEGHFGHHFGGGGGGGGGGFGGGGGGGGGFGGGGGGGGGGGFABFFF?FFFFFFFFFFFFFF?DBFFFFFFFFFFFFFFCFFFAFFB -@SRR33138449.71 71 length=248 -CACTATGTCCGGTAAAATGACTGGTATCGTAAAATGGTTCAACGCTGACAAAGGCTTCGGCTTCATCACTCCTGACGATGGCTCTAAAGACGTGTTCGTACACTTCTCTGCTATCCAGAACGATGGTTACAAATCTCTGGACGAAGGTCAGAAAGTGTCCTTCACCATCGAAAGCGGTGCTAAAGGCCCGGCAGCTGGTAACGTAACCAGCCTGTAATCTCTGCTTAACAGCACAGAATCTAAGATCC -+SRR33138449.71 71 length=248 -AAABBFFFFFBBGGGGGGGGGGGGGGFBEGHHHBGHHBFFFGHD?EE2FGGHHBGHFFGGGGGGHHHHGHGGHH5GGGGAFEHHHHGHHGGGFFHHHGFGGFHHHHHHFHEHHGHHFFHHFGEHGFHHFHFHHGHHHHHFHHGG?DG2ADFFFF.ADFFFBFDFAFDFBBB-BFFFFABD=FFFBBBFBBBFFFBEF -@SRR33138449.77 77 length=152 -ACCAAACGCAGGGATGTTCGGGAGCAGTCAAAGAGATTATTCAACAGAAAGGTGTTATTGCAGGCTGGGGCGAGATACGTCAGCGTTTTGCAGATTGCCGACTTGCGGCACAGACAATCAGGCAGAAGCAACACCTGAACGAGGTAATACGT -+SRR33138449.77 77 length=152 -CCDDDDDDDDCDGGGGGGGGGGGGGHHHHHHHHHHHHHHHHHHHHHHHHHHHBFGGHHHHHHHHGHHGGGGGGGGGHHGHHGGHGGGGGHHHHHHHHHHGGGGGHHGGGGGHHHHHHHHHHHHHGGHHHHHHHGGHHHHHHGGGEHHHHHHH -@SRR33138449.77 77 length=152 -ACGTATTACCTCGTTCAGGTGTTGCTTCTGCCTGATTGTCTGTGCCGCAAGTCGGCAATCTGCAAAACGCTGACGTATCTCGCCCCAGCCTGCAATAACACCTTTCTGTTGAATAATCTCTTTGACTGCTCCCGAACATCCCTGCGTTTGGT -+SRR33138449.77 77 length=152 -ABBABFFFFFFFGGGGGGGGGGGHHGHHHHHHHHHHHHHHHHHGGHGGGGGHHGGGGGHHHHHHHHHHGGGGEHGFHHHHHHGGGGGGHGHHGHHHHHHHHGHHHGHHHHHHHHHGHHHHHHHHHHHHHHHHGGFGCGHHGHHHHGGGGGGH -@SRR33138449.78 78 length=250 -ATCCAGAACGATGGTTACAAATCTCTGGACGAAGGTCAGAAAGTGTCCTTCACCATCGAAAGCGGTGCTAAAGGCCCGGCAGCTGGTAACGTAACCAGCCTGTAATCTCTGCTTAAAAGCACAGAATCTAAGATCCCTGCCATTTGGCGGGGATTTTTTTATTGGATAAACCGCTTTATATGTAGAGAAGAACAGTCCCAGGAACAATCCAGATCCGGCACCGGAATGTCGCAATCACGACAGACCGCAC -+SRR33138449.78 78 length=250 -CCCDCFFFFEEEGGGGGGGGGGHHHHHHHHGGGGGFHHHHHHHHHHHHHHHHHHHHHGHHGGHGGGGGHHHHHHHHGGGGGGGHHHHHHHGHHHGHHHHGHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHGGGG?FHHHHGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGFFFFFFFFFFFFFFFFFFFFFFFFFFAFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF -@SRR33138449.78 78 length=250 -ACGCAGGGATGTTCGGGAGCAGTCAAAGAGATTATTCAACAGAAAGGTGTTATTGCAGGCTGGGGCGAGATACGTCAGCGTTTTGCAGATTGCCGACTTGCGGCACAGACAATCAGGCAGAAGCAACACCTGAACGAGGTAATACGTAAAAAGAAGAATAATAATTGTGATTGTTCCGGGTGCGATGGTGCGGTCTGTCGTGATTGCGACATTCCGGTGCCGGATCTGGATTGTTCCTGGGACTGTTCTT -+SRR33138449.78 78 length=250 -AAAAADDDBFAFGGGGGGGGGGHHHHHHHHHHHHHHHHHGHHHGHHFBFFFHHHHGHHHGHHGGGGGGGGGHHGHHGGGGGEGGHHHHHFGHHHGGGGGHHGGGGGHHGHHHHHHHGHHGGEHHHHHHGGHFHHHHGGGCGHHHHHHHHHHHGGHHHGHHHFHHHHHGGHGHHGGGGGGA;BDGFGGE9FFGFFF?FFFEEFFFFFFFAFFFFFFFCDFFFFFFFFFFFFFFFFFFFBFFFFFFFFFFEF -@SRR33138449.79 79 length=251 -GGATAAACCGCTTTATATGTAGAGAAGAACAGTCCCAGGAACAATCCAGATCCGGCACCGGAATGTCGCAATCACGACAGACCGCACCATCGCACCCGGAACAATCACAATTATTATTCTTCTTTTTACGTATTACCTCGTTCAGGTGTTGCTTCTGCCTGATTGTCTGTGCCGCAAGTCGGCAATCTGCAAAACGCTGACGTATCTCGCCCCAGCCTGCAATAACACTTTTCTGTTGAATAATCTCTTTG -+SRR33138449.79 79 length=251 -AA3A3@FFFBDB?FBGGGGGF6FFHGHBE4GGHF3GHHCHHAAB3BGG33EGFE00EEGGGCEFGHGEG@EEFFHEEEG/???/>EEEHHHGGEEEE0G1<1>GGHB1=GHDDFHF/D<111B333B3D3A133BBA3333D3DFGH1FA00AEEGB200A//1////0F1DGE12B1ABBFFH21A22DF10////0DDB112B10BE///>C1B2>222210?011>F1B21111>@>A111>B111BAB131B1A0FAAA0B0D112100000AD2FBFF21AFD22D12D222F2D2AB1B2D1E0AE/>//111/////0011B21@B1F11@B?//?//??//?//>GF/0F1FGG12212BB1@00@/>>D2@212211<11?1?FF0/011<1<<--//000/00<0D0=00000;:00000;0000;000//--:.90;;000;.-;-;.:9.;ABF/BF/////;/9 -@SRR33138449.81 81 length=251 -GGGCGAGATACGTCAGCGTTTTGCAGATTGCCGACTTGCGGCACAGACAATCAGGCAGAAGCAACACCTGAACGAGGTAATACGTAAAAAGAAGAATAATAATTGTGATTGTTCCGGGTGCGATGGTGCGGTCTGTCGTGATTGCGACATTCCGGTGCCGGATCTGGATTGTTCCTGGGACTGTTCTTCTCTACATATAAAGCGGTTTATCCAATAAAAAAATCCCCGCCAAATGGCAGGGATCTTAGATT -+SRR33138449.81 81 length=251 -BBBBBBBBBFFBFEFGGGEEGGHGHHHHHHHHGGGGGHHGEGEGHHGHHHHHHFFHHGGHHHHGGHEGHHGHHHEFEEGHHGHHBGHHHFGGHHHHHFHHGGHHHGHHHHHHHHHGGGGGGGDFFGHHFGGGGGHHHGFGHHHHHGGGGGGHFFGGGGHGGGG?FGGGGGGGGGGGGGGGGFGGGGGFGGGGGGFBFFGGFGGFFFFFFFFFFFFFFFFEFFFFFFFBCFFCFFFFFFFAFFFFFFFFFFF -@SRR33138449.81 81 length=251 -GACAAAGGCTTCGGCTTCATCACTCCTGACGATGGCTCTAAAGACGTGTTCGTACACTTCTCTGCTATCCAGAACGATGGTTACAAATCTCTGGACGAAGGTCAGAAAGTGTCCTTCACCATCGAAAGCGGTGCTAAAGGCCCGGCAGCTGGTAACGTAACCAGCCTGTAATCTCTGCTTAAAAGCACAGAATCTAAGATCCCTGCCATTTGGCGGGGATTTTTTTATTGGATAAACCGCTTTATATGTAA -+SRR33138449.81 81 length=251 -11>1>BC1AFFFFAEFGFGGGGG1FFFFHHGCCC0FCFHHHHFFHGEECGHFEGHGFHHFFFFAFGFFHGHGBCEA/GEFGHHFFHHEBGHGHGHHEGGEFHGFGDFFGFHFFHHHHEFFF1GEHHGGHG?C@CHGDFGHEFACCCGGCBGH0FG1>FCFAB1..;ABEFFFFFFFFFFFFFFFFFBFFFFFFFFBFFFFFFFFFF/:0 -@SRR33138449.83 83 length=251 -GGACGAAGGTCAGAAAGTGTCCTTCACCATCGAAAGCGGTGCTAAAGGCCCGGCAGCTGGTAACGTAACCAGCCTGTAATCTCTGCTTAAAAGCACAGAATCTAAGATCCCTGCCATTTGGCGGGGATTTTTTTATTGGATAAACCGCTTTATATGTAGAGAAGAACAGTCCCAGGAACAATCCAGATCCGGCACCGGAATGTCGCAATCACGACAGACCGCACCATCGCACCCGGAACAATCACAATTAT -+SRR33138449.83 83 length=251 -CCCCCCCCCCFFGGGGGGGGGGHHHHHHHHHGHHGGGGGEEGHHHHHHHHGGGGGGGHHHGGHHGHHHGHHHHGHHHHHHHHHHHHHHHHHHHGGGHHHHHHHHHHGHHHHHHHHFHHHHGHGGGGCEHHHHGGHHHHHFHGHHFGGFGGGHHFHHHHHGGGGGGGFFGBFGFGBFEGFFGFFGGFFFGFFFAFAFFFFFFFFFFEFFFFFFFADFFEFE>;BCFFBFDFFFFFFA;DAFFBFFFFFFFFE -@SRR33138449.83 83 length=251 -GAACGAGGTAATACGTAAAAAGAAGAATAATAATTGTGATTGTTCCGGGTGCGATGGTGCGGTCTGTCGTGATTGCGACATTCCGGTGCCGGATCTGGATTGTTCCTGGGACTGTTCTTCTCTACATATAAAGCGGTTTATCCAATAAAAAAATCCCCGCCAAATGGCAGGGATCTTAGATTCTGTGCTTTTAAGCAGAGATTACAGGCTGGTTACGTTACCAGCTTCCGGGCCTTTAGCACCGCTTTCGA -+SRR33138449.83 83 length=251 -A1AAAAAD1BBFFGFGFFGGGGGHHFFHHGHHHGHHFHFHHHGGHFCEGEEGGEGEEFHHGGGGGFGFFEEHHGGEFGGEGHHGHCEGGFEGCEEFHGFFBFHHHHHHGFHHHGHHGFHHHGGHHHGHHHHFHEE@?@EBGDDFFGHHGHHGGHHHFFCCCCC/GF0CGCGCEFGGGBFGGGBGGGF0BFBBFFF0BBFEFFBFFG09.F99;/B9A:B-A//9-///99@@->EFF/;BFFA-=@9:/-9 -@SRR33138449.84 84 length=251 -ATTGTGATTGTTCCGGGTGCGATGGTGCGGTCTGTCGTGATTGCGACATTCCGGTGCCGGATCTGGATTGTTCCTGGGACTGTTCTTCTCTACATATAAAGCGGTTTATCCAATAAAAAAATCCCCGCCAAATGGCAGGGATCTTAGATTCTGTGCTTTTAAGCAGAGATTACAGGCTGGTTACGTTACCAGCTGCCGGGCCTTTAGCACCGCTTTCGATGGTGAAGGACACTTTCTGACCTTCGTCCAGA -+SRR33138449.84 84 length=251 -ABCCAFFFFFFFGGGGGEGGGGHGGHHHGGGGGHHHGGGHGHHHGGGGGHHHHGGGGHGGFGGHHHGHHHHHHHHHHGHHGHHHHHHHHHHHHHHHHGFFHFGFFGGGHHHHHHHHGHHGGHHHHGGGGGGHHHHGHGGGGHGHHHHHGHHHHHHGHHHHHHHHHHHHHHHHHHHFGHHGGGGGGGGGGGGGGGGGGGGGGFFFFFFFFFFFFFFFFFFFFFBBFFFFFFFFFFFFFEFFFFFFFFFFFF/ -@SRR33138449.84 84 length=250 -ATCCAGAACGATGGTTACAAATCTCTGGACGAAGGTCAGAAAGTGTCCTTCACCATCGAAAGCGGTGCTAAAGGCCCGGCAGCTGGTAACGTAACCAGCCTGTAATCTCTGCTTAAAAGCACAGAATCTAAGATCCCTGCCATTTGGCGGGGATTTTTTTATTGGATAAACCGCTTTATATGTAGAGAAGAACAGTCCCAGGAACAATCCAGATCCGGCACCGGAATGTCGCAATCACGACAGACCGAAC -+SRR33138449.84 84 length=250 -BBBBBFBFFB@BGGGGGGGGGGGHHCHHHHGGGGGGFHHHHHHHHHHHHHHHGHHHGGHGGGGGFGGGGHHGHGGHGGGGGGGFFHHHHHGHHH?GDFGEGHHHHHHHFFHFHHEHGHGHGGHHGHGFHHHGGHGHHGHHHHHHHHGHGGGGC?GGHGGFEEGGEFFBFCB/?FBGGBBBFB9BFEF0;F/C.FF/;BFFFFEFFFF//BBB9/BB?@===EAA-D.B/BAADFAFFBF.-;-:AF=-:@ -@SRR33138449.85 85 length=249 -GGCACACTTAATTATTAAAGGTAATACACTATGTCCGGTAAAATGACTGGTATCGTAAAATGGTTCAACGCTGACAAAGGCTTCGGCTTCATCACTCCTGACGATGGCTCTAAAGACGTGTTCGTACACTTCTCTGCTATCCAGAACGATGGTTACAAATCTCTGGACGAAGGTCAGAAAGTGTCCTTCACCATCGAAAGCGGTGCTAAAGGCCCGGCAGCTGGTAACGTAACCAGCCTGTAATCTCTG -+SRR33138449.85 85 length=249 -AABAAFFFFBFFBGFGGGFGGBFGHHHHHHHHHHHHDFEEGFFHHGHHHHHHHBEBGHHHGHHHHHHHHGGGCGCEGH32CEGH@A?C/GGHFHH3FHHGFHG11??2BGHGHHHHHGGGFHHGGHHGHHHHHGHFFHHFHHHGGEFGGHGCGHFFFHHHHHGHHFHHFGCGGHHHFGFCCFA1A11B1EFCGGB3BB02FHFFF0EE/GGDA/A/EBEC/E/1FB11B>12FGGDGCFG0GFH2GHHHHGBFBFHEHD2GBE//>EGFHHG1FCH111//B1FGACCGC/0111@0E??/0AAA3DFFFFFFGGGGGGGGGGHGGEEFGHHFGHHHHFFHHGHFHHHHHHHHHHHHGGGGGFHFFGHHGHHEGGGGHHHHHHHHHHHHHGGGGCHHHHHHEHGHGGGHHHGGHHGHHHHHHHGGHFHHHGHGFHGDHGGHHGHHHEHHHGHFHGHCCCDGFHEHF1GHBDGFFHHHGHGGEHGHHGDC.CAFGHG0C:CCG.AAG??EFBFFFGBDGAFA0C?EEFFFBFFFFEFFFFFF///;@9@@F.. -@SRR33138449.88 88 length=251 -GTAATACACTATGTCCGGTAAAATGACTGGTATCGTAAAATGGTTCAACGCTGACAAAGGCTTCGGCTTCATCACTCCTGACGATGGCTCTAAAGACGTGTTCGTACACTTCTCTGCTATCCAGAACGATGGTTACAAATCTCTGGACGAAGGTCAGAAAGTGTCCTTCACCATCGAAAGCGGTGCTAAAGGCCCGGCAGCTGGTAACGTAACCAGCCTGTAATCTCTGCTTAAAAGCACAGAATCTAAGA -+SRR33138449.88 88 length=251 -BA?ABFFFFFFFGGGGGGGGGGHHHFHHHHHHHHGHHHHHHHHHHHHHHGGGGGHHHHHFEHHHGGGGGHHHHHHHHFHHGGFGGGGHHHHGHHHGHFF?HHFGGHHGHHHHHHHHHHHHFHHHFGHGEHFDGGHHHHHHHHHHHHHHGGGFFHHHHFHHHHHGHHHHHHGHHHHGHHGGHGGDFGFHHHFHHHGGGGGGBFGGFFFFDFGFDBFGGGFGGGGGGGGFFFFFFFFFFFFFFFFFFFFFFFF -@SRR33138449.88 88 length=250 -GATCTGGATTGTTCCTGGGACTGTTCTTCTCTACATATAAAGCGGTTTATCCAATAAAAAAATCCCCGCCAAATGGCAGGGATCTTAGATTCTGTGCTTTTAAGCAGAGATTACAGGCTGGTTACGTTACCAGCTGCCGGGCCTTTAGCACCGCTTTCGATGGTGAAGGACACTTTCTGACCTTCGTCCAGAGATTTGTACCTATCGTTCTGGAGCGCAGTGAAGTCTACGAACACGTTTTTATAGCCAT -+SRR33138449.88 88 length=250 -B3AAADFFFFFFGGFGGGAGGFGGFFHHCDGGHHHH6D65A5FEE2AEBGD55CGH3F5B0EGFFFFEEGG02FFFFB1//FGGHHFHHHH4F4GDFFHGFEFHH33?/0FDFFFFG/?F/EDDGGHFHGGFH/11FGB?11>F1CA1FFGDFBGGGGCA3FBB3GHHHHHGGGGGFHHHHGHEF1GH?/FABG/F/A//FDG1F0?EEFGAGHHH1FHFHFHHHEFF1BBE1FGEGFGHFBG10>B0E0FGHCGG0E111/FGFHGG?CAA1>FFFFFFFGG1C11FFFGHGF1FFEAE0BGAEBGGFFBFFFHFGHHEHHDBHBFGG/FCGHHGFBGHHHGHBG1FEEEFCFGFGFFHHHBFF1BGHGHHHFHGEFE/GG?GGGHHEFFGGFGGGGGGCBHGHHGGF9BF/FFFFF//BB/B//BFFF::BFFFA -@SRR33138449.91 91 length=251 -GAACGAGGTAAGACGTAATAAGAAGAATAATAATTGTGATTTATCCGGGCGCGATGGTGCGGTCTGTCGTGTTTGTGACATTCCGGTCCCGGATCTGGAGTGTTCCTGGGACTGTTCTGCTCTACATATAAAGCGGTTTATCCAATAACAAAATCCACGCCAAATGGCTGGGTTCTTAGATTTTGTGCTTTTAAGCAGAGATTACAGTCTGCTCACGTTACCAGCAGCCGGGCCGTTAGCACTGCTTTCGT -+SRR33138449.91 91 length=251 -A?11>11>1B3131AF1003311DA10B1A2A2BD2FGHB221221//A/AEA/A//B11>@?//0B2?/B/0B00BFG1FDG2>/>/>0/////11B10BF1>2>210?00111D2FG211/FF0FFG11.<..-<000//.;./;:C0:0;:000://0;CFFF0;00//.;0;0000;0900000;..:;.;B/A-:FE-9@-9--;9-/:9/9/;FB/;- -@SRR33138449.92 92 length=251 -GTTCTTCTCTACATATAAAGCGGTTTATCCAATAAAAAAATCCCCGCCAAATGGCAGGGATCTTAGATTCTGTGCTTTTAAGCAGAGATTACAGGCTGGTTACGTTACCAGCTGCCGGGCCTTTAGCACCGCTTTCGATGGTGAAGGACACTTTCTGACCTTCGTCCAGAGATTTGTAACCATCGTTCTGGATAGCAGAGAAGTGTACGAACACGTCTTTAGAGCCATCGTCAGGAGTGATGAAGCCGAAG -+SRR33138449.92 92 length=251 -CCCCCFFFFFFFGGGGGGCGGGGGGGHHHHHHHGHHHGGGHHHHGGGGGGFHHHHHGGGGHGHHHHHHHHHHHHHHHHHHHGHHHFHFHHHHHHHGHHGHHHHGHHHGHHGHHHGHGGGGGGHHHHHHHHGGGGGHGFHHGFHHHHHHHHHHHHHHHHHHHHHGHGGGHGHHHHEDGHHHHHHHGHHHGHHFHHHGGGGGGGGFFFFGGGGGGGGFFFEGGBBFFFFFFFFFE?BBFFFFFFFFFFFFF;- -@SRR33138449.92 92 length=249 -CCTTTAACGCTTCAAAATCTGTAAAGCACGCCATATCGCCGAAAGGCACACTTAATTATTAAAGGTAATACACTATGTCCGGTAAAATGACTGGTATCGTAAAATGGTTCAACGCTGACAAAGGCTTCGGCTTCATCACTCCTGACGATGGCTCTAAAGACGTGTTCGTACACTTCTCTGCTATCCAGAACGATGGTTACAAATCTCTGGACGAAGGTCAGAAAGTGTCCTTCACCATCGAAAGCGGTG -+SRR33138449.92 92 length=249 -BBBBBBFFBBBBGG5FGGGGGGHGCHHHH?FFAFHGHGGGGCGGFHHHHHHHHHHHHHHHHFGFHHGFHHHHHHHHHHHHGGGGFFHHHHHHHHHHHHGFFHHHHHHHHHGHHGGGGGHHHHHHGHHHFDGGDHHHHHHHHGHFHHGAECDFHGHEFHEFHGFDHFF=1A111111B1BBB1EFEC11DFDF13G10A?0EE010DFDGBGHF/0AE/A/F110B0//F1AFG1B@@/?>//BBB11FB1B2F//?//B/B>>FB//B0/B21>BGC0/>>G2GHHGGGG//??CA0>?CCACFFGFFGACGFAE?@FHHHFHGFHG/BDGHGEHFD1@FGHBGHEHFFFHHH11<FG>/>0FCF@11@F>/EE0G/ECGAHHDDGGHBBDBGGGGFDCHG/FC?11BGDGHHFBG@>1GFGCCC/111>1>A11A10AAAAE1B000///AEE//E/2F20/AEEGEGC/E1F@11B0BBGBD1>E00>>>2BG2>>>G1B112B>2211/1FFEF/C00000=0D000=0=0=000000;/./;CFH00C.::CBCF0E//99C0;./90/.-9@?-9/-----9--99-/:-9/-;9:/;////;///////////////9-9/-/;BF/B////// -@SRR33138449.98 98 length=251 -TCCAATGCATGAGCTGTTGAGCAGCCTGGAACAGATTGTTTTTAAAGATGAAACGCAGAAAATTACCCTGACGCACAGAACAACATCCTGTACCGAAATTGAGCAGTTACGAAAAGGAACAGGGTTAAAAATCGATGATTTCGCCCGGGTTTTGGGCGTATCAGTCGCCATGGTAAAGGAATGGGAATCCAGACGCGTGAAGCCTTCAAGTGCCGAACTAAAATTGATGCGTTTGATTCAAGCCAACCCGG -+SRR33138449.98 98 length=251 -BCCBBFFFFFFFGGGGGGGGGGHHHHHHHCFHHHGHHHFGHHGHHGHHHHHHHHFGGGGHHHHHHHHHHGHHGEGGGHHHHHFHHHHHHHHHHHGGGGHHHHHHHHHHHHGGHGFHGFHHEHEGFHHFHHHHHGHHHHHHHHHGGGGGGFGGGHGGHGGGGGHHFHGGGGGHHHHHFHGGHHHFFHGGHGHHFE?DFGGG?FGGFFGGGFFGGFGFFFFFFFFFFFFBFFFFFFFBFFFFFEFFFFFFFFF -@SRR33138449.98 98 length=251 -ATTCTGTGCTTTTAAGCAGAGATTACAGGCTGGTTACGTTACCAGCTGCCGGGCCTTTAGCACCGCTTTCGATGGTGAAGGACACTTTCTGACCTTCGTCCAGAGATTTGTAACCATCGTTCTGGATAGCAGAGAAGTGTACGAACACGTCTTTAGAGCCATCGTCAGGAGTGATGAAGCCGAAGCCTTTGTCAGCGTTGAACCATTTTACGATACCAGTCATTTTACCGGACATAGTGTATGACCTGTAA -+SRR33138449.98 98 length=251 -AAAB@FDDFFFFFGGGGGFFGGHFDGCGHCHA2FDEGFAGFGGFHEBACFGCC0EFHHHHGHHFGGG?BFEFFHG@DGHDGGFGGHBGHGBGBCHHHHHGGGHBEGEHHEDGHHHHGHCFGGGHGGFHBDGHFC2223F?FHHGA//@GCFDFHEHA1>DFDDBGGFGCCHEGHF;000CE:.;.;C..;90CFGGAE?@CBF009CBC0;BFBD-9@FFB99B/B/;///.;9/ -@SRR33138449.99 99 length=251 -ATTCTGTGCTTTTAAGCAGAGATTACAGGCTGGTTACGTTACCAGCTGCCGGGCCTTTAGCACCGCTTTCGATGGTGAAGGACACTTTCTGACCTTCGTCCAGAGATTTGTAACCATCGTTCTGGATAGCAGAGAAGTGTACGAACACGTCTTTAGAGCCATCGTCAGGAGTGATGAAGCCGAAGCCTTTGTCAGCGTTGAACCATTTTACGATACCAGTCATTTTACCGGACATAGTGTATTACCTTTAA -+SRR33138449.99 99 length=251 -BCBCCFFFFFFFGGGGGGGGGGHHHHHHGGHHGHHHHGHGHGHHHHGHHHGGGGGGHHHHHHHHGGGGGHGFHHGHHHHHHHHHHHHHHHHHHHHHHHHGGGHHHGFFHHFHGHHHHHGHHHGHHHHHHGHHHHHGGFGHHHHGHGGGGHHHGHHHGHHGHHHGHHHGHHGGGGHHFHHHFFDGGGHHGHHGHHHHGEGGHGGGGGGGGGGGGGGGGGGGGGGGGGGFFGGFFFFFFFFFFFFFFFFFFF/ -@SRR33138449.99 99 length=250 -CATCAACGGTTTGACGTACAGACCATTAAAGCAGTGTAGTAAGGCAAGTCCCTTCAAGAGTTATCGTTGATACCCCTCGTAGTGCACATTCCTTTAACGCTTCAAAATCTGTAAAGCACGCCATATCGCCGAAAGGCACACTTAATTATTAAAGGTAATACACTATGTCCGGTAAAATGACTGGTATCGTAAAATGGTTCAACGCTGACAAAGGCTTCGGCTTCATCACTCCTGACGATGGCTCTAAAGA -+SRR33138449.99 99 length=250 -CBBCCFFCCAABGGGGFGGGGGHHHGHHHHHHGHDGGGHFHHHFGHHGHHHHHHHHHHGHHHFHHHGHHFHHHHHHGGGHCGFGHHHHGHHHHHHGHHGGGGGHFHEHHHHHHGFHHHHGGGGGGHGGGGGDGGDHHHHHHGHHGHHHHHFHEFGGGHGHGHHHHHHD1GDGGFEGHHHGHHHHEHHGEFGGHHGGGHFHHHHGGDGDHGEFFFGGFFGGGGGGGGGGGGGGGGGGGEEFFFFFFFFFFF -@SRR33138449.100 100 length=251 -AGATTACAGGCTGGTTACGTTACCAGCTGCCGGGCCTTTAGCACCGCTTTCGATGGTGAAGGACACTTTCTGACCTTCGTCCAGAGATTTGTAACCATCGTTCTGGATAGCAGAGAAGTGTACGAACACGTCTTTAGAGCCATCGTCAGGAGTGATGAAGCCGAAGCCTTTGTCAGCGTTGAACCATTTTACGATACCAGTCATTTTACCGGACATAGTGTATTACCTTTAATAATTAAGTGTGCCTTTCG -+SRR33138449.100 100 length=251 -ABBABFFFFC?FGGGGGGGGGGFFHHHGHHHGGGGGGHGHGHHHHGGFGGHGGHHFHFDGHFHHGHHHHHFHHHFHHFHFGGEFHHHHFHHFGHHGFFHGGHHGHHHHGHHHGHHHHHBGHHEEGECFHDGFFFHFGHGFFHHHCEHGFGHGHHHHGHEHFHGGGFGHHHGHHHFHHGGGCHFFHHHHHHHGGGHECGGFGGFGGGGGG0E-DAGGGGCFBFFEFFFGFFFFFFFFFFFFBB/FFFFFFFD -@SRR33138449.100 100 length=251 -GTGTAGTAAGGCAAGTCCCTTCAAGAGTTATCGTTGATACCCCTCGTAGTGCACATTCCTTTAACGCTTCAAAATCTGTAAAGCACGCCATATCGCCGAAAGGCACACTTAATTATTAAAGGTAATACACTATGTCCGGTAAAATGACTGGTATCGTAAAATGGTTCAACGCTGACAAAGGCTTCGGCTTCATCACTCCTGACGATGGCTCTAAAGACGTGTTCGTACACTTCTCTGCTATCCAGAACGAT -+SRR33138449.100 100 length=251 -AAAAAFDFFFFFGGGGGGGGBGHHGHHBFFDGHHGFAGGGFBCFGEHEGFFFFHHHFFHHFFHFBE?EFGFHHHHFFHHBGHHGHHGG1E1GHHFDEGGCGF1FHFGHHHHBGGHHHHHHHB3BDGGFF3BFGEDD311ADGF1FFG11F0F1FEEA0EEC1AFGB1FCE/AAFFA/EEA1A11AF0FFCCG1D2@2@0EF1/0/EEGHEHF1DB22BB1B1GC/>FFHHDGFCGBG0F0F0FFHGF/FG/>0A<<0FH2D1110@1>1>131B31>1A1111AAEGG3113333D2DA211111B10/BA01D11/ABF2F///AA//A12D22B///B0?EECG1212222B2F21110/?FE/1B2B21>11121B1BG1121B1B110///////>222@G//B/////-<1>CGG..<111./00/000...<.;00/;0;;;...:..--9/;...;/000;0;/.;9.0000/////-9:--:/9->9-@-9AF//;///9/////9/// -@SRR33138449.102 102 length=251 -TTACGGTCCTGATGACAGGACCGTTTTCCAACCGATTAATCATAAATATGAAAAATAATTGTTGCATCACCCGCCAATGCGTGGCTTAATGCACATCAACGGTTTGACGTACAGACCATTAAAGCAGTGTAGTAAGGCAAGTCCCTTCAAGAGTTATCGTTGATACCCCTCGTAGTGCACATTCCTTTAACGCTTCAAAATCTGTAAAGCACGCCATATCGCCGAAAGGCACACTTAATTATTAAAGGTAA -+SRR33138449.102 102 length=251 -BCCCBCFCCFFFGGGGGGGGGGGGGHGHHHHHGGGGGGHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHGGGGGHHHGGGGGHHHHHHHHHHHHHHHGGGGGGHHGFHHGHHHHHHHHHHHHHHGGHHHHHGHHHHHGHHHHHHHHHHHHHHHHHHHHHHHHHGHHGGGHGGGHHHHHHHGHHHHHHHGGGGGHHGHHHHHHHHGGGGGGGGGGGGGGGGGAGFDFEFFFFFFFFFFFFFFFFFFBFF -@SRR33138449.102 102 length=251 -GATTACAGGCTGGTTACGTTACCAGCTGCCGGGCCTTTAGCACCGCTTTCGATGGTGAAGGACACTTTCTGACCTTCGTCCAGAGATTTGTAACCATCGTTCTGGATAGCAGAGAAGTGTACGAACACGTCTTTAGAGCCATCGTCAGGAGTGATGAAGCCGAAGCCTTTGTCAGCGTTGAACCATTTTACGATACCAGTCATTTTACCGGACATAGTGTATTACCTTTAATAATTAAGTGTGCCTTTCGG -+SRR33138449.102 102 length=251 -AAAABFFFB@FFFGGFGGGGGGHHHHHHGHGGGGFGHGHHHHHHGGGGEFG1GHGFHHFHHHHHHHHHHHHHGHHGHHFGGGHHHEHHHHHGHGHGHFG?FGAGGHHHGHHFHHGHHHHHHHHGGGGHGGFHGHHHGHBGFHFGHHGGHFFHHHHHGHHHHGGGGGHHHGH=GHHHGGDGEHG.C0CCHHGGEEGCFGGGGGEBFFGGBE?-?@FFG;FFFFFCFFCBFFBFF99;;FF///9;/F/BBDD -@SRR33138449.103 103 length=251 -GGTTACGTTACCAGCTGCCGGGCCTTTAGCACCGCTTTCGATGGTGAAGGACACTTTCTGACCTTCGTCCAGAGATTTGTAACCATCGTTCTGGATAGCAGAGAAGTGTACGAACACGTCTTTAGAGCCATCGTCAGGAGTGATGAAGCCGAAGCCTTTGTCAGCGTTGAACCATTTTACGATACCAGTCATTTTACCGGACATAGTGTATTACCTTTAATAATTAAGTGTGCCTTTCGGCGATATGGCGT -+SRR33138449.103 103 length=251 -BCBBBFCFFFCFGGGGGGGGGGGGGHHHHHHHHGGGGGHGGHHGHHHHHGHHHHHHHHHHHHHHHHHHGGGHGHHHHHHHHHHHHHHGHHHGHHHHHHGHHHHHHHHHHHHGGHGGHGHHHGHHHHHHHHHHGHHHGHHGHHGHHHHHHHGGGGGHHHHHHHHHHGGGGHHHHHHHHHHHGGHGGHHHHGHHHHHHHHCCGGGGGGGGGGGGGGGGGGGGGGGGGGGGFGGGFFFFFFFCFFFFFFFFFF. -@SRR33138449.103 103 length=251 -TGCGTGGCTTAATGCACATCAACGGTTTGACGTACAGACCATTAAAGCAGTGTAGTAAGGCAAGTCCCTTCAAGAGTTATCGTTGATACCCCTCGTAGTGCACATTCCTTTAACGCTTCAAAATCTGTAAAGCACGCCATATCGCCGAAAGGCACACTTAATTATTAAAGGTAATACACTATGTCCGGTAAAATGACTGGTATCGTAAAATGGTTCAACGCTGACAAAGGCTTCGGCTTCATCACTCCTGA -+SRR33138449.103 103 length=251 -BBCCABCCFFFFGGGGGGGGGGHGGFGGGHHGGGHGHHHHHHHHHHHHHHHHHHHHHHHHHHHGGGHHHHHHHHHHGHHHGHHHHGHHHGHGGCGHGGFHHHHHHHHHHHHHHHGCGGGGHHHHHGHHHHHHHHHGGGGGHEHGGGG/CGGHHHHHHHHHHHGHHHFGFFEFFFHHHHHHHHHHHHGGGFGHHHHHHHHHFGHEEHHGHHHGGFFGGGG?DGDDFGBFGFFFGFC.;EFFFFFFFFFFFF; -@SRR33138449.104 104 length=251 -CTGCCGGGCCTTTAGCACCGCTTTCGATGGTGAAGGACACTTTCTGACCTTCGTCCAGAGATTTGTAACCATCGTTCTGGATAGCAGAGAAGTGTACGAACACGTCTTTAGAGCCATCGTCAGGAGTGATGAAGCCGAAGCCTTTGTCAGCGTTGAACCATTTTACGATACCAGTCATTTTACCGGACATAGTGTATTACCTTTAATAATTAAGTGTGCCTTTCGGCGATATGGCGTGCTTTACAGATTTT -+SRR33138449.104 104 length=251 -BBBCCCDCDDDFGGGGGGGGGGGGHGGHHGGHHHHHHHHHHHHHHHHHHHHHHHGGGHBGGHHHHHHHHHHHHGHHHGHHHHHHHHHHHHHHHHHHHHGHGGHGHHHGHHHHHHHHHHGHHHGHBFGGHHHHHHHHGGGGGHHHHHHHGHHGGGGGHHGHHHHHHHGGHGGHHHHHHHHHHHHGGGGGGHHHHHHHGGHHGHHGHGGGGGGGFGGGGGGGGGGFFGGG;DFFFFFFFFFFFFFFFFFFFFF -@SRR33138449.104 104 length=251 -CCCTCGTAGTGCACATTCCTTTAACGCTTCAAAATCTGTAAAGCACGCCATATCGCCGAAAGGCACACTTAATTATTAAAGGTAATACACTATGTCCGGTAAAATGACTGGTATCGTAAAATGGTTCAACGCTGACAAAGGCTTCGGCTTCATCACTCCTGACGATGGCTCTAAAGACGTGTTCGTACACTTCTCTGCTATCCAGAACGATGGTTACAAATCTCTGGACGAAGGTCAGAAAGTGTCCTTCA -+SRR33138449.104 104 length=251 -BACCCCC@BAFFGGGGGGGGGGHHHGGCGGHGHGHHHHHHHHCHHHGGGGGHHHGGGGAFGGHHGHHHHHHHHHGHHHGHHHGGHHHHHHGHHHHGHGGGEGGFHHHGHHHHFHHGHHHHHHHHHHHHHGGGGGGHHGFHHHHHHGFCGGHHHHGHGHHHHEHGEGGGHHFHHHHHGHGGGHGGFGEHGHHHHGHHHHHHHGHHHEGGGBGE.:0;BFFFGGGGGGGFFGAAB?BFFFEFFFFFFFFFFF/ -@SRR33138449.105 105 length=251 -CTGCCGGGCCTTTAGCACCGCTTTCGATGGTGAAGGACACTTTCTGACCTTCGTCCAGAGATTTGTAACCATCGTTCTGGATAGCAGAGAAGTGTACGAACACGTCTTTAGAGCCATCGTCAGGAGTGATGAAGCCGAAGCCTTTGTCAGCGTTGAACCATTTTACGATACCAGTCATTTTACCGGACATAGTGTATTACCTTTAATAATTAAGTGTGCCTTTCGGCGATATGGCGTGCTTTACAGATTTT -+SRR33138449.105 105 length=251 -AB>ABCCCCCCFGGGGGGGGGGGGHGFHHGHHHHHFHHHHHHHHHHHHHHHHHHGGGHHHHHHHHHHHHHHHHGHHHGHHHHHHHHFHHFHHHHHHHHGHGGHGGHHGHGHHHHHHHHGHHHGHHGGHFHHHHGHHGGGGGHHHHHGFBGHGGGGHHHHHHHHHHHGGHGGHHHHHHHHHHHHHGCCFGHHHHFHHGGGGGGGGFGGGGGGGGGGGFGGGGGGGFAGFDFFFFFFFFFFFFFFFFFFFFFF -@SRR33138449.105 105 length=251 -GCTTAATGCACATCAACGGTTTGACGTACAGACCATTAAAGCAGTGTAGTAAGGCAAGTCCCTTCAAGAGTTATCGTTGATACCCCTCGTAGTGCACATTCCTTTAACGCTTCAAAATCTGTAAAGCACGCCATATCGCCGAAAGGCACACTTAATTATTAAAGGTAATACACTATGTCCGGTAAAATGACTGGTATCGTAAAATGGTTCAACGCTGACAAAGGCTTCGGCTTCATCACTCCTGACGATGG -+SRR33138449.105 105 length=251 -A3AA@BFFFFFFGGGFGGGGGGFGHGGHHCHHFHHHGHHHFHFH5FGHHGFHHHFHEFHGGGGFHHHHHHFFHHHHGHHFHHHHBHGFGGGGCGHHHHFHHHHGHHHHGGGGGGHGHHHHHHHHHFHHHGGGGGHHHGFGGGG?DHHGHHGGHGHHHHGHHHHGFFFHHHHHHGHHHHHHGGGGGGFFHGGHHADDGHEGHAGHHFFHGHFGHF-;DDFFBFFGGGFBBGG.DGFGGGBGF.;//BD?.AA -@SRR33138449.106 106 length=251 -CCATTAAAGCAGTGTAGTAAGGCAAGTCCCTTCAAGAGTTATCGTTGATACCCCTCGTAGTGCACATTCCTTTAACGCTTCAAAATCTGTAAAGCACGCCATATCGCCGAAAGGCACACTTAATTATTAAAGGTAATACACTATGTCCGGTAAAATGACTGGTATCGTAAAATGGTTCAACGCTGACAAAGGCTTCGGCTTCATCACTCCTGACGATGGCTCTAAAGACGTGTTCGTACACTTCTCTGCTA -+SRR33138449.106 106 length=251 -CDCCDFFFFFFFGGGGGGGGGGHHHGHHHHHHHHHHHHHHHHHHHHHHHHHHHHGGGHGGGHHHHHHHHHHHHHHHGGGGGHHHHHHHHHHHHHHHHGGGGGHHHGGGGGGGGHHHHHHHHHHHHHHHHHHHHHGGHHHHHHHHHHHHGGGGGHHHHHHHFHHHHHGHHHHHHHHHHHHHHGGGGGHHHHHHHHHHGGGGGHHHHGHHHGHHHGGGGGGGGGGHGGGGGGGGGGGGGGGFFFFFHFHFFH1 -@SRR33138449.106 106 length=251 -CTTTAGCACCGCTTTCGATGGTGAAGGACACTTTCTGACCTTCGTCCAGAGATTTGTAACCATCGTTCTGGATAGCAGAGAAGTGTACGAACACGTCTTTAGAGCCATCGTCAGGAGTGATGAAGCCGAAGCCTTTGTCAGCGTTGAACCATTTTACGATACCAGTCATTTTACCGGACATAGTGTATTACCTTTAATAATTAAGTGTGCCTTTCGGCGATATGGCGTGCTTTACAGATTTTGAAGCGTTA -+SRR33138449.106 106 length=251 -CCCCCFFFFCCCGGGGGGGGGGHHHHHHHHGHHHHHHHHHHHHHHGGGHHGHHFHHHHHHHHGHGHHHGHHHHHHHHHHHGHGFHHHHHGHGGHGHHGGHHHHFHHHHHGHHHGHHGFHHEHHHHGGGGGGGHHHHHHHHHHGGGGEGHHHHGHHHHGGHGGFHGHGHHHHHHHHGGGGGHGHHGGHHHHHHHHHHFFH:GHHHHHHHHGHHHGGGAAGGGGGGGGGFFFFFFFFFFBFEFFFFFBFBDF. -@SRR33138449.107 107 length=251 -GCTTTCGATGGTGAAGGACACTTTCTGACCTTCGTCCAGAGATTTGTAACCATCGTTCTGGATAGCAGAGAAGTGTACGAACACGTCTTTAGAGCCATCGTCAGGAGTGATGAAGCCGAAGCCTTTGTCAGCGTTGAACCATTTTACGATACCAGTCATTTTACCGGACATAGTGTATTACCTTTAATAATTAAGTGTGCCTTTCGGCGATATGGCGTGCTTTACAGATTTTGAAGCGTTAAAGGAATGTG -+SRR33138449.107 107 length=251 -CCCCCFBBFFBFGGGGCGGGGGHHHHHHHHHHHHHGGGHGGCGGFHHFDHHBGHGFHHGHHHHGHHFHGHFHHEEGGHEFEG@HGGHHFHHHGGHGFHHGGHFFFCFGFFHHBFHHHGFEGGHHFHHGHFHHGGDGHHHHGHHHHHHGGHGGHHHHHFHHHHHHHGGGGGHEHGGDHHHHHHHHHHHHGHHHHHGFHHHHHHHHHGGGGGGGGGGGGADFGGGGGBFBFGGGGFFFFFDFFEFFFFFFFFB -@SRR33138449.107 107 length=251 -CCATTAAAGCAGTGTAGTAAGGCAAGTCCCTTCAAGAGTTATCGTTGATACCCCTCGTAGTGCACATTCCTTTAACGCTTCAAAATCTGTAAAGCACGCCATATCGCCGAAAGGCACACTTAATTATTAAAGGTAATACACTATGTCCGGTAAAATGACTGGTATCGTAAAATGGTTCAACGCTGACAAAGGCTTCGGCTTCATCACTCCTGACGATGGCTCTAAAGACGTGTTCGTACACTTCTCTGCTA -+SRR33138449.107 107 length=251 -BBABBFFDFBFFGGGFGGGGGGGHHGHHHHGFGHHGHHHHHHHGGGBEHHHCGHGGGHGFEHHHHHHHHHHHHHHHGGGGGHHFFHHHHHHEFGHFHGGGGGHHHGGGG/@/EGHHHCHHHGFHGHHHHBFHEFFGFHHGHHHHHHHHGCCGFF2GHFHHFGFGGFFFHHHGHHHHHHHHHG?.ADGD0/A>A4AABFC>AGGGGGCBGGGBHHHGFBFFCFHD555B3EGFHHGHAGEFFAG2FAAGHHHHFGEEEC1BBEGFHFHBGHHHFEE?EFGHFHHGCDGHFHHHFHHGEF?FHHGGGFGGCGFGFFHHHHHHHHBDDFGGBHH1FGHFGFFDFGG@FHGGDGEHHHGGBBFC=FHEEFFH.=1FGG=DGFG?C?D.CFHFFHGGFHCGCCB/FFGF0FFBFBFGG?EADBC0;0;9C09CA./:FFAFFF -@SRR33138449.108 108 length=251 -GTGAAGGACACTTTCTGACCTTCGTCCAGAGATTTGTAACCATCGTTCTGGATAGCAGAGAAGTGTACGAACACGTCTTTAGAGCCATCGTCAGGAGTGATGAAGCCGAAGCCTTTGTCAGCGTTGAACCATTTTACGATACCAGTCATTTTACCGGACATAGTGTATTACCTTTAATAATTAAGTGTGCCTTTCGGCGATATGGCGTGCTTTACAGATTTTGAAGCGTTAAAGGAATGTGAACTACGAGG -+SRR33138449.108 108 length=251 -AA1>1BC1>CCCGCGC33GFCDFCHFCGF1F01AFGHA2FGHBBE0G0/D11AFF11F0/FGH1BGBHEGAEEHGFHHGFBGFF10BD1EG///BEFBG1DE2B@G>E>@E/CGH1F>BBGF/EG?/211>A11BDFD>EEGFGGCGGBFGHFGGGGEE??FBFBFD21E1BBAFGG2FFBEHBD1FEEAFGHGHFGFH2FHBBHHHGFHDFGFEEGHH11GHFFGFGE@AEHGHGHEFA/EEGGHHHHHBEDHFGHE2BGEGHHG2GGHFHHHF1<@0EE?EEEFEDG4DGGGFH?FGFEFBG3DCHGHHBFDHGB4FFG?GGF211F1BFHGGEAGG/DF111>>DFFBDD@F1EAFBF1F1DD11110001000AB1A00E/0A0/B/FBE11DFGE1A/F/A0A/0012D2111>/////0BB1@@22BF/?/>/1B00BFF22B//?//>1BBG1BFEF21DEC///1FB2@2222@GD11112@2@@DH1?1111B3@1111111B313B3000001B1A1112AD1AA00/1A1GG1D0//0A/22221BA/0/B1DD221A111@//////0B@2BB1101B@G2B22111B1B@B@2@@2F0/?/EF1/0B00122>2>211?F2>21/?0/011>00>1000=00=..-FCEEH>EEEHHHEGF?EGFHH1FHHHGHFHH1FC1GHHHHFF/ECCCG/GCF1>C?GHHFHFDGEF/E.A@@?C-G00C;CFBBBF9FBFGGGFBBBBF0C9CBFFGFGFFBFFG009;CFFB/9A-9--/99BFBFF/;FFE?BBB-//99A -@SRR33138449.119 119 length=251 -TCTTTAGAGCCATCGTCAGGAGTGATGAAGCCGAAGCCTTTGTCAGCGTTGAACCATTTTACGATACCAGTCATTTTACCGGACATAGTGTATTACCTTTAATAATTAAGTGTGCCTTTCGGCGATATGGCGTGCTTTACAGATTTTGAAGCGTTAAAGGAATGTGCACTACGAGGGGTATCAACGATAACTCTTGAAGGGACTTGCCTTACTACACTGCTTTAATGGTCTGTACGTCAAACCGTTGATGT -+SRR33138449.119 119 length=251 -CCCCCFFFFFFFGGGGGGGGGGHHHHHHHHHHGGGGGHHHHHHHHHHGGGGHHHHHHHHHHHGGHGGHHHHHHHHHHHHHGGGCGHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHGGGGGGGGHHHGGGGGHHHHHHHHHHHHGHHHGGGGGHHHHHHHHHHHHHHHGGGGGGGGHHHHHGGHGGGGGGGGGGGGGGGGGGGGGGGGGGGGGHGGGHHHHHFFFFFFFFFFFFFFFFFFFFFFHF< -@SRR33138449.119 119 length=249 -ATTCAAGCCAACCCGGCATTAAGTAAGCAGTTGATGGAATAGACTTTTATCCACTTTATTGCTGTTTACGGTCCTGATGACAGGACCGTTTTCCAACCGATTAATCATAAATATGAAAAATAATTGTTGCATCACCCGCCAATGCGTGGCTTAATGCACATCAACGGTTTGACGTACAGACCATTAAAGCAGTGTAGTAAGGCAAGTCCCTTCAAGAGTTATCGTTGATACCCCTCGTAGTGCACATTC -+SRR33138449.119 119 length=249 -CCBBCFFFCFFBGGGGGGGGGGHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHGHHHHHHHHHHGFHGGHHHHHHHHHGHHHGGGHGHHHHHGGGGGGGHHHHGGHHHHHEHHHHHHHHHHHHHHHHHHHHHGGGGGHHHGGGGGHHHHHHHHHHHHHGHGGGGGGHHGGHHGHGHHHGHHHHHHHHGHHHHHHHGGHHHGGGGGGGGGGGGGGGGGGGGGGGGFG0BE?EGEFFFFEFFFFFF -@SRR33138449.120 120 length=235 -GTACAGACCATTAAAGCAGTGTAGTAAGGCAAGTCCCTTCAAGAGTTATCGTTGATACCCCTCGTAGTGCACATTCCTTTAACGCTTCAAAATCTGTAAAGCACGCCATATCGCCGAAAGGCACACTTAATTATTAAAGGTAATACACTATGTCCGGTAAAATGACTGGTATCGTAAAATGGTTCAACGCTGACAAAGGCTTCGGCTTCATCACTCCTGACGATGGCTCTAAAGA -+SRR33138449.120 120 length=235 -BACCBFFFCFFFGGGGGGGGGGHHHHHHHHHHGHHHHHHHHGHHHGHHHHHGHHHHHHHHHGGGHGGGHHGHHHHHHGHHHHHGGGGGHHHHHHHHHHHHHHHHGGGGGHHHGGGGGGGGHHHHHHHHHHHHHHHHHHHHHGHHHHGHHHHHHHHGGDGGHHHHHHHHGFHHHGGHHHHHHHGFHHHHGGGGGHHHHHGGHHHGFGGGHHHHGGGGGGGGGGGGGGGGGGGGGGG -@SRR33138449.120 120 length=235 -TCTTTAGAGCCATCGTCAGGAGTGATGAAGCCGAAGCCTTTGTCAGCGTTGAACCATTTTACGATACCAGTCATTTTACCGGACATAGTGTATTACCTTTAATAATTAAGTGTGCCTTTCGGCGATATGGCGTGCTTTACAGATTTTGAAGCGTTAAAGGAATGTGCACTACGAGGGGTATCAACGATAACTCTTGAAGGGACTTGCCTTACTACACTGCTTTAATGGTCTGTAC -+SRR33138449.120 120 length=235 -AA?ABFFFFFFFGGGGGGGGGGGHHHHHGHGHGEGGGHHHHHHHHHHGGGGHHHHHHHHHHHGGHGGGGHHHHHHHHGHGGGCGGHHHFFFGHHHHHHGHHHHHHHHHHHHFHHHHHHHHGGGGGGGGHHHGGGGGHHGHHGHHHHGHEHHHGGGGGHHGHHHHFGHFFEHFGGGGGGGGHHHHGGEHGGGGGGGG09BFGGGGGGFFGGGGFFGGGFFGEGGGGFFFFFFFFFF -@SRR33138449.121 121 length=251 -ATGCACATCAACGGTTTGACGTACAGACCATTAAAGCAGTGTAGTAAGGCAAGTCCCTTCAAGAGTTATCGTTGATACCCCTCGTAGTGCACATTCCTTTAACGCTTCAAAATCTGTAAAGCACGCCATATCGCCGAAAGGCACACTTAATTATTAAAGGTAATACACTATGTCCGGTAAAATGACTGGTATCGTAAAATGGTTCAACGCTGACAAAGGCTTCGGCTTCATCACTCCTGACGATGGCTCTA -+SRR33138449.121 121 length=251 -3>AABFFFFBFFGGGGGGGGGGGHGHGHHCGFHBBHFHFECHFDEDFGFHFFFGDEGHHGH5GDGGFHHHFHHHGHHHGFGFEGHE1EFFHHGHHHHHHHGHH1EEGG3FGB?GGH4GF4FFGHCGGGGHHHEGF///CGHGF00FFHH1FGGHHH1<<1F1?GHGBHHHHHFGHGGGGCDHHHHHHFE>A1>?11>0>FE0EAEE0GGFFA002ABA/A/EAFG0///EB////>1@1@01>@22B@@0/0BBE@EF2G111B2G2F2>22211/>?E?1FFHH1B0BB11/0B11<1>FF003A>55DFBFFFECGGCGGGG5GFGFHFHCFHGHHGHHBGFFHGHGHHHHHHHFFDDG5FGHGHFFGHGGAHGGHEGHEHGF3?GHEEGEHGHHF4FECEEGEFFGEHFFDDFF4EEDBAFDEFHHHHHGHHBBG0F@DGG/<1@CD/C?F<1A?D3DFFDDEEGGGGCGFG3FFHHHHG00EEGHFFGF2DBE1EEBGC/FBAGFHFFFGFAA/EGEC/EEH2DFFHH12FB1>BBEGGHGGFEFFFFHHHEFBGFHHCDGGHHGH1FGB1FHFGHFB0?FGBHGAC-@CHFGFCG.G0/00;0;CAC-;@B0BF0FB0CFFAE9A?F.9/.CB..9.-//99-;BF-;B//;/--- -@SRR33138449.126 126 length=251 -GCCTTCAAGTGCCGAACTAAAATTGATGCGTTTGATTCAAGCCAACCCGGCATTAAGTAAGCAGTTGATGGAATAGACTTTTATCCACTTTATTGCTGTTTACGGTCCTGATGACAGGACCGTTTTCCAACCGATTAATCATAAATATGAAAAATAATTGTTGCATCACCCGCCAATGCGTGGCTTAATGCACATCAACGGTTTGACGTACAGACCATTAAAGCAGTGTAGTAAGGCAAGTCCCTTCAAGA -+SRR33138449.126 126 length=251 -CCCDDFFFFFFFGGGGGGGGGGHHHFHHHGGGGGHHHHHHHHGHHGGGGGGGGHHHHHHHHHHHHHHHHHHHHGHHHHHHHHHHHHHHHHHHHHHHHHHHHHHGGHGGHHHHHHHHHGHHHGGGHGHHHHHGGGGGGHHHHHHHHHHHHHHHHGHHHHHHHHHHHHHHHHHGGGGGHHHGGGGGHHHHHHGHHHHGHHHGGGGGGFGGGGGGGGGGGGGGGFGGGGFGFGGGGFFFFFFFFFFFFFFFFFB -@SRR33138449.126 126 length=251 -GCGTTGAACCATTTTACGATACCAGTCATTTTACCGGACATAGTGTATTACCTTTAATAATTAAGTGTGCCTTTCGGCGATATGGCGTGCTTTACAGATTTTGAAGCGTTAAAGGAATGTGCACTACGAGGGGTATCAACGATAACTCTTGAAGGGACTTGCCTTACTACACTGCTTTAATGGTCTGTACGTCAAACCGTTGATGTGCATTAAGCCACGCATTGGCGGGGGATGCAAACATTATTTTTCAT -+SRR33138449.126 126 length=251 -ABCCCCCFFCFFGGGGGGGGGGHHGHGHHHHHHHGGGGGGHHHGGGHHHHHHHHHHGHHHFHHHHDGGHHHFHHHGEGGGGGGFGHGGGGFHHHHHFHHHHHHAD3G?GGGGHGGGFHGHHHHEHHHGGGCGGGGHHHHHGFHGEHHGHHHBGGHGHGHGHHHHHHHHHHGHHHFGHHBDHHG:GHFGHHGHHGGHFEDGGGFFFFGFBFCGGFEFGGG?ADFGGGGF---9::/0;09;09FBB0BF/00 -@SRR33138449.127 127 length=250 -CGCGTGAAGCCTTCAAGTGCCGAACTAAAATTGATGCGTTTGATTCAAGCCAACCCGGCATTAAGTAAGCAGTTGATGGAATAGACTTTTATCCACTTTATTGCTGTTTACGGTCCTGATGACAGGACCGTTTTCCAACCGATTAATCATAAATATGAAAAATAATTGTTGCATCACCCGCCAATGCGTGGCTTAATGCACATCAACGGTTTGACGTACAGACCATTAAAGCAGTGTAGTAAGGCAAGTC -+SRR33138449.127 127 length=250 -BCCDADBCFFFFGGGGGGGGGGGGGHHHHHHHHHHHHGGGGGHHHHHHHHGHHGGGGGGGGHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHGHHHHHHGGHGGHHHHHHHHHGHHHDDGHGHHHHHGGGGGGHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHGGGGGHHHGFGGGHHHHHGHHHGGGGGGGGGGGGHGGGGGGGGGGGGGFFFFFFFFFFFFFFFFFFFFFFF -@SRR33138449.127 127 length=249 -GCGTTGAACCATTTTACGATACCAGTCATTTTACCGGACATAGTGTATTACCTTTAATAATTAAGTGTGCCTTTCGGCGATATGGCGTGCTTTACAGATTTTGAAGCGTTAAAGGAATGTGCACTACGAGGGGTATCAACGATAACTCTTGAAGGGACTTGCCTTACTACACTGCTTTAATGGTCTGTACGTCAAACCGTTGATGTGCATTAAGCCACGCATTGGCGGGGGGTGCCACAATTTTTTTTT -+SRR33138449.127 127 length=249 -BBBCCCBFFCFFGGGGGGGGGGHHHHHHHHGHHHHGGGGGFHHBFDGHHHHFHHHHHHHHHHGHHFFHHHHHHHHGGGGEGGGHHHGGGGGHHHHHHGHHHHHGGHH?EEFGHHFHHHHGHHHGHGHDGGGGGCDHHHGHGGHGGHHHHHHHHGGDHGFHHHHGHHHHHEHHEFHHHHHGHHGHFFFGHHHFHGGHHFGGGF?FGFGGBBFFGGGGGGGGGGGFGFGF;9---;./90:.0:00;@DD- -@SRR33138449.128 128 length=250 -ACATAGTGTATTACCTTTAATAATTAAGTGTGCCTTTCGGCGATATGGCGTGCTTTACAGATTTTGAAGCGTTAAAGGAATGTGCACTACGAGGGGTATCAACGATAACTCTTGAAGGGACTTGCCTTACTACACTGCTTTAATGGTCTGTACGTCAAACCGTTGATGTGCATTAAGCCACGCATTGGCGGGCGATGCAACAATTATTTTTCATATTTATGATTAATCGGTTGGAAAACGGTCCTGTCAT -+SRR33138449.128 128 length=250 -ABABBBBDDBFFGGGGGGGGGGHHHHHHHHHHHHHHHHGGGGGGGGHBFCGCFGHHHHHHHHHHHHGHFFGEG?FGHFFHHHHHHHHGFHG?EGGGEEHHGHHGGHEGHFGHHHHHHHGHGHHHHHHHHHHHHHHHGHHHHHFHHHGHHHHHHHHHGGHHGGFGGHHHGGGHHHHHGHHHGGCCFGHHHGG@-;9ADBFFFAEFF/BFFFF/9BB/FF/BF/BB/BF/9A9;-9/:/B9-9.A9/99FBF -@SRR33138449.128 128 length=251 -GCCTGGAACAGATTGTTTTTAAAGATGAAACGCAGAAAATTACCCTGACGCACAGAACAACATCCTGTACCGAAATTGAGCAGTTACGAAAAGGAACAGGGTTAAAAATCGATGATTTCGCCCGGGTTTTGGGCGTATCAGTCGCCATGGTAAAGGAATGGGAATCCAGACGCGTGAAGCCTTCAAGTGCCGAACTAAAATTGATGCGTTTGATTCAAGCCAACCCGGCATTAAGTAAGCAGTTGATGGAA -+SRR33138449.128 128 length=251 -AABAAABFFFFFFDGFGGFEGF5GHHHHGHHGGCGGHCCGGDDFHGHBG?E?E?FFHHGGEHF33DGDFHHGC00?DBBGHBCGFFHGCE?FHGHFAFHHE2?GEGBFFBEA/FBGBG4CDEGGC/?///F@DFC<@DCGFF1CCDAEF<=GGHH<0.AAFFFB/9BB9///9..BFFFB -@SRR33138449.130 130 length=251 -GGAATGGGAATCCAGACGCGTGAAGCCTTCAAGTGCCGAACTAAAATTGATGCGTTTGATTCAAGCCAACCCGGCATTAAGTAAGCAGTTGATGGAATAGACTTTTATCCACTTTATTGCTGTTTACGGTCCTGATGACAGGACCGTTTTCCAACCGATTAATCATAAATATGAAAAATAATTGTTGCATCACCCGCCAATGCGTGGCTTAATGCACATCAACGGTTTGACGTACAGACCATTAAAGCAGT -+SRR33138449.130 130 length=251 -AABBCFFFFFFFGGGGGGGGGGGGGHHHHHHHHGHHHGFGGHHHHHHHHHHHGGFGEGHHHHHHHHGHHGGGGGGGGHGHHHHHHHHHFHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHGGHGGHHHHHHHHHGHHHGGGHGHHHHHGGGGGGHHHHHHHHHHHHHHFFHHHGHHHHHHHHHHHHHHGGGGGGGGGDGGGGGGFGGGGGGGGGGGGFFFFFFFFFFFFFFFFFFFEFFFFFF/ -@SRR33138449.130 130 length=250 -GTGTATTACCTTTAATAATTAAGTGTGCCTTTCGGCGATATGGCGTGCTTTACAGATTTTGAAGCGTTAAAGGAATGTGCACTACGAGGGGTATCAACGATAACTCTTGAAGGGACTTGCCTTACTACACTGCTTTAATGGTCTGTACGTCAAACCGTTGATGTGCATTAAGCCACGCATTGGCGGGTGGGGCAACAATTTTTTTTTATATTTATGATTAATCGGGTGGGAAACGGACCTGTCATTAGGA -+SRR33138449.130 130 length=250 -ABBBBFFFFFFFGGGGGBGGGGHHHGHCHFFHHGCEEGGFGHHFGGGGGHGGHEGHHHHHHCGFGGGGGFHGHEHFHHHHHHHHHGDGGEEAFHHHHHGGHGGHFHHHHGHHGEFGHHHGHFHFHHHHHEHHFHHHHFHHHF22FHHHGGHGDFHGGGGGH11>1FFHHHHHHHHHGGFDDHHEDFF-@---.-.9../;//9@B=-:9BB/B//9//9////9.-;@--9;A/.9.9;BFB//;//;99 -@SRR33138449.131 131 length=251 -GCCTTTCGGCGATATGGCGTGCTTTACAGATTTTGAAGCGTTAAAGGAATGTGCACTACGAGGGGTATCAACGATAACTCTTGAAGGGACTTGCCTTACTACACTGCTTTAATGGTCTGTACGTCAAACCGTTGATGTGCATTAAGCCACGCATTGGCGGGTGATGCAATAATTATTTTTCATATTTTTTATTAATCGGTTGGAAAACGGTCCTTTCATCAGGACCGTAAAAAGCAATAAAGTGGATAAAA -+SRR33138449.131 131 length=251 -BBB@AFFBBDD2EEGGGGGGGEAGFFHHFDDGGHHCFFGGFGGGEHHEGHHHFHHHHHHGGDEEGEEHHEHHDFEEFGHHHHFHCFGGHGHHGFHHH?GFHHHHEHGHHHGHHGHHGHGHHHEHHGGHFGCADDCFHHHHHHHHHHEFGC@DCDDFHFGG@::A0000;0090;90;BFFB009;BF00-.9C0000.;.:-9/9BF/;:9.;///99////;/;A-9..://..;B//;F//99////:F -@SRR33138449.131 131 length=251 -ATGCATGAGCTGTTGAGCAGCCTGGAACAGATTGTTTTTAAAGATGAAACGCAGAAAATTACCCTGACGCACAGAACAACATCCTGTACCGAAATTGAGCAGTTACGAAAAGGAACAGGGTTAAAAATCGATGATTTCGCCCGGGTTTTGGGCGTATCAGTCGCCATGGTAAAGGAATGGGAATCCAGACGCGTGAAGCCTTCAAGTGCCGAACTAAAATTGATGCGTTTGATTCAAGCCAACCCGGCATT -+SRR33138449.131 131 length=251 -1>111BBDFBDB11EB11F1CFGFCC1A0F01DGHGCFCHBBGFBFFB1DFE?/EAE01BFH1BAGFFEE//EFHHHBFF/GG>BGGFHE//E/?D11B@@F@B22/?/?/CEFGHHHAEHHHHEFHAC/CEHGHGD2CEC>0GF1F?-C.CFF=D0DF.F01F1CGDGDDGFHHBDFGHHHFFGBGG0::0CA:E-E0CGFB-?9:D0FFFFBFFBFB:E-:;CEFFBAFFBBFFF0FF0BFFFFB:FFFFFBB00B0F/:/FFF0BFFBFBFFF-BFAF-D?AF=.0:B0F -@SRR33138449.132 132 length=251 -CCTGACTTCACCCCGCTGTGTCTGCTTTTCCCGACTATTCTTAATGAGCTTCGATGCAATTCACGATCCCGCAGTGTGATTTGAGGAGTTTTCAATGGAATATAAAGATCCAATGCATGAGCTGTTGAGCAGCCTGGAACAGATTGTTTTTAAAGATGAAACGCAGAAAATTACCCTGACGCACAGAACAACATCCTGTACCGAAATTGAGCAGTTACGAAAAGGAACAGGGTTAAAAATCGATGATTTCG -+SRR33138449.132 132 length=251 -AAA?ACFFFFFFGGGGGGGGGGDGBHHEGGHHGGGGGGHHHGGGFHBBGHFFGHFHGHHHHFHHHGEG1GGCEGEGFGGHHHHEEHHGFHHHHHGHFHGHHHHHHFBHHGHHHFHGHHGGHFGBGHHFHHHHGHHHGFHHHGGHHHFGHHGHHHHGHHFHHHGCCFCHHHHHHGEFHHHH?DGGGGFFGHHCGHHHHGFHFHG?C?EGFBFFFE;BFGGGEFAE.FBFGGCEFFGFFBB?FFF.9FFBBF; -@SRR33138449.133 133 length=177 -GTTTACGGTCCTGATGACAGGACCGTTTTCCAACCGATTAATCATAAATATGAAAAATAATTGTTGCATCACCCGCCAATGCGTGGCTTAATGCACATCAACGGTTTGACGTACAGACCATTAAAGCAGTGTAGTAAGGCAAGTCCCTTCAAGAGTTATCGTTGATACCCCTCGTAG -+SRR33138449.133 133 length=177 -CBCBCFCBFABFGGGGGGGGGGHHGGGHGHHHHHGEFGGGHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHGGGGGHHHGGGGGHHHHHHHHHHHHHHHGGGGGGHHGEHHGHHHHHHHHHHHHHHHHHHHHHHHHHHHGGHHHHHHHHHHHHHHHHHHHHHGGGHHHGGGHGG -@SRR33138449.133 133 length=177 -CTACGAGGGGTATCAACGATAACTCTTGAAGGGACTTGCCTTACTACACTGCTTTAATGGTCTGTACGTCAAACCGTTGATGTGCATTAAGCCACGCATTGGCGGGTGATGCAACAATTATTTTTCATATTTTTGATTAATCGGTTGGAAAACGGTCCTGTCATCAGGACCGTAAAC -+SRR33138449.133 133 length=177 -ABBCAABBCCBBGGGGGGGGGGGGGHHHHHHHGGGHHGHHGGHHHHHHHHHGHHHHHHHHFHHHHHGHHFGGHGGEFGGHHHHHHHHHHHHHHGHGGGGGHHGGGE///3D3DFCCGBD22FHHH2FGDDGF1?//@G1GF1=E.C-@1GFGHH<2@>DGH2>//?G1FG1/C/>-C1AFFFFFCBGGGGGGGGGGHHHGHGHHHHHHHHHHHHHHHHHHHHHHHHHFHHHHHIHHHGGHHGGEEGHHHHHHHHHHHHHHHHGGGGGHHGGG@/>/FHEGHGCHFGGFHHHHCDF3FGHDF?DFG2@F2FD?DCFBGFHCABG1FGFEFHGFHHF.FB2DG2?D?C@C1?DG0-<.111DDHCGHG.DDH11<...-<0D.:-9//.-.;////99/..///;/9BBB// -@SRR33138449.139 139 length=250 -CGCAGAAAATTACCCTGACGCACAGAACAACATCCTGTACCGAAATTGAGCAGTTACGAAAAGGAACAGGGTTAAAAATCGATGATTTCGCCCGGGTTTTGGGCGTATCAGTCGCCATGGTAAAGGAATGGGAATCCAGACGCGTGAAGCCTTCAAGTGCCGAACTAAAATTGATGCGTTTGATTCAAGCCAACCCGGCATTAAGTAAGCAGTTGATGGAATAGACTTTTATCCACTTTATTGCTGTTTA -+SRR33138449.139 139 length=250 -BCDDCDFFFFFFGGGGGGGGGGGGHHHHHHHHHHHHHHHHHGGGGHHHHHHHHHHHHGGHGGHGHHHHHHGHHHHHHHHHGHHHHHHHHHGGGGGGFGGGHGGHGGGGHHHHHGGGGGHHHHHHHHHHHHHHHHHHHHHHGGGGGGGGHHHHHHHHHHHHHGGGGHHHHHHHHHHHGGGGGGGGGHHGGGGGGGGGGGGGGFFHFFFFFFFFFFFFFFFFFFFFFFHFFFHHFFFFHFFFFFFFFFFFFF -@SRR33138449.139 139 length=249 -CGATAACTCTTGAAGGGACTTGCCTTACTACACTGCTTTAATGGTCTGTACGTCAAACCGTTGATGTGCATTAAGCCACGCATTGGCGGGTGATGCAACAATTATTTTTCATATTTATGATTAATCGGTTGGAAAACGGTCCTGTCATCAGGACCGTAAACAGCAATAAAGTGGATAAAAGTCTATTCCATCAACTGCTTACTTAATGCCGGGTTGGCTTGAATCAAACGCATCAATTTTAGTTCGGCA -+SRR33138449.139 139 length=249 -CCBCCCFFFFFFGGGGGGGGGGHHHHGHHHHHHHHHHHHHHHHHHHHHHHHHHHGGHHGGGGGHHHHHHHHHHHHHHHHGGGGGHHHGGE//?G4?3F3CCBGDFFHHH34F?BFFFGFFDGDD33FG/CCF2>FHB@DEDCFHHG11=11CGEG-.E/BDB3B30?FH4BEHHH34B4BGH4FF4FB4BF3C//<>1BF4GHBFHDGFDHHHH34F4FGG44B?GF4GFDAA/<<@FHFHHGGHGH1<0?FCFDFF<11<1FFHHH1>FF=GG0DFDDFHB=/DGHF0GGFHCGBHGGG.C9DFGG-9FFFGFGBAFFABFGGFGGF/;F/AFFFFEF/99BFEFB.FFE?FB=DEFEB -@SRR33138449.141 141 length=251 -CTGTTGAGCAGCCTGGAACAGATTGTTTTTAAAGATGAAACGCAGAAAATTACCCTGACGCACAGAACAACATCCTGTACCGAAATTGAGCAGTTACGAAAAGGAACAGGGTTAAAAATCGATGATTTCGCCCGGGTTTTGGGCGTATCAGTCGCCATGGTAAAGGAATGGGAATCCAGACGCGTGAAGCCTTCAAGTGCCGAACTAAAATTGATGCGTTTGATTCAAGCCAACCCGGCATTAAGTAAGCA -+SRR33138449.141 141 length=251 -3AABBFFFF4CFGGGGBGGGGGHBGGFHFGEHHFFFHHHHHGEEGC3ECGHHHHHHHGFGCGGGHHGHHHHHHHGHGFHHFGGGCHHGHHHHGHHHHGEG1EH/GFHHGGGHG/GH4FGHGA/FBHGGH0DEFGGGGGGGHGGHGGGFHFGHHFGEGGGFDGHHHHHHHHFGFHHHHHGHEGGCDGBEGFFGFGGFGFFBB@@B@FFG0FFFFBFFFBDFBFFFFFFFFFEFFFFFB/0E1@443F3GHFH44FFGH3@DEGG44B444F4G4@/E/?>E3FGH3E/??B2FFBBGD3FGGFFD//EF>GHFGHHEGGFHFFFDBGBGFHBDBGBGCGG:ECC/:G@CB0C0CFFDFGDAFFFFFFFBCGF.??-@FFFFFFFEFFFFFFFF@FFEFB/B/;B/BF9//:B -@SRR33138449.143 143 length=247 -GCTATAAGTCACGCTAAATGACTGGCTGAATCGAATCATAGCCAGAGCATGCCCTGACTTCACCCCGCTGTGTCTGCTTTTCCCGACTATTCTTAATGCGCTTCGATGCAATTCACGATCCCGCAGTGTGATTTGAGGAGTTTTCAATGGAATATAAAGATCCAATGCATGAGCTGTTGAGCAGCCTGGAACAGATTGTTTTTAAAGATGAAACGCAGACAATTACCCTGACGCACAGAACAACATC -+SRR33138449.143 143 length=247 -A3A3AFFFFFFFGGFECGGGGFEFEHHGHHHHHGHFEDFH5GHHHG2FFBGH3GFHFHGBHHHF10AEEEEHHGHFGFEGE@@FEF?EEFGGH444BF4AEEGGC/FGEGFF3?FGHEHFEH?EDCDHHHHGGH>GEFH?FGHGHHHHFHCGGHHFFFH1FGHH>GF1GGHHGFFBFHH1F0ECFHFEFGHHHHBFHGHHFGH4FGH4GHFGHFGFFGG>FEGGGHHFCG?EGHHHFFGFHFHHGH?CFFHFFFHHHFHHH1DFF1CGHHHFFFDDGHBGGHHHHGGFHHHHGFHFFGDHHGGGCCFCHHG?HHGEFFGGGGGGAGGGGGGBB9CBEGACFFFFFFFFFBFFFFFFF@>D?FFFFFFFBFFFFFFF -@SRR33138449.144 144 length=251 -GCACAGAACAACATCCTGTACCGAAATTGAGCAGTTACGAAAAGGAACAGGGTTAAAAATCGATGATTTCGCCCGGGTTTTGGGCGTATCAGTCGCCATGGTAAAGGAATGGGAATCCAGCCGCGTGAAGCCTTCAAGTGCCGAACTAAAATTGATGCGTTTGATTCAAGCCAACCCGGCATTAAGTAAGCAGTTGATGGAATAGACTTTTATCCACTTTATTGCTGTTTACGGTCCTGATGACAGGACCG -+SRR33138449.144 144 length=251 -33AABFFFFFFBCGGFEGGFGDAEE?AFFHHHDFFGBGDGF?AFGHFFBGECAEGGHF3FFGFHFFGHHGCEDAEEGEGFGA/EHG@/EGHHHHGGGGECBFFFHBGFGFGFFECHGFHG0?CGG@D/FGCGFHHGEFE?5G43G3B?3F@EGHHH3@@@GFF4@4BFGBHF4FFEF@G/?GG3?E//>?/?2?F32FG2/F?/=CH1=0?FG1?GH1=FG111555B53BGEG44EGHH34@4B4F4F44BF4B4GEA>E/>3BBG3/>//@1?1FGHF11<1G111>>1<1=<-<--<.:/;C@.<0000;0.ADAC0C90BF0/9CF.----/C00;009;/9FB..-99--;.;A/BB/;FFFFFF//;B9//9--- -@SRR33138449.146 146 length=249 -GCATTTATTAGATCGTTAGGTGGCTATAAGTCACGCTAAATGACTGGCTGAATCGAATCATAGCCAGAGCATGCCCTGACTTCACCCCGCTGTGTCTGCTTTTCCCGACTATTCTTAATGAGCTTCGATGCAATTCACGATCCCGCAGTGTGATTTGAGGAGTTTTCAATGGATTATAAAGATCCAATGCATGAGCTGTTGAGCAGCCTGATACAGATTGTTTTTAAAGATGAAACGTAGAAAATTACT -+SRR33138449.146 146 length=249 ->A?AAF3@3333BFA11AFFGF1001FDBFD3FE00BE01GHBG2GC/F/FFA1F00B/BD221A1B0A0AE1G10B0AE1FG@F10/>EGC/BG2EB1BBDFFGF>EE>E122F22>>221FG11//BGEF1GB11BA/?C/0/FGH1FF10DDFFFF//B///FB -@SRR33138449.148 148 length=251 -GCCTGGAACAGATTGTTTTTAAAGATGAAACGCAGAAAATTACCCTGACGCACAGAACAACATCCTGTACCGAAATTGAGCAGTTACGAAAAGGAACAGGGTTAAAAATCGATGATTTCGCCCGGGTTTTGGGCGTATCAGTCGCCATGGTAAAGGAATGGGAATCCAGACGCGTGAAGCCTTCAAGTGCCGAACTAAAATTGATGCGTTTGATTCAAGCCAACCCGGCATTAAGTAAGCAGTTGATGGAA -+SRR33138449.148 148 length=251 -CCCCCCFFFFFFGGGGGGGGGGHHHHHHHHHGGGGGHHHHHHHHHHHHHGGGGGHHHHHHHHHHHHHHHHHGGGGHHHHHHHHHHHHGGHGGHGHHHHHHGHHGHHHHHHGHHHHHHHHHGGGGGGFGGGHGGGGGGGHHHHHGGGGGHHHHHHHHHHHHHHHHHHHHHHGGGGGGGGHHHGGGGGGGGGGGGGGGGGGGGGGGGGGGDFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF -@SRR33138449.148 148 length=251 -ATTGGCGGGGGATGCAACAATTATTTTTCATATTTATGATTAATCGGGTGGGAAACGGGCCTGTCATCAGGACCGTAAACAGCAATAAAGTGGATAAAAGTCTATTTCATCAACTTCTTACTTAATGCCGGGTTGGCTTGAATCAAACGCATCAATTTTAGTTCGGCACTTGAAGGCTTTACGCGTCTGGATTCCCATTCCTTTACCATGGCGACTGATACGCCCAAAACCCCGGCGAAATCATCGATTTT -+SRR33138449.148 148 length=251 -AAAAAFD@A/>///01B10A1F1DFHHH022BBGG2F222D2B@2/?/B////FG0/E//EA/1B2>22111FF/?/?G1B0?GG11>AA111@DFFGBFA11110000BA0A0GGCBG1AGCA0FE1GHHE0AAA?//EFGGC//0/ECEFFFHD/?E/FC0FD21DFC0FCB0>@GGEED1?/>A3311111AE1EF11000/A/AA/112111DD2DGGGG//>/>EE/@//>B//D111?F//.1<11..<..<1>10=0000<00-<0..000.//00000;00;;/0000<0;0;C0/0000009009;0099000000/-9CE.9/;B///;//////9/9///99BF/-9@///////;///9///B///---;-9-/;;B-///;;///9B////9///---;-;;-/////////// -@SRR33138449.151 151 length=251 -GTCCTGTCATCAGGACCGTAAACAGCAATAAAGTGGATAAAAGTCTATTCCATCAACTGCTTACTTAATGCCGGGTTGGCTTGAATCAAACGCATCAATTTTAGTTCGGCACTTGAAGGCTTCACGCGTCTGGATTCCCATTCCTTTACCATGGCGACTGATACGCCCAAAACCCGGGCGAAATCATCGATTTTTAACCCTGTTCCTTTTCGTAACTGCTCAATTTCGGTACAGGATGTTGTTCTGTGCGT -+SRR33138449.151 151 length=251 -CCCECFFFFFFFGGGGGGGGGGHHHHHHHHHHHGHHHHHHHHHHHHHHHHHHHHHHHIHHHHHHHHHHHHHHGGGGGGGHHHGHHHHHHHHGGGGGHHHHHHHHHHHGGGGGHHHHHHHHHHHHHHHGGGGGHHHHHHHHHHHHHHHHHHHHHHHGGGGGHHHGGGGGGHHHHGGGGGGGGG -@SRR33138449.152 152 length=251 -CCTGTCATCAGGACCGTAAACAGCAATAAAGTGGATAAAAGTCTATTCCATCAACTGCTTACTTAATGCCGGGTTGGCTTGAATCAAACGCATCAATTTTAGTTCGGCACTTGAAGGCTTCACGCGTCTGGATTCCCATTCCTTTACCATGGCGACTGATACGCCCAAAACCCGGGCGAAATCATCGATTTTTAACCCTGTTCCTTTTCGTAACTGCTCAATTTCGGTACAGGATGTTGTTCTGTGCGTCA -+SRR33138449.152 152 length=251 -AAA1AFFF33DFFG11EGEGGG0A0GHHGBH2FBBFGHCHHFGHHHHGH2FFHFHHHHHHGHHHHHFABGE?EEEGCF1FEEEGDHHGHGG?CE1GHHHHGHGHHGECGGGHHEHFEEHHHHHEH?BCGGHHHGFFB1FGHHHHHHHHBGHHHC@@@E1GHH.ACGC.<.CFFGGGGC@9@EGGFGFGGFGFFFCFGEFF0CFGFGGFF@AB-BFBFFFFFFF/BF?A-B///-;BBFFFEFFFFFBBE@; -@SRR33138449.153 153 length=251 -CATCAGGACCGTAAACAGCAATAAAGTGGATAAAAGTCTATTCCATCAACTGCTTACTTAATGCCGGGTTGGCTTGAATCAAACGCATCAATTTTAGTTCGGCACTTGAAGGCTTCACGCGTCTGGATTCCCATTCCTTTACCATGGCGACTGATACGCCCAAAACCCGGGCGAAATCATCGATTTTTAACCCTGTTCCTTTTCGTAACTGCTCAATTTCGGTACAGGATGTTGTTCTGTGCGTCAGGGTA -+SRR33138449.153 153 length=251 -BABBBFBFAABAGCCFGFFGGGHHHHFGGHH5GFHD3FDGH5GFHHHHHHHHFHHHHHFGHHGHHGGGEFFAGGFGCFFGFHHHGCEGGHFHHFHFHGHHGGGGEHGHFFHFHGHHHHHHGGCDGHGHHHHHGEFHHHFFHHFFFHFHGCGGGEHHHGGGGFHHHFGGGGGGGGGCGGEFGGGGGGGGGEF0CFGBFEGGFFGGACFFAFFFBFFFFFFFFAAAD9BBAFFFFFBEFFFFFFFFADFFFB/ -@SRR33138449.153 153 length=251 -GTTTTTAAAGATGAAACGCAGAAAATTACCCTGACGCACAGAACAACATCCTGTACCGAAATTGAGCAGTTACGAAAAGGAACAGGGTTAAAAATCGATGATTTCGCCCGGGTTTTGGGCGTATCAGTCGCCATGGTAAAGGAATGGGAATCCAGACGCGTGAAGCCTTCAAGTGCCGAACTAAAATTGATGCGTTTGATTCAAGCCAACCCGGCATTAAGTAAGCAGTTGATGGAATAGACTTTTATCCA -+SRR33138449.153 153 length=251 -BBABBBBFDFFFFDGFGG?EGGFFFCHHHGHHCCGE?AEGBBGHHFEEGFFHFHGH51EEEFHFBGGHG55EGGGEEEG1FGGEFEFGFGHHHHHHGAGEFHFDFFEGDE/EBCDGFFDG/EGGHHGFHHFHHHHFGGFHFGGFGGGGFFHHHEBFFB;;BF@CDGGGGGGGGFFFFFCEDFDFFFFFFFEABBAD.@@DFAFFFFFFFFFFFFFFFFFFFB -@SRR33138449.157 157 length=229 -CCGTAAACAGCAATAAAGTGGATAAAAGTCTATTCCATCAACTGCTTACTTAATGCCGGGTTGGCTTGAATCAAACGCATCAATTTTAGTTCGGCACTTGAAGGCTTCACGCGTCTGGATTCCCATTCCTTTACCATGGCGACTGATACGCCCAAAACCCGGGCGAAATCATCGATTTTTAACCCTGTTCCTTTTCGTAACTGCTCAATTTCGGTACAGGATGTTGTTC -+SRR33138449.157 157 length=229 -BCCBBCCFFFFFGGGGGGGGGGHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHGGGGGGGHHHGHHHHHHHHGGGGGHHHHHHHHHHHGGGGGHHHHHHHHHHHHHHHGGGGGHHHHHHHHHHHHHHHHHHHHHHGGGGGGHHHHGGGGGHHHHGGGGGGGEFGHHHHHHHHHGHHFFGGGGGGGHGGHHGHGGGGGGGGGGGGGGGGGFFFFFFFEFFFFFFFF -@SRR33138449.157 157 length=229 -GAACAACATCCTGTACCGAAATTGAGCAGTTACGAAAAGGAACAGGGTTAAAAATCGATGATTTCGCCCGGGTTTTGGGCGTATCAGTCGCCATGGTAAAGGAATGGGAATCCAGACGCGTGAAGCCTTCAAGTGCCGAACTAAAATTGATGCGTTTGATTCAAGCCAACCCGGCATTAAGTAAGCAGTTGATGGAATAGACTTTTATCCACTTTATTGCTGTTTACGG -+SRR33138449.157 157 length=229 -A>AAAFCCFFFFGGGGGGGGGGHHGGHHHHHHHGDHGGFFHHGHGHG2GHFEGHHHGAHHHHHHHGGGGGGFEGGGHEGHGEGGGHHGBEGFEEHHHHFHHHHGFHFGHHHGHFHGFGGGGGGGHGHHBGHHFDBGHGGGGGFGHFGCFGHHHGGGGGHHHHHHHFFFHGGGGGGGGGGFGFFBFEFGGGFFBFFFBBFFFGEGGGGGGEFGFFFFFFBFFFF/9FB.A -@SRR33138449.158 158 length=250 -GCAATAAAGTGGATAAAAGTCTATTCCATCAACTGCTTACTTAATGCCGGGTTGGCTTGAATCAAACGCATCAATTTTAGTTCGGCACTTGAAGGCTTCACGCGTCTGGATTCCCATTCCTTTACCATGGCGACTGATACGCCCAAAACCCGGGCGAAATCATCGATTTTTAACCCTGTTCCTTTTCGTAACTGCTCAATTTCGGTACAGGATGTTGTTCTGTGCGTCAGTGTAATTTTCTGCGTTTCAT -+SRR33138449.158 158 length=250 -CCDCCFFFFFFFGGGGGGGGGGHHHHHHHHHHHHHHHHHHHHHHHHHHGGGGGGGHHHGHHHHHHHHGGGGGHHHHHHHHHHHGGGGGHHHHHHGHHHHHHHHGGGGGHGHHHHHHHHHHHHHHHHHHHHHGGGGGHHHHGGGGGHHHHGGGGGGGGGGHHHHHHHHHGHHHHHHHHHHHHHGHHGHGGGGGHHGGGHGGGGGGGGGGGGGGGFFFFFFHHFFFFFFFFF:FFFFFFFHFHFFFFFFFHF -@SRR33138449.158 158 length=251 -GCCTGGAACAGATTGTTTTTAAAGATGAAACGCAGAAAATTACACTGACGCACAGAACAACATCCTGTACCGAAATTGAGCAGTTACGAAAAGGAACAGGGTTAAAAATCGATGATTTCGCCCGGGTTTTGGGCGTATCAGTCGCCATGGTAAAGGAATGGGAATCCAGACGCGTGAAGCCTTCAAGTGCCGAACTAAAATTGATGCGTTTGATTCAAGCCAACCCGGCATTAAGTAAGCAGTTGATGGAA -+SRR33138449.158 158 length=251 -BCCCCCFFFFFFGGGGGGGGGGHHHHHHHHGGGGGGHHHHHHH5GHHHHGGGGGHHHHHHHHHHHHHGHHHGGGGHHHHHFHHHHHHGGHGGHGHHHHHHGHHHHHHHHHGHHHGHHHHGGGGGGGGGGGHGGHGGGGGHHHHGGGGGHHHHHHHHHHHHFHHHHHHHHHGGGGGGGGHHGHFGGFGFFGGD?DGGGFGEFGGEGFGGGFFFFFFFFBFFFFFFFFFFFFFFFBFBBBFFFFFFFFFF9F0 -@SRR33138449.159 159 length=250 -GCAATAAAGTGGATAAAAGTCTATTCCATCAACTGCTTACTTAATGCCGGGTTGGCTTGAATCAAACGCATCAATTTTAGTTCGGCACTTGAAGGCTTCACGCGTCTGGATTCCCATTCCTTTACCATGGCGACTGATACGCCCAAAACCCGGGCGAAATCATCGATTTTTAACCCTGTTCCTTTTCGTAACTGCTCAATTTCGGTACAGGATGTTGTTCTGTGCGTCAGGGTAATTTTCTGCGTTTCAT -+SRR33138449.159 159 length=250 -BBAABFFFFDFFGGGGGGGGGGHHHHHHHHHHHHHHHHHHHHHHHHHHGGGGGGGHHHGHHHHHHHHGFGGGHHHHHHHHHGFGGGGGHHHHHHGHHHHGGHHGFGGGHHHHHHHHHHHHHHHHHFHHHHHGGGGGHHGFGGGGFHHHHGGFGGGGEGGHHHHHHGHHGHHHHHGHHHHHFHFGGGHGGGGGGHGGGGGGGGGGGFGGGGGGGFFFFFFFFFFFFFFCFFFFBBFFFFFFFFFFFFFFFF -@SRR33138449.159 159 length=250 -CCAATGCATGAGCTGTTGAGCAGCCTGGAACAGATTGTTTTTAAAGATGAAACGCAGAAAATTACCCTGACGCACAGAACAACATCCTGTACCGAAATTGAGCAGTTACGAAAAGGAACAGGGTTAAAAATCGATGATTTCGCCCGGGTTTTGGGCGTATCAGTCGCCATGGTAAAGGAATGGGAATCCAGACGCGTGAAGCCTTCAAGTGCCGAACTAAAATTGATGCGTTTGATTCAAGCCAACCCGG -+SRR33138449.159 159 length=250 -A?AABFFFFFFFGGFGGGGGGGHHHHHHHHFHHHHHFGHFHGHHHHHHHHHHGGGGGGHHHHFGHGHHHHHGGGGGHGHHHHHHHGHHHHHGHGGGGHHHHHHFHGFHHGFHGGHGHHHHHHGHGHHHHHHGFHHHFHHFGHACDDGGGGFGFGGHFDG?GFHGHD-ADEFF=DGGHGHGFGGHHHHHFHHHGGGGGB?EFGFFFFF9FFBF0DFF=EFFBFBBF;BF/;DD-AFFFFFFFF.:BAD.-- -@SRR33138449.160 160 length=251 -AATGAGCTTCGATGCAATTCACGATCCCGCAGTGTGATTTGAGGAGTTTTCAATGGAATATAAAGATCCAATGCATGAGCTGTTGAGCAGCCTGGAACAGATTGTTTTTAAAGATGAAACGCAGAAAATTACCCTGACGCACAGAACAACATCCTGTACCGAAATTGAGCAGTTACGAAAAGGAACAGGGTTAAAAATCGATGATTTCGCCCGGGTTTTGGGCGTATCAGTCGCCATGGTAAAGGAATGGG -+SRR33138449.160 160 length=251 -ABBBBFFFFFBBGGGGGGGGGFGGHGGHGGGGGHHHGHHHHGGHFGHHHHHHHHHHGHHHHHHHHHHGHHHGHHHHGHHHHHHHGHFHGHHHHHHHHHHHHHHGGHGHGHGHGHHHGHHHGGGGGHHHHHHHHGHHHHDGGGGHHHGHHHGHGHHGHHHGGGGGHHHHHGHHGHHHGFHFGHGHHHHHHGHHHHHHHHHGGHHFGFGHGGGD?ADAFC?FGGFB@DFFFFFFFFFFFFFFFFFFFFFFFFF -@SRR33138449.160 160 length=251 -GGATAAAAGTCTATTCCATCAACTGCTTACTTAATGCCGGGTTGGCTTGAATCAAACGCATCAATTTTAGTTCGGCACTTGAAGGCTTCACGCGTCTGGATTCCCATTCCTTTACCATGGCGACTGATACGCCCAAAACCCGGGCGAAATCATCGATTTTTAACCCTGTTCCTTTTCGTAACTGCTCAATTTCGGTACAGGATGTTGTTCTGTGCGTCAGGGTAATTTTCTGCGTTTCATCTTTAAAAACA -+SRR33138449.160 160 length=251 -A@AAAFFFFDFFGGFGFGFBAGFHHHGHGHHGFFHHFGEGCECC?HFH?FGGFHHGHDFCDGHFFHGHHGGHHFC0>0FGGHHHHAGHHFHEFGGCFGHFHFHHHHHHHEGHHHG3GHBFFGGC/EHBGHGGGGGEAEHGDGGGGGG@?GFHFGFEGABE1F1A1G11BGGGFGGCCF00F0FHH1AFG00BAE//E0GGEBCGHEHHEH1F2B//EE/111BFDGFGDFG/>E@G?0BBGGHGHCGHHHHHHEHF21FFH2GE//BB@??G1B/G/CFD<<<0CCCCCCC.G:CFFF0BFFFGBA9A@GGGGE9CFGFGB@A-9A/B/BB/9B-9BAAA@-@?-BFEF//;;FBF::9//BBBB -@SRR33138449.161 161 length=251 -GCTTTAATGGTCTGTACGTCAAACCGTTGATGTGCATTAAGCCACGCATTGGCGGGGGACTGAACAATTTTTTTTTCTATTTATTATTCAAACGCTGGGAAATTGTCCTTCCACCCAGCACACTAACAGCAATTAAGTGGATAAAAGTCTATTTCATCCAATGGCTAATTCATGCGAGGGTGGCGTGCATCAAACCCATCTACTTTTCATCTGCACCTGAAGGCTGCAGTCGCCTGGGTTCTCTATCCTTT -+SRR33138449.161 161 length=251 -1A1AAFFDD13BFEAB3111AEBEAEAA00D321D2221FG1CAEECGC/110/A//>>/0/1BBBG11221222?22222222221///?//?=110110..<A3AADF4DDDCGGCCEGGGGGHHGBGAFHCEGGGFFB3FF3FBFBAGG1AFEE1FGGF35FGGFHFHGFHGHFGFHHEE/EDGFBHG?EGFHGEEEEGGGGG/BDCGDDGC2?2FHHHHHHGGFHHHHHHHFHG>1131111>11A0B1B1AA0001B1BD/EAB0A111////B2BD2//AC/1@FDG21B//B0>222@F0>B2//B22B22B22121B>0>?0///1?BFBF>BB222B11<1B1F2>212BFB/?<011<0@1<222>@21?CAF/<1@11110>1101=<00DD/G0..3AAB4DFFFF?CFGGGGGGEEGGCHHHHFFFHHGHHHHHHHGHHHEGGGGFHGHFGGGCFHAGGEEFGCGGGE/?FDGGHHHHGHGHHHHEGFHHFFBGHHHGHHHDGFHGGH2GHHH22GHGFHHD1A>11>>>FBGGD1FA0BG1DDFA0EF0E11AF1FGFFC0A11DGH1E/F/FGHF2FFGHFCF/EEE>EF/GDFGHFFDFF2FG//>/>1FBD2>21FG/FAGCB1<FFFHFGHGCADFBFH11FCFCCHGFHHH1FG1CDCCEBGHHHFGFCCGAFHF-GAHGHHG.A9CEFG9F?/99EBFB0;0CA;.@-99-B99BAFD-- -@SRR33138449.169 169 length=251 -AACCGGGAGTGGTAAGAGATATATCGGCATTTATCAGATCGTTAGGTGGCTATAAGTCACGCTAAATGACTGGCTGAATCGAATCATAGCCAGAGCATGCCCTGACTTCACCCCGCTGTGTCTGCTTTTCCCGACTATTCTTAATGAGCTTCGATGCAATTCACGATCCCGCAGTGTGATTTGAGGAGTTTTCAATGGAATATAAAGATCCAATGCATGAGCTGTTGAGCAGCCTGGAACAGATTGTTTTT -+SRR33138449.169 169 length=251 -A?AA@DD1A>CA1EFGBF0AEDEHHGGGGFAF2D2FFF2FF0BGFH1AE0FHHGHHHHHGAHGGA1DF112G//0/GHBGGHHGGHFFFCGH00/BEGH1GBGFHFHHHFH10EGCC?FDB1BGFHB1BBCC?CE2DBGFFFFBFDFF<@CFGCGBFFFHB<..FE0EC-@FGFEG10?EFHD/1F1?BGBEGHB2>GFECC0EBBFGHB1BF2>GFHHBG9B9: -@SRR33138449.171 171 length=251 -GATATATCGGCATTTATCAGATCGTTAGGTGGCTATAAGTCACGCTAAATGACTGGCTGAATCGAATCATAGCCAGAGCATGCCCTGACTTCACCCCGCTGTGTCTGCTTTTCCCGACTATTCTTAATGAGCTTCGATGCAATTCACGATCCCGCAGTGTGATTTGAGGAGTTTTCAATGGAATATAAAGATCCAATGCATGAGCTGTTGAGCAGCCTGGAACAGATTGTTTTTAAAGATGAAACGCAGAA -+SRR33138449.171 171 length=251 -BAABBFFFB@BDGGGGGGGGGGHHHHHGHFGGHHHHHHHHHGHGHGGGFHHFHHHHHHGHHHHHHHHGHHFHHHGHHHHGHHHHHHHHFGHGHGHHGGGGCEFDGHFHHHHHGFHGGGGGHGGFGHGHHHFHHFGGGFFGHHGHHDGHGHHGFDGCCFFFFFFFFFF/ -@SRR33138449.172 172 length=251 -ATCATAGCCAGAGCATGCCCTGACTTCACCCCGCTGTGTCTGCTTTTCCCGACTATTCTTAATGAGCTTCGATGCAATTCACGATCCCGCAGTGTGATTTGAGGAGTTTTCAATGGAATATAAAGATCCAATGCATGAGCTGTTGAGCAGCCTGGAACAGATTGTTTTTAAAGATGAAACGCAGAAAATTACCCTGACGCACAGAACAACATCCTGTACCGAAATTGAGCAGTTACGAAAAGGAACAGGGT -+SRR33138449.172 172 length=251 -BCCCCFFFFFFFGGGGGGGGGGHHHHHHHHHGGGGGGHHHHHHHHHHHHHGCGGGHHHHHHHHFHHHHHHGHHHGHHHHHHHHGHHGHGGGGGHHHEGHHHHGHHGHHHHHHHHHHGHHHHHGHHFHHHHHHHHHHHHHHHHHHHHHHHHHHHHHGHHHFHHHHGHHHGHFHHGHHHHHHGGGGCHHHHHGHHHHHGHGGGGGGHFGHHHGGGGGGFGGGG?-BFFGGFFGGGGGGGGGFFFFFFFFFFFB -@SRR33138449.172 172 length=251 -GGCGACTGATACGCCCAAAACCCGGGCGAAATCATCGATTTTTAACCCTGTTCCTTTTCGTAACTGCTCAATTTCGGTACAGGATGTTGTTCTGTGCGTCAGGGTAATTTTCTGCGTTTCATCTTTAAAAACAATCTGTTCCAGGCTGCTCAACAGCTCATGCATTGGATCTTTATATTCCATTGAAAACTCCTCAAATCACACTGCGGGATCGTGAATTGCATCGAAGCTCATTAAGAATAGTCGGGAAA -+SRR33138449.172 172 length=251 -A?A>AD@AFFDFC?2EECCGGGGEGGGE?EAGHHHHHFHHGHHHHHFHHHHHHHHHHHFEHHHGHHHHHHHHHHHGGGHGEGHFGFFHHGGHHFHHHGGGCDFHBGFGHHHGHGHGGGGHHHHHHHGHFHFDFHHHHHHHHHH10FG0FHFGFGBCGHG1GFF1FGBGBGHHHFFHEHHHDEGEGF3AFG1GEHFHCGGGCE?E2D22EAFF21AFGDEEEEAFFGHHBHF///AF?E/@EG2@FD@FGDE01/FFHHEFFHEB1@F1FFDFFG1>GF1>E1B22BFFH1BBB>BB100/BFCB111B11B1>1FGBA333111A000001110000A001B0//A//A11////A/////>/>B1FFD01B/BB1//22/0>///0221B1B111////>F0/0/?/1<011B220//>//@GC///?F1FGDGFDGGGGF?EGAEBEGHHGHGAHGGFFGHEGHHFF?GEGFHFB5FGHHHHGHGHFHHHHHHHHFFFGGFHGHGGGHEAE?>EEHFH4@GFFFHGHEHHGGGGGHGHGGC/>/?111<10?1F11>A1B1@>D>1AEC0EGFBGFB3DEGG2GGH0FBF1D0/F/AGBAEGGGFHGG/AEFFFGGGCBBFH2GF2211>>EE/FA>FBBF0FFFHFGHFFHBDFGHBF/EE?E///00/?////@CCC/...1GHBG11.-ACC-@EHFACGC:///:GGG??;DFBBBFGFF.AEEFFBFFB/BBFFBFBBFFFBF//BF -@SRR33138449.180 180 length=251 -CACGTGTCTGACGTGACAAGGATAGTCCAGGACAGGCTTGCAGGATACCCTTTCTACGTAATCTTCCTGGTAAAATCATGGAGAAAATGAAACAAGGTTAATTTGTTGTTTCATTGTTAAAAAATGATAGTGCCGTTGCTTTTTTTGTTGCCAGAGATTTACGCCTTCTCAGAACGTTAGATTTTCGAAATCTTTACAAACCGGGAGTGGTAAGAGATATATCGGCATTTATCAGATCGTTAGGTGGCTAT -+SRR33138449.180 180 length=251 -CCCCCCFFFFFFGGGGGGGGGGHHHHHHHHHHHHGGGHHGHHHHGGHHHHHHHHHHHGHHHGHHHHHHHHGHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHGHHHHHHHHHHGGGGGHHHHHGGGHHHHHHHHHHHHHHGGGGGGHHHHHHHHGHHHGHHHHHHGGHGHHHHHHHHHHHGGFGGGFGGFGGGGGGGGGGGGGGGGGGGGGGGFFFFFFFFFFFFEFFFB -@SRR33138449.180 180 length=251 -GTACAGGATGTTGTTCTGTGCGTCAGGGTAATTTTCTGCGTTTCATCTTTAAAAACAATCTGTTCCAGGCTGCTCAACAGCTCATGCATTGGATCTTTATATTCCATTGAAAACTCCTCAAATCACACTGCGGGATCGTGAATTGCATCGAAGCTCATTAAGAATAGTCGGGAAAAGCAGACACAGCGGGGTGAAGTCAGGGCATGCTCTGGCTATGATTCGATTCAGCCAGTCATTTAGCGTGACTTATA -+SRR33138449.180 180 length=251 -CCCCCFFFFFFFGGGGGGGGGFFGGGHHGHGHHHGHHHGGGGGHHHGHHHHHHGHGHGHHHHHHHHHHGHHHGGHHHGHGHHHHHHHFHHHGHHHHHHHHHHHGFHHHHGHHHHHFHHGHHFHHHHHGHHGGGGGGGGHHHGHHHHHHHGGHG?GHHHHHGHGHHHHGBD/@DGHHHHHHHHGHHHHGGFG@DEFFFGGFGGGGGGGGGGGGGEFGGGGGBEFGGFFFEFFFFFFFFF00;@DF?AFFFF0 -@SRR33138449.181 181 length=250 -ACAGGATGTTGTTCTGTGCGTCAGGGTAATTTTCTGCGTTTCATCTTTAAAAACAATCTGTTCCAGGCTGCTCAACAGCTCATGCATTGGATCTTTATATTCCATTGAAAACTCCTCAAATCACACTGCGGGATCGTGAATTGCATCGAAGCTCATTAAGAATAGTCGGGAAAAGCAGACACAGCGGGGTGAAGTCAGGGCATGCTCTGGCTATGATTCGATTCAGCCAGTCATTTAGCGTGACTTATAG -+SRR33138449.181 181 length=250 -AABAAFFCFFFFCGFGGFGGGGGEGF4FGHHHHHHFHGEGGHBGHHHHDGHHBFHHHHHGFEFHHGEEFHGGHFBGHHFFHHHFBFGFHGGHHHHGHFHHHHHHFHHGFHDCGFHHHHHFHGGHHHFFHGGGCGGGFGGGHHHHGEGA2CF/GFHHFHBGHHHHGHFCFC@-GHHHHHHHHHHHHCDDG@BEFGGFGGGEGGB.CFGGGGGFBB9CFGGGBFFFGGFFFFFFFFFFF9FC9ADEFFFFFF -@SRR33138449.181 181 length=251 -AGGTTAATTTGTTGTTTCATTGTTAAAAAATGATAGTGCCGTTGCTTTTTTTGTTGCCAGAGATTTACGCCTTCTCAGAACGTTAGATTTTCGAAATCTTTACAAACCGGGAGTGGTAAGAGATATATCGGCATTTATCAGATCGTTAGGTGGCTATAAGTCACGCTAAATGACTGGCTGAATCGAATCATAGCCAGAGCATGCCCTGACTTCACCCCGCTGTGTCTGCTTTTCCCGACTATTCTTAATGA -+SRR33138449.181 181 length=251 -11>>A@FBFFB@AFG1FGGDFGFGFAGFEEGFFHHHHGCDEEFG?GFFHHEEEFGHHHHHAGEGHHH2FE/0EHH12F@BBFHHFEDGDGH2EC0/?FFHBF1G11>/>E/>@EHFFFF11DGFH2221<>>>ACFFFBFFGGGGGGGGGGGCHGHHHHGHHHGGGGHHHHGHHHHHHHGEEHFHHDFHHHHFHEHGHHHHGHHHGHGHHHHHGHGHGHHHHHHHHHHHGHHHGGGHHHHHHHGHGHHHHHHHHHGGGGCGEHGHGHHHFFFEGCGECFGHHHHGHHHHHHFFDGGFGHHFHGHHFFHGGHGGGG@CHGGFFFEGFGEFGGFGGGGGB0FFFGGFGGGGGGGGGFFB/9/FBFFFF.A.EF//BFFBFF -@SRR33138449.183 183 length=251 -GGTAAGAGATATATCGGCATTTATCAGATCGTTAGGTGGCTATAAGTCACGCTAAATGACTGGCTGAATCGAATCATAGCCAGAGCATGCCCTGACTTCACCCCGCTGTGTCTGCTTTTCCCGACTATTCTTAATGAGCTTCGATGCAATTCACGATCCCGCAGTGTGATTTGAGGAGTTTTCAATGGAATATAAAGATCCAATGCATGAGCTGTTGAGCAGCCTGGAACAGATTGTTTTTAAAGATGAAA -+SRR33138449.183 183 length=251 -AAAABFFFFFFFFGGAEGGFGGFGHHHHHHHFHHAGGHAGHHHHFHGHHHGHAEEHHHFGHFHHHGHHFHHHGHGHHHHHHHFHHHGHFGHHHHCGFHGFFHGGGGGGEGHHFFDHHHHHFFFGEFGFGHHHGFHHHFHHFHGHGGFHHHHGFFHGEHGGGGGFAAFF5CAFFFAFBEAE?ECGAFFGFFHGGHHG2EGHGHFHHHHHGHHGEHEH5FGGFHFHHHHHGHFFCHGGHFHHHFFEEGGBHHHHHHGHHGGFGHHHHBGFDHHHBGGFFFHHHHHGHEG/>E/EFEGHHFGFFFFGGGFHG/13>AADFFFFFFGGGGGGGGGGHHHHHGGHGFHFHGHGHGGGGGHHHHHGGGHHHHHGHHCHHHHHHEEGEDFHHHHDHHGHHHGHHHHHFECFFGGHHHHHHHHEF/EGEGGHGHGGHFHHHHHHHHG?DGGHGHHHHGBGHHHGHGHHHGHHHHHGFHBGHGHGGGHHHHHHGGHGGHHHFFHBGGHEHHFHHFHHHHGGGBFBGBFF00;BFGFGGGADGGFFFFFFFFFFFFFFFCFFFFFBBFF// -@SRR33138449.186 186 length=251 -TCTTTACAAACCGGGAGTGGTAAGAGATATATCGGCATTTATCAGATCGTTAGGTGGCTATAAGTCACGCTAAATGACTGGCTGAATCGAATCATAGCCAGAGCATGCCCTGACTTCACCCCGCTGTGTCTGCTTTTCCCGACTATTCTTAATGAGCTTCGATGCAATTCACGATCCCGCAGTGTGATTTGAGGAGTTTTCAATGGAATATAAAGATCCAATGCATGAGCTGTTGAGCAGCCTGGAACAGA -+SRR33138449.186 186 length=251 -BBBBBFFFFFBBGGFGGGGGGFGHHHHHHFHHHGGGGGHHHHFHHFHHHHHAFGDHGHHHHHGBFHHHGHGGGHHHGHHHHHHGHHHHHHFHGHHHHGHHHGGHGHHHHGHHHHHGHFFHHGGGGGFHHHHFHHHHHG2GGGGG/FGFHGHHHHFHHHHHFEHHGGGHHHHHHGHHGHGCGG;GCGGFHFHHGHFFHHHGGGGFCFGGGFFFGCFFFBBFG0FGGGGGGGGGGGGGGEFFFFFFF.BFFF/ -@SRR33138449.186 186 length=249 -ATGTTGTTCTGTGCGTCAGGGTAATTTTCTGCGTTTCATCTTTAAAAACAATCTGTTCCAGGCTGCTCAACAGCTCATGCATTGGATCTTTATATTCCATTGAAAACTCCTCAAATCACACTGCGGGATCGTGAATTTCATCGAAGCTCATTAAGAATAGTCGGGAAAAGCAGACACATCGGGGTGAAGTCAGGGCATGCTCTGGCTATGATTCGATTCAGCCAGGCATTTAGCGTGACTTATAGCCAC -+SRR33138449.186 186 length=249 ->A?AAFCAFFFB5EEF?EEGAGGFHHGHHFGCGGGGCHFHHFHGF5FGE2BAGDHHFEGHGEHHHGHHBGHFBFHFGHHEFHHHGFGHHHHHHGHHHHHHHEGHHHHHGBGGHHHGHHHHFHEF//E/EEEFG3/FD4BFG2BGFACGFC1FGFGFBDG12F/E.@-<-.D0=00CBGGGHHHHBBGFFHHFG00FB?FFFGGBCFF?.99CFF0CB-9A.909////9B/; -@SRR33138449.187 187 length=251 -GTGCGTCAGGGTAATTTTCTGCGTTTCATCTTTAAAAACAATCTGTTCCAGGCTGCTCAACAGCTCATGCATTGGATCTTTATATTCCATTGAAAACTCCTCAAATCACACTGCGGGATCGTGAATTGCATCGAAGCTCATTAAGAATAGTCGGGAAAAGCAGACACAGCGGGGTGAAGTCAGGGCATGCTCTGGCTATGATTCGATTCAGCCAGTCATTTAGCGTGACTTATAGCCACCTAACGATCTGA -+SRR33138449.187 187 length=251 -AAAAAADA?1CBF1AEFAABF1EEAAFGH3DGFFHHFFCCFHHCHFGF2FFFEHEFHHHHCGGFCGFGFHHHHHGFFFHF2GFFHHFHHHHGHHHHGGHFGHFHHH111FEGGF?EE/EFGBGH/FHDD1FFAB0FGHHGBHHFHHHHFFHHGCCGGHFFGHCGGFHCG.@GCCACHHH::0CGAFGGFCFFGFEFEFGFB0CBEFFG0C;FBFF99BCFBFF0AFFFFFFFFFFFFFFFFFBFBAAAFF/ -@SRR33138449.187 187 length=251 -ACGTAATCTTCCTGGTAAAATCATGGAGAAAATGAAACAAGGTTAATCTGTTGTTTCATTGTTAAAAAATGATAGTGCCGTTGCTTTTGTTGTTGCCAGAGATTTATGCCTTCTCAGAACGTTAGATTTTCGAAATCTTTACAAACCGGGAGTGGTAAGAGATATATCGGCATTTATCAGATCGTTAGGCGGCTATAAGTCACGCTAAATGACTGGCTGAATCGAATCATAGCCAGAGCATGCCCTGACTT -+SRR33138449.187 187 length=251 -111>111B3B3D11ABFGGG3GEF11BFGHGBGFGF110B0A1BEEA2DFGFDGG02GFHHHFGEA1EAE1AA2FGGFG?//AE@GGH1F00B/0111B0/BB1F122BFGG11B@GBBG/E00F122FGDE/0EG1F2>GGBFGHHH11A1>DBFA?F1FF3EC1FCGB1FG1FDGGHCDGFFHEGHFHHHBBHHHFHGHHHHHHHGBEFGC1BG1AEAE/AFAGH0?GHF11FGGBBFEFFFFFHF1@FG1B@F/EC?EACCFFH000GGHHHGGGH.AAAAAAAF?AGGFEGGGGFGFGH4G44GHHCAGHHGBFBEDFD3GHHHHEHBGB3GBCFFGHHHHHEGFFE33FGFHHFHHFFHFGFF5GHHHF@EGCFHGFFHFE4FF>EEGFHBFHGGGFHE4?E2BC2FGHHH//BGDFHHGH@FFFCBFCFHHFGD@/<.FBGFF11G1ABBFB@A?DBGGGGGGFFGGGCAFGGHFFGGGGHHHHHHHHHHHHFFHFFHBGHHHGHHGGHHHGGGGGGCGHHGHHHHHHHFFGHFGHEHHHGHHHHFGHGGBFGGGGGGFFHHEACHGG>FFGHGG.FGHHGGCCDGGGGGGGFFFEFFF0CGFHGCG.;0...;CFFDABEGFFEBEGB0FEBFBFFFFE/B9/B:BFEFFDF.A/B/BFFFFB/:9BBF//;BBBFD;--;9//:F//..;B9. -@SRR33138449.194 194 length=250 -CTGCTCAACAGCTCATGCATTGGATCTTTATATTCCATTGAAAACTCCTCAAATCACACTGCGGGATCGTGAATTGCATCGAAGCTCATTAAGAATAGTCGGGAAAAGCAGACACAGCGGGGTGAAGTCAGGGCATGCTCTGGCTATGATTCGATTCAGCCAGTCATTTAGCGTGACTTATAGCCACCTAACGATCTGATAAATGCCGATATATCTCTTATCACTCCCGGTTTGTAAAGATTTCGAAAAT -+SRR33138449.194 194 length=250 -3AABBFFFFFFFGGFFGGGFGGGFHHDGHFHH6FEG6GBG5FB5GFFHHHHHGHHHHGCAEBEEE?AFEHFFGBG3BG5@EGHE?GFGHFHFDGBFFHDBFEGEEEHH?GHFHEBGFFGCGFCDHGGHHGHFHGGFF1GHHFHHHBGFGGFEFHF11DFG0CH0DHHBFGHHGACDFHHHGHGHBGGFHHFHGE?EGGFFFFGGGGFGGGGGGGFGEGBGGFFFFFBEFF-DDDFFFFFFFFFFFFFDFF -@SRR33138449.194 194 length=248 -GGATAGTCAAGGACAGGCTTGCAGGATACCCTTTCTACGTAATCTTCCTGGTAAAATCATGGAGAAAATGAAACAAGGTTAATTTGTTGTTTCATTGTTAAAAAATGATAGTGCCGTTGCTTTTTTTGTTGCCAGAGATTTACGCCTTCTCAGAACGTTAGATTTTCGAAATCTTTACAAACCGGGAGTGATAAGAGATATATCGGCATTTATCAGATCGTTAGGTGGCTATAAGTCACGCTAAATGA -+SRR33138449.194 194 length=248 ->A11>@D33@BCGGGG1A11A0BE00111B110BFGFDF0A0FDFF21GHF1AF1A2B1FFF1FFHGG1FBHFGGAGA11BGGHGEGGFCCG2GG2FFFD22B1>0B1@EFGGGHCE>EEFBGHH/E?0B0>1B10F01>BD2FCC0/1<<>@1FC/<00?11F1G@@AF.<1=G/-9-;B/9 -@SRR33138449.200 200 length=251 -GGCTTGCAGGATACCCTTTCTACGTAATCTTCCTGGTAAAATCATGGAGAAAATGAAACAAGGTTAATTTGTTGTTTCATTGTTAAAAAATGATAGTGCCGTTGCTTTTTTTGTTGCCAGAGATTTACGCCTTCTCAGAACGTTAGATTTTCGAAATCTTTACAAACCGGGAGTGGTAAGAGATATATCGGCATTTATCAGATCGTTAGGTGGCTATAAGTCACGCTAAATGACTGGCTGAATCGAATCAT -+SRR33138449.200 200 length=251 -AAAAADAFC1111FGEDF1AGGFFBGCFHFHA1DB0GBFDGDCFBGFAA1B1BGAA1FH0AFFGF1ADBBAHCAGCGA2FDEAFB2F1FGHHHHFD2211/>/F/FB@DF/EEHBAFB1GEGHHHH1B///>B10?FAAFFFFF3D3BAAGGGBEBBA1FECFHFGE1A0AFE0A0//AABG/AG1DGHBE00FEGGGGH2111BGGFAF?/>@/GGHHHHCE>0F0/E//>E/G1F>1F>0B/FBB/FBFF9FFDFFDFFBFFFFBFFF -@SRR33138449.201 201 length=249 -TTATATTCCATTGAAAACTCCTCAAATCACACTGCGGGATCGTGAATTGCATCGAAGCTCATTAAGAATAGTCGGGAAAAGCAGACACAGCGGGGTGAAGTCAGGGCATGCTCTGGCTATGATTCGATTCAGCCAGTCATTTAGCGTGACTTATAGCCACCTAACGATCTGATAAATGCCGATATATCTCTTACCACTCCCGGCTTGTAAAGATTTCGAACATCTAACGTTCTGAGAAGGCGTAAATCT -+SRR33138449.201 201 length=249 -AA?AAFFFFFBFGBGBBGGGGBBGEGGHGHF44ADA22EEGCBG32EFHHHFGF331AGGHHGF55553FGHFAEGC>1BCHEHBGHFBGECE?EEGGHG?F23B0CECHHHFHH22FDGGHHEGF2@GGHHHF0FFHHH1GGG1@EF..9BB -@SRR33138449.202 202 length=251 -GGCTTGCAGGATACCCTTTCTACGTAATCTTCCTGGTAAAATCATGGAGAAAATGAAACAAGGTTAATTTGTTGTTTCATTGTTAAAAAATGATAGTGCCGTTGCTTTTTTTGTTGCCAGAGATTTACGCCTTCTCAGAACGTTAGATTTTCGAAATCTTTACAAACCGGGAGTGGTAAGAGATATATCGGCATTTATCAGATCGTTAGGTGGCTATAAGTCACGCTAAATGACTGGCTGAATCGAATCAT -+SRR33138449.202 202 length=251 -DCCCDDEFFFDFGGGGGGGGGGHGHHHGHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHGHHHHHHHHHHHHHHHHHHHHHHHHHGHHHHHHGHHHGGGGGHHHHHGGGHHHHHHHHHHHHHHHGGGHGHHHHHHHHGHHHGHHHHHHGFHGHHHHHHHHHHHGGGGGGEHGHHHHHHHHHGHHHGGGGGHHHHHHHHHHHGGGGGFGFGGGGGGGGGGGGGGGGGGGGGGGFFFFFFFFFFFFFF1 -@SRR33138449.202 202 length=249 -TTATATTCCATTGAAAACTCCTCAAATCACACTGCGGGATCGTGAATTGCATCGAAGCTCATTAAGAATAGTCGGGAAAAGCAGACACAGCGGGGTGAAGTCAGGGCATGCTCTGGCTATGATTCGATTCAGCCAGTCATTTAGCGTGACTTATAGCCACCTAACGATCTGATAAATGCCGATATATCTCTTACCACTCCCGGTTTGTAAAGATTTCGAAAATCTAACGTTCTGAGAAGGCGTAAATCT -+SRR33138449.202 202 length=249 -CCCCCFFFFFFFGGGGFGGGGGHHHHHHHHHHHHHGGGGGGGHHHGHHEGGFGEFBFEHHHGHHGHHGHHHGHGGGGGHHHHHHHHHHHHGFGGGEEGFHGHHGHHGGHHHHGHHHHBGDGHBGHGHHHHHEHHHHGHHHGGHFGGGGGGHGGHGFBFHHHHHHGGGHHHHHHHBHFFFHGGGCDGGFFFGGGFFFFGGFFFGBDGGGGG0FGFFFFFEFD@FFFFFFDFFFFFFFFFFFEDDFFFFF9 -@SRR33138449.203 203 length=251 -AGCGAGAGCAGGACTTTCCTGTCGCCATTTTCTTACTCCTTTCACCCAAAAAAAGCGCACAACGACACACACGTGTCTGACGTGACAAGGATAGTCCAGGACAGGCTTGCAGGATACCCTTTCTACGTAATCTTCCTGGTAAAATCATGGAGAAAATGAAACAAGGTTAATTTGTTGTTTCATTGTTAAAAAATGATAGTGCCGTTGCTTTTTTTGTTGCCAGAGATTTACGCCTTCTCAGAACGTTAGAT -+SRR33138449.203 203 length=251 -A@BB@DDDBFFBGGGGGGGGGFHGHGGGHHHHHHHGHHHGHHHHHHHGHHHHGGGHFFCFGHGGEGGGHGGGGFHHHHHFGDGECFHHHHGHHGGFFFGFHHHGGGGHGAGHHFHHHFHHHFHHHBFFHFFGGFHHHHHHGFFHHEHHFHFHHFFFHHFFHHFFCCDDFHHHFHBCHHHHHGHFFHHGEHHHGHHGBFGFFEGGGGG?FGGGGGGGFFFFFFFFAFFFFFBEFCFFFFFBF/:BDEFFEFF -@SRR33138449.203 203 length=249 -TTATATTCCATTGAAAACTCCTCAAATCACACTGCGGGATCGTGAATTGCATCGAAGCTCATTAAGAATAGTCGGGAAAAGCAGACACAGCGGGGTGAAGTCAGGGCATGCTCTGGCTATGATTCGATTCAGCCAGTCATTTAGCGTGACTTATAGCCACCTAACGATCTGATAAATGCCGATATATCTCTTACCACTCCCGGTTTGTAAAGATTTCGAATATCTAACGTTCTGTGAAGGCGTAAATCT -+SRR33138449.203 203 length=249 -?ABAABFFFFFFFF5BGGGBGFEF44F4B44B444E2A22FEGCGEHH35DD5E3AA0ACGG555B5FFHHHHGG00>F23?3FFHCFFAF/>//<DFHG1F0<AAAAFF@FF44GBFEDGGFGGGEEGGEHHBFGFFHFGGFBFF?GHHHBHHG55FFAAABEAAEE??FECGHGHFGF?//EGEFHFBGBG0?EFHHGBGDGFHFHHFGHHFFFGHHFHHHFHFDFGHFHEDDB1F?EEGG>1111>F01FDFGFAE0?>DGHDDFC@/A-1FDFFDBE1E00FEBA0FGG1D12DE0BFACAADAG1FFGG1B2DEEE/EC0FAGHEA>>FF/E///>>/1F1>0F/EEGEFG1>>BF1FFHH2D2FF22D1>DGCGFGFBF22@@@@/?//?FGBFF11100GFBDF3AF11AGGB1B3GD11FG1DFE00E?AFGG1000BAFCE/AE?/>>@GD1FGBG/BFG1BFFGHHEFH122@2>2FG1EDB0GFGHBEF22212221FFGFCC//@<<>F1FD1F0G11<<..--ACAAFBBF3GFBGAFEGG0AAFAGFGGB10AADDGHCGG?EHHHGEH21DFCHEAFHHF/EGHA@G2DDEF?G/E?GGHBGHD>GC>FFFGF001?G2GFB1B>F/BGGF11C1BGDGF -@SRR33138449.214 214 length=127 -GGTGAAGTCAGGGCATGCTCTGGCTATGATTCGATTCAGCCAGTCATTTAGCGTGACTTATAGCCACCTAACGATCTGATAAATGCCGATATATCTCTTACCACTCCCGGTTTGTAAAGATTTCGAA -+SRR33138449.214 214 length=127 -BABBBFFFFFF4GGGGAGGGGGBG4ADDGHEGF6AGGFFHHGFEGGBGFHHHGEGGFHFBDHHFHEFHHHHGE1G1GF55FBF5DGHECCGGHHHHHGFEGHFFFHFHD@/?EDHBGHHHHHHHFGF -@SRR33138449.215 215 length=251 -TAACCATACAGAATCGCGTAATCAGAGACAAAAGGACCAAACTGTTTTTTCTGATCGGCAGTCAGGGTTGGTAGACGTTCACCGCTACGCATCACTGTATTCTGCAGGAAGTCGATAAACGCTTTACGCTGATCGCCTTCTTTATCAAAACACCCACTCAGGCTAACTACCATTAATAACGCCGCAAGAGGCGCAAACCAGCGAGAGCAGGACTTTCCTGTCGCCATTTTCTTACTCCTTTCACCCAAAAA -+SRR33138449.215 215 length=251 -CCCCCFFFFFFFGGGGGGGGGGHFHHHHHHGHFGGHHHHHHHGHHHHHGGHHHHHHGFGGGHHHHGHHHGGGHHHHGGGHHHHGGGGGFEEGGHHHHHFHHGHHHHHHFHHHHGGGHGHHGGGGGHHGGGGGHHGGGGGHHHHHHHGHHHHHHGGGGHHGHHHHHHGHHHHHHHHHHHHHGGGGGGGGGGGGGGGFFFFFFFFFFFFFFFFEEFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE?EE -@SRR33138449.215 215 length=251 -AAGTCCGGGCATGCTCTGGCTATGATTCGATTCAGCCAGTCATTTAGCGTGACTTATAGCCACCTAACGATCTGATAAATGCCGATATATCTCTTACCACTCCCGGTTTGTAAAGATTTCGAAAATCTAACGTTCTGAGAAGGCGTAAATCTCTGGCAACAAAAAAAGCAACGGCACTATCATTTTTTAACAATGAAACAACAAATTAACCTTGTTTCATTTTCTCCATGATTTTACCAGGAAGATTACGT -+SRR33138449.215 215 length=251 -111AA1>1ADDFGGGGGGGGGGHHHHHHGGHHHH2GHGHFFFGHHGBGGEEGGHHGHHFHHHHHHEFHG/EGHHBFBG1FGBGG/>>EDGHHHHFHHEDHHHHHGCEFGFHHEGHEHGGHEAG?/FGHHFGGFHEGH211>FGFGE/CEGGHHHFBGHHGGEF0A-AEGHCGGCCCCFFHGHHCGHHGGBFFEFGEB0FFEGEA.;BFBF0BFBFFFGFGFFGGGFFFF;/BFFFFFBB/-AF?FFFBBE; -@SRR33138449.216 216 length=251 -GGCTTGCAGGATACCCTTTCTACGTAATCTTCCTGGTAAAATCATGGAGAAAATGAAACAAGGTTAATTTGTTGTTTCATTGTTAAAAAATGATAGTGCCGTTGCTTTTTTTGTTGCCAGAGATTTACGCCTTCTCAGAACGTTAGATTTTCGAAATCTTTACAAACCGGGAGTGGTAAGAGATATATCGGCATTTATCAGATCGTTAGGTGGCTATAAGTCACGCTAAATGACTGGCTGAATCGAATCAT -+SRR33138449.216 216 length=251 -DCCDDDCFFFCFGGGGGGGGGGHGHHHGHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHIHHHHGHHHHHHHHHHHHHHHHHHHHHHHHHGHHHHHHHHHHGGGGGHHHHHGGGHHHHHHHHHHHHHHHGGGHGHHHHHHHHGHHHGHHHHHHGGHGHHHHHHHHHHHGGGGGGGHGHHHHHHHHHHHHHGGGGGHHHHHHHHHHHGGGGGGGGGGGGGGHGGGGGGGGEFGGGGGGFFFFFFFHFFFFFF0 -@SRR33138449.216 216 length=251 -AGTCAGGGCATGCTCTGGCTATGATTCGATTCAGCCAGTCATTTAGCGTGACTTATAGCCACCTAACGATCTGATAAATGCCGATATATCTCTTACCACTCCCGGTTTGTAAAGATTTCGAAAATCTAACGTTCTGAGAAGGCGTAAATCTCTGGCAACAAAAAAAGCAACGGCACTATCATTTTTTAACAATGAAACAACAAATTAACCTTGTTTCATTTTCTCCATGATTTTACCAGGAAGATTACGTA -+SRR33138449.216 216 length=251 -CCCCCFFFBCFFGGGGGGGGGGHHHHHGHCHHHHHHHHHHHHHHHHHGGGGGHGHHHGGHHHHHHHHGGHHHHHHHHGHHHHGGGGGHHHHHHHHHHHHHGHGGGGGGGHHHHHHHHHHGFCEEHHHHHHGHHHGHHGHGHHHGGGGHHHHHHHGGHHHFHHHHGGGHHHHGGFGGGEFHHHHGHHGHHHHHFHHGGGGGGGGGGFGGGGGGGFFGFGGGGGGGGGGGGGGGGFFFFFFFFFFABFFFEB9 -@SRR33138449.217 217 length=251 -GTCAGGGCATGCTCTGGCTATGATTCGATTCAGCCAGTCATTTAGCGTGACTTATAGCCACCTAACGATCTGATAAATGCCGATATATCTCTTACCACTCCCGGTTTGTAAAGATTTCGAAAATCTAACGTTCTGAGAAGGCGTAAATCTCTGGCAACAAAAAAAGCAACGGCACTATCATTTTTTAACAATGAAACAACAAATTAACCTTGTTTCATTTTCTCCATGATTTTACCAGGAAGATTACGTAG -+SRR33138449.217 217 length=251 -BBAABFFBBFFFGGGGGGGGGFHHGGFFHHHHGHHGFHHFHHHGHHG2AFFHDHHHHGHHHHHHFHGFHHHHHFHFHFGFHGGGGGHHHHGHGHHHHHGHHHGGEGEFBG?FHHHHHGHGHEFHHHHHHGHHHGHFGHHHGGGADDHHFBGHHHFFFGHGHGHGGGGFHCCGC?CHHGHGHFGHHGHHGHHHECFGGFEEGGGGGFGGEFBBC;BBFFBFGGGGEGFGFGBFGFFBBBEAFFFFFFFFFFF -@SRR33138449.217 217 length=248 -GGATAGTCCAGGACAGGCTTGCAGGATACCCTTTCTACGTAATCTTCCTGGTAAAATCATGGAGAAAATGAAACAAGGTTAATTTGTTGTTTCATTGTTAAAAAATGATAGTGCCGTTGCTTTTTTTGTTGCCAGAGATTTACGCCTTCTCAGAACGTTAGATTTTCGAAATCTTTACAAACCGGGAGTGGTAAGAGATATATCGGCATTTATCAGATCGTTAGGTGGCTATAAGTCACGCTAAATGA -+SRR33138449.217 217 length=248 -AA?ABBFFFFFFGGGCCFC4EGHHHGFG4GFHHHFHFBCFFFGHHHHHHGF3DFGFEHFHGHHHFHHFFG5AFFHGFGAGHFBGHHHECGHFHHHHHHGHHHHEEHHHHHF@443E1EEGGHHHHGGG?GHBBF?FGFGHHGHG/?EEGGBHGD1122BF0/>GHHHDGGHFHEFGF1B00/CC/GF2@1@01/AAFFF1DF1FCGGGFGFGFCBFFFDFBGHHFBGHGHFHHCEA?FFA1FDEHGBEHHC1FE/FGHB2FDHFHFHHEEEEE2FE22F@GFHGEHHHHGGEEGCHGHHF1BGHFGC0/>FBGHFHGHHHGFF>>>C0F@AC/?GHEDGHHHFHHF/F0GGC?GHHHGGGCGFFHFHHHGBHGHHHGFFHHHHFHEGEEFFGBFFBFFFCFGGGGGFGFGGFFF0CFGFGGGFFG?EFFFFBFF-BFFFFB9 -@SRR33138449.220 220 length=251 -GGGCATGCTCTGGCTATGATTCGATTCAGCCAGTCATTTAGCGTGACTTATAGCCACCTAACGATCTGATAAATGCCGATATATCTCTTACCACTCCCGGTTTGTAAAGATTTCGAAAATCTAACGTTCTGAGAAGGCGTAAATCTCTGGCAACAAAAAAAGCAACGGCACTATCATTTTTTAACAATGAAACAACAAATTAACCTTGTTTCATTTTCTCCATGATTTTACCAGGAAGATTACGTAGAAAG -+SRR33138449.220 220 length=251 -B?AABFFFFFFFGGGGGGGGGGGHHHHHHHHHHHHFHHHHFFFGGGGHHHHHHHGGHHHHHHGGHHHHHHFGHHHHHGGGGGGHFHHHHHHHHGHHHGGGFGGGHHHFGHHHHGHGHGGHHHHHHGHHHGHHHHHFGHGGGGHHHHHHHHHHHHHHHHHGGGHHHHGGCGGHHHHGHHHHHGHHHHGHHFHHHGHHGGFFGGGGFGGGGG/CFGGGGGFGGGFGFFFGGGGEGGFFFFFFFFFF9FFEFFB -@SRR33138449.220 220 length=251 -GTCCAGGACAGGCTTGCAGGATACCCTTTCTACGTAATCTTCCTGGTAAAATCATGGAGAAAATGAAACAAGGTTAATTTGTTGTTTCATTGTTAAAAAATGATAGTGCCGTTGCTTTTTTTGTTGCCAGAGATTTACGCCTTCTCAGAACGTTAGATTTTCGAAATCTTTACAAACCGGGAGTGGTAAGAGATATATCGGCATTTATCAGATCGTTAGGTGGCTATAAGTCACGCTAAATGACTGGCTGA -+SRR33138449.220 220 length=251 -AB3A?CB4AFBBGCGE4FBGFBGGHHGFHGHHHGHHHDEHHGFHHHHHHF5FFHGBFGHEGHFHGFG5FGEGHGGHFFHHHEGFGAGHHHEHHHGH5FF/G3GGEGFFBGE?EFGGH?GFEGCGHABHFFFH0FGFHDAECFD1/EEEHHFEFGHFHHHHHBGFHG/FHHHGHFFHFHFHHFHHHGHGHHFHC@DGGGGGGHGGGG?CHGGGCGCGHHHFFGGGGGGBFEAFGFCFFFFGEEFEGGGG@FFFF.BFFFFFBFFFFFFFFFFEFFFFFFFBFFFFFFFFFFFF/FFFFFFFFFFFFF/BFFFFF -@SRR33138449.221 221 length=251 -GGCATGCTCTGGCTATGATTCGATTCAGCCAGTCATTTAGCGTGACTTATAGCCACCTAACGATCTGATAAATGCCGATATATCTCTTACCACTCCCGGTTTGTAAAGATTTCGAAAATCTAACGTTCTGAGAAGGCGTAAATCTCTGGCAACAAAAAAAGCAACGGCACTATCATTTTTTAACAATGAAACAACAAATTAACCTTGTTTCATTTTCTCCATGATTTTACCAGGAAGATTACGTAGAAAGG -+SRR33138449.221 221 length=251 -A?AAAFFFFFFBC1BF1FGGG1BFGFFFCFFHFGDFG3GFH?AE0FGHFHB2GCFFFF1FHCFFHFE2FEGFH2EGCE/>EHHHDFFH1GHBCHCFFGG>>EGFG222B11BDFF0/EF1F2F1E1CCG1>11B33BB1C1AE1B1AA00AA00E00AFAE0E011BEF//A/AF1112212A11B///0B01111221@B1G1F121@11B?EE/>??/0B///>/>/BFAG//////<<0111D2@D1111?FGE?/F1FGF1>1F<1<1>>1>1FF0/<<0<-<-;---/.;.:--/9.;-9.:;99F0.;;..0;.-9---///BF/-A/---;-@E/B--////://////;-9---B//9////9//9////9 -@SRR33138449.224 224 length=251 -GTTTTAGCGCAGAGTGCGCAGCATCGGCCTGCAACTTCGCATTTTGCAGCTGTTGCGCCAGTACGCCCAGAGAACCGTTCATTTCACGCAGTGGGCCGCTTTGCGTAACGTAATCCTGCGGTACGCGAATTGCGTTAACGCTGTCTACGACCGGGCGCAGACCGGAATCCATCGCCTGATTCACCTGCTGAGAATAACCATACAGAATCGCGTAATCAGAGACAAAAGGACCAAACTGTTTTTTCTGATCG -+SRR33138449.224 224 length=251 -A11A>3CFDDDD1AFEGECCGGHFH00E?FGF0EEBHHGGGGGHHEDDGGHEHHDHGGGGGHGGF?@FFC//BFFGEE>FGHHHF>BEG>E?CF/?EEEGGGHHGCFG?A?FGGGGHHFGGF?C/<@GC@CGHGCCGEH.CFFA0GHHG?GG@@?@9?A?-?9-@--EFBBFFBBBFFFFFFEF9BF?FFFFFFFFFFFBBFFFFFB/B--9@@FEFF//9-9FFFFFFFFFFFF-BF/BBFE?FFBBF/- -@SRR33138449.224 224 length=251 -ACTTATAGCCACCTAACGATCTGATAAATGCCGATATATTTCTTACCACTCCCGGTTTGTAAAGATTTCGAAAATCTAACGTTCTGAGAAGGCGTAAATCTCTGGCAATAATAAAATCAACGGCACTATCATTTTTTAACAATGAAACAACAAATTAACCTTGTTTCATTTTCTCCATGATTTTACCAGGAAGATTACGTAGAAAGGTAATCCTGTAAGCCTGTCCTGGTCTATCCTTGTCACGTCAGATA -+SRR33138449.224 224 length=251 -A1AA1DDFB1111BC11EAGCFG33AB1G311A0AEEFG2DGF1210F1A0B/AAAE/EGHFB21FGBGH/0/>FGHHFFC>F0?@21BBCCCEA//?FBH@GE11>11B122B112B11BGBHFEG0FGHBDGF<1F<><<>GDDFHEFH1D0D=DDGDGH0../.FF1BDGGFEGGEFEHH1@GGHFFHHHHHHFFH>BBFE2FGEA1>F2FHDGEGA/ECHHHHHGGCGG01DF1?/0??F11.A@HAGHHB1<=/111111>1111331B1A311B1EG0AEC1D2BFDBG22AA///1DAG1E1/AEA/FC/EE>/>010001011B2FFBG12222BB001>112221?//E?///F/?/F<0.>>CGF0D0@.B.EHEB=C0::GCGB::..;0;.C//00090=0;00;-;.-.9:9/0;09;F0;-9:-/9/;/;B//;9/9///9///////B/-A-9//B;-///////;///; -@SRR33138449.226 226 length=251 -ACACACACGTGTCTGACGTGACAAGGATAGTCCAGGACAGGCTTGCAGGATACCCTTTCTACGTAATCTTCCTGGTAAAATCATGGAGAAAATGAAACAAGGTTAATTTGTTGTTTCATTGTTAAAAAATGATAGTGCCGTTGCTTTTTTTGTTGCCAGAGATTTACGCCTTCTCAGAACGTTAGATTTTCGAAATCTTTACAAACCGGGAGTGGTAAGAGATATATCGGCATTTATCAGATCGTTAGGTG -+SRR33138449.226 226 length=251 -1A1A1AAAD>>CGGFGGGFCFAG1CA0FGHAAGGHBFFGFGC0FEGHGA/1ECFHH0CABF2/0BFA2FEB1FGHFFG@EGF1EBFBE1BFE1EBGGDE/0/F1FGBGGHHB0FGHHFEFHHHHFG11>>11B1111B11B31F111B1BD1DDEFF111ABAFDH2A2D2DGH2222A1B11A11FGHHF1A10A1E///AB/AAA/2A12222//B/>11@1@2BGB1B1E@2/>////B1111>>1/?0///////0>F2>22122@2222212111->.ECG111D<00000=0=/.-..;:0000;000;-.:/;C0;0099000009000.0/.9;;0C000000 -@SRR33138449.227 227 length=251 -ACTGTATTCTGCAGGAAGTCGATAAACGCTTTACGCTGATCGCCTTCTTTATCAAAACACCCACTCAGGCTAACTACCATTAATAACGCCGCAAGAGGCGCAAACCAGCGAGAGCAGGACTTTCCTGTCGCCATTTTCTTACTCCTTTCACCCAAAAAAAGCGCACAACGACACACACGTGTCTGACGTGACAAGGATAGTCCAGGACAGGCTTGCAGGATACCCTTTCTACGTAATCTTCCTGGTAAAAT -+SRR33138449.227 227 length=251 -BBCCBFFFFFFFGGGGGGGGGGGHGHHGGGGGHHGGGGGHHGGGGGHHHHHHHHHHHHHHGGGGHHHHHHHHHHHHHHHHHHHHHHHGGGGGGGGHHHGGGGGGHGHHGGGGGGHHHHGHHHHHHHHHHGHGGGHEHHHHHHGHHHHGHHHHHGHHHHGGGHGGGGGHGGGGGGGGGGGFFGGGGGGGFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF9FFFF0 -@SRR33138449.227 227 length=250 -ACCTAACGATCTGATAAATGCCGATATATCTCTTACCACTCCCGGTTTGTAAAGATTTCGAAAATCTAACGTTCTGAGAAGGCGTAAATCTCTGGCAACAAAAAAAGCAACGGCACTATCATTTTTTAACAATGAAACAACAAATTAACCTTGTTTCATTTTCTCCATGATTTTACCAGGAAGATTACGTAGAAAGGGTATCCTGCAAGCCTGTCCTGGACTATCCTTGTCACGTCAGACACGTGTGTGT -+SRR33138449.227 227 length=250 -AABBBFFCCBCFGGGGGGGGGGGGGGGHHHHHHHHHHHHCHHHGGEEGGHFGHHGHHGHHGHGCHHHHGFGGHHGHHHHHHHHGGGGHHHHHGHHHHHHHHHHHGFGHGHHGEGGGHHHHHHHHHHGGHGHHHHHHHHHHGHHHHHHHHHHHHHHHHHHHHHGHHHHHHHHGHHHHBHGEACGHHHGHGFHAEBGHHGFFFGGGGFGFFEEGFFGGFGGGGGGGEBFBFFFGFDAEFFFFFF?:A;AAAA -@SRR33138449.228 228 length=251 -CGACACACACGTGTCTGACGTGACAAGGCTAGTCCAGGACAGGCTTGCAGGATACCCTTTCTACGTAATCTTCCTGGTAAAATCATGGAGAAAATGAAACAAGGTTAATTTGTTGTTTCATTGTTAAAAAATGATAGTGCCGTTGCTTTTTTTGTTGCCAGAGATTTACGCCTTCTCAGAACGTTAGATTTTCGAAATCTTTACAAACCGGGAGTGGTAAGTGCTATATCGGCATTTATCAGCTCGTTAGG -+SRR33138449.228 228 length=251 -AAAAADFAAAAFGGGG111AAACFF1GC0AAAFHHFHF/B0FCFFHGGEHGGHBHHHHHHHHHGE0GAFGDHFHEFB1GGHH21FGGE1B1FC0BEG1FGG/0/BBBGHHEFHHFHG0GFG22>2>2FG1FGFHHHHC/CG/@GGHHGC??G01?D1<<0>DD1F..AGGHFH1>1>B31131B1111FE11100BF0B00000A//A//A0/BB1101B11B12FA22210/00111@11100/01>BCEFG//10B1BDD?/?//////<00011=1000AA111FAAEG1GGH3B33FDD000EEA0EE0AGGGGHHHHHC1DBGF2FA1E/A/EE//>0//>E@/>>/>/FE/>/EC10F//FGG>F1D1B/?D>GFF1<<@GBG1FFCGG/ACC0.CCCGHCGG@-::EA.A..;/CGFBA9CE.;099.9BB9;009C../..CFG?E//-9BB/B9FFFBF/F-9B-ABFF/BFF9BFB/;/9;://9F9ABFFFF/BFFABABFFFFB -@SRR33138449.229 229 length=251 -GCCGATATATCTCTTACCACTCCCGGTTTGTAAAGCTTTCGAAAATCTAACGTTCTGAGAAGGCGTAAATCTCTGGCAACAAAAAAAGCAACGGCACTATCATTTTTTAACAATGAAACAACAAATTAACCTTGTTTCATTTTCTCCATGATTTTACCAGGAAGACTACGTAGAAAGGGTATCCTGCAAGCCTGTCCTGGACTATCCTTGTCACGTCAGACACGTGTGTGTCGTTGTGCGCTTTTTTTGGG -+SRR33138449.229 229 length=251 ->1>A1>1>3@@@33AFGG1AA0AEEEA0EEFHGFF1AAAGE/AAA112D11A0BBFDBBAF00AEGEEGG22BGHHHBBGG/B0B>E/GFFF?//EE0FHH222>>G@CGDGFFA.>AH000<..CG.==/.<.:CG00GBGH0.;/G0/0=:/;.CBFGFBF09;0BAF?ACCB/CFFGGFFFFF-EAB-A/B-;--;F@;;@A -@SRR33138449.230 230 length=251 -ATATATCTCTTACCACTCCCGGTTTGTAAAGATTTCGAAAATCTAACGTTCTGAGAAGGCGTAAATCTCTGGCAACAAAAAAAGCAACGGCACTATCATTTTTTAACAATGAAACAACAAATTAACCTTGTTTCATTTTCTCCATGATTTTACCAGGAAGATTACGTAGAAAGGGTATCCTGCAAGCCTGTCCTGGACTATCCTTGTCACGTCAGACACGTGTGTGTCGTTGTGCGCTTTTTTTGGGTGAA -+SRR33138449.230 230 length=251 -CDDCDFFFFFFFGGGGGGGGGGGGGGHHHHHHHHHHHGHGGHHHHHHGHHHGHHHHHHHHGGGGHHHHGHHHHHHHHHHHHGGGHHHHGGGGGHHHHHHHHHHGHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHGHHHHHHHHHGGHHHHHHHGGHHGHHHHGGGFHHHHHHHGHGHHHGHHHHHHHHHHHHHHGGGGGGGGGGGGGGFGGGGGGGGGGGFFFFFFFFFFFFFFFFFEF -@SRR33138449.230 230 length=251 -GGGTTGGTAGACGTTCACCGCTACGCATCACTGTATTCTGCAGGAAGTCGATAAACGCTTTACGCTGATCGCCTTCTTTATCAAAACACCCACTCAGGCTAACTACCATTAATAACGCCGCAAGAGGCGCAAACCAGCGAGAGCAGGACTTTCCTGTCGCCATTTTCTTACTCCTTTCACCCAAAAAAAGCGCACAACGACACACACGTGTCTGACGTGACAAGGATAGTCCAGGACAGGCTTGCAGGATA -+SRR33138449.230 230 length=251 -AABABBBBFFFFGFGGGGGGGGGFGGGGGHHHHGHGHGHHHFHHGHFGFFGGHGHHGGGGGHHGGGGGHGGGGGGHHHHHHHGHHHHHHGGGGHHHGFHFFHHGHHGHHHHHHHHHGGGGGGFGGG/DFFCGCHGHGFCGGGCGHHHGEGFHHHFHGGEHCDGHHHHGHHCBFGGFGGEFFFGGF.C?AGGGFFFFFFFFAFFFFFFFFFF.FFBF?AAEEFFEEFFFB;BBFFF?FFEFEFBAFFF?./9 -@SRR33138449.231 231 length=251 -AGTCAGGGTTGGTAGACGTTCACCGCTACGCATCACTGTATTCTGCAGGAAGTCGATAAACGCTTTACGCTGATCGCCTTCTTTATCAAAACACCCACTCAGGCTAACTACCATTAATAACGCCGCAAGAGGCGCAAACCAGCGAGAGCAGGACTTTCCTGTCGCCATTTTCTTACTCCTTTCACCCAAAAAAAGCGCACAACGACACACACGTGTCTGACGTGACAAGGATAGTCCAGGACAGGCTTGCA -+SRR33138449.231 231 length=251 -BCBBCFFFFFCCGGGGGGGGGGHHGGGGGGGGGGHHHHHHHHHHHHHHHGHHHHGGGHGHHGGGGGHHGGGGGHHGGGGGHHHHHHHHHHHHHHGGGGHGHHHHHHHHHHHHHGHHHHHHHGGGGGGGGFHHGGGGGGHGHGHGFGGGHHHHGHHHHHHHHHHGHGGGHHGGGGGGGGGGGGGGGGGGGGGGGGFF?BFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFFFFFFFFFFFFFFFFEFFFF0 -@SRR33138449.231 231 length=251 -TCTCTTACCACTCCCGGTTTGTAAAGATTTCGAAAATCTAACGTTCTGAGAAGGCGTAAATCTCTGGCAACAAAAAAAGCAACGGCACTATCATTTTTTAACAATGAAACAACAAATTAACCTTGTTTCATTTTCTCCATGATTTTACCAGGAAGATTACGTAGAAAGGGTATCCTGCAAGCCTGTCCTGGACTATCCTTGTCACGTCAGACACGTGTGTGTCGTTGTGCGCTTTTTTTGGGTGAAAGGAG -+SRR33138449.231 231 length=251 -CCCCCFBFFFFFGGGGGFGAEGGHHHHFFHHGGHGGHH5GHGFHHHGGFEFHHGGGFGGHHHHHHFFFHGHHGHHHGFGHHHGGGGGGHHHHHHHHHHGHHHHHHHHHHHHGHHHHHHHHHHHHHHHHGHHHHHHHHHEHHHHHHHHGGHGGEHFHHHHHGFHFGHFHGACCHHHHHGHHGGG/GGHHHHHFHHFHHHHFGHFHHGFEFGGGFGGFGFE.FAFDAAFEGGFF;AFFB@9CDBDFA/BF.FA -@SRR33138449.232 232 length=251 -TACCACTCCCGGTTTGTAAAGATTTCGAAAATCTAACGTTCTGAGAAGGCGTAAATCTCTGGCAACAAAAAAAGCAACGGCACTATCATTTTTTAACAATGAAACAACAAATTAACCTTGTTTCATTTTCTCCATGATTTTACCAGGAAGATTACGTAGAAAGGGTATCCTGCAAGCCTGTCCTGGACTATCCTTGTCACGTCAGACACGTGTGTGTCGTTGTGCGCTTTTTTTGGGTGAAAGGAGTAAGA -+SRR33138449.232 232 length=251 -CCCCCFFFFBBCGGGGGGGGGGHHHGHGHGGHHHHHHGHHHGHHHGHHHHGGGGHHHHHHHHHHHHFHHHHGGGGHHHGGCGGHHHHHHHHHHGHHHGHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHGHHHHHHHHHGHHHHHHHHGGHHGGHHHGHHHGHHHHHHHHGHHHHHHHHHHHHHHHHHHHGHHGHHGGGGGGGGGBEAFFGGGGGGGGGGFFFFFFDFFFFFEFFFFFFFFFFF -@SRR33138449.232 232 length=249 -CATTTTGCAGCTGTTGCGCCAGTACGCCCAGAGAACCGTTCATTTCACGCAGTGGGCCGCTTTGCGTAACGTAATCCTGCGGTACGCGAATTGCGTTAACGCTGTCTACGACCGGGCGCAGACCGGAATCCATCGCCTGATTCACCTGCTGAGAATAACCATACAGAATCGCGTAATCAGAGACAAAAGGACCAAACTGTTTTTTCTGATCGGCAGTCAGGGTTGGTAGACGTTCACCGCTACGCATCA -+SRR33138449.232 232 length=249 -ABABBFFFDFFFGGG45A?EGGHHGGGGGGHHHHHBCEGGCCHFHHHGHHEE1GFFCHEGGGGGHGE11EEGHHF4GFHFEGEG??DEE//FHEDDDDFFFCDDDHHGHGGG/DGG?DGC?CHEG?DCE0CGCHGDG?EGGGGGGGGG?FGGAB9BFGGFGFFGGGF/CBAE;BBDFFFFEFFFBF.9F?BFFE;AEFFBFFFFBFFFFFFF.;?B9/FF..;9B-./FFFF.A.;/:A--9;.-.ABF -@SRR33138449.233 233 length=250 -GTTGGTAGACGTTCACCGCTACGCATCACTGTATTCTGCAGGAAGTCGATAAACGCTTTACGCTGATCGCCTTCTTTATCAAAACACCCACTCAGGCTAACTACCATTAATAACGCCGCAAGAGGCGCAAACCAGCGAGAGCAGGACTTTCCTGTCGCCATTTTCTTACTCCTTTCACCCAAAAAAAGCGCACAACGACACACACGTGTCTGACGTGACAAGGATAGTCCAGGACAGGCTTGCAGGATAC -+SRR33138449.233 233 length=250 -CBCCCBFFFFCBGGGGGGGGGGGGGGGHHHHHHHHHHHHHHHGHHHHGGGHGHHGGGGGHHGGGGGHHGGGGGHHHHHHHHHHHHHHGGGGHHHGHHHHHHHHHHHHHHHHHHHGGGGGGGGHHHGGGGGGHGHHHGGGGGHHHHGHHHHHHHHHHGGGGGHHHHHHHHHHHHHHHGGGGGGGGGGGGGGGGFFFFFFFFFFFFFFFFFFFFFFFEFFFFEFFFFFFFFFFFFFFFFFFFFFFFFFFFFF -@SRR33138449.233 233 length=251 -TTTGTAAAGATTTCGAAAATCTAACGTTCTGAGAAGGCGTAAATCTCTGGCAACAAAAAAAGCAACGGCACTATCATTTTTTAACAATGAAACAACAAATTAACCTTGTTTCATTTTCTCCATGATTTTACCAGGAAGATTACGTAGAAAGGGTATCCTGCAAGCCTGTCCTGGACTATCCTTGTCACGTCAGACACGTGTGTGTCGTTGTGCGCTTTTTTTGGGTGAAAGGAGTAAGAAAATGGCGACAG -+SRR33138449.233 233 length=251 -BBBBBFFFFDFFGGGEGEEGGGHHHGHHHGHHFHHHGGGGGGHHHGFHEGHHHHHGHGHGGGHHHHGGGGGHHHHHHHHHHGHHHGHHHGHHGHHHHGHHFHHHGHGHHHHFHHHHHHHHFHHGGHHHHHFHHHGEHHHHGGHEGHHEFHFH/GHHHGHHGHHGGGGHHHHHHHHHHHHEHFHHHHGGFFHGDFFHHGFHFHHHHHGFGGG?F?BGGGFFGGFF?AFFFBB?.EFF/BFFFFFFFFFA-;@ -@SRR33138449.234 234 length=251 -GCTTTACGCTGATCGCCTTCTTTATCAAAACACCCACTCAGGCTAACTACCATTAATAACGCCGCAAGAGGCGCAAACCAGCGAGAGCAGGACTTTCCTGTCGCCATTTTCTTACTCCTTTCACCCAAAAAAAGCGCACAACGACACACACGTGTCTGACGTGACAAGGATAGTCCAGGACAGGCTTGCAGGATACCCTTTCTACGTAATCTTCCTGGTAAAATCATGGAGAAAATGAAACAAGGTTAATT -+SRR33138449.234 234 length=251 -DDDDDFFCCDDDGGGGGGGGGGHHHHHHGHGHHGGGGHHHHHHHHHHHHHHHHHHHHHHHGGGGGGGGHHHGGGGCGHGHHHGFGGGHHHHGHHHHHHHGHHGHGGGHHHHHHHHHHHHHHHHHHHGHHHHGGGGGGGGGHGGGGGGHGGGGHHHHHHGHGFGGGGGGGGGGGGGGGGGGGGGGGGGGFFFFFFFFFFFFFFFHFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF -@SRR33138449.234 234 length=249 -ATCTAACGTTCTGAGAAGGCGTAAATCTCTGGCAACAAAAAAAGCAACGGCACTATCATTTTTTAACAATGAAACAACAAATTAACCTTGTTTCATTTTCTCCATGATTTTACCAGGAAGATTACGTAGAAAGGGTATCCTGCAAGCCTGTCCTGGACTATCCTTGTCACGTCAGACACGTGTGTGTCGTTGTGCGCTTTTTTTGGGTGAAAGGAGTAAGAAAATGGCGACAGGAAAGTCCTGCTCTCG -+SRR33138449.234 234 length=249 -CCCCCFFBFFFCGGGGGGGGGGGGHHHGHHHHHHHHHHHHHGGGHHHHGGGGGHHHHHHHHHHGHHGHHHHHHHHHHHHHGHHHHGHHHHHHGHHHHHHHHHHHHGHHHHHHHHHHFAHHFHHHHGFHHGFHGH?FHHHFHHGHGHEFGHHHFHHHGHHHHGBGHHHHHHGHHGGHHHHGCEFGCGHHGEEHGGHGGGFGGGGGGADEE9F;CFGEBFBFFFFFFFFF;AFFFFFBBFBFFFFFFFFFF -@SRR33138449.235 235 length=251 -GGCGTAAATCTCTGGCAACAAAAAAAGCAACGGCACTATCATTTTTTAACAATGAAACAACAAATTAACCTTGTTTCATTTTCTCCATGATTTTACCAGGAAGATTACGCAGAAAGGCTATCCTGCAAGCCTGCCCTGGACTATCCTTGTCACTTCAGACACGTGTGTGTCGTTGTGCGCTTTTTTTGGGTGAAAGGAGTAAGAAAATGGCGACAGGAAAGTCCTGCTCTCGCTGGTTTGCGCCTCTTGCG -+SRR33138449.235 235 length=251 -A11>1AAADD3DGFCGCBF1AFFCGE?0BFCE/E//0B1DBEGHHFEGFH1BFHE1D1EF/C>F1GHHFGGHH1BFFF2BFGH12B@F@12BF12@1@BFAGHAFB>FA/>EE0B10/1B<0FGDFGG1FEHF00<10FCFHHFHFGGHDFB11111<110?CCA<<>GGFCFCHGCH.@C.EGHGGG-;AA.00;CB./9;FFFGGGEFE@--@@?-BBFFFFFBFFEFE??---BBFAF@@FFB///F@ -@SRR33138449.235 235 length=250 -TTCGGGTACTGGATGCCATTTGCGACATAGCTCACCTCAGTACCCTGCTCCGCAATGCAGTCTCCAATTATCACCACTACATCCGTGCAGGATTGTGTCGCCGGACTCAACGGCTGCACCGCCTCAGCCGGCGTCGGTACCACTTTCGTGTACGCCTGATCAATGACCGGTTTGATATCATTGTACTGCTTCAGCGCACAGTGCGCAGCCTCTGCTCGCAACTTCGCACTTCTCAGCTGCTGCGCGTGTA -+SRR33138449.235 235 length=250 ->AA1A111@BB11B1111FG3FGE00A011ADF1B0101112BA11BA111//A/A1111B1DB11111DEG2201/1001120B>F//1B/BF0F2>B//?//>/>011B///>//010///?<11@0/--<..=G0/<00..../0..::.:C00000;9F-A9CC.00;090000000;00;00;0;-9>---/:B-9-;----/////-9-:@//:A-9--;/9/999:F/B//;------ -@SRR33138449.236 236 length=249 -CCGCTACGCATCACTGTATTCTGCAGGAAGTCGATAAACGCTTTACGCTGATCGCCTTCTTTATCAAAACACCCACTCAGGCTAACTACCATTAATAACGCCGCAAGAGGCGCAAACCAGCGAGAGCAGGACTTTCCTGTCGCCATTTTCTTACTCCTTTCACCCAAAAAAAGCGCACAACGACACACACGTGTCTGACGTGACAAGGATAGTCCAGGACAGGCTTGCAGGATACCCTTTCTACGTAAT -+SRR33138449.236 236 length=249 -CDCDCCDDDCDDGGGGGGGGGGHHHHHGHHHHGGGHGHHGGGGGHHGGGGGHHGGGGGHHHHHHHHHHHHHHGGGGHHHHHHHHHHHHHHHHHHHHHHHGGGGGGGGHHHGGGGGGHGHHHGGGGGHHHHGHHHHHHHHHHGHGGGHHHHHHHHHHHHHHHHHHHGHHHHGGGGGGGGGGGGGGGGGFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFBEFFFF -@SRR33138449.236 236 length=251 -GCGTAAATCTCTGGCAACAAAAAAAGCAACGGCACTATCATTTTTTAACAATGAAACAACAAATTAACCTTGTTTCATTTTCTCCATGATTTTACCAGGAAGATTACGTAGAAAGGGTATCCTGCAAGCCTGTCCTGGACTATCCTTGTCACGTCAGACACGTGTGTGTCGTTGTGCGCTTTTTTTGGGTGAAAGGAGTAAGAAAATGGCGACAGGAAAGTCCTGCTCTCGCTGGTTTGCGCCTCTTGCGG -+SRR33138449.236 236 length=251 -BBBABCCFFFFFGGGGGGGGGGHGGGHHHHGGGGGHHHHHHHHHHGHHHHHHHGHHHHHHHHHHHHHHHHHHHHHHHHHHHGHHGHHHHHHHGHHGFHGHHHHHHHHGHHHGHHGHGFGHGHHHEHHHFGHHHHHHHHHHGHHHHGHHHGHHGHHGGHHHHGFHGHHHHHGGGHGGHGCEFEHHGGFGEAEEFFFGGGGGFFFFFGFFGFFGGGGFFFFFFFF0BFFFFFFFFF;FF?DFFFAFF0BFFF@ -@SRR33138449.237 237 length=250 -GCGTAACGTAATCCTGCGGTACGCGAAGTGCGTTAACGCTGTCTACGACCGGGCGCAGACCGGAATCCATCGCCTGATTCACCTGCTGAGAATAACCATACAGAATCGCGTAATCAGAGACAAAAGGACCAAACTGTTTTTTCTGATCGGCAGTCAGGGTTGGTAGACGTTCACCGCTACGCATAACTGTATTCTGCAGGAAGTCGATAAACGCTTTACGCTGAGCGCCTTCTTTATCAAAACACCCACT -+SRR33138449.237 237 length=250 -BAAABABBCFFBFB5BGC?AEEEEFCE2B3FAFDGGHGGGGGF5BFEC1EEEG?EG@E@GHGF??EHGFFGE?EGHGBGFFGDGHHBFDGG2GFF2GHHHGHHF1?DGGC@@DEHFHEFFCFFFFF?FF.GGHAD0:;F:CD/;0CBFEA?DGBBFFBF9A-A.=BB;.9A/;//.@ADADF9D.9;FFFFEF/BFFFFF.B?:/;.A.A/BDAAE.BFEB.A../9-;.9/BB//;/:9/9F...99.9 -@SRR33138449.237 237 length=251 -ATCTCTGGCAACAAAAAAAGCAACGGCACTATCATTTTTTAACAATGAAACAACAAATTAACCTTGTTTCATTTTCTCCATGATTTTACCAGGAAGATTACGTAGAAAGGGTATCCTGCAAGCCTGTCCTGGACTATCCTTGTCACGTCAGACACGTGTGTGTCGTTGTGCGCTTTTTTTGGGTGAAAGGAGTAAGAAAATGGCGACAGGAAAGTCCTGCTCTCGCTGGTTTGCGCCTCTTGCGGCGTTAT -+SRR33138449.237 237 length=251 -AAAA?F5@F>CAFEGGGGEGGFFHC2ECEEGGFGHHGHHGGHHHHGHFDFHHEEGHHFEGF5FF35GHBGHHHGHGBHBDHH3FGFBGHBFB?1GFHHHHHG0GHAGBGAEGFFGBGHBCFBEFHHHHFHGEFCGHHFCHFBFBGHGFHG?1FFHDCHBA?GHBEFDEFABEGC.CGDDCCDGDA.:;CFBEBF0FBFGGE9CF?B9B@EGBFFBFFFFEF9BFDFFF1111@DB111>1111ED13110B0011111122222221AE/02B2221///BA021222?/B/?2110/>000/11212B2221>00/001/11B111121B222/000/21<>B1>2B01211102111121>22222222/////?11<11111?<.-----./0.<./<.0000.:CG/0;0/..-9--.000......09/9CC/.9C.-;9//://-----////9////;////--/;-:-- -@SRR33138449.240 240 length=251 -GTATTCTGCAGGAAGTCGATAAACGCTTTACGCTGATCGCCTTCTTTATCAAAACACCCACTCAGGCTAACTACCATTAATAACGCCGCAAGAGGCGCAAACCAGCGAGAGCAGGACTTTCCTGTCGCCATCTTCTTACTCCTTTCACCCAAAAAAAGCGCACAACGACACACACGTGTCTGACGTGACAAGGATAGTCCAGGACAGGCTTGCAGGATACCCTTTCTACGTAATCTTCCTGGTAAAATCAT -+SRR33138449.240 240 length=251 -B>BBCFFFFFFFGGGGGGGGGGHHGGGGGHHGGGGGHHGGGGGHHHHHHHHHHHHHHGGGGHHHHHHHHHHHHHHHHHHHHHHHGGGGGGGGHHHGGGGGGHGHHHGGGGGHHHHGHHHHHHHHHHGHGGGBGHHHHHHHHHHHHHHHHHGHHHHGGGHGGGGGHGGGGGGGGGGGFGFGGGGGGGFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFB -@SRR33138449.240 240 length=251 -CAACAAATTAACCTTGTTTCATTTTCTCCATGATTTTACCAGGAAGATTACGTAGAAAGGGTATCCTGCAAGCCTGTCCTGGACTATCCTTGTCACGTCAGACACGTGTGTGTCGTTGTGCGCTTTTTTTGGGTGAAAGGAGTAAGAAGATGGCGACAGGAAAGTCCTGCTCTCGCTGGTTTGCGCCTCTTGCGGCGTTATTAATGGTAGTTAGCCTGAGTGGGTGTTTTGATAAAGAAGGCGATCAGCGT -+SRR33138449.240 240 length=251 -CCCCCCFFFFFFGGGGGGGGGGHHHHHHHHHHGHHHHGHHHHGHHHHHHHHGGHHGHIHHGGFHHHHGHHHHHGHHHHHGHHHHHGHGGHHHHHHHGHHGGHHHHGHHHHHHHFGFGHGGHGGGGGHHGGGGGFGHHHHHFHHHFHHHHHHGHH?EGGCGHFHHGGHHHHHHHHGGGGCGHHGGGGFGGGHHHGGGGGGGGGGFFFFBFFFFFFFFFFFF:FE.:ADFFFFFFFFFFFFFFFFCAEABFF. -@SRR33138449.241 241 length=251 -ACGCAGCATCGGCCTGCAACTTCGCATTTTGCAGCTGTTGCGCCAGTACGCCCAGAGAACCGTTCATTTCACGCAGTGGGCCGCTTTGCGTAACGTAATCCTGCGGTACGCGAATTGCGTTAACGCTGTCTACGACCGGGCGCAGACCGGAATCCATCGCCTGATTCACCTGCTGAGAATAACCATACAGAATCGCGTAATCAGAGACAAAAGGACCAAACTGTTTTTTCTGATCGGCAGTCAGGGTTGGT -+SRR33138449.241 241 length=251 -CCCCBCCFFFCCGGGGGGGGGGHGGEGGHGHHHHHHHHHHHGGGGGHHHGGGGGHHHFFHHGGGHHHHHHHHHHGGGHHHGHGGGGGHHGGGGGGHHHGHHHHHGGGGGGGGGGGHHHGGGGGHGGGGGHHHHGGGGGGGGGGGGGGGGG?GGGGGGGFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFBFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFBFFC. -@SRR33138449.241 241 length=251 -GTCCTGGACTATCCTTGTCACGTCAGACACGTGTGTGTCGTTGTGCGCTTTTTTTGGGTGAAAGGAGTAAGAAAATGGCGACAGGAAAGTCCTGCTCTCGCTGGTTTGCGCCTCTTGCGGCGTTATTAATGGTAGTTAGCCTGAGTGGGTGTTTTGATAAAGAAGGCGATCAGCGTAAAGCGTTTATCGACTTCCTGCAGAATACAGTGATGCGTAGCGGTGAACGTCTACCAACCCTGACTGCCGATCAG -+SRR33138449.241 241 length=251 -BBBBBFFFFFCFGGGGGGGGGGHHGGHFFHGGFHGHHHHGGGHGGHGGGGGHHGGGGGFGABGFFHHGGFHHEHHHHHHGGGGGGHHEHHHHHHHHHHHGGEGGFHHGHGGGGGHHHHGGGGGGGHGHHHHHGHHHHHHHHHHFHHHGHGGFGFHHEGHHGEBAFGHGAGFDFGGGGGEGGGGGGGGGG;@FFFFFFFFBFFFEFFFFFEFFFFFFFFFFFFFBFFEFF?FBBFFFFFFFFFFFFFFFFF/ -@SRR33138449.242 242 length=251 -GTCCTGGACTATCCTTGTCACGTCAGACACGTGTGTGTCGTTGTGCGCTTTTTTTGGGTGAAAGGAGTAAGAAAATGGCGACAGGAAAGTCCTGCTCTCGCTGGTTTGCGCCTCTTGCGGCGTTATTAATGGTAGTTAGCCTGAGTGGGTGTTTTGATAAAGAAGGCGATCAGCGTAAAGCGTTTATCGACTTCCTGCAGAATACAGTGATGCGTAGCGGTGAACGTCTACCAACCCTGACTGCCGATCAG -+SRR33138449.242 242 length=251 -CBBCDFFFFFFFGGGGGGGGGGHHGGHHHHGHGHHHHHHGGGHGGHGGGGGHHGGGGGFFHHHHHHHHHHHHHHHHHHHGGGGGGHGHHHHHHHHHHHHGGGGGHHHGHGGGGGHHHHGGGGGGGHHHHHHHGHHHHHHHHHHHHGHGHFGHGGHHHHHHGGGGGHHHHHHHHGGDGFHHGB>1FF3BGB331///>EE0BE0BBF4?GCGDHGGCECFGAA1BB///11@1///<<-.<>1FGF1>=GG0=0=-D?.9/9.;;.///9.9E?.;A//;B@----. -@SRR33138449.245 245 length=250 -CGGTTGCAATGCATCAGCTGGCGTCGTTACCACTTTGGTGAATGCCTGATCAAAGACCGGTTTCAAATCATCGGTCTGTTTTAGCGCAGAGTGCGCAGCATCGACCTGCAACTTCGCATTTTGCAGCTGTTGCGCCAGTACGCCCAGAGAACCGTTCATTTCACGCAGTGGGCCGCTTTGCGTAACGTAATCCTGCGGTACGCGAATTGCGTTAACGCTGTCTACGACCGGGCGCAGACCGGAATCCATC -+SRR33138449.245 245 length=250 -BABBBDFFFFFFGGGGGGGCFEGGGGGGHGHHHHHHHHGFHBGFHHHHGHHHHHHHHHGGAEGHHHHHHGHHHGEGGFHHHGFHGGGGEGEEGHEFGGGHHGG3EEFFFHHHHHHGGFGGHHGFHCFHHGFHHGGGGGGHG?DGGGHHGGHHHGGGHHHHGFGGGGGDGGGCCGCGGGGBBFDDDDGGGGGGGFFFFCAFFFFADFAA/BDFFFFFFFFDFFFFFEDDADFFFDFFFFFFFFA=ABFFFF -@SRR33138449.245 245 length=251 -GGACTATCCTTGTCACGTCAGACACGTGTGTGTCGTTGTGCGCTTTTTTTGGGTGAAAGGAGTAAGAAAATGGCGACAGGCAAGTCCTGCTCTCGCTGGTTTGCGCCTCTTGCGGCGTTATTAATGGTAGTTAGCCTGAGTGGGTGTTTTGATAAAGAAGGCGTTCAGCGTAAAGCGTTTATCGACTTCCTGCAGAATACAGTGATGCGTAGCGGTGAACGTCTACCAACCCTGACTGCCGATCAGAAAAA -+SRR33138449.245 245 length=251 ->1>>AC3FFFFFGGGB?1A1111BGEHA0FFFGHECGFF?GGGGEGHHEGEEG/EAGGFFFHHHF1GHHHGHCGEGEECF/>FF@GF21BF1GHEAEEE<1BEFG1FFHFH//FC?C1BE1E@?A?HFHHGFDDG11F1<1FCC=G.C..-AA11BBFC111A3313A111101B13A13333311DFFG10AE0//111B2A///AB22D22DA21A1FF111D011/>>//////B0/>02/1220122>>>122B1>1FH1B111B22B/B0??B1F121B2F2////12221?/1111<101??1?/>-<<..1>FD00.0=DC---:.;.0;0::-C.---.;9/009.000000.--;;-9/////--9----//////--///9//-----;- -@SRR33138449.247 247 length=249 -TTTCTGATCGGCAGTCAGGGTTGGTAGACGTTCACCGCTACGCATCACTGTATTCTGCAGGTAGTCGATAAACGCTTTACGCTGATCGCCTTCTTTATCAAAACACCCACTCAGGCTAACTACCATTAATAACGCCGCAAGAGGCGCAAACCAGCGAGAGCAGGACTTTCCTGTCGCCATTTTCTTACTCCTTTCACCCAAAAAAAGCGCACAACGACACACACGTGTCTGACGTGACAAGGATAGTCC -+SRR33138449.247 247 length=249 ->AA3ABF@FA?DEECE4F44BB2E2AGHHGGGGFHF2E2EEECGFEFHHEDBGFHH5AFFA1D33DECEEF5F?EG1?FGGGFDDHGE?EDFGDFGBFHFHHDFGADEFEG3FG0GC?CCGBHHFHHGDGHFBACGGCDGGHH..ACDC=FHBG1<<=G/<00.<AAAFBABBBBGGGGGGGGGGGAEGHHHBHGGGGHHHFHHHHHHGHGHHHHFHFHHHFHGHHHHHHHGGGGGGHHFFHHHHFHFHGHHHHHGHHHFFHHGGHHHHHHGGGGCHHHHGG?F@D>>GHGGFFHHGHGGCGGGGGFGHHHGCGHHHHGHHHHGHFCFGGGGGGGGGGEFGGGGGGGGGFFFFFFFFFFFFFFFFFFFDFFFFFFFFFFFFFFFFFFFFFFFFFFFBBBBBBBFFFDBB?BBF -@SRR33138449.248 248 length=251 -ATCCTTGTCACGTCAGACACGTGTGTGTCGTTGTGCGCTTTTTTTGGGTGAAAGGAGTAAGAAAATGGCGACAGGAAAGTCCTGCTCTCGCTGGTTTGCGCCTCTTGCGGCGTTATTAATGGTAGTTAGCCTGAGTGGGGGTTTTGATAAAGAAGGCGATCAGCGTAAAGCGTTTATCGACTTCCTGCAGAATACAGTGATGCGTAGCGGTGAACGTCTACCAACCCTGACTGCCGATCAGAAAAAACAGT -+SRR33138449.248 248 length=251 -BBBABDDFFFFBGGFGFGGBE4BAA4BGFFG2F2AFFGEFF3E0EEEEFGHGGGHGCGHFGFHHBFHH2E?EEEFABGF4DGBFFFFEGGGFFGB3FEGG@EFGGHHFGGGCACFAGHH1GEGHHHHHGHHHFBG>1111>@B@FFFG1FF1FFF?BB0A1EE?0A11A10AAAEEGCFG2D2A1AAFHHHFFFHFHDFCCEFGGHHC/FFB1FDFFGEG?EGCGEA/<>/00CCFGGHEEECGGHHHFFFF1FGF1FGDDGF11?F1FHCCGCGG1FFGHGF00C<<F/FHGFDEEFHGGGEHEFG/CGDDD3FFGFF22FDGH>22>F2FFFEFFFAFFFFFFFFFFFFFFFFFFFFFFFFFFFFF -@SRR33138449.255 255 length=251 -GTCCTGCTCTCGCTGGTTTGCGCCTCTTGCGGCGTTATTAATGGTAGTTAGCCTGAGTGGGTGTTTTGATAAAGAAGGCGATCAGCGTAAAGCGTTTATCGACTTCCTGCAGAATACAGTGATTCGTAGCGGTGAACGTCTACCAACCCTGACTGCCGATCAGAAAAAACAGTTTGGTCCTTTTGTCTCTGATTACGCGATTCTGTATGGTTATTCTCAGCAGGTGAATCAGGCGATGGATTCCGGTCTGG -+SRR33138449.255 255 length=251 -AAAAAFFFBFF1FFG1BFFEFCCE0EGHHFGEGGEEECFHCFFHAFGF2A2DCBGBGDFFHEE/?BCG/G2@FBGFCEEEG/FGBGGGGCHHHGGGE/1DGEGCAF1FFCGGFGGDGGH@FG22@/0?FFCGCCE11GGEGGF1GF?EHFG?GGHHHGHGFHGGGGCBFG/>EFHHGEFGHHHHADDDHGFBGHHHHGHFHHHHHGHHGA?DEGGEDGHHHHHHHHHHHGDGHHGHGGHFD@@DGHHHG/GBGFFCFGEE;BF0CFFGFGGGFB0FFBBEFF-BEFFFFFFFFBFFFFFFFFFFFFFFFBBFFFFBFC--.D/999FF-;DD/ -@SRR33138449.257 257 length=251 -GTTTTAGCGCAGAGTGCGCAGCATCGGCCTGCAACTTCGCATTTTGCAGCTGTTGCGCCAGTACGCCCAGAGAACCGTTCATTTCACGCAGTGGGCCGCTTTGCGTAACGTAATCCTGCGGTACGCGAATTGCGTTAACGCTGTCTACGACCGGGCGCAGACCGGAATCCATCGCCTGATTCACCTGCTGAGAATAACCATACAGAATCGCGTAATCAGAGACAAAAGGACCAAACTGTTTTTTCTGATCG -+SRR33138449.257 257 length=251 -CCCCCFFFDECDGGGGGGGGGGHHHGGGGGGHHHHHHHGGGGGHHHHHHHHHHHHHGGGGGHHHGGGGGHHHGHHHGGGHHHHHHHHHHGGGHHHGHGGGGGHHGGGGGGHHHGHHHHGGGGGGGGGGGGHHHGGGGGHGGGGGHHHHGGGGGGGGGGGFFFFFFFFFFFFFFFFFFFFFFFHFFFFFFFFFFEFFFFFFFFFFHHFFFFFFFFDFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE -@SRR33138449.257 257 length=250 -GTCCTGCTCTCGCTGGTTTGCGCCTCTTGCGGCGTTATTAATGGTAGTTAGCCTGAGTGGGTGTTTTGATAAAGAAGGCGATCAGCGTAAAGCGTTTATCGACTTCCTGCAGAATACAGTGATGCGTAGCGGTGAACGTCTACCAACCCTGACTGCCGATCAGAAAAAACAGTTTGGTCCTTTTGTCTCTGATTACGCGATTCTGTATGGTTATTCTCAGCAGGTGAATCAGGCGATGGATTCCGGTCTG -+SRR33138449.257 257 length=250 -CCCCCFFFFFFCCGGGGGGGFGGGGGHHGHGGGGGGGHHHFFHFGFFHHHHHHHHGHFGFHGGEGFHGGHHHHHGHHGHFFGGGHHGGGGHHHGGGGHHGGGGGGGHHGHHFFHHHHHHHHHHHHGEDGGGGGGGHHEGHGHHHHHGGGHHHHHHHFGCGGGHGHHBGGGGFCFGGGFGGGGGGGGGFGGGGGGGGEGFCDFFFFBFFFFFFFFFFFFBFFFFEFFFFFFFFBFC;BFABFFFFFADF.B -@SRR33138449.258 258 length=251 -GTTTGCGCCTCTTGCGGCGTTATTAATGGTAGTTAGCCTGAGTGGGTGTTTTGATAAAGAAGGCGATCAGCGTAAAGCGTTTATCGACTTCCTGCAGAATACAGTGATGCGTAGCGGTGAACGTCTACCAACCCTGACTGCCGATCAGAAAAAACAGTTTGGTCCTTTTGTCTCTGATTACGCGATTCTGTATGGTTATTCTCAGCAGGTGAATCAGGCGATGGATTCCGGTCTGCGCCCGGTCGTAGACA -+SRR33138449.258 258 length=251 -CACCDFCCCCCFGGGGGGGGGGHHHHHHHHHHHHHHHHHHEHFHGHEEGGHHHGHHHHHHHHHHGGGGGHHGGGGHHHGGGGHHHHGGGGHHHHHHHHHHHHHHHHHHGHGGGGGGGFGGHHHHHHHHHHHGGGHHHHHHHGGGGGGHHHHHHGGHHHHHHGHHHHHHHGGGGGGGGGGGGGFGDGGGGGGGGGGFGFFHFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF -@SRR33138449.258 258 length=249 -CCAGAGAACCGTTCATTTCACGCAGTGGGCCGCTTTGCGTAACGTAATCCTGCGGTACGCGAATTGCGTTAACGCTGTCTACGACCGGGCGCAGACCGGAATCCATCGCCTGATTCACCTGCTGAGAATAACCATACAGAATCGCGTAATCAGAGACAAAAGGACCAAACTGTTTTTTCTGATCGGCAGTCAGGGTTGGTAGACGTTCACCGCTACGCATCACTGTATTCTGCAGGAAGTCGATAAACG -+SRR33138449.258 258 length=249 -BCBBBCFFFCCCGGGGGGGGGGHGGGHGHGHGGGGGHHGGGGGGHHHGHFHHHGGGGGGGGGGGFHHGGGGGHGGGGGHHHHGGGFGCGGGGGGGHHGGCFGHGHHHGGGGHHGGHHHGGHHHHGHHBGHFGHFHHHHHHHHHGGGGGGGEFBFGGGGGFFGDGFEGEGFFFBFFFFFFFFFFFFF;DDB/BFFF/FCFDFFFBF?EFFFFDADDFBFDFABFFFFFFFFFF/BBFEFFFFFFFFF/9D -@SRR33138449.259 259 length=234 -GCGTTATTAATGGTAGTTAGCCTGAGTGGGTGTTTTGATAAAGAAGGCGATCAGCGTAAAGCGTTTATCGACTTCCTGCAGAATACAGTGATGCGTAGCGGTGAACGTCTACCAACCCTGACTGCCGATCAGAAAAAACAGTTTGGTCCTTTTGTCTCTGATTACGCGATTCTGTATGGTTATTCTCAGCAGGTGAATCAGGCGATGGATTCCGGTCTGCGCCCGGTCGTAGAC -+SRR33138449.259 259 length=234 -B?ABBAAFDDFFFB5GFDGEFGHGFFDFDEAAAEGHFAFGHHGHHFGGGGGGGHHGEEGHFHGEGGFHFHEE?AGGHHGHHHHHHHHHEGHFHHGEEEGGEEEEGHGFGGGFGFHGFGHFHGHFHHGGGGGHHHFHHGGHHFHGHGFFGGHEGHHHHHHHHGGHFGGGCCHHFH:GHGHCFFHGHHHHHHHGEFFGFGFFFGGGGGDAFFGGGGG;DFFFFFFFFA@BC-9EFF -@SRR33138449.259 259 length=234 -GTCTACGACCGGGCGCAGACCGGAATCCATCGCCTGATTCACCTGCTGAGAATAACCATACAGAATCGCGTAATCAGAGACAAAAGGACCAAACTGTTTTTTCTGATCGGCAGTCAGGGTTGGTAGACGTTCACCGCTACGCATCACTGTATTCTGCAGGAAGTCGATAAACGCTTTACGCTGATCGCCTTCTTTATCAAAACACCCACTCAGGCTAACTACCATTAATAACGC -+SRR33138449.259 259 length=234 ->AAAAFAAAD@D0E0EEEEGGCGGCCHFHFHGGGGGFFEGGHHHHFHHFHE11DEHHHFHHHGGGBGAFGGGGGG1GFC>GFFHAGG0?AFFFBE/9 -@SRR33138449.263 263 length=250 -GTTTGTTATATTCACTTGCCTGCTGTGACGTCGGGAACTGGATGCCATTTGCGACAAAGCTCACCTGAGTACCTTGCTGAGCAATGTAGTCTCCAACCATCACCAGTTGTTGCGTGAAAGTTTGTGCCGCCGGAATCAACGGTTGCAATGCATCAGCTGGCGTCGTTACCACTTTGGTGAATGCCTGATCAAAGACCGGTTTCAAATCATCGGTCTGTTTTAGCGCAGAGTGCGCAGCATCGGCCTGCAA -+SRR33138449.263 263 length=250 -AAABBFFFFDFFGGGGGGGGGGGHHHFHFEFGA2?C2FHHFHFCGGHHHGHHGGGGGHFHHHGHHHEHGHHBGHBFBGHDFC3GHFGGGHHH4F3GGFGHHHHCGHB43BGCBEGEEGHFFEGGH2FHGG?@CFCFHGHHGCEGGHGFHHHBGHFGHHHFGC111B3333BDFGCFCGGCG1AAECG003FBG1GHBC0E/AFE1F//AAF21ABE?/ABBAEE/FBGFHFFFFEB@D@B11DFHG/EEGGE/EFGGF//B/1F>FF//FF01B111<GHE1<<11.ACCG.0=<==GFGCFCGEB0CB0;0::;/0;GG00099-@EEB0;F0;00090;9-;->---;-A-EFFB?@--A-BB----//:-;----A@- -@SRR33138449.264 264 length=161 -GCCTGAGTGGGTGTTTTGATAAAGAAGGCGATCAGCGTAAAGCGTTTATCGACTTCCTGCAGAATACAGTGATGCGTAGCGGTGAACGTCTACCAACCCTGACTGCCGATCAGAAAAAACAGTTTGGTCCTTTTGTCTCTGATTACGCGATTCTGTATGGT -+SRR33138449.264 264 length=161 -AABBCFFDFBCAEEFGGGGGGGHHHHHHHGGGGGHHGGGGGHGGFGGHFHHGCEGHHHHGHHHHHHHHHGHGHHHGGGGGGGGGGFFHGHHHHHHHGGGHGHHHHHGFGCGGHHHHHHGGHHFHHHGFGHHHHHHHHHHHHHHHHHGGGGGHHHHHHHHHH -@SRR33138449.264 264 length=161 -ACCATACAGAATCGCGTAATCAGAGACAAAAGGACCAAACTGTTTTTTCTGATCGGCAGTCAGGGTTGGTAGACGTTCACCGCTACGCATCACTGTATTCTGCAGGAAGTCGATAAACGCTTTACGCTGATCGCCTTCTTTATCAAAACACCCACTCAGGC -+SRR33138449.264 264 length=161 -BBBC@FFFFFFFGGGGGGGGGGHHHGHFHHHHGHGHHGHHGHHHHHGGHHHHHGGGGGGFHFGEFGGGGEGHHHGFFFHHHGGGGGGGGGGFHGHHHHHHHGHHHHEHAGGFEGHGGHGGGGGGFGGGGGGHGGGFFFHHHHHHHHFFHHHCGGGHGHEFG -@SRR33138449.265 265 length=251 -GCGTAAAGCGTTTATCGACTTCCTGCAGAATACAGTGATGCGTAGCGGTGAACGTCTACCAACCCTGACTGCCGATCAGAAAAAACAGTTTGGTCCTTTTGTCTCTGATTACGCGATTCTGTATGGTTATTCTCAGCAGGTGAATCAGGCGATGGATTCCGGTCTGCGCCCGGTCGTAGACAGCGTTAACGCAATTCGCGTACCGCAGGATTACGTTACGCAAAGCGGCCCACTGCGTGAAATGAACGGTT -+SRR33138449.265 265 length=251 -BBBBBBAFFBABGCGGGGGGGGFHHHHHHGHHHGHFHBGHHGGGEEGGGGGGGHFHCCGFHHGGGHHHHHHHHGGGGDHHGHHHGGHEGHHHEHHEHHHHHHHHHHHGHHGGGGGDGFHHHHHHHHHHHHHHHHHGHHHHHHFHHHHHHHC-@DD0GGFHGCDFC/D-D?BBFB9//9;FC-D//;9A -@SRR33138449.266 266 length=127 -GCGTTTATCGACTTCCTGCAGAATACAGTGATGCGTAGCGGTGAACGTCTACCAACCCTGACTGCCGATCAGAAAAAACAGTTTGGTCCTTTTGTCTCTGATTACGCGATTCTGTATGGTTATTCTC -+SRR33138449.266 266 length=127 -AABBBCCFFBBBGGGGGGGGGGHHHGHHGHHHGHFEEGGGGEEDFHHGHHFHHFHGGGHHHHHHHHGGGGGHHHHHHGGHHGHHHGHHHHGHHHHHHHHHHHHHHGGGGGHHHGHHHHGHHHHHHHG -@SRR33138449.266 266 length=127 -GAGAATAACCATACAGAATCGCGTAATCAGAGACAAAAGGACCAAACTGTTTTTTCTGATCGGCAGTCAGGGTTGGTAGACGTTCACCGCTACGCATCACTGTATTCTGCAGGAAGTCGATAAACGC -+SRR33138449.266 266 length=127 -CCCABFFFFF@FGGGGFGGGGGGGGGGHHH3GHHHHGFHGHHHHFHHHGFHHGGFHHEHHHGGGGGHHGHGGGHGGGFHGHEG?FGHBF@GGGGGDAEG3GFHHEHHEHHHEHGHHBFAAFGGHDDG -@SRR33138449.267 267 length=250 -GCAGAATACAGTGATGCGTAGCGGTGAACGTCTACCAACCCTGACTGCCGATCAGAAAAAACAGTTTGGTCCTTTTGTCTCTGATTACGCGATTCTGTATGGTTATTCTCAGCAGGTGAATCAGGCGATGGATTCCGGTCTGCGCCCGGTCGTAGACAGCGTTAACGCAATTCGCGTACCGCAGGATTACGTTACGCAAAGCGGCCCACTGCGTGAAATGAACGGTTCTCTGGGCGTACTGGCGCAACAG -+SRR33138449.267 267 length=250 -BBBBBFFFFFFFFFGGGGGGGGGGGCGHHHHHHHHHFHGGGHHHHHHHGGCGGGHHFHGHGGHHGHHHGGHHHHHHGHHHHHHHHHBGGGG/EHHHHGHHHHHHHHHHHGHHGHGHFHHHHHHHGFC@FGGHHGFHFFGGGGHG?DDGGGGFGHGEGHHHGGGGEHDGGGGHHFBCDG?FGGGCGGGGGGFBFFFFFFFFFFFBFFFFFFFFFFFF.BFFFFFE?:AEFFFFFB?D@DFFBF?DDFB9FF -@SRR33138449.267 267 length=251 -CACCAGTTGTTGCGTGAAAGTTTGTGCCGCCGGAATCAACGGTTGCAATGCATCAGCTGGCGTCGTTACCACTTTGGTGAATGCCTGATCAAAGACCGGTTTCAAATCATCGGCCTGTTTTAGCGCAGAGTGCGCAGCATCGGCCTGCAACTTCGCATTTTGCAGCCGTTGCGCCAGTACCCCCAGAGAACCGTTCATTTCACGCATTGTGCCGCTTTTCGTAACGGAATCCTGTGGTAGGCAAATTGCTT -+SRR33138449.267 267 length=251 -A1AA1FBBD@1FGEEEEC3FGGHHBGGGGEGG/E/FGF10EEGG?0FGH11GGFHHBG@>/>/>/>EECGFHHHHHFGF1@FGFFGHHH1F>1B>>GE/EE?>>FG21BF21//<<----0F0<.<FFFFFFA -@SRR33138449.271 271 length=251 -GCGGTGAACGTCTACCAACCCTGACTGCCGATCAGAAAAAACAGTTTGGTCCTTTTGTCTCTGATTACGCGATTCTGTATGGTTATTCTCAGCAGGTGAATCAGGCGATGGATTCCGGTCTGCGCCCGGTCGTAGACAGCGTTAACGCAATTCGCGTACCGCAGGATTACGTTACGCAAAGCGGCCCACTGCGTGAAATGAACGGTTCTCTGGGCGTACTGGCGCAACAGCTGCAAAATGCGAAGTTGCAG -+SRR33138449.271 271 length=251 -AAAAA@AFFFAAGGGCCG1EECBE0FGFGGEGGGFHFHHHGGHC1EFGF/FHFHFHFF22FGHHFHG1/EGCEHGDGEHFHHBGFFHDG2GFGHHGHHEHE1FFGGE/EEGGFH1FFECCCHFGCCCGGCGG@C//DGCF@EGCFFACGGCHGGGGC@CECCCCGAFGFGF/FFFGGG@@F/A@@@@F?9FFB9BAEFEFFEB?FBFFFFFFBFF@9;@FBB?9B-@;F-AFFFFFFFBB/-99@;BF/B/ -@SRR33138449.271 271 length=251 -GAGTGCGCAGCATCGGCCTGCAACTTCGCATTTTGCAGCTGTTGCGCCAGTACGCCCAGAGAACCGTTCATTTCACGCAGTGGGCCGCTTTGCGTAACGTAATCCTGCGGTACGCGAATTGCGTTAACGCTGTCTACGACCGGGCGCAGACCGGAATCCATCGCCTGATTCACCTGCTGAGAATAACCATACAGAATCGCGTAATCAGAGACAAAAGGACCAAACTGTTTTTTCTGATCGGCAGTCAGGGT -+SRR33138449.271 271 length=251 -A1>AABDDDDDFCG10EF0AGF1FGHGGGGGGHFHBGFFCFFDGGEEGGGHHHGGGGGCAGHHBEE>EHHHBDFHFEFE??E0EFHGE@EGHGGGGAAGBBFFFFFEBFFFB9BBFFEE-AAFFBBBFE@BBFFFEEB@@?FFFFBF- -@SRR33138449.272 272 length=226 -TCTACCAACCCTGACTGCCGATCAGAAAAAACAGTTTGGTCCTTTTGTCTCTGATTACGCGATTCTGTATGGTTATTCTCAGCAGGTGAATCAGGCGATGGATTCCGGTCTGCGCCCGGTCGTAGACAGCGTTAACGCAATTCGCGTACCGCAGGATTACGTTACGCAAAGCGGCCCACTGCGTGAAATGAACGGTTCTCTGGGCGTACTGGCGCAACAGCTGCAA -+SRR33138449.272 272 length=226 -BCCBAFFFABCFGGGGGGGGGGGGHHHHHHGGHHHHHHGHHHHHHHHHHHHHHHHHHFFGGGGHHHHGHHHGHHHHHHHHHHHHHHHGHHHHHHHHGGGGGGHHHHHGGGGHHGGGGGGGGGGHGGHHHHGGGGGHGGGGGHHGGGGGGGGGGGGGHHHHGHHHGGGGGGGGGGGGGGGGFFDDFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF -@SRR33138449.272 272 length=226 -TTGCAGCTGTTGCGCCAGTACGCCCAGAGAACCGTTCATTTCACGCAGTGGGCCGCTTTGCGTAACGTAATCCTGCGGTACGCGAATTGCGTTAACGCTGTCTACGACCGGGCGCAGACCGGAATCCATCGCCTGATTCACCTGCTGAGAATAACCATACAGAATCGCGTAATCAGAGACAAAAGGACCAAACTGTTTTTTCTGATCGGCAGTCAGGGTTGGTAGA -+SRR33138449.272 272 length=226 -BBCBCFFFFFFFGGGGGGGGGGGGGGHHHHHHHGEGHHHHHHHHHHGGGHGHFHGGGGGHFEEGGGGHHHGHHHFHGGEFEGGGGGGHHGGDGGGHGGGGGGFFGGGGGGGGGGGGGGHGGGFGGHHHHHGGGGGGGGGGGGGGGGGGGGGGFGGGGGGEGGGGFBFFFFFFFFFFBFFFFFFFFFFFFFFFFFFBFBFFBFFFFFFFDCCFFFBFFFF.>B9BFF -@SRR33138449.273 273 length=251 -GTTTTAGCGCAGAGTGCGCAGCATCGGCCTGCAACTTCGCATTTTGCAGCTGTTGCGCCAGTACGCCCAGAGAACCGTTCATTTCACGCAGTGGGCCGCTTTGCGTAACGTAATCCTGCGGTACGCGAATTGCGTTAACGCTGTCTACGACCGGGCGCAGACCGGAGTCCATCGCCTGATTCACCTGCTGAGAATAACCATACATAATCGCGTAATCAGAGACAAAAGGACCAAACTGTTTTTTCTGATCG -+SRR33138449.273 273 length=251 -AAAAAFFB>AD?11GGGA?0AA0EFEAE0BAFBDGAG2//EEGHHF12FF0BABGGGC/E@G1B///EEGFE01B/>EEHFEBFF2FG>EE/EGAC////CC11/333@>1>1A331110A11AB1100000BB1B11D21/B/A/////A00>0?>/221210BB0G011B0////>01B222101>>2>1>0/1B1>F1BBF>E1EF22111C/F/120///>////1??111<11..><.1<11>1<---<.0000<.:.:;0;.C.---..-;://0000-;A..0/./0/;//;/9//9;---9//9///;9//;://///;9/9/9;F//-;--;///9//9; -@SRR33138449.274 274 length=251 -GGAATCAACGGTTGCAATGCATCAGCTGGCGTCGTTACCACTTTGGTGAATGCCTGATCAAAGACCGGTTTCAAATCATCGGTCTGTTTTAGCGCAGAGTGCGCAGCATCGGCCTGCAACTTCGCATTTTGCAGCTGTTGCGCCAGTACGCCCAGAGAACCGTTCATTTCACGCAGTGGGCCGCTTTGCGTAACGTAATCCTGCGGTACGCGAATTGCGTTAACGCTGTCTACGACCGGGCGCAGACCGGA -+SRR33138449.274 274 length=251 -BBBBBFBFFABAFFGGGGGFG6GHFHGFBEFGFFGEDGHGHHF3FHGFHHHGEHHHHHFHHFGFHHGG>FFHHHGDHHGHHGGFGHGFBHHHHGGGGEHF?GEEGGGHHHGGGGGEGGHHHHHGGG?D1DFFFFEAB5FGGFFGFHFC2EAAE55D5BGFHAA5BDDDGFHF5GEGH5EGFBGFAE00E13FBFDG0EE1?DE/1//>EFG@E/CBECGGFG//GA?CEGHEFG?CF??ACB@/FC1?FFFGHAFG?-->/A?-@DFF/A./..;9-9///99/ -@SRR33138449.275 275 length=251 -AATGAAATCCTCCGCCATATTCATTAATGCCGGACGTGGCCCGGTGGTTGATGAAAATGCACTGATTGCTGCATTGCAGAAAGGGGAAATTCACGCTGCCGGGCTGGATGTCTTCGAACAAGAGCCACTGTCTGTAGATTCGCCGTTGCTCTCAATGGCCAACGTCGTCGCAGTACCGCATATTGGATCTGCCACTCATGAGACGCGTTATGGCATGGCCGCCTGTGCCGTGGATAATTTGATTGATGCGT -+SRR33138449.275 275 length=251 -BBBBBFFFFFFFG2EGGGGGGCHHHHGHHHHGGGGFGGFGHGEE0EEEFFFHHGHHHFGGHEHHEHHHHGHGHHHGEHGFHHHHGGGGHGHHHEHGGGGEGGGGGGGHHHHGHFHGEFHCGHHGGHFBHFHHHFHGHHHHHGGDDFGGCHEHFGFGHHHFFEFGGHGGEFDDG.CB0ADGGGFGGBFFCFFBFFGEFFGFGEGGFFF@ADEFFEFFFA.9@:@FEBFEFFFDCABFFFFFFFBFFFFFFF. -@SRR33138449.275 275 length=251 -GATTACGCGATTCTGTATGGTTATTCTCAGCAGGTGAATCAGGCGATGGATTCCGGTCTGCGCCCGGTCGTAGACAGCGTTAACGCAATTCGCGTACCGCAGGATTACGTTACGCAAAGCGGCCCACTGCGTGAAATGAACGGTTCTCTGGGCGTACTGGCGCAACAGCTGCAAAATGCGAAGTTGCAGGCCGATGCTGCGCACTCTGCGCTAAAACAGACCGATGATTTGAAACCGGGCTTTGATCAGGA -+SRR33138449.275 275 length=251 -A?AAA1AADADADGGCGGGG3FDBBEDFGHHGH0FFGHHFFHCFEE?EEGFEEGH/E/AGF/AEG///>>EHEEDGBCEEEEEDE?EG/EGA/EE/@CB@CG//F1FGF0FHA@CECG0EAEFGHF0CGC.?:.:GHFB0C.@?@@.09.-C@--CBFBFGFFFFBB/9A;=-BB/FFFF-9--?B/BF;@;-@//;/==@@-/BBEBB--99=BF/9/9////-99----////9/99/ -@SRR33138449.276 276 length=250 -GCTGTAGTCCAGGCCTGATTAAACGCCTGATGCTGTGCCGGTAATGGCGCAATCAGTTTGTTATATTCACTTGCCTGCTGTGACGTCGGGAACTGGATGCCATTTGCGACAAAGCTCACCTGAGTACCTTGCTGAGCAATGTAGTCTCCAACCATCACCAGTTGTTGCGTGAAAGTTTGTGCCGCCGGAATCAACGGTTGCAATGCATCAGCTGGCGTCGTTACCACTTTGGTGAATGCCTGATCAAAGA -+SRR33138449.276 276 length=250 -CCCCCFFFFFFFGGGGGGGGGGHHGGGHHHHHHHHHHHHGGGGGHHHHGGGGGHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHHGGGGGGGGGHHHHHHHHHHHHHHGGGGGHFHGHHHHHHFHHHHHHHHHHHGHHHHHHHHGHHHHHHHHGGHHHGHHHHHHHHHHGGGGGHHHHHHHHHHGGGGGGGGGGGGGGFGGGGGGGGGGGGGGGGGGFFFFFFFFFHFFFFFFFFFFFFFFFFFFFFFFFF -@SRR33138449.276 276 length=250 -GATTACGCGATTCTGTATGGTTATTCTCAGCAGGTGAATCAGGCGATGGATTCCGGTCTGCGCCCGGTCGTAGACAGCGTTAACGCAATTCGCGTACCGCAGGATTACGTTACGCAAAGCGGCCCACTGCGTGAAATGAACGGTTCTCTGGGCGTACTGGCGCAACAGCTGCAAAATGCGAAGTTGCAGGCCGATGCTGCGCACTCTGCGCTAAAACAGACCGATGATTTGAAACCGGTCTTTGATCAGG -+SRR33138449.276 276 length=250 -BBBCCFCCCBBBGGGGGGGGGGHHHHHHHHGHHGGHHHHCFHHHGEGGGGGFHHHGGGGHHGGGGGGGGGGHGGHGHHGGGGGHGGGGGHGGGGGGGGGGGGGGHHEHGHHGGGGGGGHHGGGGFGGHHHGGGGGFHHH0GHGGGGHHHFFGGGGGGGFFEGGGGGGFGEGGFFFFEFB?DFFFEFFFBFDFFFFFFFFFFFFFFFFFF;C=BAFEFFFFF?-;FFFFFFFFFFFF@-BFDFBFFFFFFE -@SRR33138449.277 277 length=250 -ACGCGATTCTGTATGGTTATTCTCAGCAGGTGAATCAGGCGATGGATTCCGGTCTGCGCCCGGTCGTAGACAGCGTTAACGCAATTCGCGTACCGCAGGATTACGTTACGCAAAGCGGCCCACTGCGTGAAATGAACGGTTCTCTGGGCGTACTGGCGCAACAGCTGCAAAATGCGAAGTTGCAGGCCGATGCTGCGCACTCTGCGCTAAAACAGACCGATGATTTGAAACCGGTCTTTGATCAGGCATT -+SRR33138449.277 277 length=250 -CCCBBBCCFFFFGGGGGGGGGGHHHHHHHHHHHHHHHHHHGGGGGHHHHHHGGGGHHAFGGGGGGGGHGGHHGHGGGGGHGGGGGHHEGGGGGGGGGGGGHHGHGHHHGGGGGGHHGGGGGGGHHHGGGGGHHHHHHHGGGGHHHGHHHGGGGGGBCGGGGGGFFGGGGGEFGGGGFFFFFFFFFFFFFFAFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFDFFFFFFFFFFFFFFFFFFFFFFFFFFFF -@SRR33138449.277 277 length=250 -GTTTGTTATATTCACTTGCCTGCTGTGACGTCGGGAACTGGATGCCATTTGCGACAAAGCTCACCTGAGTACCTTGCTGAGCAATGTAGTCTCCAACCATCACCAGTTGTTGCGTGAAAGTTTGTGCCGCCGGAATCAACGGTTGCAATGCATCAGCTGGCGTCGTTACCACTTTGGTGAATGCCTGATCAAAGACCGGTTTCAAATCATCGGTCTGTTTTAGCGCAGAGTGCGCAGCATCGGCCTGCAA -+SRR33138449.277 277 length=250 -ACCCCFFFFFFFGGGGGGGGGGHHHHHHHAFEEGGG?CCDFFHFGHHHHHHHG?EGGFHHHHHHHFHBGGFHHHHHHFHHHHFHHHHFFGGHGHHHHGEHHHHHBHHFHBFGGEEEGFBFHHHHHHEHEGGG/?/FGDFHDGGGGHHEHGHEHFGHGHHGG-ADDCEHFFBHHHHHHGEEGHFCGHF/BFBFBFFFFFGDDGGFGFFGGGGFG9DDFFFFFFFFF;F>9.;BFFA>FBFFFF:-A.9..B -@SRR33138449.278 278 length=251 -ATTCTGTATGGTTATTCTCAGCAGGTGAATCAGGCGATGGATTCCGGTCTGCGCCCGGTCGTAGACAGCGTTAACGCAATTCGCGTACCGTAGGATTACGTTACGCAAAGCGGCCCACTGCGTGAAATGAACGGTTCTCTGGGCGTACTGGCGCAACAGCTGCAAAATGCGAAGTTGCAGGCCGATGCTGCGCACTCTGCGCTAAAACAGACCGATGATTTGAAACCGGTCTTTGATCAGGCATTCACCAA -+SRR33138449.278 278 length=251 -AABBBFFFFFFFGGGGGFGGGGGHHFHGGHHDFHHGFG?ECFFHHHGGGGHHGGGGGGGGGGHGFHGHHFFGGGHGGGCGHHAFGGGGGGGHDEDEG3GGFFHGGGDGCBGAFFCGGGGFHDEDD.<=GD1FAAEEGHHHHHHHC:DGHHH.DGGCGGFGGBFGB0BBBFGDAG9FGFG0.AAC;DDDBBFFFFFFFFFFCFFF.;/BFFF?E-BCF/FBFFFEFFFF-BDFF/B/BF/9;BFFFFFFFFA -@SRR33138449.278 278 length=250 -CGTTATGGCATCGCCGCCCTTGCCGCGGTTAATTTGATTGTTGCGTTACACGGTACGGTCGAGCACAACTGCGTGAATCCGCACGTCCCGCTCTAAGCCGCGACTGCGTGGAGTAAAGCCCCATAATCGCTCGGGCCTTTACTCTTTATTGGGTTGCAGCAACTTCTGTAGTCCAGGCCCGATTCAACGCCTGATTCTGTGCCGGTCTTGGCGTACTCTGTTTGTTTTATTCACTTCCCTGCCGTGACGC -+SRR33138449.278 278 length=250 ->1>>A111>1D11AE1AE00B0FE00/////1AAA22DF112FD/BE//11>//////B///>//000?B01??///2BG/GE/////>F1120?///>>-.>E..<<.ADFFFFFFGGGGGFGGGGHCHGGCGGGHHHHHHHGGGGHGHFHFHHHHHGGGGGHHHHGGEGGGHHH5GHGHHHHHHHHGHHHFHGHHHHHHGGGGGCGG/EGHHHGFHHHHHHHHHGGGGGHHEGFHGHHGHHHFGHGFHHHHGHGHHHHGHHFFHGHHHFGEHHHGHHHGHGHHHHGFGGGHHGHHHFHHHGCGGG?DG009BA99EDFGFF/BFFFFFFBFFBFFFF99AEBFFFFEFFEEFE/ -@SRR33138449.281 281 length=251 -GCCTGCTGCTGTGCCGGTAATGGCGCAATCAGTTTGTTATATTCACTTGCCTGCTGTGACGTCGGGAACTGGATGCCATTTGCGACAAAGCTCACCTGAGTACCTTGCTGAGCAATGTAGTCTCCAACCATCACCAGTTGTTGCGTGAAAGTTTGTGCCGCCGGAATCAACGGTTGCAATGCATCAGCTGGCGTCGTTACCACTTTGGTGAATGCCTGATCAAAGACCGGTTTCAAATCATCGGTCTGTTT -+SRR33138449.281 281 length=251 -AAAAA1BCDFF@FBBE?AEFFAEG?AEGGHGHEGGEFHHHHHHHFDEFG1FFFHCGEB2FEFFGGGGGGFFCCGBFGFBGGFG?EEGEEHGHHHHCE>BF>2FFF1FG1FDGFFFFHHHE2GFHH111DF3C1>1FGB1AFGGFB00A000BBDB0AAEEG1//A//AAEA/BFA110AEFEGFB?E/>/11//>>//EEEEGG/C1<11BFBF9/FFFFFFF./BBFFFF?EFBFBFFFFAEDF;-;-.BB/.:.-ADFFBBB.B..;EFBFB/-@9 -@SRR33138449.283 283 length=251 -GTCGTAGACAGCGTTAACGCAATTCGCGTACCGCAGGATTACGTTACGCAAAGCGGCCCACTGCGTGAAATGAACGGTTCTCTGGGCGTACTGGCGCAACAGCTGCAAAATGCGAAGTTGCAGGCCGATGCTGCGCACTCTGCGCTAAAACAGACCGATGATTTGAAACCGGTCTTTGATCAGGCATTCACCAAAGTGGTAACGACGCCAGCTGATGCATTGCAACCGTTGATTCCGGCGGCACAAACTTT -+SRR33138449.283 283 length=251 -ABABAFCCFFFCGGGGGGGGGGGHGGGGGEGGGGGGGGHHHHGHHHGGGGGGHHGGGGGGGHHGGGGGGHHHHHHHGGGGGHHHHHHGGGGHHFHCGGGGHCFFFHFGHHGHHGGGGGEHHGHHGFGGGDDDFFDGGGFF0GFGCCGGGFFEFGGEFGAAFGGFFGGGGFFF@DFFBFBBBFFFFEFFFFFFFFFFFFF9BFFFFFFFA?>E/BBBFFFFFFFFBFFFA...B:FFFFCF;CFFFFFFFFB -@SRR33138449.283 283 length=251 -GGCTTTTACTCTTTATTGGGTTGCAGTAACTGCTGTAGTCCAGGCCTGATTAAACGCCTGATGCTGTGCCGGTAATGGCGCAATCAGTTTGTTATATTCACTTGCCTGCTGTGACGTCGGGAACTGGATGCCATTTGCGACAAAGCTCACCTGAGTACCTTGCTGAGCAATGTAGTCTCCAACCATCACCAGTTGTTGCGTGAAAGTTTGTGCCGCCGGAATCAACGGTTGCAATGCATCAGCTGGCGTCG -+SRR33138449.283 283 length=251 -BABBBFBFFFFFGGGGFGGGEGGHHHGHHHHHHGHHHGGHHFHHHGHGFHHHGFFF?GHHHFHBGGHHHHGGGEFHFFHCGG?EHHHHHFHHGHGHHHHBGGHGDHHHHHHHGBGGGGGGG//FFGHHG2GGFFGHHHGCGGGGFHFCGHHHHBGGGHFFDFGHHEHH1GHHFGHFHHHGHFH?EHGHGHHACGFGAFHC:A.:;00C0FFB0BACGGFGGGEFBFFCDFF?FB///;/BFFFFF.:@AF- -@SRR33138449.284 284 length=251 -GTCGTAGACAGCGTTAACGCAATTCGCGTACCGCAGGATTACGTTACGCAAAGCGGCCCACTGCGTGAAATGAACGGTTCTCTGGGCGTACTGGCGCAACAGCTGCAAAATGCGAAGTTGCAGGCCGATGCTGCGCACTCTGCGCTAAAACAGACCGATGATTTGAAACCGGTCTTTGATCAGGCATTCACCAAAGTGGTAACGACGCCAGCTGATGCATTGCAACCGTTGATTCCGGCGGCACAAACTTT -+SRR33138449.284 284 length=251 -AAA@1BAAFFFAGAEG1A00AAE1AFE00AEEECGCCFC1FD/A0A/A///>00/EEEFE/0BF/EE?FBGGBHGH//FFFFH110?///-@-@BB?FFFF -@SRR33138449.284 284 length=251 -CTTGTGAAGAACTGTGTGAATCCGCACGTCGCGAACTAAGCCGCGCGTGCGTGGAGTAGAGCCCGCTAATAGCTCCGGCTTTTACTCACTTGTGGGTTGTAGTAACTGCTCTAGTCCAGGCCTGATTACACGTCTGATGCTGGGCCGGTACTGGCGCAATCAGCTTGCTATATTCACTTGCCTGCTGTGACGTCGGGAACTGGATGCCATTTGTGACCAAGCTCACCTGAGTACCTTGCTGAGCAATGTAG -+SRR33138449.284 284 length=251 -11>11BB@1@BC1F3F3133B3110A000AA00/AE/1BA0A/AE//>//>F////B012110?E/>//1221B11?E///01>B>F11B11B1//??//?@2222211111111?F110>AA>B3>FFF@GEEEFFGECGGGDFEEEGCFGCGGGCFHHHGEGFEFEGGGEB1?@>@DGGBGEFE?AFHHHHHHGDFGHHFEGHGGCEGHFHGG?@@DFHHHHHHBGHHGFGFCDCGEGFHF?CC??ADFD0@CCGCH0GCDGG@FFGEEGFFE?@BDBFFGBBFBCFBB=B?FFFFFFFFFFE/FFFFFE.AF/BF/BBFD?-@;-9@AFFFBFFFBFFFFFFFBF?DAFEB/B@BBBBBBBFFFF// -@SRR33138449.285 285 length=251 -GAGTAAAGCCCGATAATCGCTCGGGCTTTTACTCTTTATTGGGTTGCAGTAACTGCTGTAGTTCAGGCCTGATTAAACGCCTGATGCTGTGCCGGTAATGGCGCAATCAGTTTGTTATATTCACTTGCCTGCTGTGACGTCGGGCACTGGATGCCATTTGCGACAAAGCTCACCTGAGTACCTTGCTGAGCAATGTAGTCTCCAACCATCACCAGTTGTTGCGTGAAAGTTTGTGCCGCCGGAATCAACGG -+SRR33138449.285 285 length=251 ->1>>ADDFF11>AA1033AFG0AE?0000BB1D12AGGH2AAAEE/0GC112AFEBGH112D2D2F10BFFH1GDE1E?//F/0F1G@1BFF1//>E/GHEF/E?EEGFFGBDGHGGEGGH:CGG?EGGHHHD:BGDAFGFEGGBFGGG?FGGFGGGGGFFFFFFFFA0F0:ADAFF.EFFFFF9DFFBBBBFFFFFCABFFF.- -@SRR33138449.287 287 length=251 -GTCGTAGACAGCGTTAACGCAATTCGCGTACCGCAGGATTACGTTACGCAAAGCGGCCCACTGCGTGAAATGAACGGTTCTCTGGGCGTACTGGCGCAACAGCTGCAAAATGCGAAGTTGCAGGCCGATGCTGCGCACTCTGCGCTAAAACAGACCGATGATTTGAAACCGGTCTTTGATCAGGCATTCACCAAAGTGGTAACGACGCCAGCTGATGCATTGCAACCGTTGATTCCGGCGGCACAAACTTT -+SRR33138449.287 287 length=251 -DDDDCFCCFFFDGGGGCGGGGGGHHGGGGGGGGGGGGGHHHHGHHHGGGGGGHHGGGGGGGHHHGGGGGHHHHHHHGGGGHHHHHGHGGGGHHGHGGGGGHHHHHHHHHHHHHGGGGGHHHHHHGGGGGGGHHHGGGGGHHHHGGGGGGGGGGGGGGGGGGGGGGGGGGFFFFFFHFFFFFFFFFFFFHFFFFFFFFFFFFFFFFFFFFFFFFFFFHFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF -@SRR33138449.287 287 length=250 -GCAATGTAGTCTCCAACCATCACCAGTTGTTGCGTGAAAGTTTGTGCCGCCGGAATCAACGGTTGCAATGCATCAGCTGGCGTCGTTACCACTTTGGTGAATGCCTGATCAAAGACCGGTTTCAAATCATCGGTCTGTTTTAGCGCAGAGTGCGCAGCATCGGCCTGCAACTTCGCATTTTGCAGCTGTTGCGCCAGTACGCCCAGAGAACCGTTCATTTCACGCAGTGGGCCGCTTTGCGTAACGTAAT -+SRR33138449.287 287 length=250 -CCBCCFFFFFFFGGGGFGGGGGHHGHHFHHHHGGGGGGGHHHHGHGFHGGGGGGGGGHHHGGGGGHHHHGHHHHHHHGHGGGGGGGGGGHFHFHHGHHGHGHHHHHHGGHHHHHHHGGGGGFHHHFHGGFHHGGGGHHHHHHHGCCDGGHHHHGGGGGHHHFGGGGGHHGHHGGEGGGGGGGGGFFFFFGFFGGGGGFFFBFDFFFEFFFBAFFFFFFFFBF/EA;D.9;.;A=A-99BFFFFADABF.E -@SRR33138449.288 288 length=251 -GAGCAATGTAGTCTCCAACCATCACCAGTTGTTGCGTGAAAGTTTGTGCCGCCGGAATCAACGGTTGCAATGCATCAGCTGGCGTCGTTACCACTTTGGTGAATGCCTGATCAAAGACCGGTTTCAAATCATCGGTCTGTTTTAGCGCAGAGTGCGCAGCATCGGCCTGCAACTTCGCATTTTGCAGCTGTTGCGCCAGTACGCCCAGAGAACCGTTCATTTCACGCAGTGGGCCGCTTTGCGTAACGTAA -+SRR33138449.288 288 length=251 -BABCCFFFFFFFGGGGGGGGGGHHHHHHHHHHHHHGGFGFHHHHHHHHHHGGGGGGGGHHHHGGGGGHHHHHHHHHHHHHHHGGEGGGGHGHHHHHHHHHGHHHHHHHHHHHHHHHGHHGGDGGHHHHHHHGHHGGGGHHHHHHHHGGGGGHHFGGGGGGHHHGFGGGGHHHGGGGGGFGGGGGGGGGGGGGGGGGGGFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF?EEFFAFFFFFFFFFFFFF -@SRR33138449.288 288 length=251 -GTCGTAGACAGCGTTAACGCAATTCGCGTACCGCAGGATTACGTTACGCAAAGCGGCCCACTGCGTGAAATGAACGGTTCTCTGGGCGTACTGGCGCAACAGCTGCAAAATGCGAAGTTGCAGGCCGATGCTGCGCACTCTGCGCTAAAACAGACCGATGATTTGAAACCGGTCTTTGATCAGGCATTCACCAAAGTGGTAACGACGCCAGCTGATGCATTGCAACCGTTGATTCCGGCGGCACAAACTTT -+SRR33138449.288 288 length=251 -AAAABFCBFFFCGGGGGGGGGGGHHGGGGGGGGGGGCGHHHHGHHGGGGGGGHHGGGGGGGHHEGGGGGHHHHGHHGGGGHHHHGFHGGGGHFHEFGGFGHHHHGHHHHHHHGGGGGFHHHHFHGGGFGGGHHFCCGGCHHFEGGGGGGGGGGGGGGGGFB0BFGGGGFFFAD;FFFFBFFFFFFFFFFFFFFFFFFFFBFFBDECFAFFF?BB//B///;BFFFF?CFFBE/BFBF-@BAC-BBEFE/BB -@SRR33138449.289 289 length=251 -CAGTACCGCATATTGGATCTGCCACTCATGAGACGCGTTATGGCATGGCCGCCTGTGCCGTGGATAATTTGATTGATGCGTTACAAGGAAAGGTTGAGAAGAACTGTGTGAATCCGCACGTCGCGGACTAAGCCGCGACTGCGTGGAGTAAAGCCCGATAATCGCTCGGGCTTTTACTCTTTATTGGGTTGCAGTAACTGCTGTAGTCCAGGCCTGATTAAACGCCTGATGCTGTGCCGGTAATGGCGCAA -+SRR33138449.289 289 length=251 -BBBBBFF?DBBBCGFGGGGGGGHHHHHGFHFGHHCFGFEEHHGHHHFHFH?CEGHHHHHGDFEEHFGHHHHFHHHHHHHGGGGGHFHHGHHHFFFFCFFGHHHGHHHHHHHHHHHGGGGGAHGGGGGGGHHHHHGGG?GCDGGCFC.C:CGHFHHFGGGGGFGGGGGC?9B=FFFFFFFFFFFFFFFADEFFFFFFFFFFFFFFF9FFFFFFFFFFFFFFEFFFFFFFBFFFFFFFFFFF-BEFFFFFDCF -@SRR33138449.289 289 length=251 -GTCGTAGACAGCGTTAACGCAATTCGCGTACCGCAGGATTACGTTACGCAAAGCGGCCCACTGCGTGAAATGAACGGTTCTCTGGGCGTACTGGCGCAACAGCTGCAAAATGCGAAGTTGCAGGCCGATGCTGCGCACTCTGCGCTAAAACAGACCGATGATTTGAAACCTGTCTTTGATCAGGCATGCACCAAAGTGGTAACGACGCCAGCTGATGCATTGCAACCGTTGATTCCGTCGGCTCAAACTTT -+SRR33138449.289 289 length=251 ->AAA111>B11AEGGGCGFGE0EFGC00AAECCEEG//FA1BFBGHEECEEGGFGGCEEGGEFG/EEGG1BB111G//?FEFDD10<F11B1F.F-<@ECFBG@C-C-CGGFCGGGBF00C9/A?--:.00BB0;0;FF/9BFFF9/////BBBEF-/9;/--:;/9B;9/;AE@;@;A@ABB///;////;B9/A-BB--:9B///;9@---///;/9/ -@SRR33138449.290 290 length=251 -GACGTCGGGAACTGGATGCCATTTGCGACAAAGCTCACCTGAGTACCTTGCTGAGCAATGTAGTCTCCAACCATCACCAGTTGTTGCGTGAAAGTTTGTGCCGCCGGAATCAACGGTTGCAATGCATCAGCTGGCGTCGTTACCACTTTGGTGAATGCCTGATCAAAGACCGGTTTCAAATCATCGGTCTGTTTTAGCGCAGAGTGCGCAGCATCGGCCTGCAACTTCGCATTTTGCAGCTGTTGCGCCAG -+SRR33138449.290 290 length=251 -CBCBCCCCCCCCGGGGGGGGGGHHHGGGGGGHHHHHHHHHHFHHHHDHHGHHHHHFHHHHHGFGHFHHHGHGGHHHHFGHGHGHHHFEGGEEGHHFHHHHHHGGEGGGGGFFHFGGGGGGHHFFGFHHHGHHHHGGGGGCGHGHFGGGHDCCEHHGHGGHFFHHHHHHHHHGGFGGGGGGEFFGGGGFGGGGGGGGFGGFGGFF/BFDFFAFFFFF:BDFFFFFFFFFFDFFFFFFFFFFFFFFFFFAF?; -@SRR33138449.290 290 length=251 -GTCGTAGACAGCGTTAACGCAATTCGCGTACCGCAGGATTACGTTACGCAAAGCGGCCCACTGCGTGAAATGAACGGTTCTCTGGGCGTACTGGCGCAACAGCTGCAAAATGCGAAGTTGCAGGCCGATGCTGCGCACTCTGCGCTAAAACAGACCGATGATTTGAAACCGGTCTTTGATCAGGCATTCACCAAAGTGGTAACGACGCCAGCTGATGCATTGCAACCGTTGATTCCGGCGGCACAAACTTT -+SRR33138449.290 290 length=251 -AAA3AFAAFFFBGGGGGGGGGCGGHFGGAEGGGGGC?FHGHFFGHFGGDGGFFHGGG?EFGFHGGGGGGHHHHHGHGEA/FHHHBFFEEEDGH2FFDGGGG2GHGFFHFFB@B@F..9FFF -@SRR33138449.291 291 length=251 -CCTGAGTACCTTGCTGAGCAATGTAGTCTCCAACCATCACCAGTTGTTGCGTGAAAGTTTGTGCCGCCGGAATCAACGGTTGCAATGCATCAGCTGGCGTCGTTACCACTTTGGTGAATGCCTGATCAAAGACCGGTTTCAAATCATCGGTCTGTTTTAGCGCAGAGTGCGCAGCATCGGCCTGCAACTTCGCATTTTGCAGCTGTTGCGCCAGTACGCCCAGAGAACCGTTCATTTCACGCAGTGGGCCG -+SRR33138449.291 291 length=251 -BBBBBFFFFFFBFFGCBGGGGGHGHFHHHGHHFGFGHHHHGHFGHHFHFHGEAEFFFAFHGCGHHGGGFFGFGHFHHGGAFGGHHHHHHHHHHHGHGDFGGGGGHGFHHFHHFHHGHFHHHHHHBGGGHBGHFGDGAAE?FG?ECEFF1FECEEGCEG?FGFHGHDGBGHGFG//>>FECEC//B0GHE/?E/1F11FGAE/?FH2F>GFHEEECH2BF@CC?CFGE@@>A-AFF//9///FBE:BF-9;F/;B///:=@->--9-;--9BF -@SRR33138449.292 292 length=218 -GTCGTAGACAGCGTTAACGCAATTCGCGTACCGCAGGATTACGTTACGCAAAGCGGCCCACTGCGTGAAATGAACGGTTCTCTGGGCGTACTGGCGCAACAGCTGCAAAATGCGAAGTTGCAGGCCGATGCTGCGCACTCTGCGCTAAAACAGACCGATGATTTGAAACCGGTCTTTGATCAGGCATTCACCAAAGTGGTAACGACGCCAGCTGATGC -+SRR33138449.292 292 length=218 -BBBCBFCBFFFDGGGGGGGGGGGHHGGGEFGGGGGGGGGHHHGHHHGGGGGGHHGGGGGGGHHHGGGGGHHHHHHHGFGGHHHHHHHGGGGHHHHGGGGGHHHHHHHHHHHHHGGGGGHHHHHHGGGGGGGHHHGGGGGHHHHGGGGGGGGGGGGGGGGGGGGGGGGGGFFFFFFFFFFFHHFFFFFFFFFFFEFFFBFBFFFFFFFFFFFFFFFFFF -@SRR33138449.292 292 length=218 -GCATCAGCTGGCGTCGTTACCACTTTGGTGAATGCCTGATCAAAGACCGGTTTCAAATCATCGGTCTGTTTTAGCGCAGAGTGCGCAGCATCGGCCTGCAACTTCGCATTTTGCAGCTGTTGCGCCAGTACGCCCAGAGAACCGTTCATTTCACGCAGTGGGCCGCTTTGCGTAACGTAATCCTGCGGTACGCGAATTGCGTTAACGCTGTCTACGAC -+SRR33138449.292 292 length=218 -CCCCCFFFFFFCFEEEFGGGGGHHHFHHFFHGHHHHHHHHFHHHFHHGGGEEGHHHHEHGFHHGEEGHHHHHHHHGGGGEHGGHGGGGGHHHGGGGGGHHFHHHHGGGGGHGHHHFFFHHFHFFCGGGGGHGGFGGHHGGHHGGDGHFHHHHHGHH-@CCCGEGC?-DFGGGDDDDGGGG?FCFFFD?CDD?DCFFFFFFF.9AAFFF?A.B/BFEAD -@SRR33138449.293 293 length=196 -CTTTGGTGAATGCCTGATCAAAGACCGGTTTCAAATCATCGGTCTGTTTTAGCGCAGAGTGCGCAGCATCGGCCTGCAACTTCGCATTTTGCAGCTGTTGCGCCAGTACGCCCAGAGAACCGTTCATTTCACGCAGCGGGCCGCTTTGCGTAACGTAATCCTGCGGTACGCGAATTGAGATAACGCTGTCTACGAC -+SRR33138449.293 293 length=196 ->>AA>CAABDFBBGFGCGGFGFHHH4EEAAA55FGD3EAGHGBEECA55AFGDEEFGGHDG5E0EE>C5FFA/EEFAGB3?FGFDEEGGHHHHHFCCB3BGFCFEEGGDDDGCGE/GBG2GD/?CGFBFHHFGECC/<--@@DC:::C0:::CA:/9A9009:FGFGG?ADC?D-AF/;.;;9999AFFFFBBED? -@SRR33138449.293 293 length=196 -GTCGTAGACAGCGTTAACGCAATTCGCGTACCGCAGGATTACGTTACGCAAAGCGGCCCACTGCGTGAAATGAACGGTTCTCTGGGCGTACTGGCGCAACAGCTGCAAAATGCGAAGTTGCAGGCCGATGCTGCGCACTCTGCGCTAAAACAGACCGATGATTTGAAACCGGTCTTTGATCAGGCATTCACCAAAG -+SRR33138449.293 293 length=196 -BBBBBFBB5BF?FFCEFGGEFGGEEFAEEE2AE2E0EEHHBEFFFHBFF1EGHBAEE>EAAGHGG1E1FHHEGFDCG/?/3FGGHGHCEECF?2C/DCGGHHHHH2?GGBGHBEF@@.1FH1>0CFC?F.AFFBC@CC@GCCFCDDF?BFGG9FFGG?-90;BFF0;0BADA=-./:BBBBFFFFFFFBF/B/;A/ -@SRR33138449.294 294 length=125 -GTCGTAGACAGCGTTAACGCAATTCGCGTACCGCAGGATTACGTTACGCAAAGCGGCCCACTGCGTGAAATGAACGGTTCTCTGGGCGTACTGGCGCAACAGCTGCAAAATGCGAAGTTGCAGGC -+SRR33138449.294 294 length=125 -AAABAFBBFFFBGGGGGGGGGGGGHGGGGGGDGGGGGFHHHHGHHHGGGGGGHHGGGGGGGHHHGGGGGHHHHHHGGGGGHHHHHHHGGGGHHHHGGGGGGHHHHHHHHHHHHGGGGGHHHHHHG -@SRR33138449.294 294 length=125 -GCCTGCAACTTCGCATTTTGCAGCTGTTGCGCCAGTACGCCCAGAGAACCGTTCATTTCACGCAGTGGGCCGCTTTGCGTAACGTAATCCTGCGGTACGCGAATTGCGTTAACGCTGTCTACGAC -+SRR33138449.294 294 length=125 -CCCBCFFFFFFFFGGGGGGGGGFHGBFHGHGGGGGHHGGGGGGHHHHHHHGFGHGHHFHHHHHGGEHHHGHGGGGGHHGGGGGGFGHFHHHFHGG/EFGGGCG@FHGGBEDFFEGDGGFDGGGDG -@SRR33138449.295 295 length=251 -CCTGAGTACCTTGCTGAGCAATGTAGTCTCCAACCATCACCAGTTGTTGCGTGAAAGTTTGTGCCGCCGGAATCAACGGTTGCAATGCATCAGCTGGCGTCGTTACCACTTTGGTGAATGCCTGATCAAAGACCGGTTTCAAATCATCGGTCTGTTTTAGCGCAGAGTGCGCAGCATCGGCCTGCAACTTCGCATTTTGCAGCTGTTGCGCCAGTACGCCCAGAGAACCGTTCATTTCACGCAGTGGGCCG -+SRR33138449.295 295 length=251 -CDDDDFFFFFFFGGGGGGGGGGHHHHHHHHHHHHGGHHHHHHHHHHHHHHGGGGGHHHHHHHHHHGGGGGGGGHHHHGGGGGHHHHHHHHHHHHHHHGGGGGGGHGHHHHHHHHHGHHHHHHHHHHHHHHHHHHGGGGGHHHHHHHHHHGGGGHHHHHHHGGGGGGHHHHGGGGGHHHGGGGGGGGGGGGGGGGGGGGGGHGGGGFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFHFHHFFFFFFFFFF= -@SRR33138449.295 295 length=251 -GGATTACGTTACGCAAAGCGGCCCACTGCGTGAAATGAACGGTTCTCTGGGCGTACTGGCGCAACAGCTGCAAAATGCGAAGTTGCAGGCCGATGCTGCGCACTCTGCGCTAAAACAGACCGATGATTTGAAACCGGTCTTTGATCAGGCATTCACCAAAGTGGTAACGACGCCAGCTGATGCATTGCAACCGTTGATTCCGGCGGCACAAACTTTCACGCAACAACTGGTGATGGTTGGAGACTACATTG -+SRR33138449.295 295 length=251 -BABBBFFBFFFBGGGGGGGGGGGCGGHHHGGGGGHHHHHHHGGGGHHHHHHHGGGGFHGGEGGGGHHHHHHGHHHHHHGGGGGHHHGHHGGGGGGGHHHGGGGGHHHGGGGGGHHHHHGGHGGGGHHHHHHHHHGGGGGGHHHGHHHHHHHHHHHHHHGGG;FFFFFGDGGGGGGGGGFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFCFFFFFFFFFFFFAAFFEEFFFFFFFFFFFF>FFFFFFFFFF/ -@SRR33138449.296 296 length=244 -GGATTACGTTACGCAAAGCGGCCCACTGCGTGAAATGAACGGTTCTCTGGGCGTACTGGCGCAACAGCTGCAAAATGCGAAGTTGCAGGCCGATGCTGCGCACTCTGCGCTAAAACAGACCGATGATTTGAAACCGGTCTTTGATCAGGCATTCACCAAAGTGGTAACGACGCCAGCTGATGCATTGCAACCGTTGATTCCGGCGGCACAAACTTTCACGCAACAACTGGTGATGGTTGGAGAC -+SRR33138449.296 296 length=244 -CDCDCFFCFFFDGGGGGGGGGGGGGGHHHGGGGGHHHHHHHGGGGHHHHHHHGGGGHHHHGGGGGHHHHHHHHHHHHHGGGGGHHHHHHGGGGGGGHHHGGGGGHHHHGGGGGHHHHHHHHGGGGHHHHHHHHHHGGGGGHHHHHHHHHHHHHHHHHHGGGFGGGGGGGGGGGGGGGGFFFFFFFFFFFFFFFFFFFFFHFFFFFFFFFFFFFFHHFFFFFFFFFFFFFFFFFFFFFFFFFFFF -@SRR33138449.296 296 length=244 -GTCTCCAACCATCACCAGTTGTTGCGTGAAAGTTTGTGCCGCCGGAATCAACGGTTGCAATGCATCAGCTGGCGTCGTTACCACTTTGGTGAATGCCTGATCAAAGACCGGTTTCAAATCATCGGTCTGTTTTAGCGCAGAGTGCGCAGCATCGGCCTGCAACTTCGCATTTTGCAGCTGTTGCGCCAGTACGCCCAGAGAACCGTTCATTTCACGCAGTGGGCCGCTTTGCGTAACGTAATCC -+SRR33138449.296 296 length=244 -BCCCCFFFFBCFGGGGGGGGGGHGHGGGGGHHHHHHHHHHGGGGGGGGHHHHGGEGGHHHGGHHHHHHGHHGGGGGGGGHGHHHHHHHHHGHHHHGHHHHHHHHHHHHHGGCGGHHHHFHGHHHGGGGHGHHHHHHGGGGGGHHHG-DGGHHHGGFGGGHHHHHHHGGGGGGGGGFGGGGBFBFGGGGGBFFBFFF?A?AFFFEFFDEFBFFFFFFFFF:9;E?EFFFFFFEFDD.ADBF9EFF -@SRR33138449.297 297 length=251 -CCTGAGTACCTTGCTGAGCAATGTAGTCTCCAACCATCACCAGTTGTTGCGTGAAAGTTTGTGCCGCCGGAATCAACGGTTGCAATGCATCAGCTGGCGTCGTTACCACTTTGGTGAATGCCTGATCAAAGACCGGTTTCAAATCATCGGTCTGTTTTAGCGCAGAGTGCGCAGCATCGGCCTGCAACTTCGCATTTTGCAGCTGTTGCGCCAGTACGCCCAGAGAACCGTTCATTTCACGCAGTGGGCCG -+SRR33138449.297 297 length=251 -ABCBCFFFFFFFGGGGGGGGGGHHHHHHHHHFHGGGHHHHHGHHHHHHHHGGGEGHHHFHHHHFHGGGGGGGGHHHHGGFGGHHHHHHHHHHHGHHHGGGGGGGHGHHHHHHHHHGGHFHHHHHHHHHHHHGHHGGCGGHHHHGHHHHHGGGGHHHGHHHHGGGGGHGHHGGGGGHHHGGGGGGGGFGGGGGGGGGGGGGGGGGGFFFFFFFFFFFFFFFFFF?AFFBEFFFFFFFFFFFFFFFDFFFFF= -@SRR33138449.297 297 length=251 -ACGTTACGCAAAGCGGCCCACTGCGTGAAATGAACGGTTCTCTGGGCGTACTGGCGCAACAGCTGCAAAATGCGAAGTTGCAGGCCGATGCTGCGCACTCTGCGCTAAAACAGACCGATGATTTGAAACCGGTCTTTGATCAGGCATTCACCAAAGTGGTAACGACGCCAGCTGATGCATTGCAACCGTTGATTCCGGCGGCACAAACTTTCACGCAACAACTGGGGATGGTTGGAGACTACATTGCTCAG -+SRR33138449.297 297 length=251 -ABBBBFBB?ADDFFEE?E2EFGFHGEGCFFFHBGHHGEFGHGHHFHHGEEFHEBFGCE@EFFHGBGHHHHGHHGGFFFGDFHHHGGGGGGGHHHGC?DGHHGGGGGGGHHHFHGGCFGCDFHHHHHHGHHGCCFFHHHFFHHGEHEHFHHHGHHHHCGGFFFFG?BGGGGGGGGBFFBFGFFBFFFFDDDDEFFFFFCAFBAACFF?EFFBFFFE.9=BFFEFF/.99.;;:F?FFFB?BFB9FFB/BBFB -@SRR33138449.298 298 length=251 -ACGTTACGCAAAGCGGCCCACTGCGTGAAATGAACGGTTCTCTGGGCGTACTGGCGCAACAGCTGCAAAATGCGAAGTTGCAGGCCGATGCTGCGCACTCTGCGCTAAAACAGACCGATGATTTGAAACCGGTCTTTGATCAGGCATTCACCAAAGTGGTAACGACGCCAGCTGATGCATTGCAACCGTTGATTCCGGCGGCACAAACTTTCACGCAACAACTGGTGATGGTTGGAGACTACATTGCTCAG -+SRR33138449.298 298 length=251 -BBBABFBBBBBBGGCGFGGEFFHHDEGGGGEGGHHGGGFFHHFFBGFGEEEEFBC0EEG@FCHGFHHHGHDBGDEGGFGHHHFHGEGGGGGHFF/@CDDGHHFDGGGGHHHHGHFECCGGHHHHFHHHHFGC@@CHHFGHFHHGHHHGGHHHFGHFCGGFFGGG?CFGGGFGF/0BBFGEFFFFFFFF99;.:BFFF;;DDFFFFFFFFFFEFFFFADAFFFFFF;BF/BB9FD@FFFFFFFFBFFFFFFB -@SRR33138449.298 298 length=251 -GTTATATTCACTTGCCTGCTGTGACGTCGGGAACTGGATGCCATTTGCGACAAAGCTCACCTGAGTACCTTGCTGAGCAATGTAGTCTCCAACCATCACCAGTTGTTGCGTGAAAGTTTGTGCCGCCGGAATCAACGGTTGCAATGCATCAGCTGGCGTCGTTACCACTTTGGTGAATGCCTGATCAAAGACCGGTTTCAAATCATCGGTCTGTTTCAGCGCAGAGTGCGCAGCATCGGCCTGCAACTTCG -+SRR33138449.298 298 length=251 -?AAAAB3@DBDB1FGG11AEG31FGGG?EEG?EAGB001B01EFGEGHAEEECBF0BG1BFCGGHHF11FHGFFHFFGHHHEHHGHHHHGFHHGG1FFHCGH1FGHHHHCE?//1FGGDB0BF>EEEEECGGHHHGHGGGGGGGHGGGGACGHHGFGHEC@GCEFHGGGHFGGGGGGGGGGFFGGGGG0CEAECFFGFGGFFFFFFBBFFFFFFFFFFFFFFFFFFFFFFEFFFFFEF/BF@FFFEFFFFCD;FFFEBFFFFBFFFFFFFFFFFFF -@SRR33138449.305 305 length=250 -GTTCTCTGGGCGTACTGTCGCAACAGCTGCAAAATGCGAAGTTGCAGTCCGATGCTGCGCACTCTGCGCTAAAACAGACCGATGATTTGAAACCGGTCTTTGATCAGGCATTCACCAAAGTGGTAACGACGCCAGCTGATGCATTGCAACCGTTGATTCCGGCGGCACAAACTTTCACGCAACAACTGGTGATGGTTGGAGACTACATTGCTCAGCAAGGTACTCAGGTGAGCTTTGTCGCAAATGGCAT -+SRR33138449.305 305 length=250 -AAA1AF331B1@FEGGG3FC0E0AGFGHFFHGHFGGG/EEAFD11BE1BCAAAEA1FGEEEGGHHHHC?GEEFFFE0EHC>EC/FHFFFB1FGHCCEECHHHF2FBFBGHHHHFHHHGHGHFE12GHG//CCGGCFFCDFBBFGD0DGGFFGGGCEC0CC/--:-99@FFGGFFB0BB9A@@-AFFFFF/B/9BBB/A=9B/AB/9FBFFFFFF/B/9-9BBFFFBF9/:/ABBFFFF/-E@--9/;BBF -@SRR33138449.306 306 length=250 -ATAATCGCTCGGGCTTTTACTCTTTATTGGGTTGCAGTAACTGCTGTAGTCCAGGCCTGATTAAACGCCTGATGCTGTGCCGGTAATGGCGCAATCAGTTTGTTATATTCACTTGCCTGCTGTGACGTCGGGAACTGGATGCCATTTGCGACAAAGCTCACCTGAGTACCTTGCTGAGCAATGTAGTCTCCAACCATCACCAGTTGTTGCGTGAAAGCTTGTGCCGCCGGAATCAACGGTTGCAATGCAT -+SRR33138449.306 306 length=250 -AAAAADAA1>AD?EGGFBGBGBFFFDFGBG0FGGHBF1BFH2FFGHHHFFHHHBFEFHFHHFGDEHEEEGHCGDFHBD0FEE/EEGGHEGGFCGGHFB2F>GHHHEHHHHFHBHHHHHDFCGGHFFGGFGG/B@AF10>A111111111100B1133B33FG11D01000BAG2200B1B/110//BF/F0011D120DFF11/B///0//>/////2>B0100/>///0/////?0B1221/1001?211//////-->.>G1=F11DGH0000.<<-.<00<00;0000;C000000..;---.;/0;0;00/;A-.-;/9//;--;--;/9--;---B--;FF-9-/;/;BF///9/////-///-//;;--:--;-;F//;9B -@SRR33138449.308 308 length=251 -TACTGGCGCAACAGCTGCAAAATGCGAAGTTGCAGGCCGATGCTGCGCACTCTGCGCTAAAACAGACCGATGATTTGAAACCGGTCTTTGATCAGGCATTCACCAAAGTGGTAACGACGCCAGCTGATGCATTGCAACCGTTGATTCCGGCGGCACAAACTTTCACGCAACAACTGGTGATGGTTGGAGACTACATTGCTCAGCAAGGTACTCAGGTGAGCTTTGTCGCAAATGGCATCCAGTTCCCGACG -+SRR33138449.308 308 length=251 -CCCCCFCCCCCCGGGGGGGGGGHHGGGGGGHHHHHHGGGGGGGGHHGGGGGHHHHGGGGGHHHHHHHHGGGGGHHHHHGHHHGGGGGHHHGHHHHHHHHHHHHHHHGHHHHHHHHGEGGGGGGGHHHHHHHFHHHHHHHGGGGHHHHGHGGGGGGGGFGGGGGGGGGGGGGFFFFFFFFFFFFFFFFEFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFDFFABFFFFFFFFFFFFFFFFBFB -@SRR33138449.308 308 length=250 -CTGTAGTCCAGGCCTGATTAAACGCCTGATGCTGTGCCGGTAATGGCGCAATCAGTTTGTTATATTCACTTGCCTGCTGTGACGTCGGGAACTGGATGCCATTTGCGACAAAGCTCACCTGAGTACCTTGCTGAGCAATGTAGTCTCCAACCATCACCAGTTGTTGCGTGAAAGTTTGTGCCGCCGGAATCAACGGTTGCAATGCATCAGCTGGCGTCGTTACCACTTTGGTGAATGCCTGATCAAAGAC -+SRR33138449.308 308 length=250 -BBBBBFFFFFFFGGGGGGGGGGHCGGHGFHCHHHHGHHGGGFAFFFHGGCGEHHHHHHHHHHHHHHHHFHHHHHHHGHHHHFHGGGGGGCCEHHHEFHHHHHHGHHGG?EFHHHHHHGHHHFGHHHHHHGHFHGFCG;9A..B9BBBFFBFEFFFFFFF/BB//:BB/9 -@SRR33138449.309 309 length=250 -GGCGCAACAGCTGCAAAATGCGAAGTTGCAGGCCGATGCTGCGCACTCTGCGCTAAAACAGACCGATGATTTGAAACCGGTCTTTGATCAGGCATTCACCAAAGTGGTAACGACGCCAGCTGATGCATTGCAACCGTTGATTCCGGCGGCACAAACTTTCACGCAACAACTGGTGATGGTTGGAGACTACATTGCTCAGCAAGGTACTCAGGTGAGCTTTGTCGCAAATGGCATCCAGTTCCCGACGTCA -+SRR33138449.309 309 length=250 -AAAAABBBFFFFGGGGGGGFGGGGGGHHHHHHGGGGGGGHHHGGGGGHHHHGGGGGHHHHGFHHGGGGGHHHHHGHHHGGGGGHHHHHHHHHHHHHHHHHHEHHFHHGFHHGGGGGGGGGHHHHHHGHHHHHHHHGGGGHHHHHHGFGGGGGGGGGGGGGGGGGGGGGGGGGGBFFFFFFFFAFFFFFFFFFFFFFFFFFFEFEBFFFFFFFFFFFFFFFFFFFFC9DFFEFFFFFFB9BFBFFFFFFFF -@SRR33138449.309 309 length=251 -GTCCAGGCCTGATTAAACGCCTGATGCTGTGCCGGTAATGGCGCAATCAGTTTGTTATATTCACTTGCCTGCTGTGACGTCGGGAACTGGATGCCATTTGCGACAAAGCTCACCTGAGTACCTTGCTGAGCAATGTAGTCTCCAACCATCACCAGTTGTTGCGTGAAAGTTTGTGCCGCCGGAATCAACGGTTGCAATGCATCAGCTGGCGTCGTTACCACTTTGGTGAATGCCTGATCAAAGACCGGTTT -+SRR33138449.309 309 length=251 -BBBBBFFBBFBFGGFGGGEGGGHHHHHHHHHHHGGCGGFFFGEGGGGHFDDFHHHHFFGHHHGHHHHHHHHHHHGHHH?FGGGFEEFFHHHHHGHGHHHFFGEGGGHHHGHGFFHHCHHHHHHHHGHHDGHGHHHHHFHFHHGGHHGFHHHHHGFGGHHHFEGGGGGBFHHFFHFHHCCGFG@DEHHFFGBAEAFFFFGGGFFGGGGGGFGGFAFDFEFFFFFFFFFFBFFFFFFFBFFF/BFFFF;-9AA -@SRR33138449.310 310 length=251 -GTTATATTCACTTGCCTGCTGTGACGTCGGGAACTGGATGCCATTTGCGACAAAGCTCACCTGAGTACCTTGCTGAGCAATGTAGTCTCCAACCATCACCAGTTGTTGCGTGAAAGTTTGTGCCGCCGGAATCAACGGTTGCAATGCATCAGCTGGCGTCGTTACCACTTTGGTGAATGCCTGATCAAAGACCGGTTTCAAATCATCGGTCTGTTTTAGCGCAGAGTGCGCAGCATCGGCCTGCAACTTCG -+SRR33138449.310 310 length=251 -AAABBFFFFFFFGGGGGGGGGGGHGGGGGGGAEGHHFFHHHHHHHHHGEGEGGHHGHHHHHHHFHHHHHHHHHHFGFHFHHHFHEHHHHHHHHGGHHHHHHGHGFHGHHGGFGGFGBHHHCFGHGGGGGGFFHHHGGCDGGFHGHHGHHHHHHHHBD@A@DGGHGHH0GGFFHHGEHHHEHHGHGHGGGGBGEGGBDGGFFFFCFGGGDGGFFFFFFFFFFAF;DEFFFFFFFFFFFFAADDAFFFFFFFA -@SRR33138449.310 310 length=251 -GCGTAAAGCGTTTATCGACTTCCTGCAGAATACAGTGATGCGTAGCGGTGAACGTCTACTAACCCTGACTGCCGATCAGAAAAAATAGTTTGGTCCTTTTGTCTCTGTTTACGCGATTCTGTATGGTTATTCTCAGAAGGTGATTCAGGCGCTGGATTCCGGTCTGCGCCCTGTCGTAGACAGCGTTAACGCAATTCCCGTACCGCAGTATTTCGTTACGCACAGCGGCCCACTCCGTGTACTGATCGGTT -+SRR33138449.310 310 length=251 ->1>>1113BAAAEGD31100AFFHGFH1BEDFFG1BBBGHCEEAC//E/A/BBAFAC/12DFFGHAF0F1100>EEE?1BGFFC/0B@2FGD/>BF1FFGGFHHHH122BB1?//FFF22>>0>B1FF2B22211<@@B121?FF1E1GHGEFG1FGG?EEFHFFFG/BCC/CGHHGHFDGHFHEHC?CFEGHHG=C=CCFHG?;0;BEHHGFHHBBFFGF/FFGE90CFFBGFB/CAF0/;BFGG?@A@FFBBBBFFFB/9FF/AA@?-;EB9BEF-A?-?-//9//;FFFEFF-9///:9/BA=A?AF -@SRR33138449.311 311 length=250 -GAAGTACTGTGTGAATCCGCACGTCGCGGACTAAGCCGCCACCGCGTGGCGTAAAGCTCGATAATCGCTCGGGCTTTTACTTCTTATTGGGTCGCAGTAACTGCTGTAGTCCAGGCCTGATTAAACGCCTGATGCTGAGCCGGTACTGGCGCATTCAGTTTGTTATATTCACTTGACTGCTGTGACGTCGGGAACTGGATGCCATTTGCGACAAAGCTTACCTGAGTACCTTGCTGAGCATTGGAGTCGC -+SRR33138449.311 311 length=250 -A1AA1B333FBFFBD33G?C1EEHAA000AAAF1AGHCE//A0A//>>//E////B111//>>/FHA/E//>E@E/BEGFH21G12FG1G/C0//BEFHFECGGFHFGHHEFGCEEGHBGHFHHFHHBFGFHHHHGHGHHAFFCGFGHE>GHEFGHFDFFBGHHHHHBHHHF1-AFFEHFHFFFHHH0@D-ACCCEFGFGDG.CBFFF09F0CFGEEGGGGBF=--;AEFFFFF///.ABFBBBFFFF/FFFFFFF -@SRR33138449.313 313 length=251 -GCGTTATGGCATGGCCGCCTGTGCCGTGGATAATTTGATTGATGCGTTACAAGGAAAGGTTGAGAAGAACTGTGTGAATCCGCACGTCGCGGACTAAGCCGCGACTGCGTGGAGTAAAGCCCGATAATCGCTCGGGCTTTTACTCTTTATTGGGTTGCAGTAACTGCTGTAGTCCAGGCCTGATTAAACGCCTGATGCTGTGCCGGTAATGGCGCAATCAGTTTGTTATATTCACTTGCCTGCTGTGACGT -+SRR33138449.313 313 length=251 -ABBC@BBFFCFFGGGGGGGGGGHHHGEGGGHHHHHHHHHHGFHHHGGGGEGHGHFHFGGAFFGFHHHGHHHHHHHHHHHHHGGGGGHHGGGGGGGHHGHHGGGFGGGHGAFGGHFHHHHHGHGGGGGHHGGHEGGGFGHHHHGHHHHHHHHHGGDGGGGGGGGGGGGGGGGGFFGGGGGGGGGGGFFFFFFFFFFFFFFFFFFFFFF;EFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEBBFFFFF; diff --git a/crates/pithos_lib/tests/fs_ingest.rs b/crates/pithos_lib/tests/fs_ingest.rs new file mode 100644 index 0000000..c10bf7a --- /dev/null +++ b/crates/pithos_lib/tests/fs_ingest.rs @@ -0,0 +1,191 @@ +use pithos_lib::archive::{ + AccessKeys, Archive, ArchiveWriter, EntryKind, OpenOptions, ProcessingOptions, WriteOptions, +}; +use pithos_lib::crypto::PrivateKey; +use pithos_lib::error::PithosError; +use pithos_lib::fs::FsError; +use pithos_lib::fs::ingest::build_input_manifest; +use pithos_lib::source::MemorySource; +use std::fs; +use std::path::Path; + +fn write_manifest(manifest: &pithos_lib::fs::ingest::InputManifest) -> Archive { + let sender = PrivateKey::generate(); + let recipient = sender.public_key(); + let mut writer = ArchiveWriter::create( + Vec::new(), + WriteOptions::new(sender.duplicate(), vec![recipient]), + ) + .unwrap(); + manifest + .ingest(&mut writer, ProcessingOptions::default()) + .unwrap(); + Archive::open( + MemorySource::new(writer.finish().unwrap()), + OpenOptions::default().with_access_keys(AccessKeys::new().with_key(sender)), + ) + .unwrap() +} + +#[test] +fn manifest_ingests_files_directories_and_safe_symlinks() { + let temporary = tempfile::tempdir().unwrap(); + let source = temporary.path().join("source"); + fs::create_dir_all(source.join("nested")).unwrap(); + fs::write(source.join("nested/file.txt"), b"content").unwrap(); + #[cfg(unix)] + std::os::unix::fs::symlink("file.txt", source.join("nested/link")).unwrap(); + + let archive = write_manifest(&build_input_manifest(&[source]).unwrap()); + assert!(matches!( + archive.entry("nested").unwrap().unwrap().kind, + EntryKind::Directory + )); + assert!(matches!( + archive.entry("nested/file.txt").unwrap().unwrap().kind, + EntryKind::File { size: 7, .. } + )); + #[cfg(unix)] + assert!(matches!( + archive.entry("nested/link").unwrap().unwrap().kind, + EntryKind::Symlink { ref target } if target == "file.txt" + )); +} + +#[cfg(unix)] +#[test] +fn manifest_rejects_unsafe_and_non_utf8_symlink_targets() { + use std::os::unix::ffi::OsStringExt; + + let temporary = tempfile::tempdir().unwrap(); + let absolute = temporary.path().join("absolute"); + std::os::unix::fs::symlink("/outside", &absolute).unwrap(); + assert!(matches!( + build_input_manifest(&[absolute]), + Err(FsError::Core { + source: PithosError::InvalidSymlinkTarget { .. }, + .. + }) + )); + + let escaping = temporary.path().join("escaping"); + std::os::unix::fs::symlink("../outside", &escaping).unwrap(); + assert!(matches!( + build_input_manifest(&[escaping]), + Err(FsError::Core { + source: PithosError::InvalidSymlinkTarget { .. }, + .. + }) + )); + + let non_utf8 = temporary.path().join("non-utf8-target"); + std::os::unix::fs::symlink( + Path::new(&std::ffi::OsString::from_vec(b"bad\xff".to_vec())), + &non_utf8, + ) + .unwrap(); + assert!(matches!( + build_input_manifest(&[non_utf8]), + Err(FsError::InvalidUtf8Path { .. }) + )); +} + +#[cfg(unix)] +#[test] +fn manifest_rejects_non_utf8_archive_paths() { + use std::os::unix::ffi::OsStringExt; + + let temporary = tempfile::tempdir().unwrap(); + let input = temporary + .path() + .join(std::ffi::OsString::from_vec(b"input-\xff".to_vec())); + fs::write(&input, b"content").unwrap(); + assert!(matches!( + build_input_manifest(&[input]), + Err(FsError::InvalidUtf8Path { .. }) + )); +} + +#[test] +fn manifest_rejects_duplicate_and_conflicting_archive_paths() { + let temporary = tempfile::tempdir().unwrap(); + let left = temporary.path().join("left"); + let right = temporary.path().join("right"); + fs::create_dir_all(&left).unwrap(); + fs::create_dir_all(&right).unwrap(); + let duplicate_left = left.join("same"); + let duplicate_right = right.join("same"); + fs::write(&duplicate_left, b"left").unwrap(); + fs::write(&duplicate_right, b"right").unwrap(); + assert!(matches!( + build_input_manifest(&[duplicate_left, duplicate_right]), + Err(FsError::Core { + source: PithosError::PathOccupied(_), + .. + }) + )); + + let file = left.join("node"); + fs::write(&file, b"file").unwrap(); + fs::create_dir_all(right.join("node")).unwrap(); + fs::write(right.join("node/child"), b"child").unwrap(); + assert!(matches!( + build_input_manifest(&[file, right]), + Err(FsError::Core { + source: PithosError::InvalidArchivePath { .. }, + .. + }) + )); +} + +#[test] +fn manifest_rejects_missing_input_and_retains_preflight_opened_files() { + let temporary = tempfile::tempdir().unwrap(); + assert!(matches!( + build_input_manifest(&[temporary.path().join("missing")]), + Err(FsError::Host { .. }) + )); + + let first = temporary.path().join("first"); + let second = temporary.path().join("second"); + fs::write(&first, b"first").unwrap(); + fs::write(&second, b"second").unwrap(); + let manifest = build_input_manifest(&[first, second.clone()]).unwrap(); + fs::remove_file(second).unwrap(); + + let sender = PrivateKey::generate(); + let recipient = sender.public_key(); + let mut writer = ArchiveWriter::create( + Vec::new(), + WriteOptions::new(sender.duplicate(), vec![recipient]), + ) + .unwrap(); + manifest + .ingest(&mut writer, ProcessingOptions::default()) + .unwrap(); + let archive = Archive::open( + MemorySource::new(writer.finish().unwrap()), + OpenOptions::default().with_access_keys(AccessKeys::new().with_key(sender)), + ) + .unwrap(); + assert!(archive.entry("first").unwrap().is_some()); + let mut contents = Vec::new(); + archive.copy_to("second", &mut contents).unwrap(); + assert_eq!(contents, b"second"); +} + +#[test] +fn wide_directory_ingestion_and_archive_traversal_complete() { + let temporary = tempfile::tempdir().unwrap(); + let source = temporary.path().join("source"); + fs::create_dir(&source).unwrap(); + for index in 0..1_000 { + fs::write(source.join(format!("entry-{index:04}")), b"content").unwrap(); + } + + let archive = write_manifest(&build_input_manifest(&[source]).unwrap()); + + assert_eq!(archive.entries().len(), 1_000); + assert!(archive.entry("entry-0000").unwrap().is_some()); + assert!(archive.entry("entry-0999").unwrap().is_some()); +} diff --git a/crates/pithos_lib/tests/streaming_memory.rs b/crates/pithos_lib/tests/streaming_memory.rs new file mode 100644 index 0000000..a23cada --- /dev/null +++ b/crates/pithos_lib/tests/streaming_memory.rs @@ -0,0 +1,104 @@ +use peak_alloc::PeakAlloc; +use pithos_lib::archive::{ + ArchivePath, ArchiveWriter, CdcConfig, EntryMetadata, ProcessingOptions, WriteOptions, +}; +use pithos_lib::crypto::PrivateKey; +use std::io::{Read, Write}; + +#[global_allocator] +static PEAK_ALLOC: PeakAlloc = PeakAlloc; + +struct VirtualReader { + remaining: u64, + state: u64, +} + +impl VirtualReader { + fn new(length: u64) -> Self { + Self { + remaining: length, + state: 0x9e37_79b9_7f4a_7c15, + } + } +} + +impl Read for VirtualReader { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + let count = self.remaining.min(buffer.len() as u64) as usize; + for byte in &mut buffer[..count] { + self.state = self + .state + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add(1); + *byte = (self.state >> 56) as u8; + } + self.remaining -= count as u64; + Ok(count) + } +} + +#[derive(Debug, Default)] +struct CountingSink(u64); + +impl Write for CountingSink { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0 += bytes.len() as u64; + Ok(bytes.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +fn streamed_peak(logical_bytes: u64) -> (u64, u64) { + let sender = PrivateKey::generate(); + let recipient = sender.public_key(); + let baseline = PEAK_ALLOC.current_usage(); + PEAK_ALLOC.reset_peak_usage(); + let mut writer = ArchiveWriter::create( + CountingSink::default(), + WriteOptions::new(sender, vec![recipient]).with_cdc(CdcConfig::DEFAULT), + ) + .unwrap(); + writer + .add_file( + ArchivePath::new("streamed.bin").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(true, 3).unwrap(), + Some(logical_bytes), + VirtualReader::new(logical_bytes), + ) + .unwrap(); + let sink = writer.finish().unwrap(); + ( + PEAK_ALLOC.peak_usage().saturating_sub(baseline) as u64, + sink.0, + ) +} + +#[test] +fn streamed_writer_memory_is_bounded_by_block_working_set() { + let small_logical_bytes = CdcConfig::DEFAULT.max_size() as u64; + let large_logical_bytes = 32 * 1024 * 1024; + let (small_peak, small_output) = streamed_peak(small_logical_bytes); + let (large_peak, large_output) = streamed_peak(large_logical_bytes); + let permitted_delta = 8 * 1024 * 1024u64; + let peak_delta = large_peak.saturating_sub(small_peak); + + println!( + "{{\"schema\":\"pithos-memory-v1\",\"workload\":\"streamed_writer_memory\",\"small_logical_bytes\":{small_logical_bytes},\"large_logical_bytes\":{large_logical_bytes},\"cdc_min_bytes\":{},\"cdc_avg_bytes\":{},\"cdc_max_bytes\":{},\"small_peak_heap_bytes\":{small_peak},\"large_peak_heap_bytes\":{large_peak},\"peak_delta_bytes\":{peak_delta},\"permitted_delta_bytes\":{permitted_delta},\"small_output_bytes\":{small_output},\"large_output_bytes\":{large_output}}}", + CdcConfig::DEFAULT.min_size(), + CdcConfig::DEFAULT.avg_size(), + CdcConfig::DEFAULT.max_size(), + ); + + assert!(small_output > 0); + assert!(large_output > small_output); + // Both inputs exceed the configured maximum CDC block. Eight MiB allows the second block's + // directory/reference bookkeeping while rejecting retention of the 16 MiB logical delta. + assert!( + large_peak <= small_peak.saturating_add(permitted_delta), + "large stream peak {large_peak} exceeded small stream peak {small_peak} by more than 8 MiB" + ); +} diff --git a/crates/pithos_lib/tests/writer.rs b/crates/pithos_lib/tests/writer.rs deleted file mode 100644 index d348f0a..0000000 --- a/crates/pithos_lib/tests/writer.rs +++ /dev/null @@ -1,1546 +0,0 @@ -pub mod common; - -use crate::common::util::{ - create_pithos_writer, extract_pithos_entry, load_test_keys, minimal_ro_crate_metadata, - read_pithos_directory, write_zip_entries, -}; -use indexmap::IndexMap; -use pithos_lib::error::PithosError; -use pithos_lib::helpers::directory::DirectoryBuilder; -use pithos_lib::helpers::file_entry_map::KeyQuery; -use pithos_lib::helpers::ro_crate::{LoadedRoCrate, read_ro_crate_directory, read_ro_crate_zip}; -use pithos_lib::helpers::x25519_keys::private_key_from_pem_bytes; -use pithos_lib::io::pithosreader::{PithosReaderSimple, ReaderLimits}; -use pithos_lib::io::pithoswriter::{Content, InputFile, PithosWriter}; -use pithos_lib::model::structs::{ - BlockIndexEntry, BlockLocation, FileEntry, FileType, ProcessingFlags, Reference, -}; -use std::cell::Cell; -use std::fs::{ - File, copy, create_dir_all, read, read_dir, read_link, read_to_string, remove_file, write, -}; -use std::io::{Cursor, Read}; -use std::path::{Path, PathBuf}; -use tempfile::TempDir; -use x25519_dalek::StaticSecret; - -fn copy_directory(source: &Path, destination: &Path) { - create_dir_all(destination).unwrap(); - for item in read_dir(source).unwrap() { - let item = item.unwrap(); - let source_path = item.path(); - let destination_path = destination.join(item.file_name()); - if item.file_type().unwrap().is_dir() { - copy_directory(&source_path, &destination_path); - } else { - copy(&source_path, &destination_path).unwrap(); - } - } -} - -fn convert_ro_crate( - temp_dir: &TempDir, - loaded: &LoadedRoCrate, - cdc: Option<(usize, usize, usize)>, -) -> (PathBuf, StaticSecret) { - let (path, reader_key, mut writer) = create_pithos_writer(temp_dir, cdc); - writer.write_file_header().unwrap(); - writer.process_ro_crate(loaded).unwrap(); - writer.write_directory().unwrap(); - (path, reader_key) -} - -fn metadata_reference() -> Reference { - Reference { - target_file_id: 0, - relationship: 0, - } -} - -struct CountingReader { - cursor: Cursor>, - reads: std::rc::Rc>, -} - -impl Read for CountingReader { - fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { - self.reads.set(self.reads.get() + 1); - self.cursor.read(buffer) - } -} - -fn read_zip_member(path: &Path, name: &str) -> Vec { - let mut archive = zip::ZipArchive::new(File::open(path).unwrap()).unwrap(); - let mut member = archive.by_name(name).unwrap(); - let mut bytes = Vec::new(); - member.read_to_end(&mut bytes).unwrap(); - bytes -} - -fn assert_compression_round_trip( - payload: &[u8], - compression_level: u8, - encrypt: bool, - cdc: Option<(usize, usize, usize)>, -) { - let temp_dir = TempDir::new().unwrap(); - let source_path = temp_dir.path().join("input.bin"); - write(&source_path, payload).unwrap(); - - let (pithos_path, reader_key, mut writer) = create_pithos_writer(&temp_dir, cdc); - writer.write_file_header().unwrap(); - writer - .process_input(InputFile { - file_type: FileType::Data, - inner_path: "input.bin".to_string(), - data: Content::File(source_path.to_string_lossy().into_owned()), - metadata: None, - encrypt, - compression_level: Some(compression_level), - }) - .unwrap(); - writer.write_directory().unwrap(); - - let (directory, _) = read_pithos_directory(&pithos_path, &reader_key).unwrap(); - for block in directory.blocks.values() { - assert_eq!(block.flags.is_encrypted(), encrypt); - let stored_level = block.flags.get_compression_level(); - if compression_level == 0 { - assert_eq!(stored_level, 0); - if !encrypt { - assert_eq!(block.stored_size, block.original_size); - } - } else { - assert!(stored_level == 0 || stored_level == compression_level); - } - } - assert_eq!( - extract_pithos_entry(&pithos_path, &reader_key, "input.bin", &temp_dir), - payload - ); -} - -fn deterministic_incompressible_payload(length: usize) -> Vec { - let mut state = 0x243f_6a88_u32; - (0..length) - .map(|index| { - state = state - .wrapping_mul(1_664_525) - .wrapping_add(1_013_904_223) - .rotate_left((index % 31) as u32); - (state ^ (index as u32)).to_le_bytes()[index % 4] - }) - .collect() -} - -#[test] -fn test_compression_levels_round_trip() { - let payloads = [ - ("repeated", vec![b'A'; 16 * 1024]), - ( - "incompressible", - deterministic_incompressible_payload(16 * 1024), - ), - ]; - - for (_, payload) in payloads { - for compression_level in 0..=7 { - for encrypt in [false, true] { - assert_compression_round_trip(&payload, compression_level, encrypt, None); - } - } - } -} - -#[test] -fn test_compression_empty_level_zero_round_trip() { - for encrypt in [false, true] { - let temp_dir = TempDir::new().unwrap(); - let source_path = temp_dir.path().join("empty.bin"); - write(&source_path, []).unwrap(); - - let (pithos_path, reader_key, mut writer) = create_pithos_writer(&temp_dir, None); - writer.write_file_header().unwrap(); - writer - .process_input(InputFile { - file_type: FileType::Data, - inner_path: "empty.bin".to_string(), - data: Content::File(source_path.to_string_lossy().into_owned()), - metadata: None, - encrypt, - compression_level: Some(0), - }) - .unwrap(); - writer.write_directory().unwrap(); - - let (directory, _) = read_pithos_directory(&pithos_path, &reader_key).unwrap(); - assert!(directory.blocks.is_empty()); - assert_eq!( - extract_pithos_entry(&pithos_path, &reader_key, "empty.bin", &temp_dir), - Vec::::new() - ); - } -} - -#[test] -fn test_compression_multiblock_level_zero_round_trip() { - let payload = deterministic_incompressible_payload(16 * 1024); - for encrypt in [false, true] { - let temp_dir = TempDir::new().unwrap(); - let source_path = temp_dir.path().join("multiblock.bin"); - write(&source_path, &payload).unwrap(); - - let (pithos_path, reader_key, mut writer) = - create_pithos_writer(&temp_dir, Some((256, 512, 1024))); - writer.write_file_header().unwrap(); - writer - .process_input(InputFile { - file_type: FileType::Data, - inner_path: "multiblock.bin".to_string(), - data: Content::File(source_path.to_string_lossy().into_owned()), - metadata: None, - encrypt, - compression_level: Some(0), - }) - .unwrap(); - writer.write_directory().unwrap(); - - let (directory, _) = read_pithos_directory(&pithos_path, &reader_key).unwrap(); - assert!(directory.blocks.len() > 1); - for block in directory.blocks.values() { - assert_eq!(block.flags.get_compression_level(), 0); - assert_eq!(block.flags.is_encrypted(), encrypt); - } - assert_eq!( - extract_pithos_entry(&pithos_path, &reader_key, "multiblock.bin", &temp_dir), - payload - ); - } -} - -fn write_raw_zip(path: &Path, entries: &[(&str, &[u8], u32)], overlap_duplicates: bool) { - let mut archive = Vec::new(); - let mut central_directory = Vec::new(); - let mut offsets = Vec::with_capacity(entries.len()); - for (index, (name, content, permissions)) in entries.iter().enumerate() { - let entry_name = *name; - let name = entry_name.as_bytes(); - let content = *content; - let offset = archive.len() as u32; - offsets.push(offset); - let checksum = crc32fast::hash(content); - - archive.extend_from_slice(&0x0403_4b50u32.to_le_bytes()); - archive.extend_from_slice(&20u16.to_le_bytes()); - archive.extend_from_slice(&0u16.to_le_bytes()); - archive.extend_from_slice(&0u16.to_le_bytes()); - archive.extend_from_slice(&0u16.to_le_bytes()); - archive.extend_from_slice(&0u16.to_le_bytes()); - archive.extend_from_slice(&checksum.to_le_bytes()); - archive.extend_from_slice(&(content.len() as u32).to_le_bytes()); - archive.extend_from_slice(&(content.len() as u32).to_le_bytes()); - archive.extend_from_slice(&(name.len() as u16).to_le_bytes()); - archive.extend_from_slice(&0u16.to_le_bytes()); - archive.extend_from_slice(name); - archive.extend_from_slice(content); - - central_directory.extend_from_slice(&0x0201_4b50u32.to_le_bytes()); - central_directory.extend_from_slice(&20u16.to_le_bytes()); - central_directory.extend_from_slice(&20u16.to_le_bytes()); - central_directory.extend_from_slice(&0u16.to_le_bytes()); - central_directory.extend_from_slice(&0u16.to_le_bytes()); - central_directory.extend_from_slice(&0u16.to_le_bytes()); - central_directory.extend_from_slice(&0u16.to_le_bytes()); - central_directory.extend_from_slice(&checksum.to_le_bytes()); - central_directory.extend_from_slice(&(content.len() as u32).to_le_bytes()); - central_directory.extend_from_slice(&(content.len() as u32).to_le_bytes()); - central_directory.extend_from_slice(&(name.len() as u16).to_le_bytes()); - central_directory.extend_from_slice(&0u16.to_le_bytes()); - central_directory.extend_from_slice(&0u16.to_le_bytes()); - central_directory.extend_from_slice(&0u16.to_le_bytes()); - central_directory.extend_from_slice(&0u16.to_le_bytes()); - let central_offset = if overlap_duplicates { - entries[..index] - .iter() - .position(|(previous_name, _, _)| { - previous_name.trim_start_matches("./") == entry_name.trim_start_matches("./") - }) - .map(|position| offsets[position]) - .unwrap_or(offset) - } else { - offset - }; - central_directory.extend_from_slice(&(*permissions << 16).to_le_bytes()); - central_directory.extend_from_slice(¢ral_offset.to_le_bytes()); - central_directory.extend_from_slice(name); - } - - let central_offset = archive.len() as u32; - archive.extend_from_slice(¢ral_directory); - archive.extend_from_slice(&0x0605_4b50u32.to_le_bytes()); - archive.extend_from_slice(&0u16.to_le_bytes()); - archive.extend_from_slice(&0u16.to_le_bytes()); - archive.extend_from_slice(&(entries.len() as u16).to_le_bytes()); - archive.extend_from_slice(&(entries.len() as u16).to_le_bytes()); - archive.extend_from_slice(&(central_directory.len() as u32).to_le_bytes()); - archive.extend_from_slice(¢ral_offset.to_le_bytes()); - archive.extend_from_slice(&0u16.to_le_bytes()); - write(path, archive).unwrap(); -} - -fn write_duplicate_zip(path: &Path, entries: &[(&str, &[u8])]) { - let entries = entries - .iter() - .map(|(name, content)| (*name, *content, 0)) - .collect::>(); - write_raw_zip(path, &entries, true); -} - -#[test] -fn test_single_file() { - // Dummy file - let input_file = InputFile { - file_type: FileType::Data, - inner_path: "t8.shakespeare.txt".to_string(), - data: Content::File("tests/data/t8.shakespeare.sample.txt".to_string()), - metadata: None, - encrypt: true, - compression_level: Some(7), - }; - - // Load dummy keys - let (sender_key, r1_key, _) = load_test_keys(); - - // Prepare PithosWriter - let temp_dir = TempDir::new().unwrap(); - let outfile = File::create(temp_dir.path().join("single.pith")).unwrap(); - let mut writer = PithosWriter::new(sender_key, vec![r1_key], None, Box::new(outfile)).unwrap(); - - // Process - writer.write_file_header().unwrap(); - writer.process_input(input_file).unwrap(); - writer.write_directory().unwrap(); -} - -#[test] -fn test_append_single_file() { - // Dummy file - let input_file = InputFile { - file_type: FileType::Data, - inner_path: "t8.shakespeare.txt".to_string(), - data: Content::File("tests/data/t8.shakespeare.sample.txt".to_string()), - metadata: None, - encrypt: true, - compression_level: Some(7), - }; - - // Load dummy keys - let (sender_key, r1_key, _) = load_test_keys(); - - // Prepare PithosWriter - let temp_dir = TempDir::new().unwrap(); - let pithos_file_path = temp_dir.path().join("single.append.pith"); - let outfile = File::create(&pithos_file_path).unwrap(); - let mut writer = PithosWriter::new(sender_key, vec![r1_key], None, Box::new(outfile)).unwrap(); - - // Process - writer.write_file_header().unwrap(); - writer.process_input(input_file).unwrap(); - writer.write_directory().unwrap(); - - // Dummy file to append - let input_file = InputFile { - file_type: FileType::Data, - inner_path: "SRR33138449.sample.fastq".to_string(), - data: Content::File("tests/data/SRR33138449.sample.fastq".to_string()), - metadata: None, - encrypt: true, - compression_level: Some(3), - }; - - // Load dummy keys - let (sender_key, r1_key, _) = load_test_keys(); - - // Prepare PithosWriter from existing Pithos file - let reader_keys = vec![r1_key]; - let mut writer = - PithosWriter::new_from_file(sender_key, reader_keys, None, &pithos_file_path).unwrap(); - - writer.process_input(input_file).unwrap(); - writer.write_directory().unwrap(); - - // Read Pithos directory - let reader_secret = private_key_from_pem_bytes( - read_to_string("tests/data/keys/recipient1_private.pem") - .unwrap() - .as_bytes(), - ) - .unwrap(); - let mut reader = PithosReaderSimple::new_with_key(pithos_file_path, reader_secret).unwrap(); - let (directory, _) = reader.read_directory().unwrap(); - - assert_eq!(directory.files.len(), 2); - - let (key, entry) = directory.files.get_entry(&KeyQuery::Id(0)).unwrap(); - assert_eq!(key.id(), 0); - assert_eq!(key.path(), "t8.shakespeare.txt"); - assert_eq!(entry.file_type, FileType::Data); - assert_eq!(entry.file_size, 79463); - - let (key, entry) = directory.files.get_entry(&KeyQuery::Id(1)).unwrap(); - assert_eq!(key.id(), 1); - assert_eq!(key.path(), "SRR33138449.sample.fastq"); - assert_eq!(entry.file_type, FileType::Data); - assert_eq!(entry.file_size, 342848); -} - -#[test] -fn format_006_append_reuses_ancestor_block_without_repointing() { - let temp_dir = TempDir::new().unwrap(); - let payload = "format-006 ancestor block reuse\n".repeat(64).into_bytes(); - let (path, reader_key, mut writer) = create_pithos_writer(&temp_dir, None); - writer.write_file_header().unwrap(); - writer - .process_input(InputFile { - file_type: FileType::Data, - inner_path: "base.bin".into(), - data: Content::Raw(String::from_utf8(payload.clone()).unwrap()), - metadata: None, - encrypt: false, - compression_level: Some(0), - }) - .unwrap(); - writer.write_directory().unwrap(); - drop(writer); - - let base_bytes = read(&path).unwrap(); - let (base_directory, _) = read_pithos_directory(&path, &reader_key).unwrap(); - assert_eq!(base_directory.blocks.len(), 1); - let (hash, original_block) = base_directory.blocks.first().unwrap(); - let hash = *hash; - let original_block = original_block.clone(); - - let (writer_key, reader_public_key, _) = load_test_keys(); - let mut writer = - PithosWriter::new_from_file(writer_key, vec![reader_public_key], None, &path).unwrap(); - writer - .process_input(InputFile { - file_type: FileType::Data, - inner_path: "child.bin".into(), - data: Content::Raw(String::from_utf8(payload.clone()).unwrap()), - metadata: None, - encrypt: false, - compression_level: Some(0), - }) - .unwrap(); - - assert_eq!( - std::fs::metadata(&path).unwrap().len(), - base_bytes.len() as u64 - ); - assert!(writer.get_directory_mut().blocks.is_empty()); - writer.write_directory().unwrap(); - drop(writer); - - let final_bytes = read(&path).unwrap(); - assert!(final_bytes.starts_with(&base_bytes)); - let (directory, _) = read_pithos_directory(&path, &reader_key).unwrap(); - assert_eq!(directory.blocks.len(), 1); - assert_eq!(directory.blocks.get(&hash), Some(&original_block)); - assert!(directory.get_file_by_path("base.bin").is_some()); - assert!(directory.get_file_by_path("child.bin").is_some()); - assert_eq!( - extract_pithos_entry(&path, &reader_key, "base.bin", &temp_dir), - payload - ); - remove_file(temp_dir.path().join("extracted-entry")).unwrap(); - assert_eq!( - extract_pithos_entry(&path, &reader_key, "child.bin", &temp_dir), - payload - ); -} - -#[test] -fn format_006_merge_preserves_older_compatible_block_record() { - let hash = [7; 32]; - let older = BlockIndexEntry { - offset: 10, - stored_size: 20, - original_size: 30, - flags: ProcessingFlags::new(false, Some(0)), - location: BlockLocation::Local, - }; - let newer = BlockIndexEntry { - offset: 40, - stored_size: 50, - original_size: 30, - flags: ProcessingFlags::new(true, Some(7)), - location: BlockLocation::External { - url: "https://example.invalid/block".into(), - }, - }; - let mut older_directory = DirectoryBuilder::new() - .blocks(IndexMap::from_iter([(hash, older.clone())])) - .build() - .unwrap(); - let newer_directory = DirectoryBuilder::new() - .blocks(IndexMap::from_iter([(hash, newer)])) - .build() - .unwrap(); - - older_directory.merge(newer_directory).unwrap(); - - assert_eq!(older_directory.blocks.get(&hash), Some(&older)); -} - -#[test] -fn format_006_merge_rejects_conflicting_original_size() { - let hash = [9; 32]; - let mut older_directory = DirectoryBuilder::new() - .blocks(IndexMap::from_iter([( - hash, - BlockIndexEntry { - offset: 10, - stored_size: 20, - original_size: 30, - flags: ProcessingFlags::new(false, Some(0)), - location: BlockLocation::Local, - }, - )])) - .build() - .unwrap(); - let newer_directory = DirectoryBuilder::new() - .blocks(IndexMap::from_iter([( - hash, - BlockIndexEntry { - offset: 40, - stored_size: 50, - original_size: 31, - flags: ProcessingFlags::new(true, Some(7)), - location: BlockLocation::Local, - }, - )])) - .build() - .unwrap(); - - assert!(matches!( - older_directory.merge(newer_directory), - Err(PithosError::BlockIndexConflict { - hash: actual_hash, - existing_original_size: 30, - new_original_size: 31, - }) if actual_hash == hash - )); -} - -#[test] -fn new_from_file_with_limits_accepts_exact_terminal_directory_length() { - let temp_dir = TempDir::new().unwrap(); - let (path, sender_key, mut writer) = create_pithos_writer(&temp_dir, None); - writer.write_file_header().unwrap(); - writer - .process_input(InputFile { - file_type: FileType::Data, - inner_path: "file.txt".into(), - data: Content::Raw("payload".into()), - metadata: None, - encrypt: true, - compression_level: Some(0), - }) - .unwrap(); - writer.write_directory().unwrap(); - - let bytes = read(&path).unwrap(); - let directory_length = - u64::from_be_bytes(bytes[bytes.len() - 12..bytes.len() - 4].try_into().unwrap()); - let limits = ReaderLimits { - max_directory_bytes: directory_length, - ..ReaderLimits::default() - }; - PithosWriter::new_from_file_with_limits(sender_key, vec![], None, &path, limits).unwrap(); -} - -#[test] -fn new_from_file_with_limits_rejects_smaller_terminal_directory_length() { - let temp_dir = TempDir::new().unwrap(); - let (path, sender_key, mut writer) = create_pithos_writer(&temp_dir, None); - writer.write_file_header().unwrap(); - writer.write_directory().unwrap(); - - let bytes = read(&path).unwrap(); - let directory_length = - u64::from_be_bytes(bytes[bytes.len() - 12..bytes.len() - 4].try_into().unwrap()); - let limits = ReaderLimits { - max_directory_bytes: directory_length - 1, - ..ReaderLimits::default() - }; - let error = - match PithosWriter::new_from_file_with_limits(sender_key, vec![], None, &path, limits) { - Ok(_) => panic!("directory limit should reject the archive"), - Err(error) => error, - }; - assert!(matches!( - error, - PithosError::LimitExceeded { - field: "directory", - .. - } - )); -} - -#[test] -fn test_multiple_files() { - // Dummy files with metadata - let input_files = vec![ - InputFile { - file_type: FileType::Data, - inner_path: "SRR33138449.fastq".to_string(), - data: Content::File("tests/data/SRR33138449.sample.fastq".to_string()), - metadata: Some(Content::Raw( - r#"{ - "@id": "SRR33138449.fastq", - "@type": "File", - "name": "SRR33138449 run sequencing reads", - "description": "Something something description" - }"# - .to_string(), - )), - encrypt: true, - compression_level: Some(1), - }, - InputFile { - file_type: FileType::Data, - inner_path: "t8.shakespeare.txt".to_string(), - data: Content::File("tests/data/t8.shakespeare.sample.txt".to_string()), - metadata: Some(Content::Raw( - r#"{"@id": "t8.shakespeare.txt","@type": "File"}"#.to_string(), - )), - encrypt: true, - compression_level: Some(7), - }, - ]; - - // Load dummy keys - let (sender_key, r1_key, _) = load_test_keys(); - - // Prepare input for writer - let temp_dir = TempDir::new().unwrap(); - let outfile = File::create(temp_dir.path().join("multifile.pith")).unwrap(); - let mut writer = PithosWriter::new(sender_key, vec![r1_key], None, Box::new(outfile)).unwrap(); - - // Process - writer.write_file_header().unwrap(); - writer.process_input_files(input_files).unwrap(); - writer.write_directory().unwrap(); -} - -#[test] -fn test_append_multiple_files() { - // Dummy files - let input_files = vec![ - InputFile { - file_type: FileType::Data, - inner_path: "SRR33138449.fastq".to_string(), - data: Content::File("tests/data/SRR33138449.sample.fastq".to_string()), - metadata: None, - encrypt: true, - compression_level: Some(1), - }, - InputFile { - file_type: FileType::Data, - inner_path: "t8.shakespeare.txt".to_string(), - data: Content::File("tests/data/t8.shakespeare.sample.txt".to_string()), - metadata: None, - encrypt: true, - compression_level: Some(7), - }, - ]; - - // Load dummy keys - let (sender_key, r1_key, _) = load_test_keys(); - let reader_keys = vec![r1_key]; - - // Prepare PithosWriter - let temp_dir = TempDir::new().unwrap(); - let file_path = temp_dir.path().join("multifile.append.pith"); - let outfile = File::create(&file_path).unwrap(); - let mut writer = PithosWriter::new( - sender_key.clone(), - reader_keys.clone(), - None, - Box::new(outfile), - ) - .unwrap(); - - // Process - writer.write_file_header().unwrap(); - writer.process_input_files(input_files).unwrap(); - writer.write_directory().unwrap(); - - // Dummy files - let append_files = vec![ - InputFile { - file_type: FileType::Data, - inner_path: "conclusions.txt".to_string(), - data: Content::File("tests/data/dummy_dir/conclusions.txt".to_string()), - metadata: None, - encrypt: true, - compression_level: Some(3), - }, - InputFile { - file_type: FileType::Data, - inner_path: "dummy_results.txt".to_string(), - data: Content::File("tests/data/dummy_dir/dummy_results.txt".to_string()), - metadata: None, - encrypt: true, - compression_level: Some(3), - }, - ]; - - // Prepare PithosWriter - let mut writer = - PithosWriter::new_from_file(sender_key, reader_keys, None, &file_path).unwrap(); - - // Append files - writer.process_input_files(append_files).unwrap(); - writer.write_directory().unwrap(); - - // Read directory and validate - let reader_secret = private_key_from_pem_bytes( - read_to_string("tests/data/keys/recipient1_private.pem") - .unwrap() - .as_bytes(), - ) - .unwrap(); - let mut reader = PithosReaderSimple::new_with_key(file_path, reader_secret).unwrap(); - let (directory, _) = reader.read_directory().unwrap(); - - assert_eq!(directory.files.len(), 4); - - let (key, entry) = directory.files.get_entry(&KeyQuery::Id(0)).unwrap(); - assert_eq!(key.id(), 0); - assert_eq!(key.path(), "SRR33138449.fastq"); - assert_eq!(entry.file_type, FileType::Data); - assert_eq!(entry.file_size, 342848); - - let (key, entry) = directory.files.get_entry(&KeyQuery::Id(1)).unwrap(); - assert_eq!(key.id(), 1); - assert_eq!(key.path(), "t8.shakespeare.txt"); - assert_eq!(entry.file_type, FileType::Data); - assert_eq!(entry.file_size, 79463); - - let (key, entry) = directory.files.get_entry(&KeyQuery::Id(2)).unwrap(); - assert_eq!(key.id(), 2); - assert_eq!(key.path(), "conclusions.txt"); - assert_eq!(entry.file_type, FileType::Data); - assert_eq!(entry.file_size, 91); - - let (key, entry) = directory.files.get_entry(&KeyQuery::Id(3)).unwrap(); - assert_eq!(key.id(), 3); - assert_eq!(key.path(), "dummy_results.txt"); - assert_eq!(entry.file_type, FileType::Data); - assert_eq!(entry.file_size, 558); -} - -#[test] -fn test_directory() { - // Load dummy keys - let (sender_key, r1_key, _) = load_test_keys(); - - // Prepare PithosWriter - let input_directory = "tests/data/dummy_dir"; - let temp_dir = TempDir::new().unwrap(); - let file_path = temp_dir.path().join("directory.pith"); - let outfile = File::create(&file_path).unwrap(); - - let mut writer = PithosWriter::new(sender_key, vec![r1_key], None, Box::new(outfile)).unwrap(); - writer.write_file_header().unwrap(); - writer.process_directory(input_directory).unwrap(); - writer.write_directory().unwrap(); - - // Read directory and validate - let reader_secret = private_key_from_pem_bytes( - read_to_string("tests/data/keys/recipient1_private.pem") - .unwrap() - .as_bytes(), - ) - .unwrap(); - let mut reader = PithosReaderSimple::new_with_key(file_path, reader_secret).unwrap(); - let (directory, _) = reader.read_directory().unwrap(); - - assert_eq!(directory.files.len(), 8); - - for (id, path, filetype, size) in [ - (0, "dataset", FileType::Directory, 0), - ( - 1, - "dataset/brain_gene_expression_data", - FileType::Directory, - 0, - ), - (2, "literature", FileType::Directory, 0), - (3, "conclusions.txt", FileType::Data, 91), - ( - 4, - "dataset/brain_gene_expression_data/brain_data_assession_number.txt", - FileType::Data, - 29, - ), - (5, "dummy_results.txt", FileType::Data, 558), - ( - 6, - "literature/Hodges06_human_brain_Affy.pdf", - FileType::Data, - 359786, - ), - (7, "ro-crate-metadata.json", FileType::Data, 2953), - ] { - let (key, entry) = directory.files.get_entry(&KeyQuery::Id(id)).unwrap(); - assert_eq!(key.id(), id); - assert_eq!(key.path(), path); - assert_eq!(entry.file_type, filetype); - - if entry.file_type != FileType::Directory { - assert_eq!(entry.file_size, size); - } - } -} - -#[test] -fn test_rocrate_directory_conversion() { - let temp_dir = TempDir::new().unwrap(); - let loaded = read_ro_crate_directory("tests/data/dummy_dir").unwrap(); - let (pithos_path, reader_key) = convert_ro_crate(&temp_dir, &loaded, None); - let (directory, entries) = read_pithos_directory(&pithos_path, &reader_key).unwrap(); - - assert_eq!(entries.len(), 8); - assert_eq!( - entries - .iter() - .map(|(path, _, _, _)| path.as_str()) - .collect::>(), - vec![ - "conclusions.txt", - "dataset", - "dataset/brain_gene_expression_data", - "dataset/brain_gene_expression_data/brain_data_assession_number.txt", - "dummy_results.txt", - "literature", - "literature/Hodges06_human_brain_Affy.pdf", - "ro-crate-metadata.json", - ] - ); - - let metadata = directory - .get_file_by_path("ro-crate-metadata.json") - .unwrap(); - assert_eq!( - directory.files.get_id_by_path("ro-crate-metadata.json"), - Some(0) - ); - assert_eq!(metadata.file_type, FileType::Metadata); - assert_eq!(metadata.file_size, 2953); - assert_eq!( - extract_pithos_entry( - &pithos_path, - &reader_key, - "ro-crate-metadata.json", - &temp_dir - ), - read("tests/data/dummy_dir/ro-crate-metadata.json").unwrap() - ); - - for path in [ - "dataset", - "dataset/brain_gene_expression_data", - "literature", - ] { - let entry = directory.get_file_by_path(path).unwrap(); - assert_eq!(entry.file_type, FileType::Directory); - assert!(entry.references.is_empty()); - } - for path in [ - "conclusions.txt", - "dataset/brain_gene_expression_data/brain_data_assession_number.txt", - "dummy_results.txt", - "literature/Hodges06_human_brain_Affy.pdf", - ] { - let entry = directory.get_file_by_path(path).unwrap(); - assert_eq!(entry.file_type, FileType::Data); - assert!(entry.references.contains(&metadata_reference())); - } - for (path, size) in [ - ("conclusions.txt", 91), - ( - "dataset/brain_gene_expression_data/brain_data_assession_number.txt", - 29, - ), - ("dummy_results.txt", 558), - ("literature/Hodges06_human_brain_Affy.pdf", 359786), - ] { - assert_eq!(directory.get_file_by_path(path).unwrap().file_size, size); - } -} - -#[test] -fn test_rocrate_directory_links_unlisted_files() { - let temp_dir = TempDir::new().unwrap(); - let source = temp_dir.path().join("crate"); - copy_directory(Path::new("tests/data/dummy_dir"), &source); - write(source.join("unlisted.txt"), b"not in @graph").unwrap(); - create_dir_all(source.join("nested")).unwrap(); - write( - source.join("nested/ro-crate-metadata.json.backup"), - b"ordinary data", - ) - .unwrap(); - - let loaded = read_ro_crate_directory(&source).unwrap(); - let (pithos_path, reader_key) = convert_ro_crate(&temp_dir, &loaded, None); - let (directory, _) = read_pithos_directory(&pithos_path, &reader_key).unwrap(); - - for path in ["unlisted.txt", "nested/ro-crate-metadata.json.backup"] { - let entry = directory.get_file_by_path(path).unwrap(); - assert_eq!(entry.file_type, FileType::Data); - assert!(entry.references.contains(&metadata_reference())); - } - assert_eq!( - extract_pithos_entry(&pithos_path, &reader_key, "unlisted.txt", &temp_dir), - b"not in @graph" - ); -} - -#[test] -fn test_rocrate_directory_requires_exact_metadata_path() { - let temp_dir = TempDir::new().unwrap(); - let source = temp_dir.path().join("crate"); - copy_directory(Path::new("tests/data/dummy_dir"), &source); - remove_file(source.join("ro-crate-metadata.json")).unwrap(); - create_dir_all(source.join("wrapper")).unwrap(); - write( - source.join("wrapper/ro-crate-metadata.json"), - minimal_ro_crate_metadata(&[]), - ) - .unwrap(); - - let error = read_ro_crate_directory(&source).unwrap_err(); - assert!(matches!(error, PithosError::MissingRoCrateMetadata(_))); - - let empty = TempDir::new().unwrap(); - let error = read_ro_crate_directory(empty.path()).unwrap_err(); - assert!(matches!(error, PithosError::MissingRoCrateMetadata(_))); -} - -#[test] -fn test_rocrate_zip_conversion() { - let temp_dir = TempDir::new().unwrap(); - let loaded = read_ro_crate_zip("tests/data/ro-crate.zip").unwrap(); - let (pithos_path, reader_key) = convert_ro_crate(&temp_dir, &loaded, None); - let (directory, entries) = read_pithos_directory(&pithos_path, &reader_key).unwrap(); - - assert_eq!(entries.len(), 7); - let metadata = directory - .get_file_by_path("ro-crate-metadata.json") - .unwrap(); - assert_eq!( - directory.files.get_id_by_path("ro-crate-metadata.json"), - Some(0) - ); - assert_eq!(metadata.file_type, FileType::Metadata); - assert_eq!(metadata.file_size, 15509); - assert_eq!( - extract_pithos_entry( - &pithos_path, - &reader_key, - "ro-crate-metadata.json", - &temp_dir - ), - read_zip_member( - Path::new("tests/data/ro-crate.zip"), - "ro-crate-metadata.json" - ) - ); - - for (path, entry) in directory.files.iter().map(|(_, path, entry)| (path, entry)) { - if entry.file_type == FileType::Data { - assert!(entry.references.contains(&metadata_reference()), "{path}"); - } else if entry.file_type == FileType::Directory { - assert!(entry.references.is_empty(), "{path}"); - } - } -} - -#[test] -fn test_rocrate_zip_synthesizes_parent_directories() { - let metadata = minimal_ro_crate_metadata(&["nested/file.txt"]); - let options = zip::write::SimpleFileOptions::default(); - - for explicit_directory in [false, true] { - let temp_dir = TempDir::new().unwrap(); - let zip_path = temp_dir.path().join("crate.zip"); - let mut entries = vec![ - ("ro-crate-metadata.json", metadata.as_bytes(), options), - ("nested/file.txt", b"payload".as_slice(), options), - ]; - if explicit_directory { - entries.push(("nested/", b"".as_slice(), options)); - } - write_zip_entries(&zip_path, &entries); - - let loaded = read_ro_crate_zip(&zip_path).unwrap(); - let (pithos_path, reader_key) = convert_ro_crate(&temp_dir, &loaded, None); - let (directory, entries) = read_pithos_directory(&pithos_path, &reader_key).unwrap(); - - assert_eq!(entries.len(), 3); - assert_eq!( - entries - .iter() - .filter( - |(path, file_type, _, _)| path == "nested" && *file_type == FileType::Directory - ) - .count(), - 1 - ); - assert_eq!( - directory - .get_file_by_path("nested/file.txt") - .unwrap() - .file_size, - 7 - ); - } -} - -#[test] -fn test_rocrate_zip_preserves_and_extracts_symlink() { - let temp_dir = TempDir::new().unwrap(); - let zip_path = temp_dir.path().join("symlink.zip"); - let target = PathBuf::from("target"); - let target_string = "target"; - let metadata = minimal_ro_crate_metadata(&[]); - write_raw_zip( - &zip_path, - &[ - ("ro-crate-metadata.json", metadata.as_bytes(), 0), - ("link", target_string.as_bytes(), 0o120777), - ], - false, - ); - - let loaded = read_ro_crate_zip(&zip_path).unwrap(); - let (pithos_path, reader_key) = convert_ro_crate(&temp_dir, &loaded, None); - let (directory, _) = read_pithos_directory(&pithos_path, &reader_key).unwrap(); - let entry = directory.get_file_by_path("link").unwrap(); - - assert_eq!(entry.file_type, FileType::Symlink); - assert_eq!(entry.symlink_target.as_deref(), Some(target_string)); - assert!(entry.references.is_empty()); - - let output_dir = temp_dir.path().join("extracted"); - create_dir_all(&output_dir).unwrap(); - let mut reader = PithosReaderSimple::new_with_key(&pithos_path, reader_key).unwrap(); - let (directory, _) = reader.read_directory().unwrap(); - reader - .read_file("link", &directory, Some(&output_dir), None) - .unwrap(); - - assert_eq!(read_link(output_dir.join("link")).unwrap(), target); -} - -#[test] -fn test_writer_rejects_unsafe_paths_before_block_output() { - let temp_dir = TempDir::new().unwrap(); - let (path, _key, mut writer) = create_pithos_writer(&temp_dir, None); - writer.write_file_header().unwrap(); - let before = std::fs::metadata(&path).unwrap().len(); - let result = writer.process_input(InputFile { - file_type: FileType::Data, - inner_path: "../outside".into(), - data: Content::Raw("payload".into()), - metadata: None, - encrypt: false, - compression_level: Some(0), - }); - assert!(matches!( - result, - Err(PithosError::InvalidArchivePath { .. }) - )); - assert_eq!(std::fs::metadata(&path).unwrap().len(), before); -} - -#[test] -fn test_writer_rejects_candidate_ancestor_before_metadata_output() { - let temp_dir = TempDir::new().unwrap(); - let (path, _key, mut writer) = create_pithos_writer(&temp_dir, None); - writer.write_file_header().unwrap(); - writer - .process_input(InputFile { - file_type: FileType::Data, - inner_path: "a".into(), - data: Content::Raw("existing".into()), - metadata: None, - encrypt: false, - compression_level: Some(0), - }) - .unwrap(); - let before = std::fs::metadata(&path).unwrap().len(); - assert!( - writer - .process_input(InputFile { - file_type: FileType::Data, - inner_path: "a/child".into(), - data: Content::Raw("child".into()), - metadata: Some(Content::Raw("metadata".into())), - encrypt: false, - compression_level: Some(0), - }) - .is_err() - ); - assert_eq!(std::fs::metadata(&path).unwrap().len(), before); -} - -#[test] -fn test_writer_duplicate_main_preflight_is_atomic() { - let temp_dir = TempDir::new().unwrap(); - let (path, _key, mut writer) = create_pithos_writer(&temp_dir, None); - writer.write_file_header().unwrap(); - writer - .process_input(InputFile { - file_type: FileType::Data, - inner_path: "duplicate".into(), - data: Content::Raw("existing".into()), - metadata: None, - encrypt: false, - compression_level: Some(0), - }) - .unwrap(); - let before_bytes = read(&path).unwrap(); - let before_directory = writer.get_directory_mut().clone(); - - let error = writer - .process_input(InputFile { - file_type: FileType::Data, - inner_path: "duplicate".into(), - data: Content::Raw("non-empty data".into()), - metadata: Some(Content::Raw("metadata".into())), - encrypt: false, - compression_level: Some(0), - }) - .unwrap_err(); - - assert!(matches!(error, PithosError::PathOccupied(_))); - assert_eq!(read(&path).unwrap(), before_bytes); - assert_eq!(*writer.get_directory_mut(), before_directory); -} - -#[test] -fn test_writer_generated_metadata_duplicate_preflight_is_atomic() { - let temp_dir = TempDir::new().unwrap(); - let (path, _key, mut writer) = create_pithos_writer(&temp_dir, None); - writer.write_file_header().unwrap(); - writer - .process_input(InputFile { - file_type: FileType::Metadata, - inner_path: "generated.meta".into(), - data: Content::Raw("existing metadata".into()), - metadata: None, - encrypt: false, - compression_level: Some(0), - }) - .unwrap(); - let before_bytes = read(&path).unwrap(); - let before_directory = writer.get_directory_mut().clone(); - - let error = writer - .process_input(InputFile { - file_type: FileType::Data, - inner_path: "generated".into(), - data: Content::Raw("data".into()), - metadata: Some(Content::Raw("new metadata".into())), - encrypt: false, - compression_level: Some(0), - }) - .unwrap_err(); - - assert!(matches!(error, PithosError::PathOccupied(_))); - assert_eq!(read(&path).unwrap(), before_bytes); - assert_eq!(*writer.get_directory_mut(), before_directory); -} - -#[test] -fn test_writer_process_file_entry_duplicate_preflight_is_atomic() { - let temp_dir = TempDir::new().unwrap(); - let (path, _key, mut writer) = create_pithos_writer(&temp_dir, None); - writer.write_file_header().unwrap(); - writer - .process_input(InputFile { - file_type: FileType::Data, - inner_path: "duplicate".into(), - data: Content::Raw("existing".into()), - metadata: None, - encrypt: false, - compression_level: Some(0), - }) - .unwrap(); - let before_bytes = read(&path).unwrap(); - let before_directory = writer.get_directory_mut().clone(); - let mut entry = - FileEntry::new_from_content(FileType::Data, &Content::Raw("data".into())).unwrap(); - let original_entry = entry.clone(); - let reads = std::rc::Rc::new(Cell::new(0)); - - let error = writer - .process_file_entry( - "duplicate", - &mut entry, - &ProcessingFlags::new(false, Some(0)), - CountingReader { - cursor: Cursor::new(b"unconsumed data".to_vec()), - reads: reads.clone(), - }, - ) - .unwrap_err(); - - assert!(matches!(error, PithosError::PathOccupied(_))); - assert_eq!(reads.get(), 0); - assert_eq!(entry, original_entry); - assert_eq!(read(&path).unwrap(), before_bytes); - assert_eq!(*writer.get_directory_mut(), before_directory); -} - -#[test] -fn test_writer_append_active_segment_duplicate_preflight_preserves_base_archive() { - let temp_dir = TempDir::new().unwrap(); - let (path, reader_key, mut writer) = create_pithos_writer(&temp_dir, None); - writer.write_file_header().unwrap(); - writer - .process_input(InputFile { - file_type: FileType::Data, - inner_path: "base".into(), - data: Content::Raw("base data".into()), - metadata: None, - encrypt: false, - compression_level: Some(0), - }) - .unwrap(); - writer.write_directory().unwrap(); - drop(writer); - - let before_bytes = read(&path).unwrap(); - let (writer_key, reader_public_key, _) = load_test_keys(); - let mut writer = - PithosWriter::new_from_file(writer_key, vec![reader_public_key], None, &path).unwrap(); - writer - .process_input(InputFile { - file_type: FileType::Directory, - inner_path: "duplicate".into(), - data: Content::Raw(String::new()), - metadata: None, - encrypt: false, - compression_level: Some(0), - }) - .unwrap(); - let staged_directory = writer.get_directory_mut().clone(); - - let error = writer - .process_input(InputFile { - file_type: FileType::Data, - inner_path: "duplicate".into(), - data: Content::Raw("non-empty data".into()), - metadata: Some(Content::Raw("metadata".into())), - encrypt: false, - compression_level: Some(0), - }) - .unwrap_err(); - assert!(matches!(error, PithosError::PathOccupied(_))); - assert_eq!(read(&path).unwrap(), before_bytes); - assert_eq!(*writer.get_directory_mut(), staged_directory); - drop(writer); - - assert_eq!(read(&path).unwrap(), before_bytes); - assert_eq!( - extract_pithos_entry(&path, &reader_key, "base", &temp_dir), - b"base data" - ); -} - -#[test] -fn test_writer_rejects_reverse_ancestor_and_metadata_conflicts_before_output() { - let temp_dir = TempDir::new().unwrap(); - let (path, _key, mut writer) = create_pithos_writer(&temp_dir, None); - writer.write_file_header().unwrap(); - writer - .process_input(InputFile { - file_type: FileType::Data, - inner_path: "a/child".into(), - data: Content::Raw("child".into()), - metadata: None, - encrypt: false, - compression_level: Some(0), - }) - .unwrap(); - let before_ancestor = std::fs::metadata(&path).unwrap().len(); - assert!( - writer - .process_input(InputFile { - file_type: FileType::Data, - inner_path: "a".into(), - data: Content::Raw("ancestor".into()), - metadata: None, - encrypt: false, - compression_level: Some(0), - }) - .is_err() - ); - assert_eq!(std::fs::metadata(&path).unwrap().len(), before_ancestor); - - writer - .process_input(InputFile { - file_type: FileType::Data, - inner_path: "metadata.meta/child".into(), - data: Content::Raw("existing metadata".into()), - metadata: None, - encrypt: false, - compression_level: Some(0), - }) - .unwrap(); - let before_metadata = std::fs::metadata(&path).unwrap().len(); - assert!( - writer - .process_input(InputFile { - file_type: FileType::Data, - inner_path: "metadata".into(), - data: Content::Raw("data".into()), - metadata: Some(Content::Raw("metadata".into())), - encrypt: false, - compression_level: Some(0), - }) - .is_err() - ); - assert_eq!(std::fs::metadata(&path).unwrap().len(), before_metadata); -} - -#[test] -fn test_writer_rejects_unsafe_symlink_targets() { - let temp_dir = TempDir::new().unwrap(); - let (_path, _key, mut writer) = create_pithos_writer(&temp_dir, None); - let mut entry = pithos_lib::model::structs::FileEntry { - file_type: FileType::Symlink, - block_data: pithos_lib::model::structs::BlockDataState::Decrypted(vec![]), - created: 0, - modified: 0, - file_size: 0, - permissions: 0o777, - references: vec![], - symlink_target: Some("/outside".into()), - }; - assert!(matches!( - writer.process_file_entry( - "link", - &mut entry, - &pithos_lib::model::structs::ProcessingFlags(0), - std::io::Cursor::new(Vec::::new()) - ), - Err(PithosError::InvalidSymlinkTarget { .. }) - )); -} - -#[test] -fn test_writer_rejects_unsafe_rocrate_absolute_symlink_target() { - let temp_dir = TempDir::new().unwrap(); - let zip_path = temp_dir.path().join("unsafe-link.zip"); - let metadata = minimal_ro_crate_metadata(&[]); - write_raw_zip( - &zip_path, - &[ - ("ro-crate-metadata.json", metadata.as_bytes(), 0), - ("link", b"/outside", 0o120777), - ], - false, - ); - let loaded = read_ro_crate_zip(&zip_path).unwrap(); - let (_path, _key, mut writer) = create_pithos_writer(&temp_dir, None); - assert!(writer.process_ro_crate(&loaded).is_err()); -} - -#[test] -fn test_writer_rejects_unsafe_rocrate_root_escaping_symlink_target() { - let temp_dir = TempDir::new().unwrap(); - let zip_path = temp_dir.path().join("unsafe-link.zip"); - let metadata = minimal_ro_crate_metadata(&[]); - write_raw_zip( - &zip_path, - &[ - ("ro-crate-metadata.json", metadata.as_bytes(), 0), - ("link", b"../outside", 0o120777), - ], - false, - ); - let loaded = read_ro_crate_zip(&zip_path).unwrap(); - let (_path, _key, mut writer) = create_pithos_writer(&temp_dir, None); - assert!(writer.process_ro_crate(&loaded).is_err()); -} - -#[test] -fn test_rocrate_zip_rejects_unsafe_path() { - let temp_dir = TempDir::new().unwrap(); - let zip_path = temp_dir.path().join("unsafe.zip"); - let metadata = minimal_ro_crate_metadata(&[]); - write_zip_entries( - &zip_path, - &[ - ( - "ro-crate-metadata.json", - metadata.as_bytes(), - zip::write::SimpleFileOptions::default(), - ), - ( - "../escape.txt", - b"escape", - zip::write::SimpleFileOptions::default(), - ), - ], - ); - - let error = read_ro_crate_zip(&zip_path).unwrap_err(); - assert!(matches!(error, PithosError::UnsafeZipPath(_))); -} - -#[test] -fn test_rocrate_zip_rejects_duplicate_path() { - let temp_dir = TempDir::new().unwrap(); - let zip_path = temp_dir.path().join("duplicate.zip"); - let metadata = minimal_ro_crate_metadata(&[]); - write_duplicate_zip( - &zip_path, - &[ - ("ro-crate-metadata.json", metadata.as_bytes()), - ("duplicate.txt", b"one"), - ("./duplicate.txt", b"two"), - ], - ); - - let error = read_ro_crate_zip(&zip_path).unwrap_err(); - assert!(matches!(error, PithosError::OverlappingZipEntries(_))); -} - -#[test] -fn test_rocrate_zip_rejects_path_conflict() { - let temp_dir = TempDir::new().unwrap(); - let zip_path = temp_dir.path().join("conflict.zip"); - let metadata = minimal_ro_crate_metadata(&[]); - write_zip_entries( - &zip_path, - &[ - ( - "ro-crate-metadata.json", - metadata.as_bytes(), - zip::write::SimpleFileOptions::default(), - ), - ("nested", b"file", zip::write::SimpleFileOptions::default()), - ( - "nested/file.txt", - b"payload", - zip::write::SimpleFileOptions::default(), - ), - ], - ); - - let error = read_ro_crate_zip(&zip_path).unwrap_err(); - assert!(matches!(error, PithosError::ZipPathConflict(path) if path == "nested")); -} - -#[test] -fn test_rocrate_directory_zip_parity() { - let temp_dir = TempDir::new().unwrap(); - let directory_source = temp_dir.path().join("directory-crate"); - create_dir_all(directory_source.join("nested")).unwrap(); - let metadata = minimal_ro_crate_metadata(&["nested/file.txt"]); - write(directory_source.join("ro-crate-metadata.json"), &metadata).unwrap(); - write(directory_source.join("nested/file.txt"), b"nested payload").unwrap(); - write(directory_source.join("unlisted.txt"), b"unlisted payload").unwrap(); - - let zip_path = temp_dir.path().join("parity.zip"); - write_zip_entries( - &zip_path, - &[ - ( - "ro-crate-metadata.json", - metadata.as_bytes(), - zip::write::SimpleFileOptions::default(), - ), - ( - "nested/file.txt", - b"nested payload", - zip::write::SimpleFileOptions::default(), - ), - ( - "unlisted.txt", - b"unlisted payload", - zip::write::SimpleFileOptions::default(), - ), - ], - ); - - let directory_loaded = read_ro_crate_directory(&directory_source).unwrap(); - let zip_loaded = read_ro_crate_zip(&zip_path).unwrap(); - let (directory_pithos, directory_key) = convert_ro_crate(&temp_dir, &directory_loaded, None); - let (zip_pithos, zip_key) = convert_ro_crate(&temp_dir, &zip_loaded, None); - let (directory, directory_entries) = - read_pithos_directory(&directory_pithos, &directory_key).unwrap(); - let (zip_directory, zip_entries) = read_pithos_directory(&zip_pithos, &zip_key).unwrap(); - - assert_eq!(directory_entries, zip_entries); - for path in ["ro-crate-metadata.json", "nested/file.txt", "unlisted.txt"] { - let directory_output = TempDir::new().unwrap(); - let zip_output = TempDir::new().unwrap(); - let directory_bytes = - extract_pithos_entry(&directory_pithos, &directory_key, path, &directory_output); - let zip_bytes = extract_pithos_entry(&zip_pithos, &zip_key, path, &zip_output); - assert_eq!(directory_bytes, zip_bytes, "{path}"); - } - assert_eq!( - directory.get_file_by_path("nested").unwrap().file_type, - FileType::Directory - ); - assert_eq!( - zip_directory.get_file_by_path("nested").unwrap().file_type, - FileType::Directory - ); -} - -#[test] -fn test_rocrate_zip_streams_multichunk_file() { - let temp_dir = TempDir::new().unwrap(); - let zip_path = temp_dir.path().join("large.zip"); - let metadata = minimal_ro_crate_metadata(&["large.bin"]); - let payload = (0..(1024 * 1024 + 123)) - .map(|index| (index % 251) as u8) - .collect::>(); - write_zip_entries( - &zip_path, - &[ - ( - "ro-crate-metadata.json", - metadata.as_bytes(), - zip::write::SimpleFileOptions::default(), - ), - ( - "large.bin", - &payload, - zip::write::SimpleFileOptions::default(), - ), - ], - ); - - let loaded = read_ro_crate_zip(&zip_path).unwrap(); - let (pithos_path, reader_key) = convert_ro_crate(&temp_dir, &loaded, Some((1024, 4096, 8192))); - let (directory, _) = read_pithos_directory(&pithos_path, &reader_key).unwrap(); - let entry = directory.get_file_by_path("large.bin").unwrap(); - assert_eq!(entry.file_size, payload.len() as u64); - assert!(entry.references.contains(&metadata_reference())); - assert_eq!( - extract_pithos_entry(&pithos_path, &reader_key, "large.bin", &temp_dir), - payload - ); -} diff --git a/crates/pithos_lib/tests/writer_configuration.rs b/crates/pithos_lib/tests/writer_configuration.rs new file mode 100644 index 0000000..48bdb31 --- /dev/null +++ b/crates/pithos_lib/tests/writer_configuration.rs @@ -0,0 +1,94 @@ +mod common; + +use common::writer::options; +use pithos_lib::archive::{ArchiveWriter, CdcConfig, WriteOptions}; +use pithos_lib::crypto::PrivateKey; +use pithos_lib::error::PithosError; + +#[test] +fn creation_rejects_empty_recipients_without_losing_sink() { + let sender = PrivateKey::generate(); + let error = match ArchiveWriter::create(Vec::new(), WriteOptions::new(sender, Vec::new())) { + Ok(_) => panic!("empty recipients must be rejected"), + Err(error) => error, + }; + assert!(matches!( + error.error(), + PithosError::WriterRequiresRecipient + )); + assert!(error.into_incomplete().is_empty()); +} + +#[test] +fn invalid_cdc_is_rejected_before_writer_creation() { + assert!(matches!( + CdcConfig::new(32, 16, 64), + Err(PithosError::InvalidCdcConfig { .. }) + )); + for (min, avg, max) in [(63, 256, 1024), (64, 255, 1024), (64, 256, 1023)] { + assert!(matches!( + CdcConfig::new(min, avg, max), + Err(PithosError::InvalidCdcConfig { .. }) + )); + } +} + +#[test] +fn cdc_accepts_every_fastcdc_boundary_and_rejects_outside_values_without_panicking() { + use fastcdc::v2020::{ + AVERAGE_MAX, AVERAGE_MIN, MAXIMUM_MAX, MAXIMUM_MIN, MINIMUM_MAX, MINIMUM_MIN, + }; + + assert!(CdcConfig::new(MINIMUM_MIN, AVERAGE_MIN, MAXIMUM_MIN).is_ok()); + assert!(CdcConfig::new(MINIMUM_MAX, AVERAGE_MAX, MAXIMUM_MAX).is_ok()); + for (min, avg, max) in [ + (MINIMUM_MIN - 1, AVERAGE_MIN, MAXIMUM_MIN), + (MINIMUM_MAX + 1, AVERAGE_MAX, MAXIMUM_MAX), + (MINIMUM_MIN, AVERAGE_MIN - 1, MAXIMUM_MIN), + (MINIMUM_MIN, AVERAGE_MAX + 1, MAXIMUM_MAX), + (MINIMUM_MIN, AVERAGE_MIN, MAXIMUM_MIN - 1), + (MINIMUM_MIN, AVERAGE_MIN, MAXIMUM_MAX + 1), + ] { + assert!(matches!( + CdcConfig::new(min, avg, max), + Err(PithosError::InvalidCdcConfig { .. }) + )); + } +} + +#[test] +fn duplicate_recipients_are_rejected_before_header_output() { + let sender = PrivateKey::generate(); + let recipient = sender.public_key(); + let error = match ArchiveWriter::create( + Vec::new(), + WriteOptions::new(sender, vec![recipient, recipient]), + ) { + Ok(_) => panic!("duplicate recipients must be rejected"), + Err(error) => error, + }; + assert!(matches!(error.error(), PithosError::DuplicateRecipientKey)); + assert!(error.into_incomplete().is_empty()); +} + +#[test] +fn finish_writes_one_header_and_one_terminal_directory_for_empty_archive() { + let bytes = ArchiveWriter::create(Vec::new(), options()) + .unwrap() + .finish() + .unwrap(); + assert_eq!(&bytes[..4], b"PITH"); + assert_eq!( + bytes + .windows(8) + .filter(|window| *window == b"PITHOSDR") + .count(), + 1 + ); +} + +#[test] +fn open_writer_cannot_be_recovered_as_incomplete() { + let writer = ArchiveWriter::create(Vec::new(), options()).unwrap(); + assert!(writer.into_incomplete().is_err()); +} diff --git a/crates/pithos_lib/tests/writer_failures.rs b/crates/pithos_lib/tests/writer_failures.rs new file mode 100644 index 0000000..09077a0 --- /dev/null +++ b/crates/pithos_lib/tests/writer_failures.rs @@ -0,0 +1,429 @@ +mod common; + +use common::writer::options; +use pithos_lib::archive::{ + AccessKeys, Archive, ArchivePath, ArchiveWriter, CdcConfig, EntryKind, EntryMetadata, + OpenOptions, ProcessingOptions, WriteOptions, WriterError, +}; +use pithos_lib::crypto::PrivateKey; +use pithos_lib::error::PithosError; +use pithos_lib::source::MemorySource; +use std::cell::RefCell; +use std::io::{self, Cursor, Read, Write}; +use std::rc::Rc; + +struct SensitiveDebugSink { + writes: usize, +} +impl std::fmt::Debug for SensitiveDebugSink { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str("SINK_PLAINTEXT_SENTINEL") + } +} +impl Write for SensitiveDebugSink { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.writes += 1; + if self.writes > 2 { + return Err(io::Error::other("sink failure")); + } + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +#[test] +fn writer_errors_redact_caller_owned_sink_debug_output() { + let sender = PrivateKey::generate(); + let create = match ArchiveWriter::create( + SensitiveDebugSink { writes: 0 }, + WriteOptions::new(sender, Vec::new()), + ) { + Ok(_) => panic!("empty recipients must fail"), + Err(error) => error, + }; + assert!(!format!("{create:?}").contains("SINK_PLAINTEXT_SENTINEL")); + let finish = ArchiveWriter::create(SensitiveDebugSink { writes: 0 }, options()) + .unwrap() + .finish() + .unwrap_err(); + assert!(!format!("{finish:?}").contains("SINK_PLAINTEXT_SENTINEL")); + assert!( + std::error::Error::source(&create) + .and_then(|source| source.downcast_ref::()) + .is_some() + ); + assert!( + std::error::Error::source(&finish) + .and_then(|source| source.downcast_ref::()) + .is_some() + ); +} + +#[test] +fn pre_stream_validation_error_leaves_writer_usable_and_actual_size_is_published() { + let sender = PrivateKey::generate(); + let reader = sender.duplicate(); + let recipient = sender.public_key(); + let mut writer = + ArchiveWriter::create(Vec::new(), WriteOptions::new(sender, vec![recipient])).unwrap(); + writer + .add_directory( + ArchivePath::new("parent").unwrap(), + EntryMetadata::new(0, 0, 0o755), + ) + .unwrap(); + assert!( + writer + .add_directory( + ArchivePath::new("parent").unwrap(), + EntryMetadata::new(0, 0, 0o755) + ) + .is_err() + ); + writer + .add_file( + ArchivePath::new("data").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::default(), + None, + Cursor::new(b"actual size"), + ) + .unwrap(); + let archive = Archive::open( + MemorySource::new(writer.finish().unwrap()), + OpenOptions::default().with_access_keys(AccessKeys::new().with_key(reader)), + ) + .unwrap(); + assert!(matches!( + archive.entry("data").unwrap().unwrap().kind, + EntryKind::File { size: 11, .. } + )); +} + +#[test] +fn actual_size_is_derived_and_expected_size_mismatch_poisons() { + let mut writer = ArchiveWriter::create(Vec::new(), options()).unwrap(); + let error = writer + .add_file( + ArchivePath::new("data").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(false, 0).unwrap(), + Some(9), + Cursor::new(b"actual"), + ) + .unwrap_err(); + assert!(matches!( + error, + WriterError::Pithos(PithosError::WriterExpectedSizeMismatch { + expected: 9, + actual: 6 + }) + )); + assert!(matches!( + writer.finish().unwrap_err().error(), + PithosError::WriterPoisoned + )); +} + +struct FailingReader; +impl Read for FailingReader { + fn read(&mut self, _: &mut [u8]) -> io::Result { + Err(io::Error::other("source failure")) + } +} +struct ReaderAfterBytes { + bytes: Vec, + served: bool, +} +impl Read for ReaderAfterBytes { + fn read(&mut self, output: &mut [u8]) -> io::Result { + if self.served { + return Err(io::Error::other("source failure after blocks")); + } + self.served = true; + let length = output.len().min(self.bytes.len()); + output[..length].copy_from_slice(&self.bytes[..length]); + Ok(length) + } +} + +#[test] +fn post_stream_input_failure_poisons_and_recovers_only_incomplete_sink() { + let mut writer = ArchiveWriter::create(Vec::new(), options()).unwrap(); + assert!( + writer + .add_file( + ArchivePath::new("data").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::default(), + None, + FailingReader + ) + .is_err() + ); + assert!(matches!( + writer.add_directory( + ArchivePath::new("later").unwrap(), + EntryMetadata::new(0, 0, 0o755) + ), + Err(WriterError::Poisoned) + )); + let bytes = writer.into_incomplete().unwrap(); + assert_eq!(&bytes[..4], b"PITH"); + assert!(!bytes.windows(8).any(|window| window == b"PITHOSDR")); +} + +#[test] +fn input_failure_between_emitted_blocks_poisons_the_writer() { + let sender = PrivateKey::generate(); + let recipient = sender.public_key(); + let mut state = 1u32; + let bytes = (0..4096) + .map(|_| { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + (state >> 24) as u8 + }) + .collect(); + let mut writer = ArchiveWriter::create( + Vec::new(), + WriteOptions::new(sender, vec![recipient]).with_cdc(CdcConfig::new(64, 256, 1024).unwrap()), + ) + .unwrap(); + assert!( + writer + .add_file( + ArchivePath::new("data").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::default(), + None, + ReaderAfterBytes { + bytes, + served: false + } + ) + .is_err() + ); + assert!(matches!( + writer.add_directory( + ArchivePath::new("later").unwrap(), + EntryMetadata::new(0, 0, 0o755) + ), + Err(WriterError::Poisoned) + )); + let bytes = writer.into_incomplete().unwrap(); + assert!(bytes.windows(4).any(|window| window == b"BLCK")); + assert!(!bytes.windows(8).any(|window| window == b"PITHOSDR")); +} + +#[derive(Debug)] +struct FailAfterHeader { + bytes: Vec, + writes: usize, +} +impl Write for FailAfterHeader { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.writes += 1; + if self.writes > 2 { + return Err(io::Error::other("sink failure")); + } + self.bytes.extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +#[test] +fn sink_failure_after_stream_start_poisoned_writer_returns_incomplete_sink() { + let sink = FailAfterHeader { + bytes: Vec::new(), + writes: 0, + }; + let mut writer = ArchiveWriter::create(sink, options()).unwrap(); + assert!( + writer + .add_file( + ArchivePath::new("data").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::default(), + None, + Cursor::new(b"payload") + ) + .is_err() + ); + let sink = writer.into_incomplete().unwrap(); + assert_eq!(&sink.bytes[..4], b"PITH"); +} + +#[test] +fn block_marker_failure_poisons_and_returns_the_incomplete_sink() { + let sink = FailAfterHeader { + bytes: Vec::new(), + writes: 0, + }; + let mut writer = ArchiveWriter::create(sink, options()).unwrap(); + assert!( + writer + .add_file( + ArchivePath::new("data").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::default(), + None, + Cursor::new(b"payload") + ) + .is_err() + ); + let sink = writer.into_incomplete().unwrap(); + assert_eq!(&sink.bytes[..4], b"PITH"); + assert!(!sink.bytes.windows(4).any(|window| window == b"BLCK")); +} + +#[test] +fn directory_write_failure_returns_an_incomplete_sink() { + let sink = FailAfterHeader { + bytes: Vec::new(), + writes: 0, + }; + let error = ArchiveWriter::create(sink, options()) + .unwrap() + .finish() + .unwrap_err(); + let mut source = std::error::Error::source(error.error()); + let mut found_sink_failure = false; + while let Some(error) = source { + found_sink_failure |= error.to_string().contains("sink failure"); + source = error.source(); + } + assert!(found_sink_failure, "the underlying sink failure was lost"); + assert!(matches!(error.error(), PithosError::Serialization(_))); + let sink = error.into_incomplete(); + assert_eq!(&sink.bytes[..4], b"PITH"); + assert!(!sink.bytes.windows(8).any(|window| window == b"PITHOSDR")); +} + +#[derive(Debug)] +struct FailDuringDirectory { + bytes: Vec, + writes: usize, +} +impl Write for FailDuringDirectory { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.writes += 1; + if self.writes == 5 { + return Err(io::Error::other("mid-directory failure")); + } + self.bytes.extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} +#[test] +fn mid_directory_failure_returns_an_incomplete_sink() { + let error = ArchiveWriter::create( + FailDuringDirectory { + bytes: Vec::new(), + writes: 0, + }, + options(), + ) + .unwrap() + .finish() + .unwrap_err(); + let sink = error.into_incomplete(); + assert_eq!(&sink.bytes[..4], b"PITH"); + assert!(sink.bytes.windows(8).any(|window| window == b"PITHOSDR")); +} + +#[derive(Debug)] +struct FailPayload { + bytes: Vec, + writes: usize, +} +impl Write for FailPayload { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.writes += 1; + if self.writes > 3 { + return Err(io::Error::other("payload failure")); + } + self.bytes.extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} +#[test] +fn payload_write_failure_poisons_the_writer() { + let sink = FailPayload { + bytes: Vec::new(), + writes: 0, + }; + let mut writer = ArchiveWriter::create(sink, options()).unwrap(); + assert!( + writer + .add_file( + ArchivePath::new("data").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::default(), + None, + Cursor::new(b"payload") + ) + .is_err() + ); + let sink = writer.into_incomplete().unwrap(); + assert!(sink.bytes.windows(4).any(|window| window == b"BLCK")); +} + +#[derive(Debug)] +struct FlushFail(Vec); +impl Write for FlushFail { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.0.extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Err(io::Error::other("flush failure")) + } +} +#[test] +fn flush_failure_returns_an_incomplete_sink_without_claiming_durability() { + let error = ArchiveWriter::create(FlushFail(Vec::new()), options()) + .unwrap() + .finish() + .unwrap_err(); + assert!(matches!(error.error(), PithosError::Io(_))); + assert!( + error + .into_incomplete() + .0 + .windows(8) + .any(|window| window == b"PITHOSDR") + ); +} + +#[derive(Clone, Debug)] +struct SharedSink(Rc>>); +impl Write for SharedSink { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.0.borrow_mut().extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} +#[test] +fn dropping_an_open_writer_cannot_publish_a_complete_archive() { + let bytes = Rc::new(RefCell::new(Vec::new())); + { + let _writer = ArchiveWriter::create(SharedSink(Rc::clone(&bytes)), options()).unwrap(); + } + let bytes = bytes.borrow(); + assert_eq!(&bytes[..4], b"PITH"); + assert!(!bytes.windows(8).any(|window| window == b"PITHOSDR")); +} diff --git a/crates/pithos_lib/tests/writer_processing.rs b/crates/pithos_lib/tests/writer_processing.rs new file mode 100644 index 0000000..3818b48 --- /dev/null +++ b/crates/pithos_lib/tests/writer_processing.rs @@ -0,0 +1,120 @@ +mod common; + +use pithos_lib::archive::{ + AccessKeys, Archive, ArchivePath, ArchiveWriter, CdcConfig, EntryMetadata, OpenOptions, + ProcessingOptions, WriteOptions, +}; +use pithos_lib::crypto::PrivateKey; +use pithos_lib::source::MemorySource; +use std::io::{self, Cursor, Write}; + +#[test] +fn repeated_identical_chunks_round_trip_in_order() { + let sender = PrivateKey::generate(); + let reader = sender.duplicate(); + let data = vec![0u8; 2 * 1024]; + let mut writer = ArchiveWriter::create( + Vec::new(), + WriteOptions::new(sender, vec![reader.public_key()]) + .with_cdc(CdcConfig::new(64, 256, 1024).unwrap()), + ) + .unwrap(); + writer + .add_file( + ArchivePath::new("repeated.bin").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(true, 0).unwrap(), + Some(data.len() as u64), + Cursor::new(&data), + ) + .unwrap(); + let archive = Archive::open( + MemorySource::new(writer.finish().unwrap()), + OpenOptions::default().with_access_keys(AccessKeys::new().with_key(reader)), + ) + .unwrap(); + let mut full = Vec::new(); + archive.copy_to("repeated.bin", &mut full).unwrap(); + assert_eq!(full, data); + let mut range = Vec::new(); + archive + .copy_range_to("repeated.bin", 900..1150, &mut range) + .unwrap(); + assert_eq!(range, data[900..1150]); +} + +#[test] +fn committed_actual_size_and_descriptor_are_readable_through_archive() { + let sender = PrivateKey::generate(); + let reader = sender.duplicate(); + let recipient = sender.public_key(); + let mut writer = + ArchiveWriter::create(Vec::new(), WriteOptions::new(sender, vec![recipient])).unwrap(); + writer + .add_file( + ArchivePath::new("data").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(false, 0).unwrap(), + None, + Cursor::new(b"descriptor-backed payload"), + ) + .unwrap(); + let archive = Archive::open( + MemorySource::new(writer.finish().unwrap()), + OpenOptions::default().with_access_keys(AccessKeys::new().with_key(reader)), + ) + .unwrap(); + assert!(matches!( + archive.entry("data").unwrap().unwrap().kind, + pithos_lib::archive::EntryKind::File { size: 25, .. } + )); + let mut copied = Vec::new(); + archive.copy_to("data", &mut copied).unwrap(); + assert_eq!(copied, b"descriptor-backed payload"); +} + +#[derive(Debug, Default)] +struct ShortSink(Vec); + +impl Write for ShortSink { + fn write(&mut self, bytes: &[u8]) -> io::Result { + let accepted = bytes.len().min(3); + self.0.extend_from_slice(&bytes[..accepted]); + Ok(accepted) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +#[test] +fn short_writes_are_counted_and_completed_without_saturating_offsets() { + let sender = PrivateKey::generate(); + let reader = sender.duplicate(); + let recipient = sender.public_key(); + let mut writer = ArchiveWriter::create( + ShortSink::default(), + WriteOptions::new(sender, vec![recipient]), + ) + .unwrap(); + writer + .add_file( + ArchivePath::new("data").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(false, 0).unwrap(), + None, + Cursor::new(b"short writes"), + ) + .unwrap(); + let sink = writer.finish().unwrap(); + let archive = Archive::open( + MemorySource::new(sink.0), + OpenOptions::default().with_access_keys(AccessKeys::new().with_key(reader)), + ) + .unwrap(); + assert!(archive.entry("data").unwrap().is_some()); + let mut copied = Vec::new(); + archive.copy_to("data", &mut copied).unwrap(); + assert_eq!(copied, b"short writes"); +} From f101b409f828d508c9ce6517c673ab04f8bb7519 Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Tue, 11 Aug 2026 23:00:44 +0200 Subject: [PATCH 008/166] refactor: rebuild direct archive append --- crates/pithos_lib/src/archive/append.rs | 56 ++ crates/pithos_lib/src/archive/snapshot.rs | 467 ++++++++++++ crates/pithos_lib/src/fs/append.rs | 707 +++++++++++++++++++ crates/pithos_lib/tests/append_durability.rs | 155 ++++ crates/pithos_lib/tests/append_workflow.rs | 185 +++++ crates/pithos_lib/tests/grant_readers.rs | 344 +++++++++ 6 files changed, 1914 insertions(+) create mode 100644 crates/pithos_lib/src/archive/append.rs create mode 100644 crates/pithos_lib/src/archive/snapshot.rs create mode 100644 crates/pithos_lib/src/fs/append.rs create mode 100644 crates/pithos_lib/tests/append_durability.rs create mode 100644 crates/pithos_lib/tests/append_workflow.rs create mode 100644 crates/pithos_lib/tests/grant_readers.rs diff --git a/crates/pithos_lib/src/archive/append.rs b/crates/pithos_lib/src/archive/append.rs new file mode 100644 index 0000000..ae19183 --- /dev/null +++ b/crates/pithos_lib/src/archive/append.rs @@ -0,0 +1,56 @@ +use super::{CdcConfig, ProcessingOptions, WriteOptions}; +use crate::crypto::{PrivateKey, PublicKey}; +use crate::error::PithosError; + +/// Durability requested after a child directory has been flushed. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum AppendDurability { + /// Flush the terminal directory to the supplied file handle. + Flush, + /// Flush the terminal directory and request `sync_all` from the filesystem. + SyncAll, +} + +/// Options for a direct filesystem append. +pub struct AppendOptions { + pub(crate) sender: PrivateKey, + pub(crate) recipients: Vec, + pub(crate) cdc: CdcConfig, + pub(crate) durability: AppendDurability, + pub(crate) processing: ProcessingOptions, +} + +impl AppendOptions { + pub fn new(sender: PrivateKey, recipients: Vec) -> Self { + Self { + sender, + recipients, + cdc: CdcConfig::default(), + durability: AppendDurability::Flush, + processing: ProcessingOptions::append_default(), + } + } + + pub fn with_cdc(mut self, cdc: CdcConfig) -> Self { + self.cdc = cdc; + self + } + + pub fn with_durability(mut self, durability: AppendDurability) -> Self { + self.durability = durability; + self + } + + pub(crate) fn validate_recipients(&self) -> Result<(), PithosError> { + WriteOptions::new(self.sender.duplicate(), self.recipients.clone()).validate() + } +} + +/// Measurements collected while validating and extending an archive. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct AppendObservation { + pub source_read_count: u64, + pub source_read_bytes: u64, + pub base_archive_bytes: u64, + pub final_archive_bytes: u64, +} diff --git a/crates/pithos_lib/src/archive/snapshot.rs b/crates/pithos_lib/src/archive/snapshot.rs new file mode 100644 index 0000000..a2e93e0 --- /dev/null +++ b/crates/pithos_lib/src/archive/snapshot.rs @@ -0,0 +1,467 @@ +use crate::archive::access::ResolvedAccess; +use crate::archive::index::{ArchiveIndex, build_effective_index}; +use crate::archive::types::{ + ArchivePath, BlockDescriptor, BlockHash, Entry, FileId, Span, ValidatedSegment, +}; +use crate::archive::validation::IndexLimits; +use crate::crypto::FileKey; +use crate::error::PithosError; +use indexmap::IndexMap; +use std::collections::{BTreeMap, HashMap}; +use std::sync::Arc; + +/// The secret-contained state needed to safely plan a direct append or grant. +/// +/// This is intentionally derived only from a fully opened archive. It retains validated +/// segments for pure prospective merge, but not the source, wire directories, or reader facade. +pub(crate) struct AppendSnapshot { + archive_len: u64, + terminal_directory: Span, + maximum_id: Option, + descriptors: IndexMap, + entries: BTreeMap, + paths: HashMap, FileId>, + hierarchy: BTreeMap, + relationships: BTreeMap>, + segments: Vec, + index_limits: IndexLimits, + access: ResolvedAccess, +} + +pub(crate) struct SnapshotEntry { + #[cfg(test)] + pub(crate) id: FileId, + #[cfg(test)] + pub(crate) path: ArchivePath, + pub(crate) kind: SnapshotEntryKind, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum SnapshotEntryKind { + File, + Metadata, + Directory, + Symlink, +} + +impl AppendSnapshot { + pub(crate) fn new( + archive_len: u64, + terminal_directory: Span, + index: ArchiveIndex, + segments: Vec, + index_limits: IndexLimits, + access: ResolvedAccess, + ) -> Self { + let relationships = index + .relationships() + .map(|(id, name)| (id.0, Arc::from(name))) + .collect(); + let (index_entries, descriptors, maximum_id) = index.into_append_snapshot_parts(); + let mut entries = BTreeMap::new(); + let mut paths = HashMap::new(); + let mut hierarchy = BTreeMap::new(); + for entry in index_entries { + let kind = snapshot_entry_kind(&entry.entry); + let id = entry.id; + let path_key = Arc::from(entry.path.as_str()); + hierarchy.insert(entry.path.clone(), id); + entries.insert( + id, + SnapshotEntry { + #[cfg(test)] + id, + #[cfg(test)] + path: entry.path, + kind, + }, + ); + paths.insert(path_key, id); + } + Self { + archive_len, + terminal_directory, + maximum_id, + descriptors, + entries, + paths, + hierarchy, + relationships, + segments, + index_limits, + access, + } + } + + pub(crate) fn archive_len(&self) -> u64 { + self.archive_len + } + + pub(crate) fn terminal_directory(&self) -> Span { + self.terminal_directory + } + + pub(crate) fn maximum_id(&self) -> Option { + self.maximum_id + } + + pub(crate) fn descriptor(&self, hash: BlockHash) -> Option<&BlockDescriptor> { + self.descriptors.get(&hash) + } + + pub(crate) fn entry(&self, id: FileId) -> Option<&SnapshotEntry> { + self.entries.get(&id) + } + + pub(crate) fn entry_at_path(&self, path: &ArchivePath) -> Option<&SnapshotEntry> { + self.paths + .get(path.as_str()) + .and_then(|id| self.entries.get(id)) + } + + pub(crate) fn ensure_path_available(&self, path: &ArchivePath) -> Result<(), PithosError> { + if self.paths.contains_key(path.as_str()) { + return Err(PithosError::PathOccupied(path.as_str().to_owned())); + } + Ok(()) + } + + pub(crate) fn ensure_id_available(&self, id: FileId) -> Result<(), PithosError> { + if self.entries.contains_key(&id) { + return Err(PithosError::DuplicateFileId(format!( + "File id already occupied: {}", + id.0 + ))); + } + Ok(()) + } + + pub(crate) fn ensure_candidate_hierarchy( + &self, + path: &ArchivePath, + is_directory: bool, + ) -> Result<(), PithosError> { + for (index, _) in path.as_str().match_indices('/') { + let ancestor = ArchivePath::new(&path.as_str()[..index])?; + if let Some(entry) = self.entry_at_path(&ancestor) + && entry.kind != SnapshotEntryKind::Directory + { + return Err(PithosError::InvalidArchivePath { + path: path.as_str().to_owned(), + reason: format!("file entry {} is an ancestor", ancestor.as_str()), + }); + } + } + if !is_directory + && let Some((successor, _)) = self.hierarchy.range(path.clone()..).next() + && path.is_ancestor_of(successor) + { + return Err(PithosError::InvalidArchivePath { + path: path.as_str().to_owned(), + reason: format!("entry is an ancestor of {}", successor.as_str()), + }); + } + Ok(()) + } + + pub(crate) fn validate_child_relationships( + &self, + relationships: &[(u64, &str)], + ) -> Result<(), PithosError> { + for (id, name) in relationships { + if let Some(existing) = self.relationships.get(id) + && existing.as_ref() != *name + { + return Err(PithosError::ConflictingRelationshipDefinition(*id)); + } + } + Ok(()) + } + + pub(crate) fn has_relationship(&self, id: u64) -> bool { + self.relationships.contains_key(&id) + } + + /// Runs the normal pure merge against the immutable ancestor chain before publication. + pub(crate) fn validate_prospective_child( + &self, + child: ValidatedSegment, + ) -> Result<(), PithosError> { + let archive_len = child.span.end(); + let mut segments = self.segments.clone(); + segments.push(child); + build_effective_index(&segments, archive_len, self.index_limits).map(|_| ()) + } + + /// Borrows an opaque, zeroizing recovered file key only for content entries. + pub(crate) fn with_file_key( + &self, + id: FileId, + operation: impl FnOnce(&FileKey) -> T, + ) -> Result { + let entry = self + .entries + .get(&id) + .ok_or(PithosError::SnapshotFileIdNotFound(id.0))?; + match entry.kind { + SnapshotEntryKind::Directory => { + return Err(PithosError::SnapshotDirectoryHasNoContent(id.0)); + } + SnapshotEntryKind::Symlink => { + return Err(PithosError::SnapshotSymlinkHasNoContent(id.0)); + } + SnapshotEntryKind::File | SnapshotEntryKind::Metadata => {} + } + self.access + .with_file_key(id, operation) + .ok_or(PithosError::SnapshotContentUnavailable(id.0)) + } +} + +fn snapshot_entry_kind(entry: &Entry) -> SnapshotEntryKind { + match entry { + Entry::File(_) => SnapshotEntryKind::File, + Entry::Metadata(_) => SnapshotEntryKind::Metadata, + Entry::Directory(_) => SnapshotEntryKind::Directory, + Entry::Symlink { .. } => SnapshotEntryKind::Symlink, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::archive::{ + AccessKeys, Archive, ArchiveWriter, EntryMetadata, OpenOptions, ProcessingOptions, + WriteOptions, + }; + use crate::crypto::{PrivateKey, parse_private_pem}; + use crate::error::PithosError; + use crate::format::limits::DeserializationLimits; + use crate::source::MemorySource; + use std::io::{Cursor, Write}; + use std::sync::Arc; + + fn private(name: &str) -> PrivateKey { + parse_private_pem(&std::fs::read(format!("tests/data/keys/{name}_private.pem")).unwrap()) + .unwrap() + } + + fn archive_with_entries() -> (Vec, PrivateKey, PrivateKey) { + let sender = private("sender"); + let sender_access = sender.duplicate(); + let recipient = private("recipient1"); + let recipient_public = recipient.public_key(); + let mut writer = ArchiveWriter::create( + Cursor::new(Vec::new()), + WriteOptions::new(sender, vec![sender_access.public_key(), recipient_public]), + ) + .unwrap(); + writer + .add_file( + ArchivePath::new("data").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(true, 0).unwrap(), + Some(4), + Cursor::new(b"data"), + ) + .unwrap(); + writer + .add_directory( + ArchivePath::new("directory").unwrap(), + EntryMetadata::new(0, 0, 0o755), + ) + .unwrap(); + writer + .add_symlink( + ArchivePath::new("link").unwrap(), + EntryMetadata::new(0, 0, 0o777), + "data", + ) + .unwrap(); + ( + writer.finish().unwrap().into_inner(), + sender_access, + recipient, + ) + } + + fn open(bytes: Vec, keys: AccessKeys) -> Archive { + Archive::open( + MemorySource::new(Arc::<[u8]>::from(bytes)), + OpenOptions::default().with_access_keys(keys), + ) + .unwrap() + } + + #[test] + fn snapshot_records_archive_length_terminal_span_and_maximum_id() { + let (bytes, _sender, recipient) = archive_with_entries(); + let snapshot = + open(bytes.clone(), AccessKeys::new().with_key(recipient)).into_append_snapshot(); + let terminal_len = + u64::from_be_bytes(bytes[bytes.len() - 12..bytes.len() - 4].try_into().unwrap()); + + assert_eq!(snapshot.archive_len(), bytes.len() as u64); + assert_eq!(snapshot.terminal_directory().len(), terminal_len); + assert_eq!(snapshot.terminal_directory().end(), snapshot.archive_len()); + assert_eq!(snapshot.maximum_id(), Some(FileId(2))); + } + + #[test] + fn snapshot_creation_consumes_archive_and_moves_recovered_access() { + fn consume(archive: Archive) -> AppendSnapshot { + archive.into_append_snapshot() + } + + let (bytes, _sender, recipient) = archive_with_entries(); + let snapshot = consume(open(bytes, AccessKeys::new().with_key(recipient))); + assert!(snapshot.with_file_key(FileId(0), |_| ()).is_ok()); + } + + #[test] + fn snapshot_checks_existing_paths_ids_and_content_kinds() { + let (bytes, _sender, recipient) = archive_with_entries(); + let snapshot = open(bytes, AccessKeys::new().with_key(recipient)).into_append_snapshot(); + + assert!(matches!( + snapshot.ensure_path_available(&ArchivePath::new("data").unwrap()), + Err(PithosError::PathOccupied(path)) if path == "data" + )); + assert!( + snapshot + .ensure_path_available(&ArchivePath::new("new").unwrap()) + .is_ok() + ); + let entry = snapshot + .entry_at_path(&ArchivePath::new("directory").unwrap()) + .unwrap(); + assert_eq!(entry.id, FileId(1)); + assert_eq!(entry.path.as_str(), "directory"); + assert_eq!(entry.kind, SnapshotEntryKind::Directory); + assert_eq!( + snapshot.entry(FileId(2)).unwrap().kind, + SnapshotEntryKind::Symlink + ); + assert!(matches!( + snapshot.ensure_id_available(FileId(0)), + Err(PithosError::DuplicateFileId(_)) + )); + assert!(snapshot.ensure_id_available(FileId(3)).is_ok()); + assert!(matches!( + snapshot.with_file_key(FileId(1), |_| ()), + Err(PithosError::SnapshotDirectoryHasNoContent(1)) + )); + assert!(matches!( + snapshot.with_file_key(FileId(2), |_| ()), + Err(PithosError::SnapshotSymlinkHasNoContent(2)) + )); + } + + #[test] + fn snapshot_keeps_the_earliest_effective_descriptor() { + let sender = private("sender"); + let recipient = private("recipient1"); + let mut writer = ArchiveWriter::create( + Vec::new(), + WriteOptions::new(sender.duplicate(), vec![recipient.public_key()]), + ) + .unwrap(); + writer + .add_file( + ArchivePath::new("data").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(true, 0).unwrap(), + Some(4), + Cursor::new(b"data"), + ) + .unwrap(); + let first = writer.finish().unwrap(); + let first_len = + u64::from_be_bytes(first[first.len() - 12..first.len() - 4].try_into().unwrap()); + let first_start = first.len() - usize::try_from(first_len).unwrap(); + let first_directory = crate::format::codec::decode_directory( + &mut Cursor::new(&first[first_start..]), + &DeserializationLimits::default(), + ) + .unwrap(); + let (hash, earliest) = first_directory.blocks.first().unwrap(); + let hash = *hash; + let earliest_stored_size = earliest.stored_size; + + let snapshot = open( + first.clone(), + AccessKeys::new().with_key(recipient.duplicate()), + ) + .into_append_snapshot(); + let child = ArchiveWriter::append( + Vec::new(), + sender, + vec![recipient.public_key()], + crate::archive::CdcConfig::default(), + snapshot, + ) + .unwrap() + .finish() + .unwrap(); + let mut bytes = first; + bytes.extend_from_slice(&child); + let len = u64::from_be_bytes(bytes[bytes.len() - 12..bytes.len() - 4].try_into().unwrap()); + let start = bytes.len() - usize::try_from(len).unwrap(); + let mut directory = crate::format::codec::decode_directory( + &mut Cursor::new(&bytes[start..]), + &DeserializationLimits::default(), + ) + .unwrap(); + let hash = BlockHash(hash); + let mut newer = earliest.clone(); + newer.stored_size -= 1; + directory.blocks.insert(hash.0, newer); + crate::format::codec::update_directory_len(&mut directory).unwrap(); + crate::format::codec::update_directory_crc(&mut directory).unwrap(); + let mut replacement = Vec::new(); + crate::format::codec::encode_directory(&directory, &mut replacement).unwrap(); + bytes.truncate(start); + bytes.write_all(&replacement).unwrap(); + + let snapshot = open(bytes, AccessKeys::new().with_key(recipient)).into_append_snapshot(); + assert_eq!( + snapshot.descriptor(hash).unwrap().stored_size, + earliest_stored_size + ); + } + + #[test] + fn snapshot_borrows_recovered_keys_for_sender_and_recipient_access() { + let (bytes, sender, recipient) = archive_with_entries(); + let sender_key = open(bytes.clone(), AccessKeys::new().with_key(sender)) + .into_append_snapshot() + .with_file_key(FileId(0), |key| *key.expose_for_protocol()) + .unwrap(); + let recipient_key = open(bytes, AccessKeys::new().with_key(recipient)) + .into_append_snapshot() + .with_file_key(FileId(0), |key| *key.expose_for_protocol()) + .unwrap(); + assert_eq!(sender_key, recipient_key); + } + + #[test] + fn snapshot_rejects_missing_wrong_and_unavailable_file_keys() { + let (bytes, _sender, _recipient) = archive_with_entries(); + let unavailable = open(bytes.clone(), AccessKeys::new()).into_append_snapshot(); + let wrong = + open(bytes, AccessKeys::new().with_key(private("recipient2"))).into_append_snapshot(); + + assert!(matches!( + unavailable.with_file_key(FileId(99), |_| ()), + Err(PithosError::SnapshotFileIdNotFound(99)) + )); + assert!(matches!( + unavailable.with_file_key(FileId(0), |_| ()), + Err(PithosError::SnapshotContentUnavailable(0)) + )); + assert!(matches!( + wrong.with_file_key(FileId(0), |_| ()), + Err(PithosError::SnapshotContentUnavailable(0)) + )); + } +} diff --git a/crates/pithos_lib/src/fs/append.rs b/crates/pithos_lib/src/fs/append.rs new file mode 100644 index 0000000..4a1367a --- /dev/null +++ b/crates/pithos_lib/src/fs/append.rs @@ -0,0 +1,707 @@ +use crate::archive::{ + AccessKeys, AppendDurability, AppendObservation, AppendOptions, AppendSnapshot, Archive, + ArchiveWriter, FileId, OpenOptions, WriterError, +}; +use crate::error::PithosError; +use crate::fs::FsError; +use crate::fs::ingest::{InputManifest, build_input_manifest}; +use crate::source::{ArchiveSource, FileSource, SourceError}; +use rustix::fs::{FlockOperation, Mode, OFlags, flock}; +use std::fs::File; +use std::io::{self, Seek, SeekFrom, Write}; +use std::os::unix::fs::MetadataExt; +use std::path::{Path, PathBuf}; +use std::sync::{ + Arc, + atomic::{AtomicBool, AtomicU64, Ordering}, +}; + +#[cfg(test)] +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum AppendFailurePoint { + AfterMarker, + AfterPayload, + DuringDirectory, + Flush, + SyncAll, + RollbackTruncate, + RollbackFlush, + MetadataQuery, +} + +struct AppendSource { + source: FileSource, + reads: Arc, + bytes: Arc, +} + +impl ArchiveSource for AppendSource { + fn len(&self) -> Result { + self.source.len() + } + + fn read_exact_at(&self, offset: u64, buffer: &mut [u8]) -> Result<(), SourceError> { + self.source.read_exact_at(offset, buffer)?; + self.reads.fetch_add(1, Ordering::Relaxed); + self.bytes.fetch_add(buffer.len() as u64, Ordering::Relaxed); + Ok(()) + } +} + +/// Metadata-first direct append inputs. Open handles remain in the manifest until execution. +struct AppendSpec { + options: AppendOptions, + files: Vec, +} + +struct AppendPlan { + manifest: InputManifest, + ids: Vec, +} + +impl AppendPlan { + fn build( + spec: &AppendSpec, + snapshot: &AppendSnapshot, + archive_device: u64, + archive_inode: u64, + ) -> Result { + let manifest = build_input_manifest(&spec.files)?; + if let Some(path) = manifest.source_with_identity(archive_device, archive_inode) { + return Err(FsError::AppendSourceIsArchive { + path: path.to_path_buf(), + }); + } + let ids = manifest + .validate_append(snapshot) + .map_err(|source| FsError::Core { + operation: "validate append plan", + path: spec.files.first().cloned().unwrap_or_default(), + source, + })?; + Ok(Self { manifest, ids }) + } +} + +/// Append filesystem entries in one child directory. +/// +/// The target is opened once, then held under a nonblocking advisory exclusive lock +/// through validation, publication, durability handling, and any rollback attempt. +/// The lock coordinates only processes that use this protocol. All knowable input +/// conflicts are validated before content is read or the archive is mutated; a +/// successful append writes one child directory and never a second header. +/// +/// [`AppendDurability::Flush`] flushes the completed child directory. [`AppendDurability::SyncAll`] +/// additionally requests filesystem durability with `sync_all`. After a mutation-time +/// failure, the implementation attempts to truncate to the original length and flush +/// while retaining the lock. A successful restoration reports `AppendRolledBack`; a +/// failed truncation or rollback flush reports `AppendRollbackFailed` with both causes. +/// Neither mode promises atomic publication, crash-universal durability, or recovery +/// after process termination, storage failure, or power loss. +pub fn append_files( + archive: &Path, + options: AppendOptions, + files: &[PathBuf], +) -> Result { + append_files_impl( + archive, + options, + files, + #[cfg(test)] + None, + ) +} + +#[cfg(test)] +fn append_files_with_injected_failure( + archive: &Path, + options: AppendOptions, + files: &[PathBuf], + failure: AppendFailurePoint, +) -> Result { + append_files_impl(archive, options, files, Some(failure)) +} + +fn append_files_impl( + archive: &Path, + options: AppendOptions, + files: &[PathBuf], + #[cfg(test)] failure: Option, +) -> Result { + if files.is_empty() { + return Err(FsError::AppendRequiresInput); + } + let spec = AppendSpec { + options, + files: files.to_vec(), + }; + + let mut locked = open_locked(archive)?; + spec.options + .validate_recipients() + .map_err(|source| append_error("validate append recipients", archive, source))?; + let archive_metadata = locked.metadata().map_err(|source| FsError::Host { + operation: "inspect archive", + path: archive.to_path_buf(), + source, + })?; + let original_len = archive_metadata.len(); + let reads = Arc::new(AtomicU64::new(0)); + let bytes = Arc::new(AtomicU64::new(0)); + let snapshot = open_snapshot( + &locked, + spec.options.sender.duplicate(), + Arc::clone(&reads), + Arc::clone(&bytes), + ) + .map_err(|source| append_error("open append snapshot", archive, source))?; + let plan = AppendPlan::build( + &spec, + &snapshot, + archive_metadata.dev(), + archive_metadata.ino(), + )?; + let mutated = Arc::new(AtomicBool::new(false)); + + let AppendSpec { options, .. } = spec; + let result = (|| { + let mut sink = AppendSink::new( + locked.try_clone()?, + #[cfg(test)] + failure, + Arc::clone(&mutated), + ); + sink.seek(SeekFrom::Start(original_len))?; + let mut writer = ArchiveWriter::append( + sink, + options.sender, + options.recipients, + options.cdc, + snapshot, + )?; + plan.manifest + .ingest_planned(&plan.ids, &mut writer, options.processing) + .map_err(writer_error)?; + let mut sink = writer.finish().map_err(|error| error.into_parts().0)?; + if options.durability == AppendDurability::SyncAll { + sink.sync_all()?; + } + let final_archive_bytes = sink.metadata()?.len(); + Ok::<(AppendSink, u64), PithosError>((sink, final_archive_bytes)) + })(); + + match result { + Ok((_, final_archive_bytes)) => Ok(AppendObservation { + source_read_count: reads.load(Ordering::Relaxed), + source_read_bytes: bytes.load(Ordering::Relaxed), + base_archive_bytes: original_len, + final_archive_bytes, + }), + Err(error) => { + let source = rollback_if_mutated( + &mut locked, + original_len, + #[cfg(test)] + failure, + mutated.load(Ordering::Relaxed), + error, + ); + Err(append_error("append archive", archive, source)) + } + } +} + +/// Add recipient access records in one metadata-only child directory. +/// +/// This follows the same opened-target advisory locking, durability, and best-effort +/// rollback contract as [`append_files`]. It validates requested entry IDs while the +/// lock is held and writes no content blocks or second archive header. +pub fn grant_readers( + archive: &Path, + options: AppendOptions, + ids: &[u64], +) -> Result { + let mut locked = open_locked(archive)?; + options + .validate_recipients() + .map_err(|source| append_error("validate grant recipients", archive, source))?; + let original_len = locked + .metadata() + .map_err(|source| FsError::Host { + operation: "inspect archive", + path: archive.to_path_buf(), + source, + })? + .len(); + let reads = Arc::new(AtomicU64::new(0)); + let bytes = Arc::new(AtomicU64::new(0)); + let snapshot = open_snapshot( + &locked, + options.sender.duplicate(), + Arc::clone(&reads), + Arc::clone(&bytes), + ) + .map_err(|source| append_error("open grant snapshot", archive, source))?; + validate_grant_ids(&snapshot, ids) + .map_err(|source| append_error("validate grant ids", archive, source))?; + let mutated = Arc::new(AtomicBool::new(false)); + + let result = (|| { + let mut sink = AppendSink::new( + locked.try_clone()?, + #[cfg(test)] + None, + Arc::clone(&mutated), + ); + sink.seek(SeekFrom::Start(original_len))?; + let mut writer = ArchiveWriter::append( + sink, + options.sender, + options.recipients, + options.cdc, + snapshot, + )?; + writer.grant_file_keys(&ids.iter().copied().map(FileId).collect::>())?; + let mut sink = writer.finish().map_err(|error| error.into_parts().0)?; + if options.durability == AppendDurability::SyncAll { + sink.sync_all()?; + } + let final_archive_bytes = sink.metadata()?.len(); + Ok::<(AppendSink, u64), PithosError>((sink, final_archive_bytes)) + })(); + + match result { + Ok((_, final_archive_bytes)) => Ok(AppendObservation { + source_read_count: reads.load(Ordering::Relaxed), + source_read_bytes: bytes.load(Ordering::Relaxed), + base_archive_bytes: original_len, + final_archive_bytes, + }), + Err(error) => { + let source = rollback_if_mutated( + &mut locked, + original_len, + #[cfg(test)] + None, + mutated.load(Ordering::Relaxed), + error, + ); + Err(append_error("grant archive access", archive, source)) + } + } +} + +fn open_locked(path: &Path) -> Result { + let file = rustix::fs::open( + path, + OFlags::RDWR | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::empty(), + ) + .map_err(io::Error::from) + .map(File::from) + .map_err(|source| FsError::Host { + operation: "open archive", + path: path.to_path_buf(), + source, + })?; + // Keep this description locked through validation, publication, and any rollback. + match flock(&file, FlockOperation::NonBlockingLockExclusive) { + Ok(()) => Ok(file), + Err(rustix::io::Errno::WOULDBLOCK) => Err(FsError::AppendLocked { + path: path.to_path_buf(), + }), + Err(error) => Err(FsError::Host { + operation: "lock archive", + path: path.to_path_buf(), + source: io::Error::from_raw_os_error(error.raw_os_error()), + }), + } +} + +fn open_snapshot( + locked: &File, + sender: crate::crypto::PrivateKey, + reads: Arc, + bytes: Arc, +) -> Result { + Archive::open( + AppendSource { + source: FileSource::from_file(locked.try_clone()?)?, + reads, + bytes, + }, + OpenOptions::default().with_access_keys(AccessKeys::new().with_key(sender)), + ) + .map(Archive::into_append_snapshot) +} + +fn validate_grant_ids(snapshot: &AppendSnapshot, ids: &[u64]) -> Result<(), PithosError> { + if ids.is_empty() { + return Err(PithosError::GrantRequiresFileId); + } + let mut unique = std::collections::HashSet::with_capacity(ids.len()); + for id in ids { + if !unique.insert(*id) { + return Err(PithosError::DuplicateRecipientFileId); + } + snapshot.with_file_key(FileId(*id), |_| ())?; + } + Ok(()) +} + +fn writer_error(error: WriterError) -> PithosError { + match error { + WriterError::Pithos(error) => error, + WriterError::Poisoned => PithosError::WriterPoisoned, + } +} + +fn append_error(operation: &'static str, path: &Path, source: PithosError) -> FsError { + FsError::Core { + operation, + path: path.to_path_buf(), + source, + } +} + +fn rollback_if_mutated( + locked: &mut File, + original_len: u64, + #[cfg(test)] failure: Option, + mutated: bool, + error: PithosError, +) -> PithosError { + #[cfg(test)] + let observed_len = if failure == Some(AppendFailurePoint::MetadataQuery) { + Err(io::Error::other( + "injected append rollback metadata query failure", + )) + } else { + locked.metadata().map(|metadata| metadata.len()) + } + .unwrap_or(original_len); + #[cfg(not(test))] + let observed_len = locked + .metadata() + .map(|metadata| metadata.len()) + .unwrap_or(original_len); + if !mutated && observed_len == original_len { + return error; + } + + // A failed child is never publication: restore the validated prefix before unlocking. + #[cfg(test)] + let rollback = if failure == Some(AppendFailurePoint::RollbackTruncate) { + Err(io::Error::other( + "injected append rollback truncation failure", + )) + } else if failure == Some(AppendFailurePoint::RollbackFlush) { + locked + .set_len(original_len) + .and_then(|()| Err(io::Error::other("injected append rollback flush failure"))) + } else { + locked.set_len(original_len).and_then(|()| locked.flush()) + }; + #[cfg(not(test))] + let rollback = locked.set_len(original_len).and_then(|()| locked.flush()); + match rollback { + Ok(()) => PithosError::AppendRolledBack { + source: Box::new(error), + }, + Err(rollback) => PithosError::AppendRollbackFailed { + source: Box::new(error), + rollback, + original_len, + observed_len, + }, + } +} + +struct AppendSink { + file: File, + mutated: Arc, + #[cfg(test)] + failure: Option, + #[cfg(test)] + after_block_marker: bool, +} + +impl AppendSink { + fn new( + file: File, + #[cfg(test)] failure: Option, + mutated: Arc, + ) -> Self { + Self { + file, + mutated, + #[cfg(test)] + failure, + #[cfg(test)] + after_block_marker: false, + } + } + + fn metadata(&self) -> io::Result { + self.file.metadata() + } + + fn sync_all(&mut self) -> Result<(), PithosError> { + #[cfg(test)] + if self.failure == Some(AppendFailurePoint::SyncAll) { + return Err(PithosError::Io(io::Error::other( + "injected append sync-all failure", + ))); + } + self.file.sync_all().map_err(PithosError::Io) + } +} + +impl Write for AppendSink { + fn write(&mut self, bytes: &[u8]) -> io::Result { + let accepted = self.file.write(bytes)?; + if accepted > 0 { + self.mutated.store(true, Ordering::Relaxed); + } + #[cfg(test)] + if accepted == bytes.len() { + let is_marker = bytes == b"BLCK"; + let is_directory = bytes == b"PITHOSDR"; + let after_payload = self.after_block_marker && !is_marker; + self.after_block_marker = is_marker; + let fail = match self.failure { + Some(AppendFailurePoint::AfterMarker) if is_marker => true, + Some( + AppendFailurePoint::AfterPayload + | AppendFailurePoint::RollbackTruncate + | AppendFailurePoint::RollbackFlush + | AppendFailurePoint::MetadataQuery, + ) if after_payload => true, + Some(AppendFailurePoint::DuringDirectory) if is_directory => true, + _ => false, + }; + if fail { + return Err(io::Error::other("injected append write failure")); + } + } + Ok(accepted) + } + + fn flush(&mut self) -> io::Result<()> { + self.file.flush()?; + #[cfg(test)] + if self.failure == Some(AppendFailurePoint::Flush) { + return Err(io::Error::other("injected append flush failure")); + } + Ok(()) + } +} + +impl Seek for AppendSink { + fn seek(&mut self, position: SeekFrom) -> io::Result { + self.file.seek(position) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::archive::{AccessKeys, ArchivePath, EntryMetadata, ProcessingOptions, WriteOptions}; + use crate::crypto::PrivateKey; + use crate::format::limits::DeserializationLimits; + use crate::source::FileSource; + use std::io::Cursor; + + struct AppendFixture { + archive: PathBuf, + sender: PrivateKey, + } + + impl AppendFixture { + fn append_source(&self, name: &str, contents: &[u8]) -> PathBuf { + let source = self.archive.parent().unwrap().join(name); + std::fs::write(&source, contents).unwrap(); + source + } + + fn open(&self) -> Archive { + Archive::open( + FileSource::open(&self.archive).unwrap(), + OpenOptions::default() + .with_access_keys(AccessKeys::new().with_key(self.sender.duplicate())), + ) + .unwrap() + } + + fn options(&self, durability: AppendDurability) -> AppendOptions { + AppendOptions::new(self.sender.duplicate(), vec![self.sender.public_key()]) + .with_durability(durability) + } + } + + fn append_fixture(temporary: &tempfile::TempDir) -> AppendFixture { + let archive = temporary.path().join("append.pith"); + let sender = PrivateKey::generate(); + let mut writer = ArchiveWriter::create( + File::create(&archive).unwrap(), + WriteOptions::new(sender.duplicate(), vec![sender.public_key()]), + ) + .unwrap(); + writer + .add_file( + ArchivePath::new("base.txt").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::append_default(), + None, + Cursor::new(b"base append payload"), + ) + .unwrap(); + drop(writer.finish().unwrap()); + AppendFixture { archive, sender } + } + + #[test] + fn conflicting_ancestor_relationship_is_rejected_before_mutation() { + let temporary = tempfile::tempdir().unwrap(); + let archive_path = temporary.path().join("relationships.pith"); + let input_path = temporary.path().join("appended.txt"); + std::fs::write(&input_path, b"appended relationship payload").unwrap(); + let sender = PrivateKey::generate(); + let recipient = sender.public_key(); + let mut writer = ArchiveWriter::create( + File::create(&archive_path).unwrap(), + WriteOptions::new(sender.duplicate(), vec![recipient]), + ) + .unwrap(); + writer + .add_file( + ArchivePath::new("base.txt").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::default(), + None, + Cursor::new(b"base relationship payload"), + ) + .unwrap(); + drop(writer.finish().unwrap()); + + let mut bytes = std::fs::read(&archive_path).unwrap(); + let directory_len = + u64::from_be_bytes(bytes[bytes.len() - 12..bytes.len() - 4].try_into().unwrap()); + let directory_start = bytes.len() - usize::try_from(directory_len).unwrap(); + let mut directory = crate::format::codec::decode_directory( + &mut Cursor::new(&bytes[directory_start..]), + &DeserializationLimits::default(), + ) + .unwrap(); + directory.relations[0].1 = "Conflicts_With_Default".into(); + crate::format::codec::update_directory_len(&mut directory).unwrap(); + crate::format::codec::update_directory_crc(&mut directory).unwrap(); + let mut replacement = Vec::new(); + crate::format::codec::encode_directory(&directory, &mut replacement).unwrap(); + bytes.truncate(directory_start); + bytes.extend_from_slice(&replacement); + std::fs::write(&archive_path, &bytes).unwrap(); + let original = bytes; + + assert!(matches!( + append_files( + &archive_path, + AppendOptions::new(sender, vec![recipient]), + &[input_path], + ), + Err(FsError::Core { + source: PithosError::ConflictingRelationshipDefinition(0), + .. + }) + )); + assert_eq!(std::fs::read(&archive_path).unwrap(), original); + } + + #[test] + fn post_mutation_failures_roll_back_to_byte_identical_original_content() { + for (point, durability) in [ + (AppendFailurePoint::AfterMarker, AppendDurability::Flush), + (AppendFailurePoint::AfterPayload, AppendDurability::Flush), + (AppendFailurePoint::DuringDirectory, AppendDurability::Flush), + (AppendFailurePoint::Flush, AppendDurability::Flush), + (AppendFailurePoint::SyncAll, AppendDurability::SyncAll), + (AppendFailurePoint::MetadataQuery, AppendDurability::Flush), + ] { + let temporary = tempfile::tempdir().unwrap(); + let fixture = append_fixture(&temporary); + let original = std::fs::read(&fixture.archive).unwrap(); + let source = fixture.append_source("rollback.txt", b"rollback append payload"); + + let error = append_files_with_injected_failure( + &fixture.archive, + fixture.options(durability), + &[source], + point, + ) + .unwrap_err(); + + assert!( + matches!( + error, + FsError::Core { + source: PithosError::AppendRolledBack { .. }, + .. + } + ), + "unexpected error at {point:?}: {error:?}" + ); + assert_eq!( + std::fs::read(&fixture.archive).unwrap(), + original, + "rollback at {point:?} did not restore the original bytes" + ); + fixture.open(); + } + } + + #[test] + fn rollback_failure_reports_both_failures_recovery_lengths_and_no_secrets() { + for point in [ + AppendFailurePoint::RollbackTruncate, + AppendFailurePoint::RollbackFlush, + ] { + let temporary = tempfile::tempdir().unwrap(); + let fixture = append_fixture(&temporary); + let original_len = std::fs::metadata(&fixture.archive).unwrap().len(); + let source = + fixture.append_source("rollback-failure.txt", b"ROLLBACK_PLAINTEXT_SENTINEL"); + + let error = append_files_with_injected_failure( + &fixture.archive, + fixture.options(AppendDurability::Flush), + &[source], + point, + ) + .unwrap_err(); + let message = error.to_string(); + let debug = format!("{error:?}"); + + assert!(matches!( + error, + FsError::Core { + source: PithosError::AppendRollbackFailed { + original_len: reported_original, + observed_len, + .. + }, + .. + } if reported_original == original_len && observed_len > original_len + )); + assert!( + message.contains("injected append write failure"), + "{message}" + ); + assert!(message.contains("injected append rollback"), "{message}"); + assert!(message.contains(&original_len.to_string()), "{message}"); + for secret in ["ROLLBACK_PLAINTEXT_SENTINEL", "PRIVATE KEY"] { + assert!(!message.contains(secret), "{message}"); + assert!(!debug.contains(secret), "{debug}"); + } + } + } +} diff --git a/crates/pithos_lib/tests/append_durability.rs b/crates/pithos_lib/tests/append_durability.rs new file mode 100644 index 0000000..329eb56 --- /dev/null +++ b/crates/pithos_lib/tests/append_durability.rs @@ -0,0 +1,155 @@ +mod common; + +use common::append::{append, append_fixture}; +use common::util::{private_key, public_key}; +use pithos_lib::archive::{AppendDurability, AppendOptions}; +use pithos_lib::fs::{FsError, append_files}; +use std::fs::OpenOptions; +use std::os::unix::fs::symlink; + +#[test] +fn append_keeps_the_existing_archive_prefix_unchanged() { + let temporary = tempfile::tempdir().unwrap(); + let fixture = append_fixture(&temporary); + let original = std::fs::read(&fixture.archive).unwrap(); + let source = fixture.append_source("appended.txt", b"appended payload"); + + append(&fixture.archive, vec![source]).unwrap(); + + let updated = std::fs::read(&fixture.archive).unwrap(); + assert!(updated.starts_with(&original)); + assert!( + fixture + .open_as("recipient1") + .entry("appended.txt") + .unwrap() + .is_some() + ); +} + +#[test] +fn append_observation_reports_actual_snapshot_reads_and_tail_bytes() { + let temporary = tempfile::tempdir().unwrap(); + let fixture = append_fixture(&temporary); + let source = fixture.append_source("observed.txt", b"observed append payload"); + let observation = append_files( + &fixture.archive, + AppendOptions::new( + private_key("sender"), + vec![public_key("sender"), public_key("recipient1")], + ), + &[source], + ) + .unwrap(); + + assert!(observation.source_read_count > 0); + assert!(observation.source_read_bytes > 0); + assert!(observation.final_archive_bytes > observation.base_archive_bytes); + assert_eq!( + observation.final_archive_bytes - observation.base_archive_bytes, + std::fs::metadata(&fixture.archive).unwrap().len() - observation.base_archive_bytes + ); +} + +#[test] +fn append_rejects_an_empty_manifest_before_mutation() { + let temporary = tempfile::tempdir().unwrap(); + let fixture = append_fixture(&temporary); + let original = std::fs::read(&fixture.archive).unwrap(); + + assert!(append(&fixture.archive, Vec::new()).is_err()); + assert_eq!(std::fs::read(&fixture.archive).unwrap(), original); +} + +#[test] +fn append_rejects_the_archive_itself_before_mutation() { + let temporary = tempfile::tempdir().unwrap(); + let fixture = append_fixture(&temporary); + let original = std::fs::read(&fixture.archive).unwrap(); + + assert!(matches!( + append(&fixture.archive, vec![fixture.archive.clone()]), + Err(FsError::AppendSourceIsArchive { .. }) + )); + assert_eq!(std::fs::read(&fixture.archive).unwrap(), original); +} + +#[test] +fn direct_append_rejects_a_symlink_archive_target_without_mutation() { + let temporary = tempfile::tempdir().unwrap(); + let fixture = append_fixture(&temporary); + let alias = temporary.path().join("archive-link.pith"); + symlink(&fixture.archive, &alias).unwrap(); + let source = fixture.append_source("through-link.txt", b"must not append"); + let before = std::fs::read(&fixture.archive).unwrap(); + + assert!(append(&alias, vec![source]).is_err()); + assert_eq!(std::fs::read(&fixture.archive).unwrap(), before); +} + +#[cfg(unix)] +#[test] +fn direct_append_and_grant_reject_lock_contention_without_mutation() { + let temporary = tempfile::tempdir().unwrap(); + let fixture = append_fixture(&temporary); + let original = std::fs::read(&fixture.archive).unwrap(); + let source = fixture.append_source("locked.txt", b"locked append payload"); + let lock = OpenOptions::new() + .read(true) + .write(true) + .open(&fixture.archive) + .unwrap(); + rustix::fs::flock(&lock, rustix::fs::FlockOperation::NonBlockingLockExclusive).unwrap(); + + assert!(matches!( + append(&fixture.archive, vec![source]), + Err(FsError::AppendLocked { path }) if path == fixture.archive + )); + assert!(matches!( + common::append::grant( + &fixture.archive, + "sender", + vec!["recipient2"], + vec![common::append::SELECTED_ID], + ), + Err(FsError::AppendLocked { path }) if path == fixture.archive + )); + assert_eq!(std::fs::read(&fixture.archive).unwrap(), original); +} + +#[test] +fn sync_all_durability_publishes_a_readable_child() { + let temporary = tempfile::tempdir().unwrap(); + let fixture = append_fixture(&temporary); + let source = fixture.append_source("durable.txt", b"durable append payload"); + + append_files( + &fixture.archive, + AppendOptions::new( + private_key("sender"), + vec![public_key("sender"), public_key("recipient1")], + ) + .with_durability(AppendDurability::SyncAll), + &[source], + ) + .unwrap(); + + let mut contents = Vec::new(); + fixture + .open_as("recipient1") + .copy_to("durable.txt", &mut contents) + .unwrap(); + assert_eq!(contents, b"durable append payload"); +} + +#[test] +fn manifest_failure_before_mutation_preserves_the_original_archive() { + let temporary = tempfile::tempdir().unwrap(); + let fixture = append_fixture(&temporary); + let present = fixture.append_source("present.txt", b"written before the failed source"); + let missing = temporary.path().join("missing.txt"); + + let original = std::fs::read(&fixture.archive).unwrap(); + assert!(append(&fixture.archive, vec![present, missing]).is_err()); + assert_eq!(std::fs::read(&fixture.archive).unwrap(), original); +} diff --git a/crates/pithos_lib/tests/append_workflow.rs b/crates/pithos_lib/tests/append_workflow.rs new file mode 100644 index 0000000..745533b --- /dev/null +++ b/crates/pithos_lib/tests/append_workflow.rs @@ -0,0 +1,185 @@ +mod common; + +use common::append::{WITHHELD_ID, append, append_fixture, append_with_cdc, archive_with_entry}; +use pithos_lib::archive::CdcConfig; +use pithos_lib::error::PithosError; +use pithos_lib::fs::FsError; + +#[test] +fn direct_append_file_workflow_adds_a_file() { + let temporary = tempfile::tempdir().unwrap(); + let fixture = append_fixture(&temporary); + let source = fixture.append_source("appended.txt", b"appended payload"); + + append(&fixture.archive, vec![source]).unwrap(); + + let archive = fixture.open_as("recipient1"); + let mut contents = Vec::new(); + archive.copy_to("appended.txt", &mut contents).unwrap(); + assert_eq!(contents, b"appended payload"); +} + +#[cfg(unix)] +#[test] +fn append_rejects_hard_link_aliases_before_mutation() { + let temporary = tempfile::tempdir().unwrap(); + let fixture = append_fixture(&temporary); + let original = std::fs::read(&fixture.archive).unwrap(); + let alias = temporary.path().join("archive-alias.pith"); + std::fs::hard_link(&fixture.archive, &alias).unwrap(); + + assert!(matches!( + append(&fixture.archive, vec![alias]), + Err(FsError::AppendSourceIsArchive { .. }) + )); + assert_eq!(std::fs::read(&fixture.archive).unwrap(), original); +} + +#[cfg(unix)] +#[test] +fn append_rejects_recursive_hard_link_aliases_before_mutation() { + let temporary = tempfile::tempdir().unwrap(); + let fixture = append_fixture(&temporary); + let original = std::fs::read(&fixture.archive).unwrap(); + let input = temporary.path().join("input"); + std::fs::create_dir(&input).unwrap(); + std::fs::write(input.join("ordinary.txt"), b"ordinary input").unwrap(); + std::fs::hard_link(&fixture.archive, input.join("archive-alias.pith")).unwrap(); + + assert!(matches!( + append(&fixture.archive, vec![input]), + Err(FsError::AppendSourceIsArchive { .. }) + )); + assert_eq!(std::fs::read(&fixture.archive).unwrap(), original); +} + +#[test] +fn append_links_the_parent_directory_to_existing_entries() { + let temporary = tempfile::tempdir().unwrap(); + let fixture = append_fixture(&temporary); + let source = fixture.append_source("appended.txt", b"appended payload"); + + append(&fixture.archive, vec![source]).unwrap(); + + let archive = fixture.open_as("recipient1"); + assert!(archive.entry("selected.txt").unwrap().is_some()); + assert!(archive.entry("appended.txt").unwrap().is_some()); +} + +#[test] +fn append_reuses_the_earliest_compatible_block_across_three_generations() { + let temporary = tempfile::tempdir().unwrap(); + let fixture = append_fixture(&temporary); + let prefix = std::fs::read(&fixture.archive).unwrap(); + let source = fixture.append_source("reused.txt", b"selected base payload"); + + append(&fixture.archive, vec![source]).unwrap(); + + let appended = std::fs::read(&fixture.archive).unwrap(); + let suffix = &appended[prefix.len()..]; + assert!(!suffix.windows(4).any(|window| window == b"BLCK")); + let archive = fixture.open_as("recipient1"); + let mut contents = Vec::new(); + archive.copy_to("reused.txt", &mut contents).unwrap(); + assert_eq!(contents, b"selected base payload"); +} + +#[test] +fn append_continues_file_id_allocation() { + let temporary = tempfile::tempdir().unwrap(); + let fixture = append_fixture(&temporary); + let source = fixture.append_source("appended.txt", b"appended payload"); + + append(&fixture.archive, vec![source]).unwrap(); + + let archive = fixture.open_as("recipient1"); + assert_eq!( + archive.entry("appended.txt").unwrap().unwrap().id, + WITHHELD_ID + 1 + ); +} + +#[test] +fn append_rejects_active_child_duplicate_paths() { + let temporary = tempfile::tempdir().unwrap(); + let fixture = append_fixture(&temporary); + let source = fixture.append_source("duplicate.txt", b"duplicate payload"); + + let original = std::fs::read(&fixture.archive).unwrap(); + assert!(matches!( + append(&fixture.archive, vec![source.clone(), source]), + Err(FsError::Core { + source: PithosError::PathOccupied(_), + .. + }) + )); + assert_eq!(std::fs::read(&fixture.archive).unwrap(), original); +} + +#[test] +fn append_rejects_ancestor_hierarchy_conflicts_in_both_directions() { + let temporary = tempfile::tempdir().unwrap(); + let archive = archive_with_entry(&temporary, "child"); + let original = std::fs::read(&archive).unwrap(); + let root = temporary.path().join("root"); + std::fs::create_dir_all(root.join("child")).unwrap(); + std::fs::write(root.join("child/leaf"), b"new").unwrap(); + assert!(append(&archive, vec![root]).is_err()); + assert_eq!(std::fs::read(&archive).unwrap(), original); + + let temporary = tempfile::tempdir().unwrap(); + let archive = archive_with_entry(&temporary, "parent/child"); + let original = std::fs::read(&archive).unwrap(); + let source = temporary.path().join("parent"); + std::fs::write(&source, b"new").unwrap(); + assert!(append(&archive, vec![source]).is_err()); + assert_eq!(std::fs::read(&archive).unwrap(), original); +} + +#[test] +fn append_streams_multiblock_content_and_writes_one_child_directory_without_a_header() { + let temporary = tempfile::tempdir().unwrap(); + let fixture = append_fixture(&temporary); + let original = std::fs::read(&fixture.archive).unwrap(); + let source = fixture.append_source( + "multiblock.bin", + &(0..8192u32).flat_map(u32::to_le_bytes).collect::>(), + ); + + append_with_cdc( + &fixture.archive, + CdcConfig::new(64, 256, 1024).unwrap(), + vec![source], + ) + .unwrap(); + + let updated = std::fs::read(&fixture.archive).unwrap(); + let suffix = &updated[original.len()..]; + assert!( + suffix + .windows(4) + .filter(|window| *window == b"BLCK") + .count() + > 1 + ); + assert_eq!( + suffix + .windows(8) + .filter(|window| *window == b"PITHOSDR") + .count(), + 1 + ); + assert_eq!( + updated + .windows(6) + .filter(|window| *window == b"PITH\x80\x02") + .count(), + 1 + ); + let mut contents = Vec::new(); + fixture + .open_as("recipient1") + .copy_to("multiblock.bin", &mut contents) + .unwrap(); + assert_eq!(contents.len(), 8192 * 4); +} diff --git a/crates/pithos_lib/tests/grant_readers.rs b/crates/pithos_lib/tests/grant_readers.rs new file mode 100644 index 0000000..9e71869 --- /dev/null +++ b/crates/pithos_lib/tests/grant_readers.rs @@ -0,0 +1,344 @@ +mod common; + +use common::append::{ + DIRECTORY_ID, SELECTED_ID, WITHHELD_ID, append_fixture, archive_with_symlink, grant, +}; +use common::util::{open, private_key, public_key}; +use pithos_lib::archive::{ + AccessKeys, AppendOptions, ArchivePath, ArchiveWriter, EntryKind, EntryMetadata, + ProcessingOptions, WriteOptions, +}; +use pithos_lib::error::PithosError; +use pithos_lib::fs::{FsError, grant_readers}; +use std::fs::File; +use std::io::Cursor; +use std::os::unix::fs::symlink; + +#[test] +fn direct_grant_rejects_a_symlink_archive_target_without_mutation() { + let temporary = tempfile::tempdir().unwrap(); + let fixture = append_fixture(&temporary); + let alias = temporary.path().join("archive-link.pith"); + symlink(&fixture.archive, &alias).unwrap(); + let before = std::fs::read(&fixture.archive).unwrap(); + + assert!(grant(&alias, "sender", vec!["recipient2"], vec![SELECTED_ID],).is_err()); + assert_eq!(std::fs::read(&fixture.archive).unwrap(), before); +} + +#[test] +fn direct_grant_reader_workflow_succeeds_for_a_selected_file() { + let temporary = tempfile::tempdir().unwrap(); + let fixture = append_fixture(&temporary); + + grant( + &fixture.archive, + "sender", + vec!["recipient2"], + vec![SELECTED_ID], + ) + .unwrap(); + + let archive = fixture.open_as("recipient2"); + assert!(matches!( + archive.entry("selected.txt").unwrap().unwrap().kind, + EntryKind::File { + available: true, + .. + } + )); +} + +#[test] +fn grant_observation_reports_actual_snapshot_reads_and_metadata_tail() { + let temporary = tempfile::tempdir().unwrap(); + let fixture = append_fixture(&temporary); + let observation = grant_readers( + &fixture.archive, + AppendOptions::new(private_key("sender"), vec![public_key("recipient2")]), + &[SELECTED_ID], + ) + .unwrap(); + + assert!(observation.source_read_count > 0); + assert!(observation.source_read_bytes > 0); + assert!(observation.final_archive_bytes > observation.base_archive_bytes); +} + +#[test] +fn newly_authorized_reader_can_access_selected_files() { + let temporary = tempfile::tempdir().unwrap(); + let fixture = append_fixture(&temporary); + + grant( + &fixture.archive, + "sender", + vec!["recipient2"], + vec![SELECTED_ID], + ) + .unwrap(); + + let mut contents = Vec::new(); + fixture + .open_as("recipient2") + .copy_to("selected.txt", &mut contents) + .unwrap(); + assert_eq!(contents, b"selected base payload"); +} + +#[test] +fn selective_grant_leaves_unselected_files_unavailable() { + let temporary = tempfile::tempdir().unwrap(); + let fixture = append_fixture(&temporary); + + grant( + &fixture.archive, + "sender", + vec!["recipient2"], + vec![SELECTED_ID], + ) + .unwrap(); + + assert!(matches!( + fixture + .open_as("recipient2") + .copy_to("withheld.txt", &mut Vec::new()), + Err(PithosError::ContentUnavailable) + )); +} + +#[test] +fn an_existing_authorized_recipient_can_regrant_but_an_unrelated_key_cannot() { + let temporary = tempfile::tempdir().unwrap(); + let fixture = append_fixture(&temporary); + + grant( + &fixture.archive, + "recipient1", + vec!["recipient2"], + vec![SELECTED_ID], + ) + .unwrap(); + assert!( + fixture + .open_as("recipient2") + .copy_to("selected.txt", &mut Vec::new()) + .is_ok() + ); + + let original = std::fs::read(&fixture.archive).unwrap(); + assert!( + grant( + &fixture.archive, + "recipient2", + vec!["sender"], + vec![WITHHELD_ID], + ) + .is_err() + ); + assert_eq!(std::fs::read(&fixture.archive).unwrap(), original); +} + +#[test] +fn grant_rejects_a_missing_file_id() { + let temporary = tempfile::tempdir().unwrap(); + let fixture = append_fixture(&temporary); + let original = std::fs::read(&fixture.archive).unwrap(); + + assert!(matches!( + grant( + &fixture.archive, + "sender", + vec!["recipient2"], + vec![u64::MAX], + ), + Err(FsError::Core { + source: PithosError::SnapshotFileIdNotFound(u64::MAX), + .. + }) + )); + assert_eq!(std::fs::read(&fixture.archive).unwrap(), original); +} + +#[test] +fn grant_rejects_duplicate_recipients() { + let temporary = tempfile::tempdir().unwrap(); + let fixture = append_fixture(&temporary); + let original = std::fs::read(&fixture.archive).unwrap(); + + assert!(matches!( + grant( + &fixture.archive, + "sender", + vec!["recipient2", "recipient2"], + vec![SELECTED_ID], + ), + Err(FsError::Core { + source: PithosError::DuplicateRecipientKey, + .. + }) + )); + assert_eq!(std::fs::read(&fixture.archive).unwrap(), original); +} + +#[test] +fn grant_rejects_duplicate_file_ids() { + let temporary = tempfile::tempdir().unwrap(); + let fixture = append_fixture(&temporary); + let original = std::fs::read(&fixture.archive).unwrap(); + + assert!(matches!( + grant( + &fixture.archive, + "sender", + vec!["recipient1"], + vec![SELECTED_ID, SELECTED_ID], + ), + Err(FsError::Core { + source: PithosError::DuplicateRecipientFileId, + .. + }) + )); + assert_eq!(std::fs::read(&fixture.archive).unwrap(), original); +} + +#[test] +fn grant_rejects_non_content_entries() { + let temporary = tempfile::tempdir().unwrap(); + let fixture = append_fixture(&temporary); + let original = std::fs::read(&fixture.archive).unwrap(); + + assert!(matches!( + grant( + &fixture.archive, + "sender", + vec!["recipient2"], + vec![DIRECTORY_ID], + ), + Err(FsError::Core { + source: PithosError::SnapshotDirectoryHasNoContent(DIRECTORY_ID), + .. + }) + )); + assert_eq!(std::fs::read(&fixture.archive).unwrap(), original); +} + +#[test] +fn grant_rejects_empty_recipients_or_ids_before_mutation() { + let temporary = tempfile::tempdir().unwrap(); + let fixture = append_fixture(&temporary); + let original = std::fs::read(&fixture.archive).unwrap(); + + assert!(matches!( + grant(&fixture.archive, "sender", vec![], vec![SELECTED_ID]), + Err(FsError::Core { + source: PithosError::WriterRequiresRecipient, + .. + }) + )); + assert_eq!(std::fs::read(&fixture.archive).unwrap(), original); + assert!(matches!( + grant(&fixture.archive, "sender", vec!["recipient2"], vec![]), + Err(FsError::Core { + source: PithosError::GrantRequiresFileId, + .. + }) + )); + assert_eq!(std::fs::read(&fixture.archive).unwrap(), original); +} + +#[test] +fn grant_rejects_unavailable_and_symlink_ids_before_mutation() { + let temporary = tempfile::tempdir().unwrap(); + let fixture = append_fixture(&temporary); + let original = std::fs::read(&fixture.archive).unwrap(); + + assert!(matches!( + grant( + &fixture.archive, + "recipient2", + vec!["sender"], + vec![SELECTED_ID], + ), + Err(FsError::Core { + source: PithosError::SnapshotContentUnavailable(SELECTED_ID), + .. + }) + )); + assert_eq!(std::fs::read(&fixture.archive).unwrap(), original); + + let archive = archive_with_symlink(&temporary); + let original = std::fs::read(&archive).unwrap(); + assert!(matches!( + grant(&archive, "sender", vec!["recipient2"], vec![0]), + Err(FsError::Core { + source: PithosError::SnapshotSymlinkHasNoContent(0), + .. + }) + )); + assert_eq!(std::fs::read(&archive).unwrap(), original); +} + +#[test] +fn grant_allows_metadata_entries() { + let temporary = tempfile::tempdir().unwrap(); + let archive = temporary.path().join("metadata.pith"); + let sender = private_key("sender"); + let mut writer = ArchiveWriter::create( + File::create(&archive).unwrap(), + WriteOptions::new(sender, vec![public_key("sender"), public_key("recipient1")]), + ) + .unwrap(); + writer + .add_metadata( + ArchivePath::new("description.json").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(true, 2).unwrap(), + None, + Cursor::new(b"metadata payload"), + ) + .unwrap(); + writer.finish().unwrap(); + + grant(&archive, "sender", vec!["recipient2"], vec![0]).unwrap(); + let mut contents = Vec::new(); + open( + &archive, + AccessKeys::new().with_key(private_key("recipient2")), + ) + .copy_to("description.json", &mut contents) + .unwrap(); + assert_eq!(contents, b"metadata payload"); +} + +#[test] +fn grant_multiple_ids_to_multiple_recipients_in_one_metadata_only_child() { + let temporary = tempfile::tempdir().unwrap(); + let fixture = append_fixture(&temporary); + let prefix = std::fs::read(&fixture.archive).unwrap(); + + grant( + &fixture.archive, + "sender", + vec!["recipient1", "recipient2"], + vec![SELECTED_ID, WITHHELD_ID], + ) + .unwrap(); + + for reader in ["recipient1", "recipient2"] { + let archive = fixture.open_as(reader); + for path in ["selected.txt", "withheld.txt"] { + assert!(archive.copy_to(path, &mut Vec::new()).is_ok()); + } + } + + let updated = std::fs::read(&fixture.archive).unwrap(); + assert!(updated.starts_with(&prefix)); + assert_eq!( + updated + .windows(6) + .filter(|window| *window == b"PITH\x80\x02") + .count(), + 1 + ); +} From 4c1710049803936eba102ed8b29729d65e14460d Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Tue, 11 Aug 2026 23:01:08 +0200 Subject: [PATCH 009/166] refactor: isolate filesystem and presentation adapters --- crates/pithos_lib/src/adapters/crypt4gh.rs | 620 +++++++ crates/pithos_lib/src/adapters/ro_crate.rs | 1536 +++++++++++++++++ crates/pithos_lib/src/fs/extraction.rs | 401 +++++ crates/pithos_lib/src/helpers/crypt4gh.rs | 514 ------ crates/pithos_lib/src/helpers/ro_crate.rs | 321 ---- crates/pithos_lib/src/io/extraction.rs | 154 -- crates/pithos_lib/tests/compile_fail.rs | 9 + crates/pithos_lib/tests/crypt4gh_export.rs | 31 + crates/pithos_lib/tests/data/README.md | 14 + .../brain_data_accession_number.txt | 2 + .../brain_data_assession_number.txt | 2 - .../literature/Hodges06_human_brain_Affy.pdf | Bin 359786 -> 0 bytes .../literature/pithos-placeholder.pdf | Bin 0 -> 327 bytes .../data/dummy_dir/ro-crate-metadata.json | 16 +- crates/pithos_lib/tests/data/ro-crate.zip | Bin 313950 -> 0 bytes crates/pithos_lib/tests/platform_fs.rs | 193 +++ .../pithos_lib/tests/ro_crate_conversion.rs | 106 ++ crates/pithos_lib/tests/ro_crate_directory.rs | 274 +++ crates/pithos_lib/tests/ro_crate_zip.rs | 471 +++++ .../ui/crypt4gh_wire_record_is_private.rs | 5 + .../ui/crypt4gh_wire_record_is_private.stderr | 11 + .../ui/incomplete_writer_cannot_resume.rs | 11 + .../ui/incomplete_writer_cannot_resume.stderr | 5 + .../tests/ui/non_copy_static_secret.rs | 7 + .../tests/ui/non_copy_static_secret.stderr | 9 + .../tests/ui/writer_finish_consumes_writer.rs | 11 + .../ui/writer_finish_consumes_writer.stderr | 15 + .../tests/ui/writer_use_after_finish.rs | 10 + .../tests/ui/writer_use_after_finish.stderr | 15 + 29 files changed, 3764 insertions(+), 999 deletions(-) create mode 100644 crates/pithos_lib/src/adapters/crypt4gh.rs create mode 100644 crates/pithos_lib/src/adapters/ro_crate.rs create mode 100644 crates/pithos_lib/src/fs/extraction.rs delete mode 100644 crates/pithos_lib/src/helpers/crypt4gh.rs delete mode 100644 crates/pithos_lib/src/helpers/ro_crate.rs delete mode 100644 crates/pithos_lib/src/io/extraction.rs create mode 100644 crates/pithos_lib/tests/compile_fail.rs create mode 100644 crates/pithos_lib/tests/crypt4gh_export.rs create mode 100644 crates/pithos_lib/tests/data/README.md create mode 100644 crates/pithos_lib/tests/data/dummy_dir/dataset/brain_gene_expression_data/brain_data_accession_number.txt delete mode 100644 crates/pithos_lib/tests/data/dummy_dir/dataset/brain_gene_expression_data/brain_data_assession_number.txt delete mode 100644 crates/pithos_lib/tests/data/dummy_dir/literature/Hodges06_human_brain_Affy.pdf create mode 100644 crates/pithos_lib/tests/data/dummy_dir/literature/pithos-placeholder.pdf delete mode 100644 crates/pithos_lib/tests/data/ro-crate.zip create mode 100644 crates/pithos_lib/tests/platform_fs.rs create mode 100644 crates/pithos_lib/tests/ro_crate_conversion.rs create mode 100644 crates/pithos_lib/tests/ro_crate_directory.rs create mode 100644 crates/pithos_lib/tests/ro_crate_zip.rs create mode 100644 crates/pithos_lib/tests/ui/crypt4gh_wire_record_is_private.rs create mode 100644 crates/pithos_lib/tests/ui/crypt4gh_wire_record_is_private.stderr create mode 100644 crates/pithos_lib/tests/ui/incomplete_writer_cannot_resume.rs create mode 100644 crates/pithos_lib/tests/ui/incomplete_writer_cannot_resume.stderr create mode 100644 crates/pithos_lib/tests/ui/non_copy_static_secret.rs create mode 100644 crates/pithos_lib/tests/ui/non_copy_static_secret.stderr create mode 100644 crates/pithos_lib/tests/ui/writer_finish_consumes_writer.rs create mode 100644 crates/pithos_lib/tests/ui/writer_finish_consumes_writer.stderr create mode 100644 crates/pithos_lib/tests/ui/writer_use_after_finish.rs create mode 100644 crates/pithos_lib/tests/ui/writer_use_after_finish.stderr diff --git a/crates/pithos_lib/src/adapters/crypt4gh.rs b/crates/pithos_lib/src/adapters/crypt4gh.rs new file mode 100644 index 0000000..a08b2a3 --- /dev/null +++ b/crates/pithos_lib/src/adapters/crypt4gh.rs @@ -0,0 +1,620 @@ +//! Crypt4GH presentation adapter. +//! +//! This module is the supported Crypt4GH header parser and serializer authority. +//! Its errors retain archive paths and operations without formatting keys, decrypted +//! packet contents, or plaintext. + +use crate::archive::{Archive, ContentOperationError, ExternalBlockResolver}; +use crate::crypto::{self, PrivateKey, PublicKey}; +use crate::error::PithosError; +use blake2::{Blake2b512, Digest}; +use byteorder::{LittleEndian, WriteBytesExt}; +use chacha20poly1305::aead::{Aead, Generate}; +use chacha20poly1305::{ChaCha20Poly1305, KeyInit, Nonce}; +use std::io::Write; +use thiserror::Error; +use x25519_dalek::PublicKey as DalekPublicKey; +use zeroize::{Zeroize, Zeroizing}; + +pub const CRYPT4GH_HEADER_MAGIC: [u8; 8] = [0x63, 0x72, 0x79, 0x70, 0x74, 0x34, 0x67, 0x68]; // "crypt4gh" +pub const CRYPT4GH_HEADER_VERSION: u32 = 1; +pub const CRYPT4GH_BLOCK_SIZE: usize = 65536; +pub const CRYPT4GH_ENCRYPTED_BLOCK_SIZE: usize = 65564; +const PACKET_LENGTH_FIELD_SIZE: usize = size_of::(); +const MIN_PACKET_BODY_LENGTH: usize = size_of::() + 32 + 12 + 16; +const MIN_PACKET_LENGTH: usize = PACKET_LENGTH_FIELD_SIZE + MIN_PACKET_BODY_LENGTH; + +fn derive_ga4gh_header_key( + shared_secret: &[u8; 32], + reader_public_key: &[u8; 32], + writer_public_key: &[u8; 32], +) -> Zeroizing<[u8; 32]> { + let mut digest = Blake2b512::new(); + digest.update(shared_secret); + digest.update(reader_public_key); + digest.update(writer_public_key); + let digest = digest.finalize(); + let mut header_key = Zeroizing::new([0; 32]); + header_key.copy_from_slice(&digest[..32]); + header_key +} + +/// Exports one verified archive entry using the established Crypt4GH framing. +/// +/// The export validates archive content before encrypting it and reports archive, +/// cryptographic, and sink failures with operation context. New headers use the +/// GA4GH KDF. +pub fn export( + archive: &Archive, + path: &str, + recipients: Vec, + sink: &mut W, +) -> Result<(), Crypt4GHError> +where + S: crate::source::ArchiveSource, + E: ExternalBlockResolver, + W: Write + ?Sized, +{ + if recipients.is_empty() { + return Err(Crypt4GHError::InvalidSpec( + "at least one recipient is required".to_string(), + )); + } + + match archive.with_crypt4gh_content(path, |id, sender, file_key| { + let packets = HeaderPacket::from_pithos( + sender, + recipients + .into_iter() + .map(PublicKey::into_dalek_public_key) + .collect(), + file_key.expose_for_protocol(), + )?; + Crypt4GHHeader::new(packets)?.serialize(sink)?; + + let mut pending = Zeroizing::new(Vec::with_capacity(CRYPT4GH_BLOCK_SIZE)); + archive + .for_each_verified_file_block(id, |plaintext| { + let mut offset = 0; + while offset < plaintext.len() { + let available = CRYPT4GH_BLOCK_SIZE - pending.len(); + let count = available.min(plaintext.len() - offset); + pending.extend_from_slice(&plaintext[offset..offset + count]); + offset += count; + if pending.len() == CRYPT4GH_BLOCK_SIZE { + sink.write_all( + &crypto::seal_crypt4gh_payload( + file_key.expose_for_protocol(), + &pending, + ) + .map_err(|source| { + Crypt4GHError::Crypto { + operation: "encrypt payload", + source, + } + })?, + ) + .map_err(|source| Crypt4GHError::Sink { + operation: "write payload", + source, + })?; + pending.zeroize(); + } + } + Ok(()) + }) + .map_err(|error| match error { + ContentOperationError::Core(source) => Crypt4GHError::Archive { + operation: "read verified archive content", + path: path.to_owned(), + source, + }, + ContentOperationError::Callback(error) => error, + })?; + if !pending.is_empty() { + sink.write_all( + &crypto::seal_crypt4gh_payload(file_key.expose_for_protocol(), &pending).map_err( + |source| Crypt4GHError::Crypto { + operation: "encrypt payload", + source, + }, + )?, + ) + .map_err(|source| Crypt4GHError::Sink { + operation: "write payload", + source, + })?; + } + Ok(()) + }) { + Ok(()) => Ok(()), + Err(ContentOperationError::Core(source)) => Err(Crypt4GHError::Archive { + operation: "access archive content", + path: path.to_owned(), + source, + }), + Err(ContentOperationError::Callback(error)) => Err(error), + } +} + +#[derive(Debug, Error)] +pub enum Crypt4GHError { + #[error("Crypt4GH archive {operation} failed for {path}: {source}")] + Archive { + operation: &'static str, + path: String, + #[source] + source: PithosError, + }, + #[error("Crypt4GH {operation} failed: {source}")] + Sink { + operation: &'static str, + #[source] + source: std::io::Error, + }, + #[error("Crypt4GH {operation} failed: {source}")] + Crypto { + operation: &'static str, + #[source] + source: crate::crypto::CryptoError, + }, + #[error("Unable to parse `{0}` from bytes")] + FromBytesError(String), + #[error("Invalid value for spec: `{0}`")] + InvalidSpec(String), + #[error("Unable to encrypt: `{0}`")] + EncryptionError(String), + #[error("Unable to serialize: {0}")] + Serialization(#[from] std::io::Error), +} + +pub(crate) struct Crypt4GHHeader { + magic: [u8; 8], + version: u32, + header_packets: Vec, +} + +impl Crypt4GHHeader { + fn new(header_packets: Vec) -> Result { + checked_packet_count(header_packets.len())?; + Ok(Crypt4GHHeader { + magic: CRYPT4GH_HEADER_MAGIC, + version: CRYPT4GH_HEADER_VERSION, + header_packets, + }) + } + + #[tracing::instrument(level = "trace", skip(self, writer))] + fn serialize(&self, writer: &mut W) -> Result<(), Crypt4GHError> { + if self.magic != CRYPT4GH_HEADER_MAGIC { + return Err(Crypt4GHError::InvalidSpec("magic bytes".to_string())); + } + if self.version != CRYPT4GH_HEADER_VERSION { + return Err(Crypt4GHError::InvalidSpec("version".to_string())); + } + let packet_count = checked_packet_count(self.header_packets.len())?; + for packet in &self.header_packets { + packet.validate_serializable()?; + } + writer.write_all(&self.magic)?; + writer.write_u32::(self.version)?; + writer.write_u32::(packet_count)?; + for packet in &self.header_packets { + packet.serialize(writer)?; + } + Ok(()) + } +} + +fn checked_packet_count(len: usize) -> Result { + u32::try_from(len).map_err(|_| Crypt4GHError::InvalidSpec("packet count".to_string())) +} + +struct HeaderPacket { + length: u32, + encryption_method: u32, // Currently only 0 (Chacha20-Poly1305) + writers_pubkey: [u8; 32], // Writer's public key + nonce: [u8; 12], // Nonce for encryption + packet_data: PacketData, + mac: [u8; 16], // Message Authentication Code (MAC) +} + +impl HeaderPacket { + #[tracing::instrument(level = "trace", skip(sender_key, reader_keys, data_key))] + fn from_pithos( + sender_key: &PrivateKey, + reader_keys: Vec, + data_key: &[u8; 32], + ) -> Result, Crypt4GHError> { + let sender_key = sender_key.as_dalek_static_secret(); + let sender_pubkey = DalekPublicKey::from(&sender_key); + let mut header_packets = vec![]; + for reader in reader_keys { + let session_key = crypto::derive_shared(sender_key.as_bytes(), reader.as_bytes()) + .map_err(|source| Crypt4GHError::Crypto { + operation: "derive header session key", + source, + })?; + let header_key = derive_ga4gh_header_key( + session_key.expose_for_protocol(), + reader.as_bytes(), + sender_pubkey.as_bytes(), + ); + let nonce = Nonce::generate(); + let mut packet_data = + PacketData::Decrypted(vec![Packet::Encryption(EncryptionPacket { + packet_type: 0, + encryption_method: 0, + encryption_key: Box::new(Zeroizing::new(*data_key)), + })]); + let mac = packet_data.encrypt(&header_key, &nonce)?; + let header_packet = HeaderPacket { + length: u32::try_from( + MIN_PACKET_LENGTH + .checked_add(packet_data.serialized_len()?) + .ok_or_else(|| { + Crypt4GHError::EncryptionError("header packet length".to_string()) + })?, + ) + .map_err(|_| Crypt4GHError::EncryptionError("header packet length".to_string()))?, + encryption_method: 0, + writers_pubkey: sender_pubkey.to_bytes(), + nonce: nonce.into(), + packet_data, + mac, + }; + + header_packets.push(header_packet); + } + + Ok(header_packets) + } + + #[tracing::instrument(level = "trace", skip(self))] + fn validate_serializable(&self) -> Result<(), Crypt4GHError> { + let PacketData::Encrypted(data) = &self.packet_data else { + return Err(Crypt4GHError::InvalidSpec( + "packet data is not encrypted".to_string(), + )); + }; + let expected_length = MIN_PACKET_LENGTH + .checked_add(data.len()) + .ok_or_else(|| Crypt4GHError::InvalidSpec("header packet length".to_string()))?; + let expected_length = u32::try_from(expected_length) + .map_err(|_| Crypt4GHError::InvalidSpec("header packet length".to_string()))?; + if self.length != expected_length { + return Err(Crypt4GHError::InvalidSpec( + "header packet length".to_string(), + )); + } + if self.encryption_method != 0 { + return Err(Crypt4GHError::InvalidSpec( + "unsupported encryption method".to_string(), + )); + } + Ok(()) + } + + #[tracing::instrument(level = "trace", skip(self, writer))] + fn serialize(&self, writer: &mut W) -> Result<(), Crypt4GHError> { + self.validate_serializable()?; + writer.write_u32::(self.length)?; + writer.write_u32::(self.encryption_method)?; + writer.write_all(&self.writers_pubkey)?; + writer.write_all(&self.nonce)?; + + match &self.packet_data { + PacketData::Encrypted(data) => { + writer.write_all(data)?; + } + PacketData::Decrypted(_) => { + return Err(Crypt4GHError::InvalidSpec( + "packet data is not encrypted".to_string(), + )); + } + } + + writer.write_all(&self.mac)?; + Ok(()) + } +} + +enum PacketData { + Encrypted(Vec), + Decrypted(Vec), +} + +impl PacketData { + #[tracing::instrument(level = "trace", skip(self))] + fn serialized_len(&self) -> Result { + match self { + Self::Encrypted(enc_data) => Ok(enc_data.len()), + Self::Decrypted(packets) => packets + .len() + .checked_mul(40) + .ok_or_else(|| Crypt4GHError::EncryptionError("packet data length".to_string())), + } + } +} + +enum Packet { + Encryption(EncryptionPacket), +} + +struct EncryptionPacket { + packet_type: u32, // 0 (Encryption) + encryption_method: u32, // 0 (Chacha20-Poly1305) + encryption_key: Box>, // 32 bytes encryption key +} + +impl TryFrom<&[u8]> for Crypt4GHHeader { + type Error = Crypt4GHError; + + #[tracing::instrument(level = "trace", skip(bytes))] + fn try_from(bytes: &[u8]) -> Result { + const PREFIX_LENGTH: usize = 16; + if bytes.len() < PREFIX_LENGTH { + return Err(Crypt4GHError::FromBytesError("header prefix".to_string())); + } + let magic: [u8; 8] = bytes[..8] + .try_into() + .map_err(|_| Crypt4GHError::FromBytesError("magic bytes".to_string()))?; + if magic != CRYPT4GH_HEADER_MAGIC { + return Err(Crypt4GHError::InvalidSpec("magic bytes".to_string())); + } + let version = u32::from_le_bytes( + bytes[8..12] + .try_into() + .map_err(|_| Crypt4GHError::FromBytesError("version".to_string()))?, + ); + if version != CRYPT4GH_HEADER_VERSION { + return Err(Crypt4GHError::InvalidSpec("version".to_string())); + } + let packet_count = u32::from_le_bytes( + bytes[12..16] + .try_into() + .map_err(|_| Crypt4GHError::FromBytesError("packet count".to_string()))?, + ); + let packet_count_usize = usize::try_from(packet_count) + .map_err(|_| Crypt4GHError::FromBytesError("packet count".to_string()))?; + let minimum_packet_bytes = packet_count_usize + .checked_mul(MIN_PACKET_LENGTH) + .ok_or_else(|| Crypt4GHError::FromBytesError("packet count".to_string()))?; + let minimum_total = PREFIX_LENGTH + .checked_add(minimum_packet_bytes) + .ok_or_else(|| Crypt4GHError::FromBytesError("packet count".to_string()))?; + if minimum_total > bytes.len() { + return Err(Crypt4GHError::FromBytesError( + "packet count exceeds header".to_string(), + )); + } + let mut header_packets = Vec::new(); + header_packets + .try_reserve_exact(packet_count_usize) + .map_err(|_| Crypt4GHError::FromBytesError("packet allocation".to_string()))?; + + let mut offset = PREFIX_LENGTH; + for _ in 0..packet_count_usize { + let length_end = offset + .checked_add(PACKET_LENGTH_FIELD_SIZE) + .ok_or_else(|| Crypt4GHError::FromBytesError("packet position".to_string()))?; + let length: [u8; PACKET_LENGTH_FIELD_SIZE] = bytes + .get(offset..length_end) + .ok_or_else(|| Crypt4GHError::FromBytesError("packet length".to_string()))? + .try_into() + .map_err(|_| Crypt4GHError::FromBytesError("packet length".to_string()))?; + let len = usize::try_from(u32::from_le_bytes(length)) + .map_err(|_| Crypt4GHError::FromBytesError("packet length".to_string()))?; + if len < MIN_PACKET_LENGTH { + return Err(Crypt4GHError::FromBytesError( + "packet length below minimum".to_string(), + )); + } + let end = offset + .checked_add(len) + .ok_or_else(|| Crypt4GHError::FromBytesError("packet length".to_string()))?; + let packet = bytes.get(length_end..end).ok_or_else(|| { + Crypt4GHError::FromBytesError("packet exceeds header".to_string()) + })?; + let mut packet_bytes = Vec::new(); + packet_bytes + .try_reserve_exact(packet.len()) + .map_err(|_| Crypt4GHError::FromBytesError("packet allocation".to_string()))?; + packet_bytes.extend_from_slice(packet); + header_packets.push(HeaderPacket::from_buf(packet_bytes, len)?); + offset = end; + } + if offset != bytes.len() { + return Err(Crypt4GHError::FromBytesError( + "trailing header bytes".to_string(), + )); + } + Ok(Crypt4GHHeader { + magic, + version, + header_packets, + }) + } +} + +impl TryInto> for Crypt4GHHeader { + type Error = Crypt4GHError; + + #[tracing::instrument(level = "trace", skip(self))] + fn try_into(self) -> Result, Self::Error> { + let capacity = self + .header_packets + .iter() + .try_fold(16usize, |size, packet| { + size.checked_add( + usize::try_from(packet.length) + .map_err(|_| Crypt4GHError::InvalidSpec("header length".to_string()))?, + ) + .ok_or_else(|| Crypt4GHError::InvalidSpec("header length".to_string())) + })?; + let mut bytes = Vec::new(); + bytes + .try_reserve_exact(capacity) + .map_err(|_| Crypt4GHError::FromBytesError("header allocation".to_string()))?; + self.serialize(&mut bytes)?; + Ok(bytes) + } +} + +impl HeaderPacket { + #[tracing::instrument(level = "trace", skip(bytes, len))] + fn from_buf(bytes: Vec, len: usize) -> Result { + let packet_bytes = len + .checked_sub(PACKET_LENGTH_FIELD_SIZE) + .ok_or_else(|| Crypt4GHError::FromBytesError("packet length".to_string()))?; + if len < MIN_PACKET_LENGTH + || bytes.len() != packet_bytes + || bytes.len() < MIN_PACKET_BODY_LENGTH + { + return Err(Crypt4GHError::FromBytesError( + "packet is too short".to_string(), + )); + } + let encryption_method = u32::from_le_bytes( + bytes[..4] + .try_into() + .map_err(|_| Crypt4GHError::FromBytesError("encryption method".to_string()))?, + ); + if encryption_method != 0 { + return Err(Crypt4GHError::InvalidSpec( + "unsupported encryption method".to_string(), + )); + } + let writers_pubkey = bytes[4..36] + .try_into() + .map_err(|_| Crypt4GHError::FromBytesError("writer's public key".to_string()))?; + let nonce = bytes[36..48] + .try_into() + .map_err(|_| Crypt4GHError::FromBytesError("nonce".to_string()))?; + let mac_start = bytes + .len() + .checked_sub(16) + .ok_or_else(|| Crypt4GHError::FromBytesError("packet MAC".to_string()))?; + let encrypted_len = mac_start + .checked_sub(48) + .ok_or_else(|| Crypt4GHError::FromBytesError("packet data".to_string()))?; + let mut encrypted_data = Vec::new(); + encrypted_data + .try_reserve_exact(encrypted_len) + .map_err(|_| Crypt4GHError::FromBytesError("packet data allocation".to_string()))?; + encrypted_data.extend_from_slice(&bytes[48..mac_start]); + let mac = bytes[mac_start..] + .try_into() + .map_err(|_| Crypt4GHError::FromBytesError("packet mac".to_string()))?; + + Ok(HeaderPacket { + length: u32::try_from(len) + .map_err(|_| Crypt4GHError::FromBytesError("header packet length".to_string()))?, + encryption_method, + writers_pubkey, + nonce, + packet_data: PacketData::Encrypted(encrypted_data), + mac, + }) + } +} + +impl PacketData { + #[tracing::instrument(level = "trace", skip(self, session_key, nonce))] + fn encrypt( + &mut self, + session_key: &[u8; 32], + nonce: &Nonce, + ) -> Result<[u8; 16], Crypt4GHError> { + let serialized_len = self.serialized_len()?; + if let Self::Decrypted(dec_data) = &self { + let mut enc_data = Zeroizing::new(Vec::new()); + enc_data.try_reserve_exact(serialized_len).map_err(|_| { + Crypt4GHError::EncryptionError("packet data allocation".to_string()) + })?; + for packet in dec_data { + match packet { + Packet::Encryption(enc_packet) => { + enc_data.extend_from_slice(&enc_packet.packet_type.to_le_bytes()); + enc_data.extend_from_slice(&enc_packet.encryption_method.to_le_bytes()); + enc_data.extend_from_slice(&enc_packet.encryption_key[..]); + } + } + } + + let encrypted = ChaCha20Poly1305::new_from_slice(session_key) + .map_err(|_| Crypt4GHError::EncryptionError("Cipher init failed".to_string()))? + .encrypt(nonce, enc_data.as_slice()) + .map_err(|_| Crypt4GHError::EncryptionError("Encrypt chunk failed".to_string()))?; + let mac_start = encrypted.len().checked_sub(16).ok_or_else(|| { + Crypt4GHError::EncryptionError("packet data MAC extraction".to_string()) + })?; + let (ciphertext, mac) = encrypted.split_at(mac_start); + let mac: [u8; 16] = mac.try_into().map_err(|_| { + Crypt4GHError::EncryptionError("packet data MAC extraction".to_string()) + })?; + *self = Self::Encrypted(ciphertext.to_vec()); + Ok(mac) + } else { + Err(Crypt4GHError::EncryptionError( + "Packet data is already encrypted".to_string(), + )) + } + } +} + +#[cfg(test)] +mod zeroization_tests { + use super::*; + use zeroize::ZeroizeOnDrop; + + fn assert_zeroize_on_drop() {} + + #[test] + fn encryption_packets_use_stable_zeroizing_key_owners() { + assert_zeroize_on_drop::>(); + let packet = EncryptionPacket { + packet_type: 0, + encryption_method: 0, + encryption_key: Box::new(Zeroizing::new([7; 32])), + }; + drop(packet); + } + + #[test] + fn packet_count_overflow_is_rejected_instead_of_saturated() { + assert!(matches!( + checked_packet_count(usize::MAX), + Err(Crypt4GHError::InvalidSpec(_)) + )); + } + + #[test] + fn ga4gh_header_kdf_uses_blake2b_512_then_truncates() { + let shared_secret = [ + 0x81, 0x17, 0x55, 0x85, 0x38, 0x36, 0x37, 0xdf, 0x2a, 0xdb, 0xed, 0xe3, 0xa0, 0x36, + 0x18, 0xf7, 0x48, 0xf8, 0x53, 0x6c, 0xb6, 0xba, 0x16, 0xa4, 0xdf, 0x1d, 0xcc, 0x77, + 0x50, 0x73, 0x52, 0x77, + ]; + let reader_public_key = [ + 0x59, 0xef, 0xb4, 0x0e, 0xa7, 0x75, 0xf8, 0x6c, 0xe8, 0x67, 0xfc, 0x3a, 0x20, 0xa1, + 0xad, 0x08, 0xe3, 0x2f, 0xd0, 0xff, 0x5b, 0x1e, 0xf0, 0xff, 0xa4, 0x9f, 0xf1, 0x03, + 0xf5, 0xca, 0x06, 0x2e, + ]; + let writer_public_key = [ + 0xda, 0x06, 0x03, 0xbe, 0x1b, 0xf9, 0x86, 0xa5, 0x73, 0x79, 0xe6, 0xe5, 0xa3, 0x63, + 0xe3, 0xb4, 0x7b, 0xf8, 0x5f, 0x81, 0xaf, 0xdd, 0x4e, 0x00, 0xf7, 0x1f, 0x63, 0x3a, + 0x94, 0x58, 0xe4, 0x59, + ]; + let expected = [ + 0x80, 0x60, 0x43, 0x6a, 0xdc, 0x0c, 0xbe, 0x9a, 0x3e, 0x6a, 0xae, 0x26, 0x39, 0x91, + 0x1f, 0x8a, 0xe0, 0x0f, 0x50, 0xb2, 0x85, 0xe3, 0x6c, 0x0b, 0x12, 0x57, 0x5d, 0x88, + 0x9f, 0x4a, 0x4d, 0xb1, + ]; + + assert_eq!( + *derive_ga4gh_header_key(&shared_secret, &reader_public_key, &writer_public_key), + expected + ); + } +} diff --git a/crates/pithos_lib/src/adapters/ro_crate.rs b/crates/pithos_lib/src/adapters/ro_crate.rs new file mode 100644 index 0000000..47030a2 --- /dev/null +++ b/crates/pithos_lib/src/adapters/ro_crate.rs @@ -0,0 +1,1536 @@ +//! RO-Crate presentation adapter. +//! +//! Upstream `ro-crate-rs` owns graph and schema parsing. This adapter separately +//! owns Pithos source identity, path, metadata, limit, and writer-conversion policy; +//! its errors retain operation, path, ZIP member, and index context. + +use crate::archive::validate_symlink_target; +use crate::archive::{ + ArchivePath, ArchiveWriter, EntryMetadata, EntryReference, ProcessingOptions, WriterError, +}; +use crate::error::PithosError; +use cap_std::fs::{ + Dir, File as CapFile, MetadataExt as CapMetadataExt, PermissionsExt as CapPermissionsExt, +}; +use cap_std::time::{SystemClock, SystemTime}; +use rocraters::ro_crate::read::CrateReadError; +use rocraters::ro_crate::read::read_crate_obj; +pub use rocraters::ro_crate::rocrate::RoCrate; +use std::collections::BTreeMap; +use std::ffi::CString; +use std::fmt; +use std::fs::File; +use std::io::{Cursor, Read, Seek, Write}; +use std::os::fd::AsRawFd; +use std::os::unix::ffi::OsStrExt; +use std::os::unix::fs::MetadataExt; +use std::path::{Component, Path, PathBuf}; +use thiserror::Error; +use zip::result::ZipError; +use zip::{CompressionMethod, ZipArchive}; + +pub const RO_CRATE_METADATA_FILE: &str = "ro-crate-metadata.json"; + +const PARSE_WITHOUT_VALIDATION: i8 = 0; +const MAX_METADATA_BYTES: u64 = 16 * 1024 * 1024; +const MAX_SYMLINK_TARGET_BYTES: u64 = 16 * 1024; + +/// Actionable failures from RO-Crate directory, ZIP, and conversion operations. +#[derive(Debug, Error)] +pub enum RoCrateError { + #[error("RO-Crate {operation} failed for {path}: {source}")] + Io { + operation: &'static str, + path: PathBuf, + #[source] + source: std::io::Error, + }, + #[error("RO-Crate {operation} failed for {path}: {source}")] + Zip { + operation: &'static str, + path: PathBuf, + #[source] + source: ZipError, + }, + #[error( + "RO-Crate {operation} failed for ZIP {archive} member {member} at index {index}: {source}" + )] + ZipMember { + operation: &'static str, + archive: PathBuf, + index: usize, + member: String, + #[source] + source: ZipError, + }, + #[error( + "RO-Crate {operation} failed for ZIP {archive} member {member} at index {index}: {source}" + )] + ZipMemberIo { + operation: &'static str, + archive: PathBuf, + index: usize, + member: String, + #[source] + source: std::io::Error, + }, + #[error("RO-Crate {operation} failed for {path}: {source}")] + DirectoryFileWriter { + operation: &'static str, + path: PathBuf, + #[source] + source: WriterError, + }, + #[error("RO-Crate {operation} failed for {path}: {source}")] + DirectoryCore { + operation: &'static str, + path: PathBuf, + #[source] + source: PithosError, + }, + #[error( + "RO-Crate {operation} failed for ZIP {archive} member {member} at index {index}: {source}" + )] + ZipMemberWriter { + operation: &'static str, + archive: Box, + index: usize, + member: Box, + #[source] + source: WriterError, + }, + #[error( + "RO-Crate {operation} failed for ZIP {archive} member {member} at index {index}: {source}" + )] + ZipMemberCore { + operation: &'static str, + archive: Box, + index: usize, + member: Box, + #[source] + source: PithosError, + }, + #[error("RO-Crate {operation} failed for ZIP {archive} synthetic path {path}: {source}")] + ZipSyntheticWriter { + operation: &'static str, + archive: Box, + path: String, + #[source] + source: WriterError, + }, + #[error("RO-Crate {operation} failed for {path}: {source}")] + Parser { + operation: &'static str, + path: PathBuf, + #[source] + source: CrateReadError, + }, + #[error( + "RO-Crate {operation} failed for ZIP {archive} member {member} at index {index}: {source}" + )] + ZipMemberParser { + operation: &'static str, + archive: Box, + index: usize, + member: Box, + #[source] + source: CrateReadError, + }, + #[error("RO-Crate core operation failed: {0}")] + Core(#[from] PithosError), + #[error("RO-Crate writer conversion failed: {0}")] + Writer(#[from] WriterError), + #[error("invalid RO-Crate source {path}: expected {expected}")] + InvalidSource { + path: PathBuf, + expected: &'static str, + }, + #[error( + "invalid RO-Crate ZIP source {archive} member {member} at index {index}: expected {expected}" + )] + ZipMemberInvalidSource { + archive: PathBuf, + index: usize, + member: String, + expected: &'static str, + }, + #[error("RO-Crate metadata file is missing from {path}")] + MissingMetadata { path: PathBuf }, + #[error("unsafe ZIP member path {member} in {archive} at index {index}")] + UnsafeZipPath { + archive: PathBuf, + index: usize, + member: String, + }, + #[error("ZIP member name in {archive} at index {index} is not valid UTF-8: {member}")] + InvalidZipEntryName { + archive: PathBuf, + index: usize, + member: String, + }, + #[error("duplicate ZIP member path {path} in {archive} member {member} at index {index}")] + DuplicateZipPath { + archive: PathBuf, + index: usize, + member: String, + path: String, + }, + #[error( + "ZIP member path {path} conflicts with a required directory in {archive} member {member} at index {index}" + )] + ZipPathConflict { + archive: PathBuf, + index: usize, + member: String, + path: String, + }, + #[error("overlapping ZIP members are not supported in {path}")] + OverlappingZipEntries { path: PathBuf }, + #[error("encrypted ZIP member is not supported in {archive} at index {index}: {member}")] + EncryptedZipEntry { + archive: PathBuf, + index: usize, + member: String, + }, + #[error("unsupported ZIP entry type or compression in {archive} at index {index}: {member}")] + UnsupportedZipEntry { + archive: PathBuf, + index: usize, + member: String, + }, + #[error("RO-Crate {operation} exceeds the {limit}-byte limit for {path}: {actual}")] + ContentLimit { + operation: &'static str, + path: PathBuf, + limit: u64, + actual: u64, + }, + #[error( + "RO-Crate {operation} exceeds the {limit}-byte limit for ZIP {archive} member {member} at index {index}: {actual}" + )] + ZipMemberLimit { + operation: &'static str, + archive: PathBuf, + index: usize, + member: String, + limit: u64, + actual: u64, + }, +} + +enum BoundedReadError { + Io(std::io::Error), + Limit { limit: u64, actual: u64 }, +} + +fn read_limited(reader: &mut R, limit: u64) -> Result, BoundedReadError> { + let capacity = usize::try_from(limit.min(64 * 1024)).unwrap_or(64 * 1024); + let mut bytes = Vec::with_capacity(capacity); + let mut buffer = [0u8; 8192]; + loop { + let remaining = limit.saturating_sub(bytes.len() as u64); + let request = usize::try_from(remaining.saturating_add(1)) + .unwrap_or(buffer.len()) + .min(buffer.len()); + let count = reader + .read(&mut buffer[..request]) + .map_err(BoundedReadError::Io)?; + if count == 0 { + return Ok(bytes); + } + bytes.extend_from_slice(&buffer[..count]); + if bytes.len() as u64 > limit { + return Err(BoundedReadError::Limit { + limit, + actual: bytes.len() as u64, + }); + } + } +} + +fn read_directory_limited( + reader: &mut R, + expected_size: u64, + limit: u64, + operation: &'static str, + path: PathBuf, +) -> Result, RoCrateError> { + if expected_size > limit { + return Err(RoCrateError::ContentLimit { + operation, + path, + limit, + actual: expected_size, + }); + } + read_limited(reader, limit).map_err(|error| match error { + BoundedReadError::Io(source) => RoCrateError::Io { + operation, + path, + source, + }, + BoundedReadError::Limit { limit, actual } => RoCrateError::ContentLimit { + operation, + path, + limit, + actual, + }, + }) +} + +fn read_zip_limited( + reader: &mut R, + expected_size: u64, + limit: u64, + operation: &'static str, + archive: PathBuf, + index: usize, + member: String, +) -> Result, RoCrateError> { + if expected_size > limit { + return Err(RoCrateError::ZipMemberLimit { + operation, + archive, + index, + member, + limit, + actual: expected_size, + }); + } + read_limited(reader, limit).map_err(|error| match error { + BoundedReadError::Io(source) => RoCrateError::ZipMemberIo { + operation, + archive, + index, + member, + source, + }, + BoundedReadError::Limit { limit, actual } => RoCrateError::ZipMemberLimit { + operation, + archive, + index, + member, + limit, + actual, + }, + }) +} + +fn open_directory_no_follow(path: &Path) -> std::io::Result

{ + rustix::fs::open( + path, + rustix::fs::OFlags::RDONLY + | rustix::fs::OFlags::DIRECTORY + | rustix::fs::OFlags::NOFOLLOW + | rustix::fs::OFlags::CLOEXEC, + rustix::fs::Mode::empty(), + ) + .map_err(std::io::Error::from) + .map(|fd| Dir::from_std_file(File::from(fd))) +} + +fn open_directory_at_no_follow(parent: &Dir, name: &Path) -> std::io::Result { + rustix::fs::openat( + parent, + name, + rustix::fs::OFlags::RDONLY + | rustix::fs::OFlags::DIRECTORY + | rustix::fs::OFlags::NOFOLLOW + | rustix::fs::OFlags::CLOEXEC, + rustix::fs::Mode::empty(), + ) + .map_err(std::io::Error::from) + .map(|fd| Dir::from_std_file(File::from(fd))) +} + +fn open_file_at_no_follow(parent: &Dir, name: &Path) -> std::io::Result { + rustix::fs::openat( + parent, + name, + rustix::fs::OFlags::RDONLY | rustix::fs::OFlags::NOFOLLOW | rustix::fs::OFlags::CLOEXEC, + rustix::fs::Mode::empty(), + ) + .map_err(std::io::Error::from) + .map(|fd| CapFile::from_std(File::from(fd))) +} + +fn same_cap_identity(left: &cap_std::fs::Metadata, right: &cap_std::fs::Metadata) -> bool { + left.dev() == right.dev() && left.ino() == right.ino() +} + +fn read_symlink_at_limited(parent: &Dir, name: &Path) -> Result, std::io::Error> { + let name = CString::new(name.as_os_str().as_bytes())?; + let mut bytes = vec![0; MAX_SYMLINK_TARGET_BYTES as usize + 1]; + // The fixed limit + 1 buffer prevents a raced replacement from growing an + // otherwise inspected symlink target beyond the adapter's allocation bound. + let count = unsafe { + libc::readlinkat( + parent.as_raw_fd(), + name.as_ptr(), + bytes.as_mut_ptr().cast(), + bytes.len(), + ) + }; + if count < 0 { + return Err(std::io::Error::last_os_error()); + } + bytes.truncate(count as usize); + Ok(bytes) +} + +fn archive_path(path: &Path) -> Result { + let display_path = path.to_str().ok_or_else(|| RoCrateError::InvalidSource { + path: path.to_path_buf(), + expected: "UTF-8 archive path", + })?; + let mut components = Vec::new(); + for component in path.components() { + let Component::Normal(component) = component else { + return Err(PithosError::InvalidArchivePath { + path: display_path.to_owned(), + reason: "archive paths must contain only normal components".into(), + } + .into()); + }; + components.push( + component + .to_str() + .ok_or_else(|| RoCrateError::InvalidSource { + path: path.to_path_buf(), + expected: "UTF-8 archive path", + })?, + ); + } + Ok(ArchivePath::new(components.join("/"))?) +} + +fn host_metadata(metadata: &cap_std::fs::Metadata) -> EntryMetadata { + let timestamp = |value: Result| { + value + .ok() + .and_then(|time| time.duration_since(SystemClock::UNIX_EPOCH).ok()) + .map_or(0, |duration| duration.as_secs()) + }; + EntryMetadata::new( + timestamp(metadata.created()), + timestamp(metadata.modified()), + metadata.permissions().mode() & 0o7777, + ) +} + +/// The kind of retained physical source used to load an RO-Crate. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RoCrateSource { + Directory, + Zip, +} + +/// An upstream RO-Crate graph together with the exact inspected physical source. +/// +/// This value deliberately cannot be cloned: conversion consumes the retained +/// file handles or ZIP archive rather than reopening the supplied path. +pub struct LoadedRoCrate { + ro_crate: RoCrate, + source: LoadedSource, +} + +impl fmt::Debug for LoadedRoCrate { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("LoadedRoCrate") + .field("source_kind", &self.source_kind()) + .finish_non_exhaustive() + } +} + +impl LoadedRoCrate { + /// Inspect the graph produced by the upstream RO-Crate parser. + pub fn ro_crate(&self) -> &RoCrate { + &self.ro_crate + } + + /// Return the kind of source retained for conversion. + pub fn source_kind(&self) -> RoCrateSource { + match &self.source { + LoadedSource::Directory(_) => RoCrateSource::Directory, + LoadedSource::Zip(_) => RoCrateSource::Zip, + } + } +} + +enum LoadedSource { + Directory(LoadedDirectory), + Zip(LoadedZip), +} + +struct LoadedDirectory { + // Keep the opened root capability as the identity of the inspected tree. + _root: Dir, + manifest: RoCrateDirectoryManifest, +} + +struct LoadedZip { + archive: ZipArchive, + manifest: RoCrateZipManifest, +} + +struct MetadataDescriptor { + inner_path: String, + metadata: EntryMetadata, + expected_size: u64, + bytes: Vec, + source: MetadataSource, +} + +enum MetadataSource { + // Metadata is emitted from retained bytes, but retain its opened handle so + // the directory manifest owns every regular source it inspected. + Directory { _source: CapFile, path: PathBuf }, + Zip(ZipEntrySource), +} + +enum DirectoryEntryKind { + Directory, + File { expected_size: u64, source: CapFile }, + Symlink { target: String }, +} + +struct DirectoryEntryDescriptor { + inner_path: String, + metadata: EntryMetadata, + source_path: PathBuf, + kind: DirectoryEntryKind, +} + +struct RoCrateDirectoryManifest { + metadata: MetadataDescriptor, + entries: Vec, +} + +#[derive(Clone)] +struct ZipEntrySource { + archive_index: usize, + member: String, +} + +#[derive(Clone, PartialEq, Eq)] +enum ZipEntryKind { + Directory, + File { expected_size: u64 }, + Symlink { target: String }, +} + +#[derive(Clone)] +struct ZipEntryDescriptor { + inner_path: String, + metadata: EntryMetadata, + source: Option, + kind: ZipEntryKind, +} + +struct RoCrateZipManifest { + source: PathBuf, + metadata: MetadataDescriptor, + entries: Vec, +} + +fn directory_kind_rank(kind: &DirectoryEntryKind) -> u8 { + match kind { + DirectoryEntryKind::Directory => 0, + DirectoryEntryKind::File { .. } => 1, + DirectoryEntryKind::Symlink { .. } => 2, + } +} + +fn zip_kind_rank(kind: &ZipEntryKind) -> u8 { + match kind { + ZipEntryKind::Directory => 0, + ZipEntryKind::File { .. } => 1, + ZipEntryKind::Symlink { .. } => 2, + } +} + +/// Convert a loaded RO-Crate through the typed writer boundary. +/// +/// Conversion consumes `loaded`, ensuring content always comes from the +/// directory handles or ZIP archive retained at load time. +pub fn write_ro_crate( + writer: &mut ArchiveWriter, + loaded: LoadedRoCrate, + processing: ProcessingOptions, +) -> Result<(), RoCrateError> { + match loaded.source { + LoadedSource::Directory(directory) => { + write_ro_crate_directory(writer, directory.manifest, processing)?; + } + LoadedSource::Zip(mut zip) => { + write_ro_crate_zip_archive(writer, &mut zip.archive, zip.manifest, processing)?; + } + } + Ok(()) +} + +fn write_metadata( + writer: &mut ArchiveWriter, + metadata: MetadataDescriptor, + processing: ProcessingOptions, +) -> Result { + Ok(writer + .add_metadata( + ArchivePath::new(&metadata.inner_path)?, + metadata.metadata, + processing, + Some(metadata.expected_size), + Cursor::new(metadata.bytes), + )? + .id) +} + +fn write_ro_crate_directory( + writer: &mut ArchiveWriter, + manifest: RoCrateDirectoryManifest, + processing: ProcessingOptions, +) -> Result<(), RoCrateError> { + let metadata_path = match &manifest.metadata.source { + MetadataSource::Directory { path, .. } => path.clone(), + MetadataSource::Zip(_) => unreachable!("directory manifests retain directory metadata"), + }; + let metadata_id = write_metadata(writer, manifest.metadata, processing).map_err(|source| { + RoCrateError::DirectoryFileWriter { + operation: "convert retained directory metadata", + path: metadata_path, + source, + } + })?; + for DirectoryEntryDescriptor { + inner_path, + metadata, + source_path, + kind, + } in manifest.entries + { + let path = ArchivePath::new(&inner_path)?; + match kind { + DirectoryEntryKind::Directory => { + writer.add_directory(path, metadata).map_err(|source| { + RoCrateError::DirectoryFileWriter { + operation: "convert retained directory entry", + path: source_path, + source, + } + })?; + } + DirectoryEntryKind::File { + expected_size, + source, + } => { + writer + .add_file( + path, + metadata.with_references(vec![EntryReference { + target_file_id: metadata_id, + relationship: 0, + }]), + processing, + Some(expected_size), + source, + ) + .map_err(|source| RoCrateError::DirectoryFileWriter { + operation: "convert retained directory file", + path: source_path, + source, + })?; + } + DirectoryEntryKind::Symlink { target } => { + writer + .add_symlink(path, metadata, target) + .map_err(|source| RoCrateError::DirectoryFileWriter { + operation: "convert retained directory entry", + path: source_path, + source, + })?; + } + } + } + Ok(()) +} + +fn write_ro_crate_zip_archive( + writer: &mut ArchiveWriter, + archive: &mut ZipArchive, + manifest: RoCrateZipManifest, + processing: ProcessingOptions, +) -> Result<(), RoCrateError> { + let RoCrateZipManifest { + source, + metadata, + entries, + } = manifest; + let metadata_source = match &metadata.source { + MetadataSource::Directory { .. } => unreachable!("ZIP manifests retain ZIP metadata"), + MetadataSource::Zip(source) => source.clone(), + }; + let metadata_id = write_metadata(writer, metadata, processing).map_err(|writer_error| { + RoCrateError::ZipMemberWriter { + operation: "convert retained ZIP metadata", + archive: Box::new(source.clone()), + index: metadata_source.archive_index, + member: Box::new(metadata_source.member), + source: writer_error, + } + })?; + for ZipEntryDescriptor { + inner_path, + metadata, + source: entry_source, + kind, + } in entries + { + let path = ArchivePath::new(&inner_path)?; + match kind { + ZipEntryKind::Directory => { + writer.add_directory(path, metadata).map_err( + |writer_error| match entry_source { + Some(entry_source) => RoCrateError::ZipMemberWriter { + operation: "convert retained ZIP entry", + archive: Box::new(source.clone()), + index: entry_source.archive_index, + member: Box::new(entry_source.member), + source: writer_error, + }, + None => RoCrateError::ZipSyntheticWriter { + operation: "convert synthetic ZIP parent", + archive: Box::new(source.clone()), + path: inner_path.clone(), + source: writer_error, + }, + }, + )?; + } + ZipEntryKind::File { expected_size } => { + let entry_source = + entry_source.expect("retained ZIP files retain their source member"); + let zip_member = archive + .by_index(entry_source.archive_index) + .map_err(|error| RoCrateError::ZipMember { + operation: "open retained ZIP member", + archive: source.clone(), + index: entry_source.archive_index, + member: entry_source.member.clone(), + source: error, + })?; + writer + .add_file( + path, + metadata.with_references(vec![EntryReference { + target_file_id: metadata_id, + relationship: 0, + }]), + processing, + Some(expected_size), + zip_member, + ) + .map_err(|writer_error| RoCrateError::ZipMemberWriter { + operation: "convert retained ZIP member", + archive: Box::new(source.clone()), + index: entry_source.archive_index, + member: Box::new(entry_source.member), + source: writer_error, + })?; + } + ZipEntryKind::Symlink { target } => { + writer + .add_symlink(path, metadata, target) + .map_err(|writer_error| match entry_source { + Some(entry_source) => RoCrateError::ZipMemberWriter { + operation: "convert retained ZIP entry", + archive: Box::new(source.clone()), + index: entry_source.archive_index, + member: Box::new(entry_source.member), + source: writer_error, + }, + None => RoCrateError::Writer(writer_error), + })?; + } + } + } + Ok(()) +} + +/// Load a directory RO-Crate, retaining all inspected regular-file handles. +pub fn read_ro_crate_directory(path: impl AsRef) -> Result { + let source = path.as_ref().to_path_buf(); + let inspected = + std::fs::symlink_metadata(&source).map_err(|source_error| RoCrateError::Io { + operation: "inspect directory RO-Crate", + path: source.clone(), + source: source_error, + })?; + let root = open_directory_no_follow(&source).map_err(|source_error| RoCrateError::Io { + operation: "open directory RO-Crate", + path: source.clone(), + source: source_error, + })?; + let opened = root + .symlink_metadata(".") + .map_err(|source_error| RoCrateError::Io { + operation: "inspect opened directory RO-Crate", + path: source.clone(), + source: source_error, + })?; + if !inspected.file_type().is_dir() + || !opened.file_type().is_dir() + || inspected.dev() != opened.dev() + || inspected.ino() != opened.ino() + { + return Err(RoCrateError::InvalidSource { + path: source, + expected: "stable directory", + }); + } + let mut manifest = inspect_ro_crate_directory_manifest(&root, &source)?; + let metadata_path = source.join(RO_CRATE_METADATA_FILE); + let json = std::str::from_utf8(&manifest.metadata.bytes).map_err(|error| RoCrateError::Io { + operation: "decode RO-Crate metadata", + path: metadata_path.clone(), + source: std::io::Error::new(std::io::ErrorKind::InvalidData, error), + })?; + let ro_crate = read_crate_obj(json, PARSE_WITHOUT_VALIDATION).map_err(|source_error| { + RoCrateError::Parser { + operation: "parse RO-Crate metadata", + path: metadata_path, + source: source_error, + } + })?; + + // The parser only borrows these bytes; retain them for metadata-first output. + manifest.metadata.bytes.shrink_to_fit(); + Ok(LoadedRoCrate { + ro_crate, + source: LoadedSource::Directory(LoadedDirectory { + _root: root, + manifest, + }), + }) +} + +/// Load a ZIP RO-Crate, retaining its opened archive and validated member indexes. +pub fn read_ro_crate_zip(path: impl AsRef) -> Result { + let source = path.as_ref().to_path_buf(); + let inspected = + std::fs::symlink_metadata(&source).map_err(|source_error| RoCrateError::Io { + operation: "inspect ZIP archive", + path: source.clone(), + source: source_error, + })?; + let file = rustix::fs::open( + &source, + rustix::fs::OFlags::RDONLY | rustix::fs::OFlags::NOFOLLOW | rustix::fs::OFlags::CLOEXEC, + rustix::fs::Mode::empty(), + ) + .map_err(std::io::Error::from) + .map(File::from) + .map_err(|source_error| RoCrateError::Io { + operation: "open ZIP archive", + path: source.clone(), + source: source_error, + })?; + let opened = file.metadata().map_err(|source_error| RoCrateError::Io { + operation: "inspect opened ZIP archive", + path: source.clone(), + source: source_error, + })?; + if !inspected.file_type().is_file() + || !opened.file_type().is_file() + || inspected.dev() != opened.dev() + || inspected.ino() != opened.ino() + { + return Err(RoCrateError::InvalidSource { + path: source, + expected: "ZIP file", + }); + } + let mut archive = ZipArchive::new(file).map_err(|zip_error| RoCrateError::Zip { + operation: "open ZIP archive", + path: source.clone(), + source: zip_error, + })?; + let mut manifest = inspect_ro_crate_zip_archive_manifest(&mut archive, source.clone())?; + let metadata_source = match &manifest.metadata.source { + MetadataSource::Directory { .. } => unreachable!("ZIP manifests retain ZIP metadata"), + MetadataSource::Zip(metadata_source) => metadata_source.clone(), + }; + let json = std::str::from_utf8(&manifest.metadata.bytes).map_err(|error| { + RoCrateError::ZipMemberIo { + operation: "decode RO-Crate metadata", + archive: source.clone(), + index: metadata_source.archive_index, + member: metadata_source.member.clone(), + source: std::io::Error::new(std::io::ErrorKind::InvalidData, error), + } + })?; + let ro_crate = read_crate_obj(json, PARSE_WITHOUT_VALIDATION).map_err(|source_error| { + RoCrateError::ZipMemberParser { + operation: "parse RO-Crate metadata", + archive: Box::new(source.clone()), + index: metadata_source.archive_index, + member: Box::new(metadata_source.member), + source: source_error, + } + })?; + + manifest.metadata.bytes.shrink_to_fit(); + Ok(LoadedRoCrate { + ro_crate, + source: LoadedSource::Zip(LoadedZip { archive, manifest }), + }) +} + +fn inspect_ro_crate_directory_manifest( + root: &Dir, + source: &Path, +) -> Result { + let mut explicit_entries = BTreeMap::new(); + collect_directory_entries(root, source, Path::new(""), &mut explicit_entries)?; + let metadata = explicit_entries + .remove(RO_CRATE_METADATA_FILE) + .ok_or_else(|| RoCrateError::MissingMetadata { + path: source.to_path_buf(), + })?; + let (expected_size, bytes, retained_source) = match metadata.kind { + DirectoryEntryKind::File { + expected_size, + source: mut metadata_source, + .. + } => { + let metadata_path = source.join(RO_CRATE_METADATA_FILE); + let bytes = read_directory_limited( + &mut metadata_source, + expected_size, + MAX_METADATA_BYTES, + "read RO-Crate metadata", + metadata_path.clone(), + )?; + let actual_size = + u64::try_from(bytes.len()).map_err(|_| RoCrateError::DirectoryCore { + operation: "verify RO-Crate metadata size", + path: metadata_path.clone(), + source: PithosError::WriterSizeOverflow, + })?; + if actual_size != expected_size { + return Err(RoCrateError::DirectoryCore { + operation: "verify RO-Crate metadata size", + path: metadata_path, + source: PithosError::WriterExpectedSizeMismatch { + expected: expected_size, + actual: actual_size, + }, + }); + } + (expected_size, bytes, Some(metadata_source)) + } + DirectoryEntryKind::Directory | DirectoryEntryKind::Symlink { .. } => { + return Err(RoCrateError::InvalidSource { + path: source.join(RO_CRATE_METADATA_FILE), + expected: "regular metadata file", + }); + } + }; + let metadata = MetadataDescriptor { + inner_path: metadata.inner_path, + metadata: metadata.metadata, + expected_size, + bytes, + source: MetadataSource::Directory { + _source: retained_source.expect("directory metadata is a retained regular file"), + path: source.join(RO_CRATE_METADATA_FILE), + }, + }; + let mut entries = explicit_entries.into_values().collect::>(); + entries.sort_by(|left, right| { + directory_kind_rank(&left.kind) + .cmp(&directory_kind_rank(&right.kind)) + .then_with(|| { + left.inner_path + .to_lowercase() + .cmp(&right.inner_path.to_lowercase()) + }) + .then_with(|| left.inner_path.cmp(&right.inner_path)) + }); + Ok(RoCrateDirectoryManifest { metadata, entries }) +} + +fn collect_directory_entries( + directory: &Dir, + source_root: &Path, + prefix: &Path, + entries: &mut BTreeMap, +) -> Result<(), RoCrateError> { + let mut pending = vec![( + (directory.try_clone()).map_err(|source| RoCrateError::Io { + operation: "retain RO-Crate directory", + path: source_root.join(prefix), + source, + })?, + prefix.to_path_buf(), + )]; + while let Some((directory, prefix)) = pending.pop() { + let directory_path = source_root.join(&prefix); + let mut children = directory + .read_dir(".") + .map_err(|source| RoCrateError::Io { + operation: "read RO-Crate directory", + path: directory_path.clone(), + source, + })? + .collect::, _>>() + .map_err(|source| RoCrateError::Io { + operation: "read RO-Crate directory entry", + path: directory_path, + source, + })?; + children.sort_by_key(|entry| entry.file_name()); + let mut child_directories = Vec::new(); + for child in children { + let relative = prefix.join(child.file_name()); + let source_path = source_root.join(&relative); + let path = archive_path(&relative).map_err(|error| match error { + RoCrateError::Core(source) => RoCrateError::DirectoryCore { + operation: "validate RO-Crate path", + path: source_path.clone(), + source, + }, + RoCrateError::InvalidSource { expected, .. } => RoCrateError::InvalidSource { + path: source_path.clone(), + expected, + }, + error => error, + })?; + let inner_path = path.as_str().to_owned(); + let host = directory + .symlink_metadata(child.file_name()) + .map_err(|source| RoCrateError::Io { + operation: "inspect RO-Crate entry", + path: source_path.clone(), + source, + })?; + let kind = if host.file_type().is_dir() { + let child_directory = + open_directory_at_no_follow(&directory, Path::new(&child.file_name())) + .map_err(|source| RoCrateError::Io { + operation: "open RO-Crate directory", + path: source_path.clone(), + source, + })?; + let opened = + child_directory + .symlink_metadata(".") + .map_err(|source| RoCrateError::Io { + operation: "inspect opened RO-Crate directory", + path: source_path.clone(), + source, + })?; + if !opened.file_type().is_dir() || !same_cap_identity(&host, &opened) { + return Err(RoCrateError::InvalidSource { + path: source_path, + expected: "stable directory", + }); + } + child_directories.push((child_directory, relative)); + DirectoryEntryKind::Directory + } else if host.file_type().is_file() { + let source = open_file_at_no_follow(&directory, Path::new(&child.file_name())) + .map_err(|source| RoCrateError::Io { + operation: "open RO-Crate file", + path: source_path.clone(), + source, + })?; + let opened = source.metadata().map_err(|source| RoCrateError::Io { + operation: "inspect opened RO-Crate file", + path: source_path.clone(), + source, + })?; + if !opened.file_type().is_file() || !same_cap_identity(&host, &opened) { + return Err(RoCrateError::InvalidSource { + path: source_path, + expected: "stable regular file", + }); + } + DirectoryEntryKind::File { + expected_size: opened.len(), + source, + } + } else if host.file_type().is_symlink() { + if host.len() > MAX_SYMLINK_TARGET_BYTES { + return Err(RoCrateError::ContentLimit { + operation: "read RO-Crate symlink", + path: source_path, + limit: MAX_SYMLINK_TARGET_BYTES, + actual: host.len(), + }); + } + let target_bytes = + read_symlink_at_limited(&directory, Path::new(&child.file_name())).map_err( + |source| RoCrateError::Io { + operation: "read RO-Crate symlink", + path: source_path.clone(), + source, + }, + )?; + if target_bytes.len() as u64 > MAX_SYMLINK_TARGET_BYTES { + return Err(RoCrateError::ContentLimit { + operation: "read RO-Crate symlink", + path: source_path, + limit: MAX_SYMLINK_TARGET_BYTES, + actual: target_bytes.len() as u64, + }); + } + let target = + String::from_utf8(target_bytes).map_err(|source| RoCrateError::Io { + operation: "decode RO-Crate symlink", + path: source_path.clone(), + source: std::io::Error::new(std::io::ErrorKind::InvalidData, source), + })?; + let current = directory + .symlink_metadata(child.file_name()) + .map_err(|source| RoCrateError::Io { + operation: "inspect read RO-Crate symlink", + path: source_path.clone(), + source, + })?; + if !current.file_type().is_symlink() || !same_cap_identity(&host, ¤t) { + return Err(RoCrateError::InvalidSource { + path: source_path, + expected: "stable symlink", + }); + } + validate_symlink_target(&inner_path, &target).map_err(|source| { + RoCrateError::DirectoryCore { + operation: "validate RO-Crate symlink target", + path: source_path.clone(), + source, + } + })?; + DirectoryEntryKind::Symlink { target } + } else { + return Err(RoCrateError::InvalidSource { + path: source_path, + expected: "regular file, directory, or symlink", + }); + }; + let descriptor = DirectoryEntryDescriptor { + inner_path: inner_path.clone(), + metadata: host_metadata(&host), + source_path: source_path.clone(), + kind, + }; + if entries.insert(inner_path.clone(), descriptor).is_some() { + return Err(RoCrateError::InvalidSource { + path: source_path, + expected: "unique archive path", + }); + } + } + pending.extend(child_directories.into_iter().rev()); + } + Ok(()) +} + +fn raw_zip_path(name: &str, is_directory: bool) -> Result { + if name.is_empty() + || name.starts_with(['/', '\\']) + || name.contains('\\') + || name.as_bytes().get(1).is_some_and(|byte| *byte == b':') + || name.as_bytes().contains(&0) + { + return Err(()); + } + let path = if is_directory { + name.strip_suffix('/').ok_or(())? + } else { + name + }; + if path.is_empty() + || path.ends_with('/') + || path + .split('/') + .any(|component| component.is_empty() || matches!(component, "." | "..")) + { + return Err(()); + } + Ok(path.to_owned()) +} + +fn zip_timestamp(entry: &zip::read::ZipFile<'_, R>) -> u64 { + let Some(date_time) = entry.last_modified() else { + return 0; + }; + if !date_time.is_valid() { + return 0; + } + let year = date_time.year(); + let mut days = 0i64; + for current_year in 1970u16..year { + days += if current_year % 4 == 0 && (current_year % 100 != 0 || current_year % 400 == 0) { + 366 + } else { + 365 + }; + } + const DAYS_BEFORE_MONTH: [i64; 12] = [0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334]; + days += DAYS_BEFORE_MONTH[usize::from(date_time.month() - 1)]; + if date_time.month() > 2 && (year % 4 == 0 && (year % 100 != 0 || year % 400 == 0)) { + days += 1; + } + days += i64::from(date_time.day() - 1); + let timestamp = days * 86_400 + + i64::from(date_time.hour()) * 3_600 + + i64::from(date_time.minute()) * 60 + + i64::from(date_time.second()); + u64::try_from(timestamp).unwrap_or(0) +} + +fn inspect_ro_crate_zip_archive_manifest( + archive: &mut ZipArchive, + source: PathBuf, +) -> Result { + if archive + .has_overlapping_files() + .map_err(|error| RoCrateError::Zip { + operation: "inspect ZIP overlap", + path: source.clone(), + source: error, + })? + { + return Err(RoCrateError::OverlappingZipEntries { path: source }); + } + let mut explicit_entries = BTreeMap::new(); + for archive_index in 0..archive.len() { + let descriptor = { + let mut entry = + archive + .by_index(archive_index) + .map_err(|error| RoCrateError::ZipMember { + operation: "inspect ZIP member", + archive: source.clone(), + index: archive_index, + member: "".to_string(), + source: error, + })?; + let entry_name = std::str::from_utf8(entry.name_raw()) + .map_err(|_| RoCrateError::InvalidZipEntryName { + archive: source.clone(), + index: archive_index, + member: String::from_utf8_lossy(entry.name_raw()).into_owned(), + })? + .to_owned(); + if entry.encrypted() { + return Err(RoCrateError::EncryptedZipEntry { + archive: source.clone(), + index: archive_index, + member: entry_name, + }); + } + if !matches!( + entry.compression(), + CompressionMethod::Stored | CompressionMethod::Deflated + ) { + return Err(RoCrateError::UnsupportedZipEntry { + archive: source.clone(), + index: archive_index, + member: entry_name, + }); + } + let inner_path = raw_zip_path(&entry_name, entry.is_dir()).map_err(|_| { + RoCrateError::UnsafeZipPath { + archive: source.clone(), + index: archive_index, + member: entry_name.clone(), + } + })?; + ArchivePath::new(&inner_path).map_err(|_| RoCrateError::UnsafeZipPath { + archive: source.clone(), + index: archive_index, + member: entry_name.clone(), + })?; + let kind = if entry.is_dir() { + ZipEntryKind::Directory + } else if entry.is_symlink() { + let expected_size = entry.size(); + let target = String::from_utf8(read_zip_limited( + &mut entry, + expected_size, + MAX_SYMLINK_TARGET_BYTES, + "read ZIP symlink target", + source.clone(), + archive_index, + entry_name.clone(), + )?) + .map_err(|error| RoCrateError::ZipMemberIo { + operation: "decode ZIP symlink target", + archive: source.clone(), + index: archive_index, + member: entry_name.clone(), + source: std::io::Error::new(std::io::ErrorKind::InvalidData, error), + })?; + validate_symlink_target(&inner_path, &target).map_err(|source_error| { + RoCrateError::ZipMemberCore { + operation: "validate ZIP symlink target", + archive: Box::new(source.clone()), + index: archive_index, + member: Box::new(entry_name.clone()), + source: source_error, + } + })?; + ZipEntryKind::Symlink { target } + } else if entry.is_file() { + ZipEntryKind::File { + expected_size: entry.size(), + } + } else { + return Err(RoCrateError::UnsupportedZipEntry { + archive: source.clone(), + index: archive_index, + member: entry_name, + }); + }; + let default_permissions = match kind { + ZipEntryKind::Directory => 0o755, + ZipEntryKind::File { .. } | ZipEntryKind::Symlink { .. } => 0o644, + }; + ZipEntryDescriptor { + inner_path, + metadata: EntryMetadata::new( + zip_timestamp(&entry), + zip_timestamp(&entry), + entry.unix_mode().unwrap_or(default_permissions) & 0o7777, + ), + source: Some(ZipEntrySource { + archive_index, + member: entry_name, + }), + kind, + } + }; + if explicit_entries + .insert(descriptor.inner_path.clone(), descriptor.clone()) + .is_some() + { + return Err(RoCrateError::DuplicateZipPath { + archive: source.clone(), + index: descriptor + .source + .as_ref() + .expect("explicit ZIP entries retain their source") + .archive_index, + member: descriptor + .source + .as_ref() + .expect("explicit ZIP entries retain their source") + .member + .clone(), + path: descriptor.inner_path, + }); + } + } + let paths = explicit_entries.keys().cloned().collect::>(); + for path in paths { + let mut parent = path.as_str(); + while let Some((parent_path, _)) = parent.rsplit_once('/') { + if let Some(existing) = explicit_entries.get(parent_path) { + if existing.kind != ZipEntryKind::Directory { + return Err(RoCrateError::ZipPathConflict { + archive: source.clone(), + index: existing + .source + .as_ref() + .expect("explicit ZIP entries retain their source") + .archive_index, + member: existing + .source + .as_ref() + .expect("explicit ZIP entries retain their source") + .member + .clone(), + path: parent_path.to_string(), + }); + } + } else { + explicit_entries.insert( + parent_path.to_string(), + ZipEntryDescriptor { + inner_path: parent_path.to_string(), + metadata: EntryMetadata::new(0, 0, 0o755), + source: None, + kind: ZipEntryKind::Directory, + }, + ); + } + parent = parent_path; + } + } + let metadata = explicit_entries + .remove(RO_CRATE_METADATA_FILE) + .ok_or_else(|| RoCrateError::MissingMetadata { + path: source.clone(), + })?; + let (metadata_source, expected_size) = match (metadata.kind, metadata.source) { + (ZipEntryKind::File { expected_size }, Some(source)) => (source, expected_size), + (ZipEntryKind::Directory | ZipEntryKind::Symlink { .. }, Some(member_source)) => { + return Err(RoCrateError::ZipMemberInvalidSource { + archive: source, + index: member_source.archive_index, + member: member_source.member, + expected: "regular metadata file", + }); + } + (ZipEntryKind::Directory | ZipEntryKind::Symlink { .. }, None) => { + unreachable!("ZIP metadata is an explicit member") + } + (ZipEntryKind::File { .. }, None) => unreachable!("ZIP metadata is an explicit member"), + }; + let mut metadata_member = archive + .by_index(metadata_source.archive_index) + .map_err(|error| RoCrateError::ZipMember { + operation: "open RO-Crate metadata member", + archive: source.clone(), + index: metadata_source.archive_index, + member: metadata_source.member.clone(), + source: error, + })?; + let bytes = read_zip_limited( + &mut metadata_member, + expected_size, + MAX_METADATA_BYTES, + "read RO-Crate metadata member", + source.clone(), + metadata_source.archive_index, + metadata_source.member.clone(), + )?; + let actual_size = u64::try_from(bytes.len()).map_err(|_| RoCrateError::ZipMemberCore { + operation: "verify RO-Crate metadata size", + archive: Box::new(source.clone()), + index: metadata_source.archive_index, + member: Box::new(metadata_source.member.clone()), + source: PithosError::WriterSizeOverflow, + })?; + if actual_size != expected_size { + return Err(RoCrateError::ZipMemberCore { + operation: "verify RO-Crate metadata size", + archive: Box::new(source.clone()), + index: metadata_source.archive_index, + member: Box::new(metadata_source.member.clone()), + source: PithosError::WriterExpectedSizeMismatch { + expected: expected_size, + actual: actual_size, + }, + }); + } + let metadata = MetadataDescriptor { + inner_path: metadata.inner_path, + metadata: metadata.metadata, + expected_size, + bytes, + source: MetadataSource::Zip(metadata_source), + }; + let mut entries = explicit_entries.into_values().collect::>(); + entries.sort_by(|left, right| { + zip_kind_rank(&left.kind) + .cmp(&zip_kind_rank(&right.kind)) + .then_with(|| { + left.inner_path + .to_lowercase() + .cmp(&right.inner_path.to_lowercase()) + }) + .then_with(|| left.inner_path.cmp(&right.inner_path)) + }); + Ok(RoCrateZipManifest { + source, + metadata, + entries, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::archive::{CdcConfig, WriteOptions}; + use crate::crypto::PrivateKey; + use std::cell::Cell; + use std::io::{Cursor, SeekFrom}; + use std::rc::Rc; + + struct BoundedReader { + inner: Cursor>, + maximum_request: usize, + largest_request: Rc>, + reads: Rc>, + } + + impl Read for BoundedReader { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + if buffer.len() > self.maximum_request { + return Err(std::io::Error::other( + "ZIP member read exceeded stream bound", + )); + } + self.largest_request + .set(self.largest_request.get().max(buffer.len())); + self.reads.set(self.reads.get() + 1); + self.inner.read(buffer) + } + } + + impl Seek for BoundedReader { + fn seek(&mut self, position: SeekFrom) -> std::io::Result { + self.inner.seek(position) + } + } + + #[test] + fn zip_member_conversion_reads_large_content_in_bounded_chunks() { + let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new())); + let options = zip::write::SimpleFileOptions::default(); + zip.start_file("ro-crate-metadata.json", options).unwrap(); + zip.write_all(b"{}").unwrap(); + zip.start_file("large.bin", options).unwrap(); + let mut state = 1u32; + let payload = (0..(1024 * 1024 + 123)) + .map(|_| { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + (state >> 24) as u8 + }) + .collect::>(); + zip.write_all(&payload).unwrap(); + let bytes = zip.finish().unwrap().into_inner(); + let largest_request = Rc::new(Cell::new(0)); + let reads = Rc::new(Cell::new(0)); + let reader = BoundedReader { + inner: Cursor::new(bytes), + maximum_request: 64 * 1024, + largest_request: Rc::clone(&largest_request), + reads: Rc::clone(&reads), + }; + let mut archive = ZipArchive::new(reader).unwrap(); + let manifest = + inspect_ro_crate_zip_archive_manifest(&mut archive, PathBuf::from("bounded.zip")) + .unwrap(); + let sender = PrivateKey::generate(); + let mut writer = ArchiveWriter::create( + Vec::new(), + WriteOptions::new(sender.duplicate(), vec![sender.public_key()]) + .with_cdc(CdcConfig::new(64, 256, 1024).unwrap()), + ) + .unwrap(); + write_ro_crate_zip_archive( + &mut writer, + &mut archive, + manifest, + ProcessingOptions::new(false, 0).unwrap(), + ) + .unwrap(); + writer.finish().unwrap(); + assert!(reads.get() > 1); + assert!(largest_request.get() <= 64 * 1024); + } +} diff --git a/crates/pithos_lib/src/fs/extraction.rs b/crates/pithos_lib/src/fs/extraction.rs new file mode 100644 index 0000000..cc7bb22 --- /dev/null +++ b/crates/pithos_lib/src/fs/extraction.rs @@ -0,0 +1,401 @@ +use super::FsError; +use crate::archive::{Archive, EntryKind, ExternalBlockResolver}; +use crate::source::ArchiveSource; +use cap_std::fs::Dir; +use rustix::fs::{AtFlags, Mode, OFlags, linkat, openat}; +use std::io::{self, Write}; +use std::path::{Component, Path, PathBuf}; + +fn open_dir_no_follow(parent: &Dir, component: &Path) -> io::Result { + rustix::fs::openat( + parent, + component, + OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::empty(), + ) + .map_err(io::Error::from) + .map(|fd| Dir::from_std_file(std::fs::File::from(fd))) +} + +/// Extract one validated archive entry through a capability-confined destination. +pub fn extract( + archive: &Archive, + archive_path: &str, + destination: &Path, +) -> Result<(), FsError> +where + S: ArchiveSource, + E: ExternalBlockResolver, +{ + let entry = archive + .entry(archive_path) + .map_err(|source| FsError::Archive { + operation: "look up", + archive_path: archive_path.to_owned(), + destination: destination.to_path_buf(), + source: Box::new(source), + })? + .ok_or_else(|| FsError::Archive { + operation: "look up", + archive_path: archive_path.to_owned(), + destination: destination.to_path_buf(), + source: Box::new(crate::error::PithosError::FileNotFound( + archive_path.to_owned(), + )), + })?; + let root = ExtractionRoot::open(destination, true, archive_path)?; + match entry.kind { + EntryKind::File { .. } | EntryKind::Metadata { .. } => { + let pending = root.pending_file(archive_path)?; + let mut writer = pending.writer(archive_path)?; + archive + .copy_to(archive_path, &mut writer) + .map_err(|source| root.archive_error("copy", archive_path, source))?; + pending.commit(archive_path)?; + } + EntryKind::Directory => root.create_dir(archive_path)?, + EntryKind::Symlink { target } => root.create_symlink(archive_path, &target)?, + } + Ok(()) +} + +pub(crate) struct ExtractionRoot { + root: Dir, + destination: PathBuf, +} + +impl ExtractionRoot { + pub(crate) fn open(path: &Path, create: bool, archive_path: &str) -> Result { + let base = if path.is_absolute() { + Path::new("/") + } else { + Path::new(".") + }; + let mut root = rustix::fs::open( + base, + OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::empty(), + ) + .map_err(io::Error::from) + .map(|fd| Dir::from_std_file(std::fs::File::from(fd))) + .map_err(|source| FsError::Operation { + operation: "open destination", + archive_path: archive_path.to_owned(), + destination: path.to_path_buf(), + source, + })?; + for component in path.components() { + let Component::Normal(component) = component else { + if matches!(component, Component::ParentDir) { + return Err(FsError::Operation { + operation: "open destination", + archive_path: archive_path.to_owned(), + destination: path.to_path_buf(), + source: io::Error::new( + io::ErrorKind::InvalidInput, + "destination path contains parent traversal", + ), + }); + } + continue; + }; + let component = Path::new(component); + root = match open_dir_no_follow(&root, component) { + Ok(directory) => directory, + Err(error) if create && error.kind() == io::ErrorKind::NotFound => { + match root.create_dir(component) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} + Err(source) => { + return Err(FsError::Operation { + operation: "create destination", + archive_path: archive_path.to_owned(), + destination: path.to_path_buf(), + source, + }); + } + } + open_dir_no_follow(&root, component).map_err(|source| FsError::Operation { + operation: "open destination", + archive_path: archive_path.to_owned(), + destination: path.to_path_buf(), + source, + })? + } + Err(source) => { + return Err(FsError::Operation { + operation: "open destination", + archive_path: archive_path.to_owned(), + destination: path.to_path_buf(), + source, + }); + } + }; + } + Ok(Self { + root, + destination: path.to_path_buf(), + }) + } + + fn operation(&self, operation: &'static str, archive_path: &str, source: io::Error) -> FsError { + FsError::Operation { + operation, + archive_path: archive_path.to_owned(), + destination: self.destination.clone(), + source, + } + } + + fn archive_error( + &self, + operation: &'static str, + archive_path: &str, + source: crate::error::PithosError, + ) -> FsError { + FsError::Archive { + operation, + archive_path: archive_path.to_owned(), + destination: self.destination.clone(), + source: Box::new(source), + } + } + + fn collision(&self, archive_path: &str, reason: &'static str) -> FsError { + FsError::ExtractionCollision { + archive_path: archive_path.to_owned(), + destination: self.destination.clone(), + reason, + } + } + + fn parents(&self, path: &str) -> Result<(Dir, String), FsError> { + let mut components = path.split('/').peekable(); + let final_name = components + .next_back() + .ok_or_else(|| self.collision(path, "empty final component"))?; + let mut dir = self + .root + .try_clone() + .map_err(|source| self.operation("open destination parent", path, source))?; + for component in components { + if component.is_empty() || component == "." || component == ".." { + return Err(self.collision(path, "invalid component")); + } + match open_dir_no_follow(&dir, Path::new(component)) { + Ok(child) => dir = child, + Err(error) if error.kind() == io::ErrorKind::NotFound => { + match dir.create_dir(component) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} + Err(source) => { + return Err(self.operation("create destination parent", path, source)); + } + } + dir = open_dir_no_follow(&dir, Path::new(component)).map_err(|source| { + self.operation("open destination parent", path, source) + })?; + } + Err(source) if source.raw_os_error() == Some(libc::ELOOP) => { + return Err(self.collision(path, "parent is a symlink")); + } + Err(source) if source.kind() == io::ErrorKind::NotADirectory => { + return Err(self.collision(path, "parent is not a directory")); + } + Err(source) => return Err(self.operation("open destination parent", path, source)), + } + } + Ok((dir, final_name.to_string())) + } + + pub(crate) fn create_dir(&self, path: &str) -> Result<(), FsError> { + let (parent, name) = self.parents(path)?; + match parent.symlink_metadata(&name) { + Ok(_) => Err(self.collision(path, "final entry already exists")), + Err(error) if error.kind() == io::ErrorKind::NotFound => parent + .create_dir(&name) + .map_err(|source| self.operation("create directory", path, source)), + Err(source) => Err(self.operation("inspect destination entry", path, source)), + } + } + + pub(crate) fn create_symlink(&self, path: &str, target: &str) -> Result<(), FsError> { + let (parent, name) = self.parents(path)?; + match parent.symlink_metadata(&name) { + Ok(_) => return Err(self.collision(path, "final entry already exists")), + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(source) => return Err(self.operation("inspect destination entry", path, source)), + } + parent + .symlink(target, &name) + .map_err(|source| self.operation("create symlink", path, source)) + } + + pub(crate) fn pending_file(&self, path: &str) -> Result { + let (parent, name) = self.parents(path)?; + match parent.symlink_metadata(&name) { + Ok(_) => return Err(self.collision(path, "final entry already exists")), + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(source) => return Err(self.operation("inspect destination entry", path, source)), + } + let file = openat( + &parent, + ".", + OFlags::WRONLY | OFlags::TMPFILE | OFlags::CLOEXEC, + Mode::RUSR | Mode::WUSR | Mode::RGRP | Mode::WGRP | Mode::ROTH | Mode::WOTH, + ) + .map_err(io::Error::from) + .map(std::fs::File::from) + .map(cap_std::fs::File::from_std) + .map_err(|source| self.operation("stage file", path, source))?; + Ok(PendingFile { + parent, + final_name: name, + file, + destination: self.destination.clone(), + }) + } +} + +pub(crate) struct PendingFile { + parent: Dir, + final_name: String, + file: cap_std::fs::File, + destination: PathBuf, +} + +impl PendingFile { + pub(crate) fn writer(&self, archive_path: &str) -> Result { + self.file.try_clone().map_err(|source| FsError::Operation { + operation: "open staged file", + archive_path: archive_path.to_owned(), + destination: self.destination.clone(), + source, + }) + } + + pub(crate) fn commit(self, archive_path: &str) -> Result<(), FsError> { + self.file.sync_all().map_err(|source| FsError::Operation { + operation: "sync staged file", + archive_path: archive_path.to_owned(), + destination: self.destination.clone(), + source, + })?; + match linkat( + &self.file, + "", + &self.parent, + &self.final_name, + AtFlags::EMPTY_PATH, + ) { + Ok(()) => Ok(()), + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => { + Err(FsError::ExtractionCollision { + archive_path: archive_path.to_owned(), + destination: self.destination.clone(), + reason: "final entry already exists", + }) + } + Err(source) => Err(FsError::Operation { + operation: "publish staged file", + archive_path: archive_path.to_owned(), + destination: self.destination.clone(), + source: source.into(), + }), + } + } +} + +impl Write for PendingFile { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.file.write(buf) + } + + fn flush(&mut self) -> io::Result<()> { + self.file.flush() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn staged_file_is_removed_after_sink_failure() { + let temporary = tempfile::tempdir().unwrap(); + let root = ExtractionRoot::open(temporary.path(), false, "data").unwrap(); + let mut pending = root.pending_file("data").unwrap(); + pending.file = cap_std::fs::File::from_std(std::fs::File::open("/dev/null").unwrap()); + + assert!( + pending + .writer("data") + .unwrap() + .write_all(b"content") + .is_err() + ); + drop(pending); + + assert!(!temporary.path().join("data").exists()); + assert_eq!(std::fs::read_dir(temporary.path()).unwrap().count(), 0); + } + + #[test] + fn staged_file_is_removed_after_commit_collision() { + let temporary = tempfile::tempdir().unwrap(); + let root = ExtractionRoot::open(temporary.path(), false, "data").unwrap(); + let pending = root.pending_file("data").unwrap(); + pending + .writer("data") + .unwrap() + .write_all(b"content") + .unwrap(); + std::fs::write(temporary.path().join("data"), b"sentinel").unwrap(); + + assert!(matches!( + pending.commit("data"), + Err(FsError::ExtractionCollision { .. }) + )); + + assert_eq!( + std::fs::read(temporary.path().join("data")).unwrap(), + b"sentinel" + ); + assert_eq!(std::fs::read_dir(temporary.path()).unwrap().count(), 1); + } + + #[test] + fn staged_file_publication_uses_the_retained_anonymous_file() { + let temporary = tempfile::tempdir().unwrap(); + let root = ExtractionRoot::open(temporary.path(), false, "data").unwrap(); + let pending = root.pending_file("data").unwrap(); + pending + .writer("data") + .unwrap() + .write_all(b"verified") + .unwrap(); + assert_eq!(std::fs::read_dir(temporary.path()).unwrap().count(), 0); + + pending.commit("data").unwrap(); + + assert_eq!( + std::fs::read(temporary.path().join("data")).unwrap(), + b"verified" + ); + } + + #[test] + fn destination_root_rejects_parent_traversal() { + let Err(error) = ExtractionRoot::open(Path::new(".."), true, "data") else { + panic!("parent traversal unexpectedly opened as a destination root"); + }; + + assert!(matches!( + error, + FsError::Operation { + operation: "open destination", + .. + } + )); + } +} diff --git a/crates/pithos_lib/src/helpers/crypt4gh.rs b/crates/pithos_lib/src/helpers/crypt4gh.rs deleted file mode 100644 index e52af34..0000000 --- a/crates/pithos_lib/src/helpers/crypt4gh.rs +++ /dev/null @@ -1,514 +0,0 @@ -use crate::model::serialization::SerializationError; -use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; -use chacha20poly1305::aead::{Aead, Generate}; -use chacha20poly1305::{ChaCha20Poly1305, KeyInit, Nonce}; -use std::io::{Cursor, Read, Write}; -use thiserror::Error; -use x25519_dalek::{PublicKey, StaticSecret}; - -pub const CRYPT4GH_HEADER_MAGIC: [u8; 8] = [0x63, 0x72, 0x79, 0x70, 0x74, 0x34, 0x67, 0x68]; // "crypt4gh" -pub const CRYPT4GH_HEADER_VERSION: u32 = 1; -pub const CRYPT4GH_BLOCK_SIZE: usize = 65536; -pub const CRYPT4GH_ENCRYPTED_BLOCK_SIZE: usize = 65564; -const PACKET_LENGTH_FIELD_SIZE: usize = size_of::(); -const MIN_PACKET_BODY_LENGTH: usize = size_of::() + 32 + 12 + 16; -const MIN_PACKET_LENGTH: usize = PACKET_LENGTH_FIELD_SIZE + MIN_PACKET_BODY_LENGTH; - -#[derive(Debug, Error)] -pub enum Crypt4GHError { - #[error("Unable to parse `{0}` from bytes")] - FromBytesError(String), - #[error("Invalid value for spec: `{0}`")] - InvalidSpec(String), - #[error("Unable to decrypt: `{0}`")] - DecryptionError(String), - #[error("Decryption failed")] - DecryptionFailed, - #[error("Unable to encrypt: `{0}`")] - EncryptionError(String), -} - -pub struct Crypt4GHHeader { - pub magic: [u8; 8], // Magic string to identify Crypt4GH format - pub version: u32, // Version of the format currently le 1 - pub packet_count: u32, // Size of the encrypted header - pub header_packets: Vec, -} - -impl Crypt4GHHeader { - pub fn new(header_packets: Vec) -> Self { - Crypt4GHHeader { - magic: CRYPT4GH_HEADER_MAGIC, - version: CRYPT4GH_HEADER_VERSION, - packet_count: header_packets.len() as u32, - header_packets, - } - } - - #[tracing::instrument(level = "trace", skip(self, writer))] - pub fn serialize(&self, writer: &mut W) -> Result<(), SerializationError> { - writer.write_all(&self.magic)?; - writer.write_u32::(self.version)?; - writer.write_u32::(self.packet_count)?; // Actually writes the 4 bytes - writer.write_u32::(self.header_packets.len() as u32)?; - for packet in &self.header_packets { - packet.serialize(writer)?; - } - Ok(()) - } -} - -pub struct HeaderPacket { - pub length: u32, // Length of the packet - encryption_method: u32, // Currently only 0 (Chacha20-Poly1305) - writers_pubkey: [u8; 32], // Writer's public key - nonce: [u8; 12], // Nonce for encryption - pub packet_data: PacketData, // Encryption or editlist packet - mac: [u8; 16], // Message Authentication Code (MAC) -} - -impl HeaderPacket { - #[tracing::instrument(level = "trace", skip(sender_key, reader_keys, data_key))] - pub fn from_pithos( - sender_key: &StaticSecret, - reader_keys: Vec, - data_key: &[u8; 32], - ) -> Result, Crypt4GHError> { - let sender_pubkey = PublicKey::from(sender_key); - let mut header_packets = vec![]; - for reader in reader_keys { - let session_key = sender_key.diffie_hellman(&reader); - let nonce = Nonce::generate(); - let mut packet_data = - PacketData::Decrypted(vec![Packet::Encryption(EncryptionPacket { - packet_type: 0, - encryption_method: 0, - encryption_key: *data_key, - })]); - let mac = packet_data.encrypt(session_key.as_bytes(), &nonce)?; - let header_packet = HeaderPacket { - length: u32::try_from(MIN_PACKET_LENGTH + packet_data.get_len()).map_err(|_| { - Crypt4GHError::EncryptionError("header packet length".to_string()) - })?, - encryption_method: 0, - writers_pubkey: sender_pubkey.to_bytes(), - nonce: nonce.into(), - packet_data, - mac, - }; - - header_packets.push(header_packet); - } - - Ok(header_packets) - } - - #[tracing::instrument(level = "trace", skip(self, writer))] - pub fn serialize(&self, writer: &mut W) -> Result<(), SerializationError> { - writer.write_u32::(self.length)?; - writer.write_u32::(self.encryption_method)?; - writer.write_all(&self.writers_pubkey)?; - writer.write_all(&self.nonce)?; - - match &self.packet_data { - PacketData::Encrypted(data) => { - writer.write_all(data)?; - } - PacketData::Decrypted(_) => { - return Err(SerializationError::Other( - "Packet data not encrypted".to_string(), - )); - } - } - - writer.write_all(&self.mac)?; - Ok(()) - } -} - -pub enum PacketData { - Encrypted(Vec), - Decrypted(Vec), -} - -impl PacketData { - #[tracing::instrument(level = "trace", skip(self))] - pub fn get_len(&self) -> usize { - match self { - Self::Encrypted(enc_data) => enc_data.len(), - Self::Decrypted(dec_data) => { - let mut len = 0; - for packet in dec_data { - match packet { - Packet::Encryption(_) => { - len += 4 + 4 + 32; - } - Packet::EditList(edit_packet) => { - len += 4 + 4 + 8 * edit_packet.num_length as usize; - } - } - } - len - } - } - } -} - -pub enum Packet { - Encryption(EncryptionPacket), - EditList(EditListPacket), -} - -pub struct EncryptionPacket { - packet_type: u32, // 0 (Encryption) - encryption_method: u32, // 0 (Chacha20-Poly1305) - encryption_key: [u8; 32], // 32 bytes encryption key -} - -impl EncryptionPacket { - #[tracing::instrument(level = "trace", skip(self))] - pub fn get_encryption_key(&self) -> &[u8; 32] { - &self.encryption_key - } -} - -pub struct EditListPacket { - packet_type: u32, // 1 - num_length: u32, // Number of edits - edits: Vec, // List of edits -} - -impl TryFrom<&[u8]> for Crypt4GHHeader { - type Error = Crypt4GHError; - - #[tracing::instrument(level = "trace", skip(bytes))] - fn try_from(bytes: &[u8]) -> Result { - let mut header = Crypt4GHHeader { - magic: [0; 8], - version: 0, - packet_count: 0, - header_packets: Vec::new(), - }; - let mut cursor = Cursor::new(bytes); - cursor - .read_exact(&mut header.magic) - .map_err(|_| Crypt4GHError::FromBytesError("magic bytes".to_string()))?; - if header.magic != CRYPT4GH_HEADER_MAGIC { - return Err(Crypt4GHError::InvalidSpec("magic bytes".to_string())); - } - header.version = cursor - .read_u32::() - .map_err(|_| Crypt4GHError::FromBytesError("version".to_string()))?; - if header.version != CRYPT4GH_HEADER_VERSION { - return Err(Crypt4GHError::InvalidSpec("version".to_string())); - } - header.packet_count = cursor - .read_u32::() - .map_err(|_| Crypt4GHError::FromBytesError("header size".to_string()))?; - let packet_count = usize::try_from(header.packet_count) - .map_err(|_| Crypt4GHError::FromBytesError("packet count".to_string()))?; - let position = usize::try_from(cursor.position()) - .map_err(|_| Crypt4GHError::FromBytesError("packet position".to_string()))?; - let remaining = bytes.len().saturating_sub(position); - let minimum_packet_bytes = packet_count - .checked_mul(MIN_PACKET_LENGTH) - .ok_or_else(|| Crypt4GHError::FromBytesError("packet count".to_string()))?; - if minimum_packet_bytes > remaining { - return Err(Crypt4GHError::FromBytesError( - "packet count exceeds header".to_string(), - )); - } - header - .header_packets - .try_reserve_exact(packet_count) - .map_err(|_| Crypt4GHError::FromBytesError("packet allocation".to_string()))?; - - for _ in 0..packet_count { - let len = usize::try_from( - cursor - .read_u32::() - .map_err(|_| Crypt4GHError::FromBytesError("packet length".to_string()))?, - ) - .map_err(|_| Crypt4GHError::FromBytesError("packet length".to_string()))?; - if len < MIN_PACKET_LENGTH { - return Err(Crypt4GHError::FromBytesError( - "packet length below minimum".to_string(), - )); - } - let packet_bytes = len - PACKET_LENGTH_FIELD_SIZE; - let position = usize::try_from(cursor.position()) - .map_err(|_| Crypt4GHError::FromBytesError("packet position".to_string()))?; - let remaining = bytes.len().saturating_sub(position); - if packet_bytes > remaining { - return Err(Crypt4GHError::FromBytesError( - "packet exceeds header".to_string(), - )); - } - let mut buf = Vec::new(); - buf.try_reserve_exact(packet_bytes) - .map_err(|_| Crypt4GHError::FromBytesError("packet allocation".to_string()))?; - buf.resize(packet_bytes, 0); - cursor - .read_exact(&mut buf) - .map_err(|_| Crypt4GHError::FromBytesError("packet data".to_string()))?; - header - .header_packets - .push(HeaderPacket::from_buf(buf, len)?); - } - - Ok(header) - } -} - -impl TryInto> for Crypt4GHHeader { - type Error = Crypt4GHError; - - #[tracing::instrument(level = "trace", skip(self))] - fn try_into(self) -> Result, Self::Error> { - let mut bytes = Vec::new(); - bytes.extend_from_slice(&self.magic); - bytes.extend_from_slice(&self.version.to_le_bytes()); - bytes.extend_from_slice(&self.packet_count.to_le_bytes()); - for packet in self.header_packets { - bytes.extend_from_slice(&packet.length.to_le_bytes()); - bytes.extend_from_slice(&packet.encryption_method.to_le_bytes()); - bytes.extend_from_slice(&packet.writers_pubkey); - bytes.extend_from_slice(&packet.nonce); - match packet.packet_data { - PacketData::Encrypted(enc_data) => { - bytes.extend_from_slice(&enc_data); - } - PacketData::Decrypted(_) => { - "packet data is not encrypted".to_string(); - } - } - bytes.extend_from_slice(&packet.mac); - } - Ok(bytes) - } -} - -impl HeaderPacket { - #[tracing::instrument(level = "trace", skip(packets))] - pub fn new(packets: Vec) -> Self { - HeaderPacket { - length: 0, - encryption_method: 0, - writers_pubkey: [0; 32], - nonce: [0; 12], - packet_data: PacketData::Decrypted(packets), - mac: [0; 16], - } - } - - #[tracing::instrument(level = "trace", skip(bytes, len))] - pub fn from_buf(bytes: Vec, len: usize) -> Result { - if len < MIN_PACKET_LENGTH || bytes.len() < MIN_PACKET_BODY_LENGTH { - return Err(Crypt4GHError::FromBytesError( - "packet is too short".to_string(), - )); - } - let mut bytes = Cursor::new(bytes); - let encryption_method = bytes - .read_u32::() - .map_err(|_| Crypt4GHError::FromBytesError("encryption method".to_string()))?; - let mut writers_pubkey = [0; 32]; - bytes - .read_exact(&mut writers_pubkey) - .map_err(|_| Crypt4GHError::FromBytesError("writer's public key".to_string()))?; - let mut nonce = [0; 12]; - bytes - .read_exact(&mut nonce) - .map_err(|_| Crypt4GHError::FromBytesError("nonce".to_string()))?; - - let mut packet_data = Vec::new(); - bytes - .read_to_end(&mut packet_data) - .map_err(|_| Crypt4GHError::FromBytesError("packet data and mac".to_string()))?; - let mac_start = packet_data - .len() - .checked_sub(16) - .ok_or_else(|| Crypt4GHError::FromBytesError("packet MAC".to_string()))?; - let (enc, mac) = packet_data.split_at(mac_start); - let mut encrypted_data = Vec::new(); - encrypted_data - .try_reserve_exact(enc.len()) - .map_err(|_| Crypt4GHError::FromBytesError("packet data allocation".to_string()))?; - encrypted_data.extend_from_slice(enc); - let encrypted_packet_data = PacketData::Encrypted(encrypted_data); - - Ok(HeaderPacket { - length: u32::try_from(len) - .map_err(|_| Crypt4GHError::FromBytesError("header packet length".to_string()))?, - encryption_method, - writers_pubkey, - nonce, - packet_data: encrypted_packet_data, - mac: mac - .try_into() - .map_err(|_| Crypt4GHError::FromBytesError("packet mac".to_string()))?, - }) - } - - #[tracing::instrument(level = "trace", skip(self, readers_private_key))] - pub fn decrypt(&mut self, readers_private_key: &StaticSecret) -> Result<(), Crypt4GHError> { - let writers_pub_key = PublicKey::from(self.writers_pubkey); - let session_key = readers_private_key.diffie_hellman(&writers_pub_key); - self.packet_data - .decrypt(session_key.as_bytes(), &self.nonce, &self.mac)?; - Ok(()) - } - - #[tracing::instrument(level = "trace", skip(self, readers_pubkey, writers_private_key))] - pub fn encrypt( - &mut self, - readers_pubkey: PublicKey, - writers_private_key: Option, - ) -> Result<(), Crypt4GHError> { - let sender_key = writers_private_key.unwrap_or_else(StaticSecret::random); - - let session_key = sender_key.diffie_hellman(&readers_pubkey); - let nonce = Nonce::generate(); - - self.mac = self.packet_data.encrypt(session_key.as_bytes(), &nonce)?; - self.writers_pubkey = PublicKey::from(&sender_key).to_bytes(); - self.nonce = nonce.into(); - self.length = (MIN_PACKET_LENGTH + self.packet_data.get_len()) - .try_into() - .map_err(|_| Crypt4GHError::EncryptionError("header packet length".to_string()))?; - Ok(()) - } -} - -impl PacketData { - #[tracing::instrument(level = "trace", skip(self, session_key, nonce))] - pub fn encrypt( - &mut self, - session_key: &[u8; 32], - nonce: &Nonce, - ) -> Result<[u8; 16], Crypt4GHError> { - if let Self::Decrypted(dec_data) = &self { - let mut enc_data = Vec::new(); - for packet in dec_data { - match packet { - Packet::Encryption(enc_packet) => { - enc_data.extend_from_slice(&enc_packet.packet_type.to_le_bytes()); - enc_data.extend_from_slice(&enc_packet.encryption_method.to_le_bytes()); - enc_data.extend_from_slice(&enc_packet.encryption_key); - } - Packet::EditList(edit_packet) => { - enc_data.extend_from_slice(&edit_packet.packet_type.to_le_bytes()); - enc_data.extend_from_slice(&edit_packet.num_length.to_le_bytes()); - for edit in &edit_packet.edits { - enc_data.extend_from_slice(&edit.to_le_bytes()); - } - } - } - } - - let encrypted = ChaCha20Poly1305::new_from_slice(session_key) - .map_err(|_| Crypt4GHError::EncryptionError("Cipher init failed".to_string()))? - .encrypt(nonce, enc_data.as_slice()) - .map_err(|_| Crypt4GHError::EncryptionError("Encrypt chunk failed".to_string()))?; - *self = Self::Encrypted(encrypted[..encrypted.len() - 16].to_vec()); - let mac: [u8; 16] = encrypted[encrypted.len() - 16..].try_into().map_err(|_| { - Crypt4GHError::EncryptionError("Packet data MAC extraction failed".to_string()) - })?; - Ok(mac) - } else { - Err(Crypt4GHError::EncryptionError( - "Packet data is already encrypted".to_string(), - )) - } - } - - #[tracing::instrument(level = "trace", skip(self, session_key, nonce, mac))] - pub fn decrypt( - &mut self, - session_key: &[u8; 32], - nonce: &[u8; 12], - mac: &[u8; 16], - ) -> Result<(), Crypt4GHError> { - if let Self::Encrypted(enc_data) = &self { - let dec_data = [enc_data.as_slice(), mac.as_slice()].concat(); - - let decrypted_bytes = ChaCha20Poly1305::new_from_slice(session_key) - .map_err(|e| { - Crypt4GHError::DecryptionError(format!("initialize decryptor: {}", e)) - })? - .decrypt(nonce.into(), dec_data.as_slice()) - .map_err(|_| Crypt4GHError::DecryptionFailed)?; - - *self = Self::Decrypted(Self::packet_from_bytes(&decrypted_bytes)?); - } else { - return Err(Crypt4GHError::DecryptionError( - "packet data is not encrypted".to_string(), - )); - } - Ok(()) - } - - #[tracing::instrument(level = "trace", skip(bytes))] - pub fn packet_from_bytes(bytes: &[u8]) -> Result, Crypt4GHError> { - let mut cursor = Cursor::new(bytes); - let mut packets = Vec::new(); - let mut found_edit = false; - while cursor.position() < bytes.len() as u64 { - let packet_type = cursor - .read_u32::() - .map_err(|_| Crypt4GHError::FromBytesError("packet type".to_string()))?; - - match packet_type { - 0 => { - let encryption_method = cursor.read_u32::().map_err(|_| { - Crypt4GHError::FromBytesError("encryption method".to_string()) - })?; - if encryption_method != 0 { - return Err(Crypt4GHError::InvalidSpec( - "unsupported encryption method".to_string(), - )); - } - let mut encryption_key = [0; 32]; - cursor - .read_exact(&mut encryption_key) - .map_err(|_| Crypt4GHError::FromBytesError("encryption key".to_string()))?; - packets.push(Packet::Encryption(EncryptionPacket { - packet_type, - encryption_method, - encryption_key, - })); - } - 1 => { - if found_edit { - return Err(Crypt4GHError::InvalidSpec( - "multiple edit lists not allowed".to_string(), - )); - } - let num_length = cursor.read_u32::().map_err(|_| { - Crypt4GHError::FromBytesError("number of edits".to_string()) - })?; - let mut edits = Vec::new(); - for _ in 0..num_length { - edits.push( - cursor - .read_u64::() - .map_err(|_| Crypt4GHError::FromBytesError("edit".to_string()))?, - ); - } - packets.push(Packet::EditList(EditListPacket { - packet_type, - num_length, - edits, - })); - found_edit = true; - } - _ => { - return Err(Crypt4GHError::FromBytesError( - "invalid packet type".to_string(), - )); - } - } - } - Ok(packets) - } -} diff --git a/crates/pithos_lib/src/helpers/ro_crate.rs b/crates/pithos_lib/src/helpers/ro_crate.rs deleted file mode 100644 index e92b51d..0000000 --- a/crates/pithos_lib/src/helpers/ro_crate.rs +++ /dev/null @@ -1,321 +0,0 @@ -use crate::error::PithosError; -use rocraters::ro_crate::read::{read_crate, read_crate_obj}; -pub use rocraters::ro_crate::rocrate::RoCrate; -use std::collections::BTreeMap; -use std::fs::File; -use std::io::Read; -use std::path::{Component, Path, PathBuf}; -use zip::{CompressionMethod, ZipArchive}; - -pub const RO_CRATE_METADATA_FILE: &str = "ro-crate-metadata.json"; - -const VALIDATE_AND_WARN: i8 = 1; - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum RoCrateSource { - Directory(PathBuf), - Zip(PathBuf), -} - -#[derive(Debug, Clone)] -pub struct LoadedRoCrate { - pub ro_crate: RoCrate, - pub source: RoCrateSource, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum ZipEntryKind { - Directory, - File, - Symlink, -} - -#[derive(Debug, Clone)] -pub(crate) struct ZipEntryDescriptor { - pub archive_index: Option, - pub inner_path: String, - pub kind: ZipEntryKind, - pub uncompressed_size: u64, - pub timestamp: u64, - pub permissions: u32, -} - -#[derive(Debug)] -pub(crate) struct RoCrateZipManifest { - pub metadata: ZipEntryDescriptor, - pub entries: Vec, -} - -pub fn read_ro_crate_directory(path: impl AsRef) -> Result { - let source = path.as_ref().to_path_buf(); - let source_metadata = std::fs::symlink_metadata(&source)?; - if !source_metadata.is_dir() { - return Err(PithosError::InvalidRoCrateSource { - path: source, - expected: "directory", - }); - } - - let metadata_path = source.join(RO_CRATE_METADATA_FILE); - let metadata = - std::fs::symlink_metadata(&metadata_path).map_err(|error| match error.kind() { - std::io::ErrorKind::NotFound => PithosError::MissingRoCrateMetadata(source.clone()), - _ => PithosError::Io(error), - })?; - if !metadata.file_type().is_file() { - return Err(PithosError::InvalidRoCrateSource { - path: metadata_path, - expected: "regular metadata file", - }); - } - - let ro_crate = read_crate(&metadata_path, VALIDATE_AND_WARN)?; - Ok(LoadedRoCrate { - ro_crate, - source: RoCrateSource::Directory(source), - }) -} - -pub fn read_ro_crate_zip(path: impl AsRef) -> Result { - let source = path.as_ref().to_path_buf(); - let source_metadata = std::fs::symlink_metadata(&source)?; - if !source_metadata.file_type().is_file() { - return Err(PithosError::InvalidRoCrateSource { - path: source, - expected: "ZIP file", - }); - } - - let manifest = inspect_ro_crate_zip_manifest(&source)?; - let metadata_index = manifest - .metadata - .archive_index - .ok_or_else(|| PithosError::MissingRoCrateMetadata(source.clone()))?; - - let mut archive = ZipArchive::new(File::open(&source)?)?; - let mut metadata = archive.by_index(metadata_index)?; - let mut json = String::new(); - metadata.read_to_string(&mut json)?; - let ro_crate = read_crate_obj(&json, VALIDATE_AND_WARN)?; - - Ok(LoadedRoCrate { - ro_crate, - source: RoCrateSource::Zip(source), - }) -} - -fn normalize_zip_path(path: &Path) -> Result { - let mut components = Vec::new(); - - for component in path.components() { - match component { - Component::Normal(component) => { - components.push(component.to_str().ok_or(())?.to_string()); - } - Component::CurDir => {} - Component::ParentDir => { - if components.pop().is_none() { - return Err(()); - } - } - Component::RootDir | Component::Prefix(_) => return Err(()), - } - } - - if components.is_empty() { - return Err(()); - } - - Ok(components.join("/")) -} - -fn has_forbidden_zip_root_or_prefix(path: &Path, name: &str) -> bool { - if path - .components() - .any(|component| matches!(component, Component::RootDir | Component::Prefix(_))) - { - return true; - } - - // ZIP paths use Windows-style syntax even when the host is Unix. - name.starts_with('/') - || name.starts_with('\\') - || name.as_bytes().get(1).is_some_and(|byte| *byte == b':') -} - -fn zip_timestamp(entry: &zip::read::ZipFile<'_, R>) -> u64 { - let Some(date_time) = entry.last_modified() else { - return 0; - }; - if !date_time.is_valid() { - return 0; - } - - // ZIP DOS timestamps have no timezone. Treating them as UTC makes the - // stored value stable across machines and extraction environments. - let year = date_time.year(); - let mut days = 0i64; - for current_year in 1970u16..year { - days += if current_year % 4 == 0 && (current_year % 100 != 0 || current_year % 400 == 0) { - 366 - } else { - 365 - }; - } - - const DAYS_BEFORE_MONTH: [i64; 12] = [0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334]; - days += DAYS_BEFORE_MONTH[usize::from(date_time.month() - 1)]; - if date_time.month() > 2 && (year % 4 == 0 && (year % 100 != 0 || year % 400 == 0)) { - days += 1; - } - days += i64::from(date_time.day() - 1); - - let timestamp = days * 86_400 - + i64::from(date_time.hour()) * 3_600 - + i64::from(date_time.minute()) * 60 - + i64::from(date_time.second()); - if timestamp < 0 { 0 } else { timestamp as u64 } -} - -fn zip_entry_kind_rank(kind: ZipEntryKind) -> u8 { - match kind { - ZipEntryKind::Directory => 0, - ZipEntryKind::File => 1, - ZipEntryKind::Symlink => 2, - } -} - -pub(crate) fn inspect_ro_crate_zip_manifest( - path: impl AsRef, -) -> Result { - let source = path.as_ref().to_path_buf(); - let file = File::open(&source)?; - let mut archive = ZipArchive::new(file)?; - - if archive.has_overlapping_files()? { - return Err(PithosError::OverlappingZipEntries(source)); - } - - let mut explicit_entries = BTreeMap::new(); - let mut metadata = None; - - for archive_index in 0..archive.len() { - let descriptor = { - let entry = archive.by_index(archive_index)?; - let entry_name = entry.name().to_string(); - - if entry.encrypted() { - return Err(PithosError::EncryptedZipEntry(entry_name)); - } - if !matches!( - entry.compression(), - CompressionMethod::Stored | CompressionMethod::Deflated - ) { - return Err(PithosError::UnsupportedZipEntry(entry_name)); - } - - let enclosed_name = entry - .enclosed_name() - .ok_or_else(|| PithosError::UnsafeZipPath(entry_name.clone()))?; - if entry_name.as_bytes().contains(&0) - || has_forbidden_zip_root_or_prefix(&enclosed_name, &entry_name) - { - return Err(PithosError::UnsafeZipPath(entry_name.clone())); - } - let inner_path = normalize_zip_path(&enclosed_name) - .map_err(|_| PithosError::UnsafeZipPath(entry_name.clone()))?; - - let kind = if entry.is_dir() { - ZipEntryKind::Directory - } else if entry.is_symlink() { - ZipEntryKind::Symlink - } else if entry.is_file() { - ZipEntryKind::File - } else { - return Err(PithosError::UnsupportedZipEntry(entry_name)); - }; - - let default_permissions = match kind { - ZipEntryKind::Directory => 0o755, - ZipEntryKind::File | ZipEntryKind::Symlink => 0o644, - }; - ZipEntryDescriptor { - archive_index: Some(archive_index), - inner_path, - kind, - uncompressed_size: if matches!(kind, ZipEntryKind::Directory) { - 0 - } else { - entry.size() - }, - timestamp: zip_timestamp(&entry), - permissions: match entry.unix_mode() { - Some(permissions) => permissions, - None => default_permissions, - }, - } - }; - - if explicit_entries - .insert(descriptor.inner_path.clone(), descriptor.clone()) - .is_some() - { - return Err(PithosError::DuplicateZipPath(descriptor.inner_path)); - } - - if descriptor.inner_path == RO_CRATE_METADATA_FILE { - if descriptor.kind == ZipEntryKind::File { - metadata = Some(descriptor); - } else { - return Err(PithosError::InvalidRoCrateSource { - path: source.join(RO_CRATE_METADATA_FILE), - expected: "regular metadata file", - }); - } - } - } - - let explicit_paths: Vec = explicit_entries.keys().cloned().collect(); - for path in explicit_paths { - let mut parent = path.as_str(); - while let Some((parent_path, _)) = parent.rsplit_once('/') { - if let Some(existing) = explicit_entries.get(parent_path) { - if existing.kind != ZipEntryKind::Directory { - return Err(PithosError::ZipPathConflict(parent_path.to_string())); - } - } else { - explicit_entries.insert( - parent_path.to_string(), - ZipEntryDescriptor { - archive_index: None, - inner_path: parent_path.to_string(), - kind: ZipEntryKind::Directory, - uncompressed_size: 0, - timestamp: 0, - permissions: 0o755, - }, - ); - } - parent = parent_path; - } - } - - let metadata = metadata.ok_or_else(|| PithosError::MissingRoCrateMetadata(source.clone()))?; - - let mut entries: Vec<_> = explicit_entries - .into_values() - .filter(|entry| entry.inner_path != RO_CRATE_METADATA_FILE) - .collect(); - entries.sort_by(|left, right| { - zip_entry_kind_rank(left.kind) - .cmp(&zip_entry_kind_rank(right.kind)) - .then_with(|| { - left.inner_path - .to_lowercase() - .cmp(&right.inner_path.to_lowercase()) - }) - .then_with(|| left.inner_path.cmp(&right.inner_path)) - }); - - Ok(RoCrateZipManifest { metadata, entries }) -} diff --git a/crates/pithos_lib/src/io/extraction.rs b/crates/pithos_lib/src/io/extraction.rs deleted file mode 100644 index e330a13..0000000 --- a/crates/pithos_lib/src/io/extraction.rs +++ /dev/null @@ -1,154 +0,0 @@ -use crate::error::PithosError; -use cap_std::fs::{Dir, OpenOptions}; -use std::io::{self, Write}; -use std::path::Path; -use std::sync::atomic::{AtomicU64, Ordering}; - -static TEMP_COUNTER: AtomicU64 = AtomicU64::new(0); - -fn collision(path: &str, reason: impl Into) -> PithosError { - PithosError::ExtractionCollision { - path: path.into(), - reason: reason.into(), - } -} - -pub(crate) struct ExtractionRoot { - root: Dir, -} - -impl ExtractionRoot { - pub(crate) fn open(path: &Path, create: bool) -> Result { - if create { - std::fs::create_dir_all(path)?; - } - Ok(Self { - root: Dir::open_ambient_dir(path, cap_std::ambient_authority())?, - }) - } - - fn parents(&self, path: &str) -> Result<(Dir, String), PithosError> { - let mut components = path.split('/').peekable(); - let final_name = components - .next_back() - .ok_or_else(|| collision(path, "empty final component"))?; - let mut dir = self.root.try_clone()?; - for component in components { - if component.is_empty() || component == "." || component == ".." { - return Err(collision(path, "invalid component")); - } - match dir.symlink_metadata(component) { - Ok(metadata) if metadata.file_type().is_symlink() => { - return Err(collision(path, "parent is a symlink")); - } - Ok(metadata) if !metadata.is_dir() => { - return Err(collision(path, "parent is not a directory")); - } - Ok(_) => dir = dir.open_dir(component)?, - Err(error) if error.kind() == io::ErrorKind::NotFound => { - dir.create_dir(component)?; - dir = dir.open_dir(component)?; - } - Err(error) => return Err(error.into()), - } - } - Ok((dir, final_name.to_string())) - } - - pub(crate) fn create_dir(&self, path: &str) -> Result<(), PithosError> { - let (parent, name) = self.parents(path)?; - match parent.symlink_metadata(&name) { - Ok(_) => Err(collision(path, "final entry already exists")), - Err(error) if error.kind() == io::ErrorKind::NotFound => { - parent.create_dir(&name).map_err(Into::into) - } - Err(error) => Err(error.into()), - } - } - - pub(crate) fn create_symlink(&self, path: &str, target: &str) -> Result<(), PithosError> { - let (parent, name) = self.parents(path)?; - match parent.symlink_metadata(&name) { - Ok(_) => return Err(collision(path, "final entry already exists")), - Err(error) if error.kind() == io::ErrorKind::NotFound => {} - Err(error) => return Err(error.into()), - } - parent.symlink(target, &name).map_err(Into::into) - } - - pub(crate) fn pending_file(&self, path: &str) -> Result { - let (parent, name) = self.parents(path)?; - match parent.symlink_metadata(&name) { - Ok(_) => return Err(collision(path, "final entry already exists")), - Err(error) if error.kind() == io::ErrorKind::NotFound => {} - Err(error) => return Err(error.into()), - } - let pid = std::process::id(); - loop { - let temp = - format! {".pithos-tmp-{pid}-{}", TEMP_COUNTER.fetch_add(1, Ordering::Relaxed)}; - let mut options = OpenOptions::new(); - options.write(true).create_new(true); - match parent.open_with(&temp, &options) { - Ok(file) => { - return Ok(PendingFile { - parent, - temp, - final_name: name, - file, - }); - } - Err(error) if error.kind() == io::ErrorKind::AlreadyExists => continue, - Err(error) => return Err(error.into()), - } - } - } -} - -pub(crate) struct PendingFile { - parent: Dir, - temp: String, - final_name: String, - file: cap_std::fs::File, -} - -impl PendingFile { - pub(crate) fn writer(&self) -> Result { - Ok(self.file.try_clone()?) - } - - pub(crate) fn commit(mut self) -> Result<(), PithosError> { - self.file.sync_all()?; - match self - .parent - .hard_link(&self.temp, &self.parent, &self.final_name) - { - Ok(()) => { - self.parent.remove_file(&self.temp)?; - self.temp.clear(); - Ok(()) - } - Err(error) if error.kind() == io::ErrorKind::AlreadyExists => { - Err(collision(&self.final_name, "final entry already exists")) - } - Err(error) => Err(error.into()), - } - } -} - -impl Write for PendingFile { - fn write(&mut self, buf: &[u8]) -> io::Result { - self.file.write(buf) - } - fn flush(&mut self) -> io::Result<()> { - self.file.flush() - } -} - -impl Drop for PendingFile { - fn drop(&mut self) { - if !self.temp.is_empty() { - let _ = self.parent.remove_file(&self.temp); - } - } -} diff --git a/crates/pithos_lib/tests/compile_fail.rs b/crates/pithos_lib/tests/compile_fail.rs new file mode 100644 index 0000000..d3cdf3f --- /dev/null +++ b/crates/pithos_lib/tests/compile_fail.rs @@ -0,0 +1,9 @@ +#[test] +fn secret_ownership_and_writer_typestate_are_compile_time_enforced() { + let cases = trybuild::TestCases::new(); + cases.compile_fail("tests/ui/non_copy_static_secret.rs"); + cases.compile_fail("tests/ui/writer_finish_consumes_writer.rs"); + cases.compile_fail("tests/ui/writer_use_after_finish.rs"); + cases.compile_fail("tests/ui/incomplete_writer_cannot_resume.rs"); + cases.compile_fail("tests/ui/crypt4gh_wire_record_is_private.rs"); +} diff --git a/crates/pithos_lib/tests/crypt4gh_export.rs b/crates/pithos_lib/tests/crypt4gh_export.rs new file mode 100644 index 0000000..a854fa1 --- /dev/null +++ b/crates/pithos_lib/tests/crypt4gh_export.rs @@ -0,0 +1,31 @@ +mod common; + +use common::util::{fixture, open, private_key, public_key}; +use pithos_lib::adapters::crypt4gh; +use pithos_lib::archive::AccessKeys; + +#[test] +fn crypt4gh_export_writes_only_its_header_before_a_source_block_fails() { + let temporary = tempfile::tempdir().unwrap(); + let path = fixture(&temporary, "recipient1"); + let mut bytes = std::fs::read(&path).unwrap(); + let marker = bytes + .windows(4) + .position(|window| window == b"BLCK") + .expect("fixture must contain a block"); + bytes[marker + 4] ^= 1; + std::fs::write(&path, bytes).unwrap(); + let archive = open(&path, AccessKeys::new().with_key(private_key("recipient1"))); + let mut output = Vec::new(); + assert!( + crypt4gh::export( + &archive, + "data", + vec![public_key("recipient2")], + &mut output + ) + .is_err() + ); + let header_len = 16 + u32::from_le_bytes(output[16..20].try_into().unwrap()) as usize; + assert_eq!(output.len(), header_len); +} diff --git a/crates/pithos_lib/tests/data/README.md b/crates/pithos_lib/tests/data/README.md new file mode 100644 index 0000000..4947dc3 --- /dev/null +++ b/crates/pithos_lib/tests/data/README.md @@ -0,0 +1,14 @@ +# Test fixtures + +This directory is for repository-owned integration-test inputs. The fixtures are deliberately small and are not examples of production data. + +- `dummy_dir/` is a hand-authored RO-Crate 1.2 directory fixture consumed by + `ro_crate_directory.rs`. It covers upstream graph parsing and Pithos directory conversion. Its + `literature/pithos-placeholder.pdf` is a minimal repository-owned PDF retained solely to cover + a non-text `application/pdf` member and `encodingFormat` handling. +- `keys/` contains test-only PEM key pairs consumed through `tests/common/keys.rs` by archive, + filesystem, adapter, and CLI integration tests. They are public repository test material, not + operational credentials. +- ZIP RO-Crates are generated deterministically by `tests/common/ro_crate.rs::write_raw_zip` for + `ro_crate_zip.rs`, `ro_crate_conversion.rs`, and the adapter benchmark. No committed ZIP fixture + is required. diff --git a/crates/pithos_lib/tests/data/dummy_dir/dataset/brain_gene_expression_data/brain_data_accession_number.txt b/crates/pithos_lib/tests/data/dummy_dir/dataset/brain_gene_expression_data/brain_data_accession_number.txt new file mode 100644 index 0000000..185a0aa --- /dev/null +++ b/crates/pithos_lib/tests/data/dummy_dir/dataset/brain_gene_expression_data/brain_data_accession_number.txt @@ -0,0 +1,2 @@ +#Accession number +E-GEOD-3790 diff --git a/crates/pithos_lib/tests/data/dummy_dir/dataset/brain_gene_expression_data/brain_data_assession_number.txt b/crates/pithos_lib/tests/data/dummy_dir/dataset/brain_gene_expression_data/brain_data_assession_number.txt deleted file mode 100644 index 56369c2..0000000 --- a/crates/pithos_lib/tests/data/dummy_dir/dataset/brain_gene_expression_data/brain_data_assession_number.txt +++ /dev/null @@ -1,2 +0,0 @@ -#Assesion number -E-GEOD-3790 diff --git a/crates/pithos_lib/tests/data/dummy_dir/literature/Hodges06_human_brain_Affy.pdf b/crates/pithos_lib/tests/data/dummy_dir/literature/Hodges06_human_brain_Affy.pdf deleted file mode 100644 index 50fff2b4cce42f672601e24dbece71d6de652b7d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 359786 zcmbq*1zc2J(>UE-(yW960?P)CbcfQ7)Gi_2h=grrN5ElQd9RiO0yG$GgAQ1ffxeg>8{?~XS z!Xk)2+lj!1|AJWr@iz`cB*aDj>?Rjb96;8eY>tFHA&2_%E16#l`+A6B85pb6z+?_%A%e#Srj6X%`_ZBKqfg z5iq#upKC+FB>uty0uK9&o&hoy`D;A5=-=ash>QLOizxgr`a_5TSopagK3+CXE@&@u z5`bao1H1wPLyACkywH9ao**s`)pK|Efk*%thv-4o-R#{V62L5AUddbG*1l$eBu1b4m3 zf4@#ZAkefXx_xrW6`MUy=q1dGef-;AfYxqKH0RSZFALpO5r*6q6Dy6=&@}G*GjV<6 zlX1FbXt8C(6mqg_*8`^Jwx(iUBeK0u5=cy_s0W1B2_lL+IcIZPhBj=5AZUSV5-GYc zi-O}9LVJ_$borWzi99 z+`-|$Z)-hPg*{v^=R*1X-au22mtyr$jP9)XJatFJBO3N$vf6Zx%=L@u0&cV>5Z{|U z;l~Ts6PB^v9g(~bZi{Z-Vkw+;Pjvd7d+5VMH?R*7k3A!HrfXuWgi=e1i&zWN)>fQElFkIOxXE) zLnHu(3(@;2Gyy65gJc!m-F(n)KHla4!v$MGU@*)GK#J)FqyW3YlmkZ{lVYX^gfln@ z&>jJZK8T*>Av_?2P#qgDU`Pnamq9okeO+xKVuwdl&)o+&5D;MiV3d=Mf_s2D*hmy6 z3=xwMv4kpmxqJLosE77;_w}*^Rt_9oaL8X*0u~%1N73EI-AmuY#tsdfL`82Apfv_! z85yVwut=a#1qPHI!l42OvH)KOQH21qhbUD+0?R#wS_SA2!^{R%5e2f}+lm3%?<0r< z`pEZE2_XA@1fV+%j=>}_ua1|yoj%&f915sHPy=*;k0n$&z(-Xd!~p=0sy+zN_s#ir zQQ!_7E(){BAoxmXZ#yq14&UhoXrsRh(Fe$ze9wu&kWhJV zJ8&z-U~ni10hkwr3yVS30kU(llXr7)0XI|8(Z&l{Cok023qZdM+TI6>8I14OMS<)5 zWl=v60o9j>*xR^xqoD=>>qY@nIQd+F>T5%Ug@ob14gk{OF9ZA+tYG#Xvvv$t1Yv;q z5ro5l9Rt-TF)`sk5u=GgjJ~L{qR0_$;QxRdF_`!d?1&)XKd~eJ3wB_l{|P&KP&Cv5 z>I8L%xINI4(bc_f(Af6px#hNs4G+r>I3!kbq6E|xED}cs65o&4Gpz(cXhRa z`uMv;{h>-w51`f^1=WZ8IC`PcP)(>a)E+n>Py^@%sJ)XP8mbD_f+|B5q25jbP&24E z+7ImpRXN-dr~=dt(7mBLP*dm`s4+AU?d1;Dh6X}4pl6|u7d#w+rXJw&#cU!bcL<6f z)E*cdfW*b!4YSS8Xdg@)aLa-GVJmO6tJ7f-K=9I}N!hzqLh2FPtE zHwWNEAD%M>8*em7c890wH==R!^7a9b0R$lo)w21X2b4dkzOSth$f?1T3>>`Qjxs=x z80HE!c0&0$dYg-4cr9EU20mvJP?K>_%Sd5>jld|*uimsO~G=o419iFE0_oIb5wVfB0&6n9}g@8^B`Pc zig^HD0Jg!*{~ebg^Z)lB(Bqdo=%FC_Ps9W4t(&}?x6=_ZL4PNrpDg(Y1&N6MLP2oE zzfllITpv*oxE>6X`}q%uZp_<{4A@8*f<%Y`G6Fy|P!HxYTM2Fp@PH`5pbOYGFb~#& zZ4XNiTYzPlalwXQ(XWGnd2rC5U>tTmYAXWZ9;^rZfq8KMF#8L>AMFpA0(F=+-}6WP z!M+BtB4`Uma&r$(+8muS%AHX%&auyiV*BrJRO6Ce9X zo%-PC^mn5LgwO3!58@5f;Hd?nL(q?@UhHn#5Gsfc<;PwMhtHcqqo~s~{Y`QWr>=cW zN?e-T{6wI3n!&6pm6TJnt%g62psfzK>WvmIlY*yl(9QS}Qi$B`#|j<`cqT3Df#=Fd z3gSFd`DT&~w(;E-^X4;so7J~;CWn`F#DW(m+VnK6llI(bvruZkmrjfVMDcEm5TM@!6(Klk%ref+4t!1%G-`TD>Q zk7b>%TFZ&`F@CL!q2By5vmJ$%J zOLJQ^Hv+KJG11XUoE(a&d|&oMP|l%%*3K zN?r1y7d|Ys=Mp_IV0vv~qUCo@xoW%%+OxX%`E1 z36YZt;nH!a5&!Ra+S%_s4GtW$BdG`j_}EXe_^(V2fzhCknA#sl z8l$2AFG17(uIQ-%hQ^=JA>luP1M>e}WFtYYqxYX+`*|Pvp}POjBmTQ6ID6(0HZ^6f zUqSm-oe&rPH`sm`27tf>nJPvQ{vQ;DAHwz*MFC{Qz;*19JA)h=BmKWic~IIOX#ikn zkT)N3@&B!+a0CRHB1Oe9QsPKcz(@>G27H$sm^qK~08F6j55fSZ;QUAJj#A7xVEvJH z0OY~-fPIc0unk-k&=|n}03tvt_`m(Y*?z+O_gLUehowjJ0VTi6Hq1@^ci9F9?k|5? zo_>>U8QSi&FskU)UHXeYI*8>L^A&co28DwPV=4+no+dQ4#4dh!v^lAFHWJ6NeH9aR zp!yS18#~v=(^5R z^RR!`-VxoGzM8}eTx#+=!*%mwu59SMmq;d9x)y&Sav+1J2BI1DsLJ1lrG6(FU$FnU zGVfH)R{eQVCIUgtD-V4X33_3Uw+ao;Uuwb&3REDvsbQ)`l@9fVJ+`geC?zJ2e5R~G z!!cpGk>1=KpvSN8xZ2F*yoVzi6ER7ge*KYjn1~sh`X@Jw`18^*Yh}cAhhaMd<>}bP zcZxgbs_n~6xj71X`!3=4AbPvGw26~=W4z5*>P5Sbw;A4)c(|M*gp@U6BDx__X-0Y6 z=Xq|l;ElU|)D}W_gcvf4D6ZDT&bdy=#C+65CNEyIaPQ2x)W3T?zWdb}b-?A;Db|J} zTa6mU3TXq&v8OB@1tX2BcxUofufOt3*&t=Qa$;(n&m(DN+x_V~ng+wF!RKN6CCP6d zS@mVzI%!y|hNUstBv`U|x6Sv#ouxdh8r+9M6sB5Jr)Z>~zSkF6$3J_=_dNBq4mS?@ zxrPcc*EA8*3iXf-T%0lI{XH{!lU{b!Wu2&aS=*tufto0%rJ7=CO?%5TRaYx-waR!F zWk1yJyv)o=nOnAiMKSP(in2L-|M5ZVNZuW#5ca#JT<6}szipp_+MEl?ioQR`?s>>l>WNU*g09lto%0?Ev}1YL@bf0ibxy;# zxD0(>&4oFcj%5#ADm<3DrX(tzxKHNeR+3UGdx6MTz@+`G30+)_Rr;N^yi&Zl6-ng8 zI(NSa^p$FoPu^AktKWiNK3^H(Pa-?HV0ewf)RTqb&MyC%Z;l=x%&C?4N%d)t)uOCC z7}}tK_}bvPn)FZN3uCEv=!W2*mb^L9eUjrhdJR2^*^j4GPLubZ=kZXM@nDZLe(K!L6MG?Gz8tFq)) z3_3~gO|^ULC0M+HFD`8;96a_Iu(!eKkTx zdGW)y=N8yD<15e}k7hofk)B57d=p*3@5gncDAk~G8tG43*meA^zDuKBIAnR6^6imN z?`zSyWZkA8hi%D3;*Z5+-|mf7AXOd0cRv1Ui%o%bOhY2p@KQq8De0xw6S7=gyX<}u z5u+0)C@k?F&td7k#l8`8{gWgawuAS~ty(5eM*MCTB@1{LVvb&z^G$d#b5t|0GZxtu zZabDt1VMhXx2vi~hiOiq@YWt<;c@p~q3n{R4FzxQd!zh)KX&#>V?7MG!o>Y zX8x{CeAEYb%7a-IFO!>BUGB5`BJ92^P{LrXZGpSZgvKJHP~UIcW;7g*v)y{uNc_@9 zV7#K8^c!;j)cfF%coisGI9_JM_LpW=?wj1})=zlKj2%|*>nge(BXqpuEk{Zfq0eJ+ z(wYCt?us25Q&&c1I*VQYbjO}5Vn4!MXf}#LXo>uV1caA0@lM#qqzoqiZ;~Hb{fhCV zZ)m9AsHXjx72;AKTXku$=UKW|x>oh>C+cAA)XIxQIqr6i<3VNGz0`h1sMBn=#8{R_ z^pLPOPZ})?aumBLAyn_V6`!~V2NH({CyGc4-+vq`-HC-2uA&ot{Fv|?P73uWtWNFU z8145J?hqR*J{o~0&1|m542nJWnP+%ouVs7FRWCNkNI~kh)pdFkU1hhfw!OWYmV1uj zX+(m@A*A`D16S#CTghH<*JSCpU(vVMZify&Ju&4=EAx#aQO2&Fv^68XxA^HEg%eveHZS9O4n6?B6dnD z=6L*?3r==}yz@z4H<{WO?UA?G6BlMXPt)ykujY^@@0MSRBULG_Pi0+Qa(4~SFJ6TV z83%8?PEN}<@tRvjNFn*~=ih!BJoZL$ek4kptbt6n$|_|#!kyY|lsO!Wp)C}L42!E| z#Ni(qz)x%8_j{iN3{?64^ve7y8+{DBye7n{o&fhg7o#}FyWF}G7nZ&93|_|E_v(Bm z(M0Fssg=Snp}N+oWI zU*0LLvBjxB8K@@^BPF}#6m3vo{>34=5_yVxwS6z6*RYG*Vl2X6QO;XQwrM3x3Vv$# z!*R>h40)m`|1oz281dld?9%$BnFZM|b|alR}76)SiDD(sgY z7FFw)vu_b>iOO@b@ku2<;aZqBMt>tPjMo$ruX%h0KBckamrNP17`b0ssgvq?N7uGB z=YbyH<(>^@($O>g8IIQw`bu#XU;Fz4FPl#%(Tx`=$m{T}`^Xmgv@+h&F>d`Nw`rab z|5@>Lsu#oO&BujiC9FyBH2Il{jbwd;%^yRAI=l1zsBHKEY3iCz&*lz{z#lr)1^LV{rhEO#U=J9eaYLsE}7ZWL0#z2ZTeTz7g z2F16E#j#hk-YI+0za~n$oAoqs^K#OQ_iMHOsC9YK)De>zaYq-)q^J||tY^jf?wrRC z&p)e0a<`@z21|74R={VK4{1V=Yu)Fb=y*j<$-jS)z*KjB^c&mws^vjdB9}?a>uvGV zH0mBaSwm{=(QSkU4Fe}`TyCbJU*P7WnhWBuk)Z#p=%f-)_>Mxh2x7;W^`t0z^V|X)a)tSR`vo-8oEyZ)*FVI!V#EJwr(c?bBc*xh@ z(xZH$oCefy6$m;40ty60RWF_{jmawRoA;sn$Zah``0C>xQ&rB>2~(*(!Y?`82gJfy z@(K-?{_#wIHzEVZgGP^Us7WTF*q~Rhb+s$nd!my&iDc-- zc@^_`nDOczyUTGkjwhW_1WaXHC2Vh&_TQ`EMk=|_(R(*%kF)A9U}xH5*F?Q*=I_1a z5G9^ix}~{_Dy0$;PZpqE9BO>vmQ$}OE^V?jPi4eEr8SYG=)i@v#|l^_di}ih%L$i$ zM%P*cS6BOFgETFkp7hxdSJ%w)&N6sd_TPz7d?7(huJa%u-5j-h=?mMk!9A4{9P z?jsRPrI2R$gVq#${FkUirQ1U}p2(~jTIYQFWgHMzUx!LpTB->Dir!7zJTH~4+Qq;Hk zh=@#wdcB1S*>pzZYvGg7e|ptsoakUn;8=83{qc`>t2MVZp0x*Mb? z`W=rIZzVF9L=&;SBvcdmf`z-3M7->XpbjH8>Ni$2yGMBbxlWYm^27P}k>wXdDH$h8 zc%5W-H0bAY$hXg(CvRdr%QjdMPKi1|i5t6ENh*C^e^^?_tJrf+iMJ(kRpV(uO3+$8 zh2c%Zk;vuH;hr?^S@jO4@Y3A_&nNHo)~j2dXHSqU%e0ITApHkFUnJ^Uds+UnR%}^} ztnTcyG6w=RmHZYwO=hp-Y+orRr*ABO;|6A#PRzPvDY$P_1a zDq}>}zDFO<;wfFOv9UV~Wyj4;S-;6kLAOM%3tx0)4mtSdM9P88RbuGS`*dk^@spbQ z$WTlq@6hI@_2sdekv9nsXrVC^#6VLNsEhtu5@uAfJp)(Dk1i}OZty9@SolFqN4C0E@>cY z>BuE5DuMjfB`xxwT+#-B+ZhdbnSBArvpe9527SyJpD^e<{_bej0sPP|fZN&|a9Cde z+{6KZE7%ZlVfz8jY!$$1edwL`09@LKuHW-e8y~=vtqpj$RiQ>02R6nnedf^heCW0I zcCqmWrpCCrl>lG0!l4ga9clsv;$FZydl$edtqj!z9Nk*Kx{^VE@b4ZSaWQav|C?qX zWY`6Ibe=4p3gRZ!9BgF8#d91|q`2iSPf?|1j>D@<#dmW5OTU7o6X~XH3c}rfN`RFk zM25BTH1}i0gEjsWOd_7g&!IT4qeolTbyb{jtWFg-sLifXNJc5#@4lIIs-be>jfm4E zNk@{w*=-B)ws6|;r8)+#OAZa`o=k%gB(EAb{8$)EnB*jzN|f_LqOUf$E<8A$CqmON zH(-mhnQb5p%E1~y5W*szBSpUkmst#c=wBhqGOyc7WlXXntu886PHm4dy#`I)TdRRN zPT5Z7=TN8X^~Y9xdZE)^9K)L*d*FhFPjN5ymf4)}PM+8j`RBVHSgce56*>j%@gq99 zDM3;m1M}Rj-flsZJf|W*3O}NG0*%PM{8*4beE>4kYk2Ef^2u@q9wA<1IPUF@MbDL* zM*NW*t#^h+iPC*v>n8g__?Z&r42`9-n@@6%1?YWv=F(J+6T*|p-^?Ho@P5*I?aupc zoFS4KHbjNO*A^+V!TP(!Yz{5&CS-1^TZbRKf4fUPKyR{%dm$gXqYKA>z;j&}!W<@` z{z2{-x6-!dy?C~TIlMBO(y?1pp3LNLE+ciONqVv1VUd-(9~yFVc+M<{uejWMWPR(M z0O?&D16e<}&Be!O8EGq5t#-?XJx+^^A-Uo4XP{#o!4w$GPyN0bb97|vQ=VVaHOBOS4Cwk zQ?((rbiEu`Te$Io_0nQmf3@dV8V#Ecr=e{niHBCl`D*j8S(j)}_mOSLguXmsx{wu@ zia+=q*(e^RW1v|Se5L7W?7WSU0nZJzXal0i`2h7u4GGMvsEPvqA z`8JKEyfplIfKac?BjS6Pj-8J3-40%P@%r>MvdJ-7PviPTdnX0^#|gJ>^Gn!R{L(#P z|0Mt6AmjgodH*wu1fn-E@mN1tB+yk53>W|cu>gO7=g;!mRjHWnT> z7B)6E6%c>*E5KkG!ww4>ga5wj-pAcc{9eids$uGcO&YL<_-I{Nz+Vv-b_^B{HZ~3$ z#ct-VycYW!2vfV*QLGq8d2i-xx!3u}{n-$bR=doL>Fxli5 zJ*&RYm1}~RW_c)^6ekMKRG*@GOBBl&mPvRJ%+sNMC5&*bvz3jnwy*s7F?sfvOAK5l zs?nZnT+WfCen|xu5uTDluM_Kyu~9U&BUJa@I|Fa>9}gYzh?Q>sP`N80Be!2E?>E}~ z!C?Oy?NHM3$x79D9XZPg`BQH`Yeu<`adiq@^ep3p++A$6NPN-y^}%qZ{~1Xm4*hYzNPWOlhLQS*5gC7p zeHHm1s_&0Tw10}y`%UbN{1p3QV*ewtkBI>TMLb3@e;2rifntXu8KVP$p=Y2%a450C z7_;9F!q40Izv(_8rh!1q9|s!-d}{F%3xJOKhw`KJ;{qV6codKKuUJ5e{W#nIL*+*Y zPM`Mco=_V*Umx^8N9Cy<#q0$hs!!mS9`5arKs-Pf12ieWBdtmmjN$_nJ3F)m zIS|NZ3@B)as+jH}^q81F6R58n3hm`>=kA3DA_z?ZMa;v+*ZV)m{h?h!mF+j`_~Y95 zTlmwDmp@~pem!~s-!`+e@dCmP0}m6eLqJ^)6IUlU03v%J`q0M-6OssoIYJK`qTGSd zL~m~=D3GzS1A|tfj=pXVHeSB2E;hb^qU!*l*BR;sg!X~huy^_GaR0Z<#y{Uze<#-; zF%Q2*0{*M(9u6L-pJ4?@-s__~D-gE~-o3!*=*|iQ?w3b*O-vo|(aZn#&U$q369L9J zdceCJcxT1jF#*2y>pc|ooP*`ygMk6K^I_TnlKLoC5qv*tcXXcx@2E%pK+%4b2g|{6 zKsdk@b=U>+0W2jq|H!So15u>PnV1K;;{;MiciB@FmV z4HI|y4|i6u-`_vX$N)5lHDDVIYyYz$xG_IL`m^>B!0$tV-!1_^VBp{A@*l36|Fg&v z<==(M5Bfxk|GWpo{?(p>5qdxA^E=*-aQUCoBkYI(A2R)TfW8=e#dn(eL07-f7`TU+ zuoa*!SP#<2uXJ@pM}N{4_>PGM{odzKy28K!j0wVZ1oJ=A6+qwrNLS#w{AKMJTgQzm)*9{YC~qZ@)jJ?hi5m*wdk?{WZq)SGDsO zD+s9B{4l41FTdMC4BQRffKYA}@YNM2?()Co}_?2^mkZa2P`I+^+<=wfQ0+d~@JTG36XQ-ngqx;|p8n@NwZ)GR9L4hP~SRXxq#%D;BIuH@We$-rYP`N`kT zw@zR8M7_t!9%;Q6`RKcPvG*QTSBHqGWE(2$jS|WXky?#2b{6R&8}h$x2P~|%Wtmb* zhbgn1(B<##6|Qipi)d^sd93NHv4&=Twevejx3@I`)hu{_(96 z4Z*|Hf6LP>Atn8#)M7*W+)|Nd6a)lZot z?%7FKAgqcitj}>*?x!@|&3-VwDtuj$bRd(+xZqyo*@pf!h5}RP&ZU;B^!U3PDfSEW z0S=PP-Ja}@6e&vFT*>$WwElq=_f9N7S5&(%#*jrXL7&zibl|9_S@_c8!>fK0QPHG^ zGg~>@7M3x4<+S;xDYH7hl$v8aW=Xfh?Dg{AjguJPq*&Z)m=&0)o9)TawXbGjcx_!v z&n-l)99lmR!F_Z4jm4l@#^&`{j+Q+H9w~)_wrMA%eUMgz?6f-Z<;hFt_l@V&7z%i^ zWs}A8;$mpUNgdc@-pX0XQFMkb;PJ>bVm%2crz;4~Ep(F8uzevAQCvgDd!dr~n-Q@> z((D|$1kE6c&J$U2ZmR{Iu2lAnN-LiZu8%?6J7(?is&$Le*rpRsCYjATh5y~ z^EX+RWy%&6HKL%yc&gT=>mvCjbJY?rA2Ah%5NoTdK5G?`R=FezeVUyW96Q>g=_r+d znI;_)cK?a-adc7GHBP>;cjBGJ7ChLzR_*Ex+)m7Ioz4+#)leT)iStbJab>h3BK*J7 zxTN#(MR147*c#5;mcOFm&=qf_ibMA?DYh5wPJt86LK;KSTnL^G#p6r zijgPRyKFX_aN?4&CSQE6L@z5$zPGb`{pQXM58KbDCR@GSDD9Eo21V!j7caghUlCqf z`H*MTN6HYjnS9sfcsyhAExZ#4*~i4{(nbz&Ws&{YYst>V*0xR_)K(rV$kyQBR7gB1 zKt-D)lIl<1CGiP$Sm^IHComB1;olcr^JU%J?ak4-b3fEw!feNRGW7WVS3&bGUiXi; zJk!69n6IIzg6OTWaK%1M{L|eO^V!(%90C)Q`QOZVJ_dbmlcH3OOFZ3NwUv%~U;UxJ zmhQA%#CmrJ+4EQ(341y-Per#BZDMswH3rdqA}QhCC_&xPc%mn5TT|{D{Svw_ z`bK0DJL>2o=k@(3J04)`i?wBNlbtlb^X zkIyfs5{@QvD2SD;(X*~fbHf&{wPZc=tGZw{O*ihle&710qHl4^0)WB%w6@R4a^rLcuM|lMzI3JB2ocaO(0hTBlCsF8iI`@>l4{@4qn3irAQN}B! z3iF$OGW9lU_3_e;t>f;s6;rPL&o8{x1|p+Y7!XW?I$>?bPz{2?gw?9~g>WMCzlZM}trEAh^8?I55*;y4IT{KT~)hVDRC9gH9ygN@~Vwh>( zwd?PkX};YNI>*de=wL6{PmGpA z$MwsFcxqa^mBn1&2|-|)!)e)OA&s(=`L%*t(fH-O^QOh>Zl2L;6X-9GH8;9ut4ORw z$rEiRmG?OoAG2S#t7IP3znX8&_>{aYVP9OU^R>8Yrhv=!=P`X)D4Wg`(a(=j6o%q4 zVm&jBzWI+Q`7bMzn8ZJS0R8KyP`?<-)Z&zqLbVb-Sv&FfUAgXX@%s69 z(ZVZg#HN<%D$H-obLsTTIs8ZGx-~)s%bq=_99E){nEy!k$@|*P`w!GY+(KMlzDS() zUi1#F+yAusWh>Y6Dz%laSTH(N_!OPg(>ej4ozLWIO2k8v%0#^62(@@A0<_}S^ zQm3RIqo%HXh%0)_YBv7dn96zSeR0{do|TKFek*5g3uYXfv(~1LLu433=_w7Ub~;6B zQZ^?~m}}1G?AhlK`mdc*JMp+Ka>abfhx5$a8H&8h%UVSFwidd%J3IJ1dJ`dY@9Ks| zpcLY@cU4PEs5(3?%GVZcBm>@8ypp%fE)eU}Oxb*?m}8rAp%xZg(@tSiuuDO0H$gx% z&|`G4=$g+)+p-jC?y1YPH+J>LXXat=k9z(UFZ>@sG$X=oxw?s?ykUEjT}zM>5uqmXV2q{^bfOV^5P zRD)Z3(G&qno*WEwx_4DR_gm=5?+|8Ve-L|A!I{uYcwXb}M8OW0hR!wPPgj=oO=H(f zE_=VU^i||^6ldM~YytJYbGq=&UD?&=@}g}0t0R=^#SW_M4fKuArbg!OO6R4f2HMPz zx$t>8)CgH*uM4fP2s|c7sFslJXd@_9n0tpmpWApKU-)!vY=Id|84et=sQ`!rOvhD znfQ;UOU!liH`xh%CHYTZHT|yC|90dO4V?_=92o8q+$O1XQsTWLbT+GwOeL~FSRyD;azkoixh zp*^7_dH?de6X!_h@Fkns<*yc)uQ_mOa-HS6r<*)<%I?^_^!&9GIB7bx*VIq)olMph z$WXaf;QEenFPzW$gTrljFv{NfVawguz1APy8z;VUaM7r~Q*B_-h+1LL7i_E&ck+Ba z7uTkla*bEh=gj7m*DdsA^mJ!UN@!J8WqhTi>(&Tw-_42VoX#0rXqNU96Qyknf*Y## zSu4B~^bbWIIv#V)lFnsX5r4%-s2uIH#nP+S_u5sc3o<& zpT3MMtIYo7jg@YfbOU2!gZm=rT#+hqba?!HSQgXrQ6r)Qn9D^7a{mLjHoj+agE<-M97oQX){La!8D9*cqIeXl`3CV zMyHe&R!XYn-5VeC6PdmvD-@@7S)0zmpiE5ENNhQyri}eGj}eFa=a06|TOz*3I@u@P z-$*jcXZtH#l-(N=FbKF)%u{BT^e|z2L8Roy+ZzsQH0HKkZCgX}7fDSo8)dQKwR}iI zp273f@xJ!{5-0Bmx&144)C}E)S7{Sp9Xri%4_Rk+Wt;MCY9ajT>sVcZbmCcyaxPHw*K~rlezeU4W@B)A-gi+ z(n@g!iK}G@61GY#QaN!qYph830xS}po##3F*;u+!>(#>gNnB4lLa{%|bS`7{Tx>8w zv|?48sv0fdd}-11(0~{BQK9GNa(Ga_Z5)HfR*kCql48FHcMJnlW|&Cv4Te*=8eXP} zWlhu;U8lrW558)A6&p8COQA?hG8Y`uH{N=EP3*$%`a41h5 zrMuGc*^gbi=bRJ++na@_FHR-XGSSSQvd?V}g+-lKB<-|*5z|Q7ynT^5*iHCmF@*4A z+Ku#1Dsl0g<{*-Lo}|Pfa#F<^#)HyOH~IcsH}0C<(N~@9a*csmljAUwrg`*I@=o>p z%5d2bbFlS)xm_wu{Y{u4t9XsSz6pxsdJRjvJ2_xnH|(_fOJKn|KiRmDWcE&rLlp zQ9bcqs;^V>N%9n})$(~M{}qpxNeN=UsvD`w*ksvf-^ZHqQc#?w*ot3SS#td-@luHT zS|9qM{u+J|i76FMX4p*lyhr*;lA+}^%_O-znfukV!Z8$GN#fLH_dVG67!BKLTuGMl zD0)|}lAmR#tSKsX_ev@9Xj|j>#}FZgLVc3bzqoRcdvG<{_duN&?n6$% zn3`j{E6HT4#m28SMLST$S8pv`uUZr7H{H0Ar62QX*tSYdzeF#t_-ggjXo&%vgJ=ChFUIY(X4AIn=at`L6x41b!tG;jIq=X&=Q zhlkE8=(0pGlQ`bwQXVAbKKN>%&U@`bsRNRCJIrv~?jS+N(1P!q_DW85ZQD2OWSCO; zYqRqON)iz)0nZGOq8TFhr=P0jT`Ab5JE?TT?M2BBoTM1qnrY`%r+AO4XmP4Ln)B>_!?K~vAh4EZGIo)r+u07UN5;J0$ z^6~g=3Wb1s(l=ing}sDvYf7$)J;=+Jv9*Nv-`st-2iJI3>kxQP5-!faYQ46uQq-H# z)*Y(e*kkbUBXoKod=*inNa^FR%-|qsb?#hB%hDa3T}ay{F_A%?wHUMVBr{UluEI~0 z&5HW(tVS4bzY|*H78$LH%~*F6NGhEB#82t1L-cvGH9Dx45G`g$C4hrX?PV;Bw10X z`|+Ha;QgUHpQkEoVe-hL0^B0&@Ju2#JlLF_4N5*Grh(xFz7X$#VdPsuDm)Q8wjM>x;8I2_a=E)IPCxGFWhnztcE{KU9X zir7uyE1JJ1?xSf413rtjqWkLGl%dIKK3!4O4aPt)0g3BkP{Er0Rt@;BlLDdh2w;8qQ1P4Mp1G;93#88MM!8u(6hI0CwOPP~y}xrm^V+ER*OYEJ z;~N43(f8KPIiclGlb20OLhkkMqN;5(>gDvT^d#8akN3qdw4tzL8elDdgXGt2s&1%oXic@Kq&Wbl!_)C#f^ z%Hq6PlZ89HEmn2GPo48lR$uTQxtXr*+}AiIuegPL+W{qrR!I8l9!n-N{#?y}`;Ex4 zxhsK5Uq7jt1{8~-SOx_3NFZUW6;k(@>kJZ4A+K=xsy-f8d|$$SaF@FfKQz=<;)T8h z4hlQ;Bpk_SDOqoBJb2ktX0daq`_hWTj@(wU zR33YSw8E?9MUlrT8MPI@;(s-GZa56~O1$cZC9~du0#&^NZ5OqAxf5@ND|U9JM=@oXz1=1gz23gneiE;vHl=SSo-jf? z+hv-BKR(aCnZmwFXk)^(;!Fn24xpcc6_#eK!aa^AofLPZK=# z(|ob;BxS02`t;x{3+F+NF<47Y&0VsEnW5ZIBd^KXw(hp!?hv!B%P#}#+) ziqqAe(QnrTMfS{mO^@^e>ilh_XjGhj zo_LpJl)n0K!A&(4f%hL9cI&oo6L9$POc|kYl)f;Wq>Vz;N4@P`myD8fzRTJAB?i^n zYgx(OdTJ`>RJ%pPXS`kIN{MhagsYuq@Ja8ofgy$`^DDD#tQKV%)}>!BR>plwjrXsI z8`;?CP{ONQt`nM0ox^#ScAt>)4lMa(1L8&; zIUgl@FkIw?S?{?Zrj)fi8*E!_E6WV@#JvR#xt;xLeP_B+h*{oPEIhMZUDO{Nd-` zjy~JXhh>5^l^0hOSGi8VFNrnT;2_P#|JvkUGMe&5Ly@sYr)JX5e51cPK$QPEl>183 z82jBGy7o3|;wr2IqE~DSn&n~hJ5PlyDIHZ8E-0X`o;-D{%+UAtZIap9`g#|7Tn5Ej zH>AC9+07TuIB0oY-?1UiB;`rihtj}|TCMw&>+_L>T*>`>50=l*+e;f>UqHgj+)jOc zR&>p={Ia=zR%4<1t(J%}e9CJq)MD3UNR!rH2}M?vYM*@1m9g5@wiu_Idht^UPb-ox zMQ|^(RHpRSDLcM|&rg`c#F7PruBaR8uMZ9x7aOlDrYrG`I2!O@Rw+$YL&4+Lwy4^D zOPqaP$=d|9);Rk`rZ?LDH*;JafE0Kl@kdIT;7~n)1rb=PF1#?U%h#r=pvu8OI;B= zL!F-5w`XK2yRjHlu@XSDZ$%BKv3l5g@*=KZT-4&*5X%XsS~=ZIjHH{PHlGr*4mSMH z#dhP6z(md7U9z9|b?HBiiheTup=YgGztUuj>|3TbMVYk`t=ajfd@s8x`r`2(duE3Y zt>HU+vh@`QiSLTi2HZ~F?3&k_In~MNY94qJciGSc%R7HL{AK5)VuLG!rus6~otAkL-cMD5E<6*gR@SpqLSFX^;JXQ>K1#^EkFJ#rz+ ziFm1IWmOTT9L8NbYgd%=IW?NLM3-=`DD7F|wB%`%h^6`Y`u4yJ8iEr7U6&`%&PT8J zNSxj*lD|Z0svkJlU}gv?J^EI~d;Sw>mJuWR_H`}q)m!aU+6_66j4#eT?8RU7FzrOu z-<(D^rodjn^RjejG}1B~YqN%jGcxr^(x01ZJgTiBPp^7->IF`!ZyLTrFGE&uZ!gp3 z^@#kh=cU^eOUrIt zOS19?F}M^pF@p4Eq}wCftIuvLZJv-cabn;>+jmaB>-=mtK+P$kr>hfX zB4EWiy$LDIe^c4)IrPr=BgB9XyOd#<&+6Isg~&cZyfmdEbG&iRg(Sv2HMcUXe>4Ob&&sj2= zUO!HMI6YkQ;X%*&Gr)hh8QCD>|JH4x7796ko5(0!MeDL(}UN zxz_TolBy2kNnAx-3;O*BQZF}7hiPp2sN#QI!IfT^P2&A_)A7{)tnMS)ci0i;UF{oP zZ;Ps=cg^Aw&P2+RHtW`x(kKL$R-WdgLJY;nFr5>0rF(gyC(+0m=4fz3zjZNVarDCV zk(Ys6djamusB`jYN+xkbhP|Raf2}<<3*L(p?u0 z_-rHG#eMQ&z=D8`x;pYz@)L<|c7 z75=q0`B)ku2RsHmY9(y3b}at8g!Ol*=q(1kPFLNz!B^{X@ddm77kO9y=5903gMiiK z{k`C)R{jU|U)C>3nn}y#yMI_&dHb+`;Jm9=Zs?_mgw|otd8CLlDk+3dv@gaKs zrqfF@6;brp&KbadoPTm=J#i~wTR*8${$y+`DJ+9$J8>;t!#?( ziPsaGVAXG;xWN(D5v0*Lts1TVD)K6CaS9<8gC{qs&k0|*$#a`@rg8S_7=2MQ=wlu4 z*!`T(zt5yvui99jUjLM*mN>1pZ}}J{#U#yx(&s$fC8Z}~Yae8Mq*Legf3_GOh2U5~ zX!}0U);>6`byL%MF4B4Qgxz&5!5Hri-w&NateeWv$0`uA1Md`FkcE(u1+obsbh3j`w3?{GEelk9=7p^)DA&CtFsNE91u zUqOJCmDl@0=x+3^J+Sbor0HgNprgDxq(nF3zy%crk{uTqy zIDLsSvOYGd%wAnXQ}F1V1^yMWcdO6fHuCc8WRgB5izb)saM_=FchmV^RTon9)Sofa z4-||Up`N~$^jf_8PxNVCz#wkjk?mM3#J@QDt+}yhUeOi5tE+F_KTPUjH@kZnhiwR_ zU-m-h{32?J=Ch?UFbnzgeZEOVtE){vE~-UJAV7~0;Ropvv0)*bqRFyY>p3(zi@*Ir zRws067VasL@388O;qOW&osok*(oU*}15n{bfP+!~8%FDSh$x(zJPGJi{q9-bT{_C8 zhoJ;%=Ev*E`aF{9J)I^6X(=}pDeMf5pk~0PIlANe=Wf-`$WaGJa2A!+26b`-k@jp4 zSSmV#082gW(SXW&02RzINTmp8Yau*JAb%w|@bxh|`E+YPtl5+xJ({uuVkH3~T7nyw zH{@k+0YKL@V$-yzV1|rG(Q^edc&y{{4d(A{SdyOO%6D&WI)d<8m(3jCdRdwv$9tAA zGpIwQZc))ST=MyEoYfz_t+x#$QdlUkIxSkFNj2%kCwcwNpT9P|`pJz8gdHtErkCfn zT!K2ha|5#NQxvDiGz52`W=jvY0wIkO<8qIn8p9bs{A(=Pq@=#Sb9Ppznv3n3V9cFMY}Zsq*ema_pDFG3X?D#@ z2kHW+)%Uj$nyeZ>IPo$-NF1QH98e7aZL2w{#k^Z$I)3TSy*5QaIsuGdvBspqhjDw} zX9@#%eNPJLnfBQ3qu^CqGVO zAuHj5C|YG+4JU%8YT$esROnw3giy4UjT!DLrkFq3;dhwJf|!QsIKisoOdFWta1Y_w zQ`sY9N+=SW$*+>!btF$hG2zAf%4FZL^nAi%5rp_+qPQZv#h_%8I5w=5xgu8l26vk? zB9!+!=fmw2X<(eOdh(ju7|L@7G&8cki4b|vYbRwCqTKnFBrDBYif#OeE9@K8n|zHv z(Ur0)v5bkAWd+FIo2DMl@3El{v)rE+B7P>g!vTW5$#4R}#+hFt-{{!u!ntMvGp0kv zj7L2k+_r`3g^IF2A)>kuW;Xu3@4r>FV;D%*`G7kd!oRq1;Ka<}Z9PdmHj~$fJnmyh z;G8jTS>R_XhK&ZtT1lNW03_sgprVV7rRPrTz1ZwKKRsTb8v}ycJP z9ynpf?WDnF>mv0zCrfz*oiiW#qg2>VM4p$fAQ(fL?2?-*dVB@>Gqb%-LA9kO<~@QB zfa46yd z3ND&L4(J9(H?$BFdnCa0V1qQw5Cb$dGHv@)do#TJngIuJPl2bSeRi2&@PhTmAQx*~6mkcWB3W^`P)!Z&UE$Ev?>!xL z--7ucH9f7ogsSi$?smSjEVSY$Zv9#)>!At#E@X}{_e$;~xF`>wWRSbX>2>CMj45}W z{xG~N1jsE(nQwsl7MYI(&DMmr0I=cofF%-8WZae%!c5=_QQ0e{{d1jdSLSieM&&vv zZXrs0B^>sSf(K$HF3?6&0^MF%6$zw*go?pqT^G066KgPFnXnbPVm8N+MZ{MvJ?>@k zmr-#cyvbeWHo!8Z(v%(s7r!Diye=I4KDg5Lq?RdtM(8F|K72SEbah}j16dcT$@y4S z=n_~UXR%I&TnZ9V=&;wp?)?W^M4chh0b_L%O%(&?ncRZEx8|+^@Uh<tS4{%KJ z?C#k5xdG-3Ynysxg^{;Zj*uQ*KV4R5=GZcJ_j`vBM8R5#ES;~&b2qx5HX%^)IG91J zg&BaX)WWhTEx`>3Lwg@jwCM!=C7G^Wu0e-@lCVKWmoKE=~rc zKI_ZSrVpE$_k`1dOU>i!9RB@*HR0*`eSeh|%Cl#fzi|t^##^qrH3{_9Riw|IQ!B1_ zNC2oi8T9Zm%j1U7qMX_M1PZp03?;^}bma2x?!xUlc$~%l-PBUMTMbb@1#gJY(8kY= zv6Z;N{+_G86T^U&t0Fr{S9M%sUuI_pZy5esex{P{vS8RB-v)ID&`ci{?wg56%mJ1M zNtBEZVxo7lPg4@x#KfF6{U}ky#1I>iRzQWA?j6KBr5jqppn`n(GpLd{cOeis&O7gSkOaWvcr_>&!k z^0mX47@|_yzzO5n>1Tx2$lpCvn$VUg{9Ztg@tkj zz44p3KgTWaJ=kgQ`?1)*RO^Wg-l^Ccbmg##rBh*rp6|XVMtmCfDI$U_AQKtL?WZhs zT+rf#i9X%xIvT$$cC3(-be}}4^FnlznX?UYw>X16X6GBAexOCFj}BWAu|dC-r)3d? zwIgP;?l1E>A*ydE^H)HZYq)R?pGkqX;26VCUBxyeVZb|nz=!Mm(WE2AB^QumCTKd5 zfwbtKEKDLfHT>=vNRivaqegF9^a&A%A#G){#26Y2z-WFH>ZN|n0%w3HE|ut0FSZ zYDb)}s4qBS)y*A{_(tf(rDP_N<({~Qjb5)lMa4fFWr~oUcBHgYOILH{uXT5S(RTyY zA-|){KD|Mxt|A(HX-!=huleXnsYc>%GfQ!nGh@iYjU;2CGZCumv+Ah^sV-E;|ujf8Ocy>d3-VI z4Kv~ovC@HR8jL*zrj{X&|7vo80mE`yi$SR#H!X8OZM1*!|# zNNW8>2>aZx|kcH~vRq_9IO)>HP?% z$*|V8pO~6gbhIUr$y^~6mq=dFshP83LONsp~H#lga)G} z3jF|$R3g}9b&=c(5vMZO^G~p?Gv8*mCit(qf4_EnU2-1rVapkfuPiUO4i629jf}iR zk5frWc=DRQSuGZH7&^}?Q_wyeGl=tOzKm+pW$%(X78(_K#K5I=TBCr+|tzO!XFd#>@U zT{P2Q7+%z$%ecy7k;?{{W*-8}64PIAdL!iiO1)O~#{Nutu6I_WwGYrsqv#~hLhDBJ zY1bCPoS;cV&Xc|yZg zdS`k@pl1iu1Bz>${+iPyLeV}l6fpg?2XvUORa%e6 zPe+Kt&lJOam=?<+G>ZhaYot@~Z@RRZKqwkU0Q>0ZPpt z2hE^NqMPLo-`Yq*%;7=n;tp!(41N_Wk6FX+gBmYVMc8w`Hn(tR+?my#D4{$PB#5vQ z8%TyBSU8UX3Yy$s2C4HdY-7n|Z@3#0MY?X>sJ}nun^H@e`+R6?bsOc$cj?t%^6m6L zbo(P)u~`^mh}5R3BtwJEk=L&5Zeiu8E|+HAi2J?0_4;xp zJJ}FD-=Y#xB6%UWBWcoVM|%7!XjnzPeO-_CRVS8(?)z+h>f~I ziG!g=xdoO(4p|X89z33r(Hx9x78>uqg8pR3^zhGz?5?fvMm$ao{7#W?bgYrS%neH9 zlNOilPSp?R;VGp433BCZ@~B@-)+8PGyIPAn7XJvfIYrve<<}p}ifLO8sujY~NK|s# z>+Wwd7x9ZBh0w}MPDkY96ooi9Yv&>1_)bY(D?l=ySA@PxRD4gy)bZ-!X*u}lsF@4XGYcwSFLvOiC6a2zzlY2&I%0L6DQH{i&y`ZXW|=k-MM^KAresL0i;Q;5bJwV;FLSXXmt5@-H&zonFyk!Sg( z+u35#r8AS76wQyRvQqtAqx$_}_wz}|=ELGc?D-+sY6urqv zdMWut3CnFxJ<*XFso12-ILc@i2xD9^=;WP@PMQHSGp8Ph5+=dR4SWy-luHsf`?A31 z(5k(GU<8nfIsOARaO3;qV(8?*U>Fm$sTvU&OON(ZThHOMwMifE=Z9-Afl`P7fal@_ zu}#`5T_x!XsGQc^>}G)+NArA(5-lj^NU68dnMI^s>_QxIY*YHuB980}kFJ$<(tja~ znxoB>t8v|Md8!Uxc`;WuyvRT{9uh^xKN{BSc8nfhcwm)>c{$#jOr7+4YYy=!ro-d| z5!+M}U?YD)Zd0BY8r2oYIb>(#mT_j z#)nY7qUH<|I&6BqFYZTnq1}yMk$Uk8zb1~CXsSt zx+R0c4(qs0vCiF0Lgwa#VWG!qfWbE?0oGBq!nfD_>vKQ|Gv7Kxc{y7Y>!18SG z>0in)!oj5BrNBPJiroR3<}-{#l4E-d#am^L$EY*`q><$8#l?1%Lz(Fisi7(#^!Q2q zB&prOEZ(gWbiNdyeW5?!^A?j}e+nR8?$_}v!9w>$1*l6tvF|NlDG#9r1=pJGh=-F} zOJDZW0f7oaT#2775Glk*2+=?YMi3z6iQO1d#Da)e7oyV@^Uw%cc|aZzYEUUaU?AV8 z1b_A*aB^>@KmYP}5}DOhvmu}J-$iE zYGBgZchD2v0Y6tY_x_yj`UVD0=OUSlL(8w5Z3DH2B`IiPu+a)-gBI*Ytn`2?}`fily1P7W4XfIw=;34^@6G4w{oXGSVkqOqZ;N`GKh9v9f6B|y7bt27d9{Ps zMGc!Hl!;7}(g?9+qv}DAy9~&-Qa;|ys|*-Xxn*Ll_nTLj)v7?kXRwd9%WY1ib3$^- zD%_8RvS?{@az2R4pU-`FT%={?(;VQ49B(o2&I|NjVgMC8Iva0nLR!9QPe9TQKXxNw zJ4~z86-6a=OBcHOAw{Xy<=J35JFn23eKWDY)#+(=X-uqDOXp*j+2LfY)dSW4c zc2a+`aZID`a5}KpE5wyd>afu2C@l~el0jqo%8n8Z$Cgm=ObmcI3$m5r8#q&q#!Q>* zrv4C&6N5k`0HMOjPUax?25(=s3qQm#7t9D6Tyvu8wWcUOoU0N>s%BCZNHm*%96G}; zjC$L4L1pajU?M3OF0QNW(`-n}EWId0&oq<+ay_SI5GD)&fXV$yIv zg7!4L&na#w!cPzSB*gJ1mYpd;0JsxG%D+!j{>|C=XPUys_;*ay|3j8S6W?Wv!iX?( z1;mG*Crp5gw2MfAunQS?41pvxozF52LIfItL$tqWb+u|m|FM3*+VPA;svr`;l!U_K zjM51X3(4|?z}#%&Van4p^PPvQg9$5kWNu-ByXC4{ujAg>*Y17z*2LBE{_S9ADt7Bp zzQ?}?y<6AY=?5Jx@b!1Odg?;)LgsScMkc-%M;6}ncb+c3fuKAhdc?Jzw$U>j0*ZK@ zv|vFxJEvHp7dt!a zf<8UHJ)MRjM}Yi6Rno<1PrE3%^3&=ZX!_^^RZW*juD;|r+!h-n=x-%Vp~xR)op8%sNyoL zS%qzu+KH!eoLAfLIF8dauOQ3Po9G?jzFP|Y5|Bq#QA=?~Gznu;8nD88bAdjWDJY66 znAvTfXWz7EL*!_pj8#wuv=Agaixz}lSrip;@`q-IPt^GD5mjFvr~hIp{pjJUP{fiqeDOV%)*h6t=?bqc=o-8_T}sGwuk20I32czaY?RtO*J@B55cyy zR2o-8sn^$7TdndrV9$4$S0rgPcPYhSfFmHaCxxJA6%WbYiSarKR!8*QI70+yUheCX zeYLj_j1DOxDyUW|#99#=VQx=h99F~(o}SYvZmQ5VsgOAXQh$nKDK2SORDsqQOV+4k ztei@b*leHaxgzIZ^ZUdA7ZD&(I8{?D8;Vf(o?O}mVr$p)Oc{k~Xv;VfB$MDO55r`8 zN%J0UPlP;{q;Q$m6K_LaTPR&$IMDU0lH)o}qFnAr4a8c2Acol$UK>aB6dfiqsMZgJ zZ!3^BrfBE@G<@LV&e1lz$8G*GlU!y(z4V5F-srG zi>()g*LVa?WUb@4;iWQnkjeBX)hGebBhOUvfLvmUdA=%b)|{3tNw8RadoK?NPPCI3 z|KnBvuPNKmj-gIIb@}mLF8WFl1BlO0Oz!x=#KfL2z`Q<~uiM{8-oF8|{F{&QC++HM z!^@wR&Oh$)e*iT8Vd?y$UokVYevv=F#C2bkmp|wJ%hLH5eUkP6lckgOzqnUlvc$j8 zzy2RAoqq*3{bAew-&t}00R{O>|KNXd?GXR)ipu}Rs`M|g-7D9t@r(Df4d^fU8N|p( z%Rj+(P=EMGmD#@ROMj5xE~WEG_s|kUoPA~IcJ<4f5R~R zPud&9f2}tg2lHRn`-_+LKUi7+i!Aon<^Gdz^dBvx%wL$r|GzC-&VRI!s-t~v&{5y; zi`Iu~(y9=tm5nP#R8&(mM${CQ5CP2(#<*_<#w=uP^~Vt8WroDSB;`^xnlH-(VFcO* z$F^@E_8IYv;3GqJs2zs>T<{(6=3`z*)pY|}0E>Q%2CqY1T8sa$M$P$YC{L^eG; zJ+oHd);(&7U2AcTGaqs3%`NiTsd>e>cc-@%aSDevxAFThBb8W%p;#FPI4yIG0huL+ zTgr_h;~eJjR0iY%1LN1$W*Fv3HG}yH6;sOJ=w!L&Mp~+Ss<~w`8?1ta=*&E#swwXNl>##L~g4&c5rGkD>sPFKu&a0eiLq>|H$@zn zum z*B!%khi-o)^mt+p8S^|;9#eE!Y>X0Cz8=3j5-LF7>AXY3O5YER2i=t)cf7Cv6g^U_ zViri~g3(V<>Dk)Tvx+$|XiLsu+)pJl&?uu54923(jo1uAK?%ZztZnaPX+n8SYXaS+x56^~}nC8vdP_ zXn@>OcRyv)>(Mse`4$mkM_cEBG#`gg0ZH?l`CwcrLchm2(zhTt#?)g19p9mW1-f9u zD=K0wsAcKWidfx+Pe<;t?L-Vqn26VABW@druJK;eXK3C^ZfHSx10+j)!f?r{Nj=$d zZ>rSW?Rwm{9{{}FxdGVIa}|S+hDj)IK31+>9}7b@R8x|yO`%p-PzXW&tXq_kXl4gt z!J_Byi3s3^h!b?A^hy?1uFv zsXD>A1Q7LC>vN4%_zqlO70C&=!LEW?NGait>9U4M?1rJ0in#>uVAR}@by<`->`S|j zyimW6ByH=BQ1}h%BIQtDjs<#!^~5LfNz6-gR%UK27b|@B^~&yNsnxlDadMT9yN#L5 z4PbJ^xk7YWsCZ|42)-S=&FA(H>hiLCxKvG`;8#xS3Z$(me{(e+|M~E zoBneE7Kl&Y)1tO(>x4A4xY@~MGBr3AF`FXHJ@byL&|O1dHOI#sLZ@|_V27z-%_fKL zYNzjVkFwXG60czzk=LF#C&&WXK#aGN+P@Gr0lcp_xl9YUa3sv#K0rT>e;!z_CA5C# zrnP8o;}QfFwDz*MBx;*t5>!40yN|-IZjgyN^bi&=ux6hH6 zP6NhP-!EtfaCG89f3b+8qo0L)v0$XRUV795eE z(7Ei;q9C(L>eleAj`+bIw)u?e3IQCe>!=7G83B&eky)h0O<|){#mS(_EBk`zMtGp3 zMz!Fp$~G9hy*UBM>$1^^Q5*A^G!LBqsr=T1jx>)W(Z|7hxUGOc^E-jr{8A;vBb%=& zgL&HPhL2NfEi;0~2IXPfK%$oXFMwtB(PNTp4<1F&Lp02HcaVWH62gL9hRV2FRe;un zja@W9(7mW!{pZpG_~y1`O)^gR?!Op1_TrgRjfgklq#M>XA>K-J-MCfRW;RfX3?uy^ zdLH~+YPEpt_PKb%!ZT0{#|)jfjyWKS>b3d}59h<00o1G3P>+s$@-C(%PSU%X34^R# z^QN=l8F1W9Sf)UbcGqu8{ZZrA!)7Pq86XO-6JRLmk3|zjpM9VVsyn58`;>LwlF-ksfPR-#kRzZZxkCGOJ93}N zOyZOK=a3(gC*33fO1Pm z5bNykr14~m3QY&m&w$Wt8bjhJmt7H&E`U5kCd=T&@9keLkNlMU<^feYNWp6D=Nsp_L#A&%wCE5e!K;5G1jOQ|*tBeeBMleXKX*-xg!kg3k>U6{zlpc{R zcx8k++Fn5dYgQu#kcb^niyxJ!`k>z|$VD7#av08|cZK1+Vety~<+q14mIHo#wQb;? za(alRwL1?lx-VwF1dr0nP0)X^-RU?tP?rm&79Ab3B%XshO-XdLb$9DNrLd~0VOBhS zAq$KKac>+N#uh@0ZfuUYK%G>~jqH;B|3sp<7`a^K|`edVyp7b9av0sO$jSK*j$;`2}Z+Ov&N%%zDP}T=h$uSLK6z z`q6y+BbNv{FOrbN&%OB*BeNe;sqLD?nxhNFj^nY~S-Qrlr8v-Kl} z#e#WB9Tcd9r&>;&?c;Yq8`RE`lIe%UcN4i@=~t=5>v$vOLh>Mpg!yIB0ZBag%qzLx zlJqzEnZ2yCI);vpE$a>p29Jt+9Cz1qdbalB)x!i!k|1*iWA$~Ik3q6~932bZHDfFU z+wQNIEUZxxwNk?B#WJT0+Y}YlY6VB#=O5fal@?jr8h6^XolS2mBkUod841p|lNjHf zC(8v+G|AEgG8$6z6QAxd+NTz@!bL5`tQnpxV_C z6=t+T&tB|dX>dpcpaU}@Ui6V(1QOOz_v(UuH8HM9k`7_w48VZ(E;CHoVdDE05T?1o zEo|guRmncpM4jdw6vr1c)-1U_^tHJ2>?O0nHe9AR0J0yEFGG5Sj6N%sF~L(c-iA*` zNTlc|@+aRn)4J59wm9}FlbGjJfYtH3Qm5V-0WAAho&Gj;X8AWU>i?2b{h6lxM@sc4 zI{J&I`Hz(9zeC|g{==gCkNo*d==(QRc#eNj;aR_Q#Q&tie<5N2no)87PZ`x8cxI6= zcJv?8i!W)$7Zcj{ix+K5ukb~a_W1H%n12zX&AyZwvR_o`KM305|Kvxjf03n?zu3{L zUqEJwFG94~Kd{XjU#My6e`2;>zVOq6|A09E1;s5&|6d$vnZGANn|s)so7nyXH|<96 z`NgJ|{32BU!GqTNC#Bl(i{tEMV)F&{w6^;XqWu?^^uJ~P9Dl3I`#+VCmBwQZSP(93 zhsVHJfy&L8-vFhb0E09bh28*!u<8u@Lb}8E2E&~aGee^jIl_P2Z$3VR3H~OP@Pg3|%S@fes3IUsL1SkKRpcmc;NN*z8Sr;_k(<}9uWn>RooVl3o=3} zoG(BV#sMyHN_cKd-RmDun5E0cl2-SOuCTw7+mtw!$W|#!pv+0+%vNTwQp$Wviyoo? zV_a0`ZWNlYPf4y^ZLfK`?E#tsSYqBVMT|P{g{-y1mqd8V$8XT)=9r4cr}~9_s{j>xXiO0F}(( zL0m3rve|=j8$mq&VuVmMTh}z!hCwtzGOvD@8(vrTJVB-)6BuL+!6*W-7GUhm!Wg^V z0m+p4LAd8}0l*nE5o)iy*<%N-!D|-WyysHr0_e>FO5g6R$5On&syQETG*_B+u7ZQZ|qrojs=(MWkCp;7)atY#4*zH)5Yqt39w3oJdzap zg?b4XT;Mn-CV`*|Sv+X2*1l;KGJ8qqBg&dL9&JQ7u)iuZfQ{23XUV4?F9Kmd}*FpzY2k1Pt`DCdL*9LU!)j ze@tTgQjN27vg*~2~@F#JdkMGa&>(`(d`{B6K8FD1rafN6%%)7U3yV>XL02} zzUkMI#g+d!^gk#6`{e%^_>U6ZSF!m2rtff8myKQHKG2 zN(3|Yl27Cgl;FIwMywy;+N?`FsMmC5n&Y@8Tp zx08G$xwF$N_0(Nv(%rx7BhzemU*78SK(80YMH|!Nj_$D-{yr{RcD3pKma1pnuUqe4 zc60PB=ksxU{rdj;y~pdx`_tt4nw*kG#Gs8X+o6)4&djB{@vOhN@)G{u;&j-x?dOu5 zOJ?2G2h?~n?If47EcTnDY9+L)fuQ<=`u)lK66vCCl`3{@$&b@oB6=0pXrJT9R`nm2 zp;KzC@7)*nWZ#Y~lM2&M-&pPnJeuC72KdJcll90=4A9}P#_Dx@EUeJ-4M5W8;G^-c zxkG;;ym+=zSyWVsZEbr=8Ro7zP+L`HD7$A!1fQPPT^JNKI~Sc|#S4+Y4ZGnPO0lYV zLzM8m1;QNXrce(!4p)tQ)rKknGNWdkjg(z7b+!A}bgCRk@@sNaAv;-C<89Wc^wSBW zG`>9Qc9{#+J^_n5&(Bs{ps`jVx3pwJZOWnfTQFhD`Q8v8G}RYxE%^PVi7Ew4@C$pF zB4)n>%EaT!F4_sv+_Rm!To zFZr-W+D7Do>hH@9PmXYPWBCo2?7ETCCl@@i7xClLiSzcW_`|;N3o3Tb9z{@fB%6vQ zKO8zz0)ssxr{uc4UKM{*s(!5Hp8^?LChxb~Yq&N5kRYCJd5;#0QUqJX&;{bEtf;>W zOGS#}`utX@tde3I}WcNEhOg7|?$x z6zn|KBYsr@dB1Gbtj3{ro6}GXOs4SD?Bh`26A=PSonoC3YehM@=xDbim@!FF>J;5m z9+$o$lA&8rvNft?dk$mqR(&~hq)l+VUkgGu{>^c|0_R$a+s=E_ z)P;M3RoHB|VVCnzTwKOak2J5x3Z_wR6lHg$Y+ce@XoNR~K4N{6+1|n;m4G!+FD@Yb zg^-K>R*Ir?4>0ReAJXYwn~3M@aQow~D2xiSVQ&8bDH79D)NGCdg|ZX%(C%WUApeTU z;8c+iWei>yX)PjxRK}WC zkxoaeFs9dk4MkkEXJnYK93Nz4qEF{(2 zKrNA1=7dRWwuNJvk~v+>=%_>xhQAancWCQe7E+D>@B}klduW9cx~0|b?xbUnXUQp^ z_fWMeOk1x#gn0!3Y_P@;uBey*rDa0-1ewvIAp9*itSl@Y^HPb)nBzFTZiZkiI~sZW ze!)I34PDIjL&4%U@~h4Cr5A~nsh?GHVBJ{M^P(U6aPFoIn3$wlgqFodK<`-1*G0-4 zA&&2dMB1bC@5<*<5PxI)dX_^EqDX0B3tCP?0MNVIM=Z*`XV*yKr+gukf)D_7%DAdc z!d#dTfpD6ga5JV_FljF7q@i>bduy2#2NT2L^a+3?fTZ-{G6)(HIZwpIxB5eAoe(lm znjxkl1u9zg=p;g0W00Q7Ni9Nfg|`xcCgs!khcLAS?>JN;fNz2IOddwQ$;8L=YXCw~ zl(9J=p9Xm}=jlkZrM%nNN7{$Mprfelx#o0fcDYDnh>1fjBJwoF$XY`l9Oa-3jGC*Q1{yh{cB;h8Qb+d39kT>SR4wN+vpw{rwhE zkBIL8emFn7~-# z9{^D$nx89gjIDAj~au*r@<}V9Kn>&mBzS8izTQW$pq}p$*%lA&D~4fJPn05oftq zb(iSYn8zenBcuT7c$k5h72Oc5^oxUhY}JM8pmh@%|4oP}`TfckRgCqTQ0Au(y;ZMF zn$QKj(JO~MdNXNc)ri)p_XDyZs96Bz3Qf?`zJ26LZ&7qB?NotCS66R6v&@gLf8N^| z1hk7V#$rdBj})~dI>j~#Q=Zqq6JysexjYFs+ZY^LM=mpS_XhzmLflS|8KFVnsW7Wl zNPoYwpeq=JF^^;@)7Up*>k&+g(v7oT9hW4^1!vK1B>l9d0_8--Zg6I{Lj~XrByeRq z#B+?r7(K&oBV7WAY@9>zG-~9IK8!8QmNO{Zh81%9<=_kc96xCdk|5&^agAENxnl}e zQCbWL?s3bi?3f8Cv)sfIUKA>>4}um{dKs#C`mHQl7H~a&`i3inii1!$ikS87r;7LG|t?@3ER9`Ts(<#!~w$KZ7#qrKtn(A@eh zOR83p!YyKx)rdoAxM*9$U1gl2X*!qSw`weYCF38CXjokOl%hza0-cj2VmZI5c^hJXRZp;H}~_k(`=Z($C)aa*d6tl@E{mm>L4kF~pT- zer7^er79XBo&jDHy!qa*v(-bxebcyoP+gMhic#Xh6oCZ;}ng= z!0<%Q-+AmiRfCjB6eypi7U_5}QqT(CfMvlF5#H$uxf1GH6cN7*R+oPw;V{U|6r(B7 zQ77u{JL)?JRtrxdf{s_Jdo(nh?`d5=EXVI7<9jPj2j)6W;4>v1ee8}oLa|tU$iCW; zMFC;cw$c(5h?)YrS0fsJl{#Vly|jv6gX>;xS-`(Sqb07RR{67sx2Mgsd-gs<`m+l& z8P}Qu*GB*oB+R{>cegP z7Wx#RASqLD&fF+82-vwI%vp#Oq`pTaOR>`2gad;!Vp-cGp`2enLW+zd7nKD`nX2`B z52FP7zW9i+?g5;P`YM;yqcEXZ5L%>(r@8;#k0I>7nbd*|U|SxSj|AKHOvbu3;@eHh zAqzpWAs`n&!>aa5c)s%|PNOQ)M?6apj^`>sl`VuZNcX;iIB_CIGkC2K zhpt4w01N|;2Y$Z*$!uh&lR|PIQ8znF>I!X*S79GIJtXcB43DnVAS^Rj-aYZ#vVplW zO=dAnm8K^FLKL1Rr+iMN&6FHuYj0Z)(*(S}7}HVB(yDn2tIwM7Ev|uqkVCWMYf!Ux zp`W)~r;uI7QLjaBlTn0zlrd zwbM>l6y>*AyKeI;y==H8S&oppb;B%9IE9*Q1U#NLWlW?mK6TlVP^D!kM`jGy-b(k) z;XX!3i2jOaLsvB125`W|y>%j0#faP^#R&}9%*)8dITbo<H@ zi1ueSdtEmoMm~$P)GS&G@MODErnLADfIm`VWy(-nKp1w{cxv{G(<-oUfC`4evSJ!K zcRQVXRTZ?ujYar4+STevaK86Hf>>x9vW%gbqBXzsosA4hRv~1lK5ezf;i^cu{hUf1 zDH8b66b^^bPPYCQpae-s+HHrgSy+fgVMus1!j9z;0}J@Xt4#H}3RV7M?diGanEkER z(mTy$Ionb4kswy)5bf8u$opt^SX0%g`Dd31!&9Aw1XjjKHA;1y;U}dF7R8? zep(bxGc4RL4nhQ`;)~zAl~#ehQ$dD{n+nwE6~wYSMCIFioPW&hoa-6v8h|-ad0|gZ zWdaonl1B<}8`9&gq^EZg+^R{rgZOk>0tPUCqP!ff0j>Gw3rwj%+f^n!H3pq1K`T$_ zbVM`Z++Fypm>%B*67N-HaL^ba?7@6Pp@6R{Vl0_?$!%z?QJ86W~t!@wBg5%2q^ zsTfC`Lw2yqifTM7VCT1CSb3dQ!@xlUx0t0`{h{>|qJK^dUNr>KGs}OI7zLPOTz-1m zrrn?qj`=jn7ZCly3m`_9w?G|3{P`W3x+eAx`o7;jWeyKx$qfkTHA+q)1OTJ!EcwaQ zN0+!?Yl%-os;@pHXHb6ue+7WleO(R>5I z7p;t>f%11H0K8P%CX~El5z+)qbAMu#S=!^{+ZE`b0aN!Yfv($6WwX4tpFD9HZ_U$! zlaL&q#znR^=jBS+O^vu!0<=ieIJm&EzdTijX}5RVcJH=r+qP}nwr$(pyKURvZQHiK ze&55HIrE)~nT)82tjdbas?0x6taV-M_TmZmrL3mKea(-?tJh>T7&;!IpzP$JQ0Aj) zA4M1+XhNmArIzzyh%AFs>rmL-ME1U3zmzT@m?sRY(535q%djYN$QnQ{*p{GuX9j)L ziPCOjSObRD`?w2l?QLFIHG$W6e>%C5o=r@R%u^}bxl%$TQIhJ0LIa>=*+HL#uS8uf zKu#jBY?@zy{fZM6nc+_1>7+i0E<{6sNdMK$9C8fs#Hx{T&znrkRo(Ix@E!Zq#uF|i zrrM=jX@=zmDR#=!iHw{>XhaeMO3^n(zWgnhEl)C&(r)|N|E+KuYHO!)#u5@Z1cYKfbfDA3dq+BRan;l~o<7R2e#t2qX_f zm1$gpLO0OI`^D}Kj;Hgqji4fqYy{^YljieL6nGMgtO*RfPt|_tL{VC9SYp2i2gR6V z*(S`WOcEW<2A3@CTP%4)v8okAp#TXdr}bwDYk|y%w0Gu&TiIW8aqhaUgF%fUXNJPv zouD0re#~~lG45R@|KOf{=-Vw>fz4Hc)setVw4!x_J#EH2an`1J;R;W^WG3_PUn&9X zr9%nXaKu@;1^$z0U?C>A8_NXF@~I6$F%zKkgwWIl*GyX9x3_y;-EN-lkB9*g86^K< zlB0q4tQjyNxAen??-j3daE_pqnkg)O;tY~-gLL;NW!)ywEerb$wYXQf3(m#T_A$?- zUoVbT?y14JHR&+`$AU{LcPJJ68-}uB!}PP>mam-i=zpZsdwY$15{M`>wR4v*?IM|L z%E=}`pm^b!J^ZBcIb@sxS@`4+y>oSc*{KJNXzfTuQK@d;1fUK{Kt+!Tl98cktDsF9=mv3(1Z=~YwG z&H0Xo-BH51D7>|C-V7X;y%sW4GMUHZfpCmXiC`Ws#LRz+d_a0JT+k?h%fB9IQiUu; zlfFf`}xQQKje zd_u)8Eckh|8Hv+?QnF3ZEX1L;z9l-zV9KL{Ivi$B1v-W$!h8ur6rZCoPjZ&@-r$+I zi`taz0mx~$*V~yNM=73O0--DEr2`8o8$rQRFSyBJbJJ`O;i)nBPn=kZu)d*qjRm+7{R zJi=|<7AlMPO1>$Z3=eE9kqYnY*+jEXW%F1x2xeLA1ODaM#aFTgEE5Bkv}a`+XjDaf zA2%0QhK~sL0Fhw?_(prbaeU1sR$Tl&Cjw-;X}Fp6BGTf(<8H*^Lg*Gk-HAjTf)>iS zUwcVjN!Y`D#jGFVRHZF6)a;x*HaY{^DiE0^7wa#M}MfjM9sMaAdhmBY~b-j%4IIoAE2 zqwf&Q_N{Lmp$T^8%}kDhxSvX=F@~zJdB~|EMr-1+rKy9nk}Hz?r>EORsy}gde^IKe z?h#~IYWfXAhP{EvYDJQT#(&pSCS(V90g6M12N99rmm_=Z5$vn3KK){m9kaKcf zpFf<6R&N*3a}PPWZcO(j##@L+Z4~U&uKQ$(eliTe!Rm$ zZV8-MyHVtoEGER`LuiWWz{Z`SwIQ71wawJ(50SK1U!j!_JX`>$vG&l{*vN>pV7mQ$ z*?fh>B-=va__IFX545Ifzf|0$TmbR=dwhbYGJwUz`H}itMhw8=-0{WGg8+wkt5G0_ zkmPficWCkRl`H2{_;-M0s8=ZtpB;GX^K5@>UZ|NPk}A#H)0c>}Jl@ksWvvMYZ0k*d5r95@7}0uvR1@an7_C# zgo`)uT zF?L+2=~)`->r}&^eFa-|noMDb;#2r&$zuW!?1;4KGuFh01jWX_teJ7IXu!UU4J*Ee z_q_Fbh2weD1#kL%CHByU;M+l;mUf4D1O+Ji22v9iA=LI9k=+yky$(Q+Pqu z{d+flA`Xx&f=M_@HGj1dlr{)vM1Fs4w|NK7>T#b&Fv)7$(W80O5Ae%YwdD?{#;GV^ zEIkS-xV}l^1C51MF5Iz~t%P>5XAQ?x|A$ips zzA>y1?K)zz|0(;U17hgsvkMppTnwSH4aw&S6 zNk!g5EfDED%eWYQqTiIVc`t+fScd(lJk-lYx!6&5t`$Y-D0giM+Vy9V2M);m4IA09 z=Lr}h^~`(2{WGpt$!r+@IYD00aeoNhB^}iz)`fh(K<_aQ?T?)Q7Q<}}xMFhVax)={ z-dk%${iQRkei6^8nS_FAcv1@1V6Pe_$~FO^!oi+)IqVuN;GwPj;Q^imUduy7vcGw-n5%}DC zF;pX4bQUM{{ccXJb?QJ)X~C)$H#T?z^m(XI$0vKq$^kB{iO%En}8P2-tr@hr*D%I>7 zF+Mx5rGg~Lp)9|dc$IgFn#x{87Gv3+-Zvl(NhfEOIhz$2JDQ;qJtrvl=iUnIw%XN1 ze361Y=P_o~#gLavcys7J3aMPXe{s) zoV^u}s(r(uEczvO_Y7g}spUBlgr55W!;#2_L9lPc>m9+$QZpwD`DaqZi;Rj(z0lLv z%F-X4R60d)+z=B&)ELdXxSBvqol`z+XNE$4A4>t3#!Tf7G+8*>1uBl54G|(Ih7IB# zq~0g$5a({6=pBKs1T$Tx@_RQnkc@EljZGevWZ(<*>i*tiXL(YxBMR~R07E+(z^5H; zyhW7@gCBpE47aCQ?xXSB;pIC9L6&LCsdLTlVzJBderE`*p0aVE!{+o3V4KeVe|^xQndx zLkryrfL>G1VhA1b2}41jwuZArkFz+2{LW$#&Wn5LzCTI9s%Uh1VCR+SDHM!;P_uF* zmhu6-W6a<}+wr1$)L8;C)^*6W=BD&0QJ~aV1l*y7>$+yb{ma?i- zyBKBK!7L!CdA3-DOhik0sM<}rK80SxMXtpjv@1uTR){`7c%Je6)&z!))9$LJqY7zY z?<*OU9|eYu|CDuumE{rf{Ft)lhwJg41S1EUYCG2S5K0lTlRf_>1-voGUjZA6C7_RJ zjjLb>V&qdjV!+5rmEYojRfhtv*l%u8%xJ|FV#p1!smLPGWU}xGr>_(@TQ(7-kRt1{ zhc&^~po{O$K_Pa$c|c%s%m9_dLXzHr@H2Mye*bmKG_hYy++9{y?$e}NX{dkZe$ogx z8M*+chg_8**~n7kON73)^I1h@PI#~^2L z5rO!UWDGI`+sCnPwkk#hI4^V>w&9}>1K%E!v~XX=y+69yu!FPEFyzbZLE?&9QdkT< zV5Vt=k>#JzqtMY28Mm(za@Xh-YS!?2rp6^?C1Yl48Fy;qoUOf(dkTStL2V^N8h(F zPor85rgQtsN0oB4WnCkoPn?rNM-T^$yS{20%^9gy0q3{R2DM$xyd>8}ec3!*xdF8pfNWFZA#0V@ zTwiKg&Hxi}i>U*%&9e%SxpK(%gohwBS7ud&{HFF@kj>E@7qt8lPSPO09vO9^^Vg5H zPJud3cB4Vy3FS)!N8DT9H$Y*PGh#$x^<>zRPCn5w3Bm*B-E(ciArdcK^Ad~a`<@F3 zK-zq6{a@iO{}b#YZ1a;bZEj;qD`akBV*JB|{4gAv^t6HuO#dlO`X4~W|G+`eGygDs z|C=z$e>2k%f6O#tIXTLI7-vj^Ov0>8|4`G=Ua+B2UB49Gu3l3>m*7_5*MHnJO2mJN zXdtjZSN%`l1^@sI1pq|PN(FZT|}=pW^fR_M|`RE^hJ)|T7wj57Hy_>6<~U#eyQp>F^GgFpWYl>D$X|I`Pm zf0DPU*_nU16#9QKCjT=6`9J>Yf7Qx<;=TV129ovP$)A5vK>x--{zSq5!*BZspF_>S z^uJ{wf9}?Q63YHT$o$JQ`gi8%Uu?MlhWYsyn&$tF`T6HL{sWu&4=?lofcg2q27UgY z>7M_^w`68w`0wbRs(;WuqX=E^qE=4oc_>jvPo%4D?$#GJ_4bHHV59n^zhV{>(!`2V zil}6L#C&Yel$lML>SO%N2>of7EBTmsN{26iFPWFOC`CeJiyt&+nhCY3oqep$q+B-ecFE3T0O|mrE~x z(~8BIflaSniJq!O_D6{QRwj#u>@{W|CCTbx*up>1XXImhc`_ryxy_}YUh;G7k7<8h z>qpBCH)E$Px@UtqF3)|wrSK{YPL&s?zkfKPdc*uw;i`*^U8IY0*>lVzbmo#nuid~x zPCOU$ya})JvHCYLQ-kopq58qff$i^rb;?IT#tB9Y zXa@_qeVVSdLMYOss?4d4B%{8Njky}ctvV_c91~z(10AxvoJX~VQo|XSdCY2!y8tRj zi{vNIs4fl{QHOyQC+b{;(H*c~aYHEUe5S%1a(FTCME3l0RnJDA+#7eRF58Rmyp5e} z6^h$U%r=o>DzP1Q%FqwhM`@^kKDMo0hll3oti#Avgks90L8~{JHPaO1Uk^ky`zlV` zI-Rnc%cn^8GF&Exxj%BGJkBhAQlKtUt$k_*{cw}rvuNF^yMVyEA71e4qP1`_GMu;L zIGI-^OBXszUnj&gkxm1aYHV7<{xtb_J|qo0CeR77q&A>dD$u-OLABuN_0)VAnANHu%(lB(}i;4%ebzuVb0+DWqmu*ap+gR+cs7?dKg}Wt{BG|v{T182Rj6&O z6Rpk*M?0C9j`0T;;N#~R=L1s3dXH!ch_W*mA+#t=V)F;fdVCv_6P1xmj^gg|MTEVr zf9&c$NL#*&QK5T%Ig#cvxGEr5!XGZwR`6Nj9p8ig0m@B)Y1q|_F#D9X?7DMVe6D0bLmTqZgc1knuh-*otw7^J!pvSPz4uynB|AbmvQgRSCOqKpr}1ngq>g;X*j8f8r;Mx;DHa5I zNSG_3gYPH+qH;xU$mHWTwgL~+wVUG}cBhS72WD&n69k4R&M^AoQ zKLbwD$>B_Y9IL@|e@aNFRsiTB@h^zn-Iu72tHw=#NUk$lo1xHz*LC>%@ydDTgu>@^ z#8+gPe*I|%JC3Ts^{A1B+`9y^G|eOAMUv80EBzFW1zDD7!d^CsT~iHVqj^;&f$nJq zNWjTF6VJZIY3avNjXxYnnZyQnjet7Jq8?W;-(vBNdFt)^M6tioMmLv)2H7a2f|vWB z-)+$EG%Hd|<<3UPe)+B>stzq5XmYRw*kKVF9b^GuW%S3{y3UBEK_Li@7rnxH)&M|{ zXis5Pgd;uX~rTB0cWS+>KZBuSi=lH0|$MtMI{tJOLzS`ul|3~$M?EuvKbuD@aFR!If5(bN*S-Qtb0W2mwrnOEfi>b!}4J|=Xe#0^Q#cidhuiUD& zt+R62qKGMWDMoLdZe%7nESRTB0Z%9j%Cvod5S-;kYeo)raGX3bA<#giznmOAc86vO zu}i)gv8L#8rKTIl7bd2jH6Y&q=;^&ja1#88Cdp?+J>UimR#+Byf^DSUDHl;2pa8O^ zkb?d=EA6$>ShhOpZ`dGgWs4SZ0XIMqE2er^>1W_x*cf3~sD`}aMgn)tja*hrU@YXl z%SJy|f9Ns4`ubuitO9_b<*p;J2A_zB^od|S;joP#*NYZSBMe=sGgA*+?nA2P;#44& zxGbj6Oh$d+#m{=$yg=rTxR|52Myv9OfFbpH|8x(%Pg!)XBP6llM>}CA%5G`D=0coC#EJTbdxByK{?fW!J{7`T7YsRYM~Qs+tOK z!D5-Er|(XIJ_~9j`hag*bHc-95Q=yBE?~sFF%cx<A(DW!$sAEQ+3f`)3In+V9A$`mz=7c;a_2>Ibq3iy4F zxcfll?YtvYXjEjSxm0D|xV8_qauhRe#JPjR6&9f!SL<+_cL-nKMgj_Vx9J=z14)L$|C_YP5`30=aWU?f`ggp? z6|Hb@CGXil(t}ZSxtyAulzbkc(PLXYvM>rG%AG~FO4lMmw2(kw4VUVXVJ;DW+Z&B> zAQLM|Mmc*vL8DW^qJNc8q&7Hrt+ z@eHbV6Yd0oX%sk6K3AP$PO8Rkk%9FeIvHZw4HX`m=H>#ulV&^b+2m|?%XWN-!>rGo zjMgs0GQ|@ZbV$b1D$=%Gxt4>@gvY-zHCgBw8jdK=piaTH8Wk&A!M7cJI8ft}pEQP1 zf*(e}dKK`CC|I@8N6NnXGV`xlN^@e)Y)hv6&=ij%f+}Wn#rb3MQgAWliV8v!U8WE! zd%a}Mxk{4;f>)FdjTmBHQ4`n{14MzRXxsV8?l@$X2OJmtj}ud*x*AoyutcZO^bDUs z>d#VbeEmW|jM}^31B%P=zJ~xNJ$%jHQ183>8lK90+J4!MDY`irNVCh_rs2CBRzEbM zV)KezJ!RUaP=BeJczsFY8dXNnmqWs9HAa(tdZkm_UBp(`Z3mhmifyoU&PmobY(EsqAwd>eMHrL2p@oOJO1 zE*5Q-D4>9+h%C@-)RHT zOUjZ&5MW+^iJ9{xfj&|~lSTnaNX<0ynnAqOWRcLKh0JI7Y1{0~);^}=Y04GFz2_ok znlp`;RkT&b3u+2DXSczHiPkfX(j(AdhJ#C!9(g?BMtdqVkN99Stt2nt!GX6)ja)&$ zzx>qeOYOr7cETgx-dsNvWaKq$H8qIf!h)dA%k=PpC5NLkEhR?*^~C2j=gjU0mC;dUAgE<>TyKRNhZhK98`I|V! z)T;qV2wsUBV)F9D*nYmK1bQEJIpH=t(4{@|5r}!j9B$6}d3HZ?u@RAxLi{Xb>UfWE zdi**BCr;^JY#(-VD27LFt7*(@Nzu{HU~esB9&e09-(U5x)iKESG#1VEy1qaz>tioC z$(wg?a=&k7IoQzyZ0&$Mk1Y)_0~u)s;@u9TNJ)UYTJ5zDLD=j2LIef~ne2Ql@Qz4^ps|p7Sa<#sg?J+Ny~78sbuwKkT)82?-_@?y zzE1OVM&Rt!c4rx2k&kA#wc;vYzwTx|NM4{+dttwVSYv)mD=Q@!h%Xbl0o4k+@BA*U zWUnqrKAA49ETdYCVhyIsB|^vzB^Wm4LBouM30EtX!+XpSbal)0kWLn)yfL&2b_jr6 zXyIj1`yDh>1;dBK#hx)&2H_~O^zv6~O=Gofi{nb$W<~s3;yOtbhBF0bNEohz%O|)F zUN+-zY@J=9>TA(wX=&?j*AVTuuelDd_4FdA&06Q~>k&*T*heG#{em+mITh!^##P&q zvth*?L*ic?g0extw1bW%)cai4L_lCQ4+z$*f$~i4 zpFlYtCG8#pnk6O*Bkd+bSSMWimjv_W+A+ysVwcnp+Lz*zK2p!J4&yFDDBUfZ6Jv-8 zpb#frCh<{R*7|Mz?+P88X0DF*-=1a_#G#;4NA)#6CuiJI#rqJlO;GBe)kmbMm-Atv zhFMW=F{2*XcQ0`x&YMeP=|0+}*1l`EV~c{TEI=$Dd@DW5Q~4<~%FU}2=LtPHPSRJi zS6YlqE$x+LQeah8D{xBE8#A#4=XjbPLI|eoQx;1%0VDEr%b-|b%4{f=n}Vd#ZHi2i znDXr?$=6aj^2A_iAbJ`*3MwaeM{D!)fU31FlpaOc8(OJjV+3jk*%P?zD#NqhJua6j zTdAuS_m9;jbiPAi5|C|p^~O!4$FHjCn>DQ3-%cpBkrL7)3;c=u;*pqwdqf-pS9RoX z{6kqAa&9#l-q1PY6&)tvPBXUo!wnr8QEJ!<%(C0(ZiqFrE_dMXbndFGQ~p;Vs5B+G z)RhAjXXmN&e&aLM_gNtw4d!G}Ck}xnEN$VyBen}EmYK*JjZQUg z)?69WD-&aHY~~@jfY&2m7v)R2FqweUhXi7eG`Y1pf$7s~c$LpAL*W+IR8w*6*%T*b zRwSoBTDB?YjpkGilDvlcmTyK>k&@(8i4g=#H$-4BcFU^KZvN*j+PdF42F^+ib+Eb- zQhbBbSE3jnQvCcYgdBBwUnEu6s1KCGB6$pSZ>;E$lIwUGl8j4Lx*G&OcG_oc~Lq`p4h=htm1qRI2}_ zoc;e+ss52hf3EZ&rRsma|F^RFf9t3I*R^W)A1C!+2haT<_^JP?Y^^41U}0?N^bhmZ z`o};0r|$ZH?W+Fs9{vwa8WuXH|E`=}q$=&WKa9|IF52q$lc5ZLAI@*1VeNi7=%T<< z>F~z^fIn<70|XraS=E;(*=JKQe9>VdbWz1TG?2fLSEtA4Uf;+0**oiW_2aZ+a$K*U zs?^u~)EU)T-?U_u(bh0+szucmt}u6LpK8Ww_T`-Go7&XS+xhvFm0L4f_0>vee`fd0 z^|AKo?eDR>N`?8RlUin#+*=m36*YD0l9K)1g7L&qt)Ft&%i$DORjD*W6H^`sRQ)coVStC+S?*;AD`NrRxA%PREs6$M&0eYc|2T89CmRFRZ>>h(qRav9X2|Tb!x(4(7>~ z!o1aW6r&)ju(8~rp+W4j6V5%w^Pmn+OHL@qlfr40%^fhAG)rO9ZW=JrRKT3n_NJMwj}EUh${Z~4bNG*oA-D)9#* z!FGa}J?|dxf#g|o)_rD%n+#fC?!agKCTZo;cH)jpa5Ez0uLOz&!}wu8Z{%S?oz1{} z`T&fkk{q;Ur|v@F79<}{dW2qSl1%s5poC7Lq>nOcu3m4dNwcss3vU3hAEkMlacp~! zJsoK!2_Nr*iAm2Y&Jlajte0C({Z--c9yHlQN^D%fG1Krw@^lCZ_kHMPlvVfrLp1*a zIDB_bw=p&rZzvFhJAr0I=@&_Ae9gI_w8QjddBA(-)I&i283w?HrqKSAh^2(zJ5-EL zeiAgjGSG3|gdkAplujl0kx;X}rc7%4yEci)5(m4YNf=?MDtJMLF#8=6xgCIg9Bv=!3RMAASU?ZxTHi8rhxfi3%3bk3w=2Zp9r$dD2 zXeB9&#A|_x#C2+TXrPBm>F(5XSp>LSmQIh zuycP4Egjc9u%k!Lt?*H1@C2XS-)UC_FwIu{YIoL1wc_bnw?VK zo}T48o?v`C9`aWs&^=}x74Ed}Uqp%yHM(9lgSj3SxtTD$06RrbUM6!-nLbN@4RGM~ zm{jKZvO@h}Eo58iV9*Tv7_27HLF&9P;;E=q-=&Ah8CX&QlaI#9LIXY4To01-y02~i zSjQ%!r*yqNz*RA^hx2#tJTaj%d6x}Cfqu+%V`z{=C480rU1)qi-`oVOb8}`hqJ*ca zSbo?HYK_kp=ra2J zUda)ZJ*T~f+~V_mS};?|#9dlqnE?1;#{CRFV|E5hWM8Q$>m6AWV=zwmodX0{=uY1+ zxk1r!V6Z3=2p^JXuo#&T?5DW|Dd$u0da$CZ_rov-Hk^||!q`nAH>=pi;r*r?>PKap z0B+y`UZRjO?&OhIF zgR`vTC#Ez;BAJ6W>po-ByJ=-mjY}^AI@aIIr}rSjl=r!NtWR!rsm zGhq}SIgLAZl4TNEvX;F^2Sd&s8i6~se#vs3=fBdWkoG6tJLh@zLDz!2``X1ir#Y70 za|h93hgiV{lFc&aGan`z3i;e5Gj%gu4b+GZZhcB-lL1U6cCd{rdjW zC0RsRf_kF!Q}>_>DmdP~O8Ww2-Q3zK-BZV!Oi^sqf$c30iS>@h@ROLsBzKjEEUEYD z>~>+kal`bEatGp{qy*v1y0ci28TSE&MrVmck04&$z1&--5IA07ahrl@`%S)QiUk2z zeaFFWs3_!@@{6SoKQ(8Q7=g}0Q|+}DLo6ur-iYlBe1)~4;~-g23_sBC)3Vw+m$O(! zff`w?=<%IQp;^2U`@9xK0}r`x>G#2xyh|%y?`r#N875v+z7IXrTh6BU4N5}3T=omS z+08`;hkKM&)3afVMxLq~Y$RAFfq-#ML1D+w-O~7^Gd$dSFUu5eAk(2uaLGffPsxB)Qw;+Ol_r8)M5!;tp+*+8%5msdg*>uA zhZx!v!EJRfc$Kf0@1>#N(2@$TCsdRUg2hzWq-GuY5qY(L<1clNpdyXeZ|{PpFTl7g zG=k?C`ueH^gaARIS`^l`$t z3eZ$@ckr;v$@C{K^;fE_6n6-NwH@kYpN1o+J?DV=o%@D-2KsE|!2ZAym|}`)83OTM z#r)EJUZw*TzbUne;dE@y~dC1ai%<`Vehn|ULB-(&qR8haMWQS`KJqn7l; zJ~ML;ts`h4LYBw@!+etg{9ydI9hU%*>-(jE&Vb@fXEgso`Wha^7dFJA0C4wflne*A zBdByi-X$~|g0=@^rfqFf{1I6!$HK9Df9KnE`YaJ?0E87SH9ceGNn|?lkUVk<%vX&K zTC2RXKpUkS%BZgbyJF%Mwk?cdp!YJpKr@q5H%iFRQs@AF3d7dhH!JixRexAx9*5=> zn7GI+cFvUL{JJl1!)KFV(S#I2Bx9(9&!a3aju_1Q-N(WRw$KKGZ#g2V_x(Asxl=oL zPU*g^>g{u z&p-1#gFhL42X_KqwSSQ%^8Xb}5k_c8t`p}wSEzv|bS?Rl>x5f2K4u*oMgEr(?Bj-a zyppKR{vhJY8NzWaB2H}-BTQ$NICVYqP7cy@A4Z5Dv0XfXgirYt{i(n7cK2iJ8^R9W zxI+*M5I^qq9?o_&Vk0gs;6cvz+UZYW!{{F1?C3syNdljRWhAz2YlF@v$eI+Oldt6bdLFsh~=jvRWF zA(CzpTFsE+`OI&(eaNJI>Fa3k()-OnA-ySy72lZbq}2y_rpx;1V)iJA&^~!0xzIA9 zOyY3Lkc>I}bvY2cR^gN~dj1&~z4z4Sliyk9T00eK}Hi6bETss!PJqdO^ENWJ=T%QpJI?I zpXk7YT@j3Enu?rtq*z(p!c}*Vy)UcVeO7A!1u)cj*323@U!7@X0?ts5NgXQCj2xFy zm74J@xOo1=ZsJ{8g-zz@enj(CDLLk?&90q@DF&Sh@swXnlFXejb=$!K= z=Q+wpj!l!Vhe<8T7~E$jZ)!1%-1_ko4RhE>3AhqeGREi6}yr;l|xZfRy&p4h6bp8+M zM_k4>CP8T}k=PZ2WCF?>RLD{TT?J%v$kxCAVc23TTcrv}Sn+CGtyZaYU8zuf{AzDw?lmYYrm^XsORc0Dg_ZZYX3>Ih0kO@G z_eB&ZkVvC)nQALV#>_cIcfTyhr85~1_YX%-#s~w!%nRcnh|z{h?!e@|SSOX=c!b2G zrFr;~K94P00H6IEb(noPB=LmnXqILo=J%6*E8zl(ETwlb8AQJ%rcsERI!pk%qPcZ> zwHEaaO)7-q+}$qmp0Q~7&3~gSmq+XqDaYr1&XSddkB&`h3p*%Ks$QFzRtzIBTX`g- z3Q?y2PRhek#)$Qz&65NTJhic$$~an6ysLhtP_=kImtT0$q;I02s#K&|i$ec8D&y5M z8c|6`@AE9CT+H+`ws-s1{M}Xp@vO}wD zFfgy|L&AxrJ9YIJ{L}HB2-yJe(8h)-BsN!48DDp&I{Ue6 z%=DY)pVaeG924JsR~bPU{0D~^VmI+r%fn+UmuzC17(n(!5q>N5HC?|{=B$`1*~M3! z75T8*XX07R1*(oBi6+Qv#L6O!khSj53k0k1e%(xl^|oi%*0R*2 zVN^ryS-6nDN?A%rwrkq?<%#~Jo{|0mxg7;7m&v4#>uri57$q_jDFfhUGyr{LB4rsY z%sOGPJ_@A!8UAwka2Q+mS?!|{&AvmqYYnRhmHo&LQTt5^vAnPHTo zOF*eazDM#=DhOzm+O(kjiK5`H9j!tt%si`LEoCNGwiXet?^^BdY1!%wR(D0hg!#+9 zI%WYqhRzWy?W0Ak(;ldcI~4^x76X~+`_l%vnnPI;Zr7PNBW=xNTQ*h`9E+!tWKsPB zBYl2MQOBTM3mkG;2!=1ZE=GH4NCN{woxu^6%1P3RDWIUP-Alv|n}hIT?*SUoEcX40 zA^_&o{xJR>z6pW2zA8uIvfZZ%UOG>@!g%5`h{4*tlAB4YXZzWAlSWx=l)8i(&V`*8 zK3F|zL^=I=uS1nz9gr~+!Bt(Oj7e|nWQb-A=#g;Q{rCJ5J>wUVt7oeyT7J#vN6qDX zmm08&J5VAS(j%xBwVHB#%2;M=#&bP(vB%;5nC3;uZS_ViX3qoVKGdk*U4m3x51?Z! zqMwBopL%?&*keO?K%?k>*OV9EI78`IGe#cwIots|c>rZuQ_og+gDE)C zwOsa*mT1< zg(yLxGR_=FfAI#(TX$GH-FQL(o~fZoWB3@c%IY9RRKHL0r=AE=5OOKdrx2XT+;t>h zzP6k_q2l-WH;UIpTVYS+?;-3QVM@dvi8*-;qKH3!V<87du_=z%ub4h1XrP235U}mz zxqL&UQcv6;iDn06s7IUN8_53D2>gh*I8jJ;LdIz~$K2-8G{kedDw~SeQ=_D4R z9%2Z+zrMHIATsbmf1YH!fJ7B1<$sn_voF>E6#R3ZJ84V+F|cCuaPH6;dvRLwS20;j z0ndbzbiyb?tqh#?z82WalDE+bOhVB`-i?mR+gQw9Y1igk(G`QL1p-+A#rZ^PxDSY^ zZA52H((@$W6~(Rqx{eEIlF4?C$NqUc8Qkd%YZ%|9siUXc_azw});?TbaCfj1e)X~n zYE&U!+?g=Fgr2wKK^_1J=hEPzbL2r)JIEbzc5)F!g-Fot5=raK0NT;0a&-SqdrQgb zor^kL{m*>z$};z%ob;~EG#n5>K&4F*t+@x2jC%}w90>Mz`rBM{Trk1NDkMM z!*=Y^LeZpuH_bZyV<3PKj~7In5?FWfmOttp{R727tn#_LzJ8Q+sh0`EL!9g6F2OAT zH5F#Rqpk;bkvGvPd8sn{z->4je_=?b+n6javovL&rmojCN33uT&-^#c6RMsqrO?$3 z9`f*>)TQGt><-dQEFxdF!@fy<8C}6KhC!b$+PIr>)k&|61weXeMLK^w|4z%Ib&VHY_@i zY0bmha@X=5Ri6o);_yEa<`-}jrSwRd`1Sndz=lA;;Vs9cJX zb%@dVs6~YCwDi_)C4w>5*|5Vv7|o2w+JC7JNigf~eWHyZDyqK;Z{%UqD$z_!9qR>*Rl#N6E-c z$Mo-ET|$CNLdHKfhcT_Hxsj6@J~JZ=t=KbB|I&l`*T%%ZD<0z3KO;kVcpRM^jPuK-~Ajcg5!0gU{9a)bcC z*8ze6z(7DiKtaGjLBSxveqK=EU|`@-(2$T&kdV;u(Er@vp<&_R;b5VWkdTp)kg&0^ zu(0v|eFOaU>lX|(3_3hKIwlGN3g*8b_&?qG?gc;q2iyTd2L?m{073u+M)-*s2Q2vc zfxmvt6F|V9oquj1puj+2fB?V1At0du0f2yj|MStG4}d{H!Txy={>ME51_lBF0{aF0 z|1tO7QBAGe_E8Z;l->j>(xf8@(z|r&By>=EM>;`@NbkKj>Am-^K%|67?@dZV6OaVN zAi^)_+&lHP`xfFh8!lUAumR2n%rY0Wq(%CN{|n z2A4omzJz=nvU(ZD_gWL{kIBuxFuA@tW|rmGPAq6Jck2^)`Y@@m@d<^Fg?s%`I{B&E0+G6O%g@(fKaY56dV_$sKNYwZ=5R@yo^ zw|yq0sAuCHo?g~AG{5tWO;}0a795dL-afpri)I&5Hn8)F%&h1bS=>9v-1v1&ls9i; z?g8u8jT?@b`@y`j|CfkwYGNYBb_q=2%dh`J`hH@aQHD&*%=P#&6Tj>m@iqe;Mvhd)AjXlMWS)P-f2$Z#y;lHcf~?+yG`A}Np-_d#79CpJ zxBUv>(8+Ti_6#WguUxE_B@Gn^tG(Z2UXKh;GfkVF>`<+&?)%|!uwjZDGO+YD}c6t$`wFK^9o=DOq*ZRUt8#ZWyxo>XjA}ob|Jx0 zP#^<=s0roStN(z<73}YI1=z|*dHrZO4A{USQ|k)40|{} z5ilxZR#o~3AoAA5(G_4=7IF05a`_4{fX|)-#<|omb+j*{FI7+X0u#bb^@s*}kNA(b*r8M7}}0 zT`qWo|6>BMZXfCnxUx4~6MJN9O8Q}$84hH_LzAL{F zf8Mu1h;}Xg6<`DpKK9*mAz)g>to-nI@=Zk!1lHW-Up!EL#{=)9{B3rh+nFz}Pf;cf zYhNck%k<29wd?0POPwb@Q0FsImCa-ln;p2(KVVk&X9QG{pcqjPb&#Qza43f)_lmuy zy_YGdt-ZAt@5L}F3B;rdJbgo2$_t{u%8s5Ld^Y2|aPEWbhh@gpA|NrRL#ibE0lJ~H zi=uH#L=LOFkF4iyvq>EVlZm}9L90~#Ua!4W#z}lA!Pkz~d`y@rL07wmV=jDQz^+i&wYnLnk#uK;K=UK@>MEq2lA|1F__7||G!l2{At{W{!r1xUIZS?Lc7 z*lI$S{%COM-J{_JY5rWm|BBxKXSMV%Ciqlv*r<MVo#b_I?r01WDz}$e`Z(cX9&COjPXYgulo+yy*a5>*kAr z24u8|SQ_Z;wnTmTAUlDKynHsYz#Wn1rCYM|B~}efatbid&C%*UB^L7p%;!~XU4T$pbHyfctd_&IXHv+WNlGG`sj`LQ% z9*gC_^JLF|Ri-!pp<&fX!IAor99WN@a|OT@^X0WVi)LWj1BV+Zm9dr{xoLiJ89Gjt z>NR@`IxYK7;C02M_4zgh=B%{7PEeSycZ`;=(#tQaE5oMSd7179H2gbVQ0gT#NXh*1 z9$~1hpWEn5xod$B*r!X?Q%%Fe-#B}TDhO=`m-_c?m)Y{^D-|yczj#<__Cvbs z7U{Ua*Ad3I;rxb6x}U1d4=$wWB4YUT5O-=+JA5Zn4WZqw@U_wICjktD2pV~* zBV5Ae;@%GssEhoDt-6M<38iK5v4)!QJz?($LrBYWlVbbw-LK~%h2$;m(o9byX}LD| zGH&y)Zv_TNt3P@#DRO$mxYI!vGMBmD)gDn3?D^7m&&0|#xMxQ59Yf;M*6~M8(I>l& zWB0^p^jo5;Y(=W0y+kbgj0{r@UOrczk-ptupTFaZMFgvOne3mh?Ef4&mv#YpaV}{* z^=Y~)sH$KUjulQL0o6Dod-p!V2`YWDeVVt!lq8!0R4`?vM5WR2r> z!>QqkP(~9qd1H{$OkOdf61{Pr(_NFlH9tT_1)AXGJ{WL4ub(R5&<&$d6I0(zGyA}F z7v9|x3Z}v{CCEWngUH<~aj`>2PIQVsc-gm6B7DxrhA8CUIGx?)t8iI-lyF}Vc0_8>>%rd`SY9GRJcV3zwDANmNY zUInh18%=w(B5HHa9q7meI#xOLy8Fr#*%iA>H6wp=vtGAj1CzU+1&E04tXfWsbvmOx z0wjt{8DRYOuWa7F=iq~|H)`se2JZNKS27@bbGnWz_IdT~R^>X!Kj#?dX=Wm4Eo3Tz zS_#C12}@*_B8TNHDTOP0ybDHl>h|{%92~}#T-fO9sfgd!qc>{FF9t6*rcdeU!I6FH zP)va^Up$-RhG$~5P#{g{o#CgXr)U#pkn{|=yOyE~MM6hHLjsmnQwM9$a3dSCe@77h z&(-GNh|zzkV5-QO86|I-GBy&5UB$Asia*hj9!{TncL%sR*W-QBj2g?RIyj**c@-d( zB^oxK`wG1N8cN(Lf3*Mf+Nb^;93b>5oXGgl$jxH##_fEjj_;!)t!cenhrUVZAVgWM zFS?R5%BBUsZ_7|^TEc(!teDodTGeoM`iL#L#7avr#qhM(KNg0*W7e=*Gr6u$#q5OTCuxE zHRH89PB}H54a|wvUrVvJ#B20AUulIZ@KoU#tWybl$EmF%R6V;U;a0` z{`b8`5D^+vOC^TE*1ke9c4GK_pa_!S;%$!+#uzY8`ypGxo{QOkzhKn7$8ySd$)jS| z&1P=b)*LIlcEhE!TA?2PWm?;z0h^v0AB4L>6rbKWf%n$m%=Uac!^z#AL;o3QZ*af# zGjTA-^==*Ad~sLpK67eLludCQWro-}eY51KA1|G0UC&evXi3#nS)cUAUIg_df3?tD ziuA)0%5_mfaEtjkbb?-P783hfv3`v@`QD*n?)~ID(Bn{{>)C#0mCZ_xY>!7lnE+PbO)SAMnZ$1< zyK{!S6!t?;M~gxg`^%o=VegLzrMDLj2nAKlZrr2i8+dgEpjE;zMw@P{3tc)b+L=6` zg7t*KSCnc#_2zz(oTb0}@m96g<6#A6nTximHMZmpGXQm)u$zoV^i!Ky%fqi<$)8NB zGB3a#?T1vt)@!4Yg*XsVa-%Yy_N1- zOpNO-aT~?2nnF_~ADnHJRsG5_$(Nq0)gQ`|I&50RKs+v4OBa9*m1lqduCqtv*iuyfR*ntlQK;4Zw@t<)T}+WNc_` z2ywG#S9j(M3$>XS5qVT*`9edQvZCZq5c|(c)|{j|nb(##$~X?}73;Qmcpd~!fYY}U z$i7_xDilSxf|a%w?r5dan_$%IHV;3{5>agroO1Y>^}dDZ`j0gBe~vLGsTzh3;m1nJy)KI)!?4k3CdeKmPDQyG3 zyWip8)<24`82yl-t0bQxIrVsvu(VfOy`*w~SR!>9w5h@>!P{=7-_AoT6V$VccBsv1 z&v^QNgZRE=%w(P*SVbC1TV4XL*w{dDsf^C?Rkx;2K8kX-@h5y2G@PLKHbyxvY#@3d zI#l>9OeKzqAKfCRiIi8}B7iwVZMnr}yzW#N|8Q8cJy*-%ScXwl4;SVDbt;M#NVEu8 zcHi8SV!2F=7HRt+KX?fHd`?h>)iSF>?MwTMh5K^H8lXV8WyQPZeLFz5} zk$Tr-CCcH9y3KZCE}?ihQJF7o+6s9HeIgKj&GY;^DEZRutjes)2oKvJe$IP|KoVUl(Ow64P&K#*!d*PRsprnr=9RE+559E@4Ahu- z%)kC#P;Bez%(nM*K>seD|7D9i~yZ+9v1sMxGt231QzOBAuQ zKckY(8Hw8NUjb-_Jz6uGzOS(^t?Yx&2*|d?xEDD*ZDx9Hm(b>i_v~M*ZQ$vHPVM;>K5ZAf3|>cGmpcNPm$uqu;aWJ`v73gXkM%4|ju8_#| zAm5{`(`&@RO)-+Q_BQK^edlu|YWJSn5D*ybu(eX7N?!&#GA!Ehzz5waWdue;Af$r8-pStta=SOjMS4;R;1pRnN z2U)d$wQqsB1b$8K*6hv}+Jezuq??s*#btSfyGQ`wR{ zYiRs(ZGTszEM6qfp_7S44%}`KEeyw}qSIBg4ak$PezB$P=HiHFn`V-m#r(OUj$u!Z zQ&|C$DOC{?f#V?Dpi%SH;6NSA`&J`8R4l60_HLyC7__0cmAE*Ns?^T(Fs4_i+nO=5 zDvOrW3kNrz=vFlWd{O3de)z-w#ENuD7+F*HcXHwupR*`M$5f+_3Z~6dAiDQ^CS=Wi9 zZndpa;^&qAgPb}{fs-!bq0Rx0g;lzt~);B^{Q&R)0>HkhZL`!n?{7oa1ykV@4Fglucg9PSqirNL?25+X+%dT zo($?v^6yqVIYRX_ob(#&0K`kQWY6Vn#2c~^vSvm!zQ~|{hFhYa$J}UUfBI#=vCc0& z61ZSMW{+R?s(mI-a+)TF=UkXbjI&<1p7Ab;D!2(0H?sooQL>tYFl54-HTJb}6)d(( zn{$|kXw3>)W4Jv?Q@PB+N;9m+M4Mj8{htVppiBGrZ^ue(cdO8jzQcqapvZ@6G88us z-rS%{XcnPGTM4;%lyaz)-c!Q*GQ~1m+SD@X^bdPET`| zBP$efa1#P9vK)3+OD>0K`3WLDj}%b3DP_9`Q#>jBOO=&7V(t9WAzgFvU9NrxFTtFa zMO?creI@CYQ!c*&6G#!Rt^Sq4@>CJ>@Q+=a&(RU!9VXKO0NGyJdee{D1%3y10*+!Hi@Hr(gV+ zu?wk<^11i`_)X;YkA}~`G*NZV$Yd0Xm!jgTMh!MT5o;-_dh0gXf7M0Y^mWlO`BSm} z8xVfynQfS6tlHLhPxH44(6NKF)aMG@atXNiGF1$sR^_}$Uf_zRMkI#(c)Fg-S3O6zTa!H63x2(TKH44Z&MwZRj!b_62Y49aS z#7$mR+ab=eX81cfGL9Tf)g+IerAZWu*}fM9!A*2%j&(ct>-V*G&Rllfq@64IE%eph zW0*J|n(Y+p#U(&Yc+hdh{s9VG1iTW}H9I7>QKd#%^Us@dEWMwq!pyDgusO!Ffx1H? z>K9vhXgcUoiTxs$ft&4Y$w#)TLn>f!Ur%NycK-9Ga>@$7bLvZ1mCF(yt}DPK+1Lff ztveCC*h>5H%mm|#6>;RDE6KceoYBKP=jmM-&rDHQ5(Q#E8ehZodrx`6ZkO>j0f;K# zT7K^~Q}3P`X6YVaJ_pmQ?3E4ps1z^_f%l%vU_O^wuj?Or1;|dk0+dHV15QFPqkRHq zH9ziN0fd<`Q*#t)lDoIJFYO@4zXik{@c%EKy#K3sod5p$to>8Z=bwq^-&?SM2K`AQ;XmNK^?%0sKd$50 z@8JA*9@`&-^M7I)|9`}j4c?bRIrv*oifzKZxlXB9_*OSYjubkQ#!nAw3_7br>xNS@m)diJV9G+j%IqarnvjDUo7iG#Fupi)8Reo*_LZ$L#6rZ6^h$5jL|kH8J)2SiZu?57+O`k%0%}F#~?w z^XS51kmjJkTJA}=^5x(t`Qu?1p;*d9ON!5pk2xJErg%F(gA*cq(*J*`;0j%YB%iIn z$rR+W=yM$fQSD7`NM^}vg6FO;F=ow55LUxTL9|M3mENF-&UkWWoyylbIT1U) zd`#Lp1=0mG#m$S4RXkj+QbL5-gChcFjK?MiKFK`9$di9>VC^5Y0>;N3i9!)Cjr!}) zUgTIlsia9-R;aR-M~WKSI>O}u4Z~lH?{)~h7u!HY?& z`mp}U8EM7zPMz0JAo@R7vq}LKfSE5f%?db30$%}GsL4PVRhS_@Te$jni(f|$UkVWJ@GFaF#=t`YSQ1GyHevm5709xqg11Vi&Nfy5;C~6U` z!W(_y8@1i$)U{o8Y08tNkR5$#KYgl`q~0}SE=ENbmt=_BHww&nZ&@2gG{-$0-v1J@ z=iWG#%bL@LhMRIrMTl>Ou(~DpDe2vqcw`BhAUE~(LE%N*SMeaQ zhS?_sSWs5#>DCZ8xI|`7>ow_cN<+RPZ~%1Arx_Va99Y(9`SkFEKjTqr4Bs*z{bw#r z&o9}++Z@t3(#u(MVyum#`*^98!sLCxwXG6$mZHLkmvTROX5U0QiZ<9{JL`opb;Ev@ z)KgtNQdC}=*_1$xx6UI#|sT7G!DJf!+C2+ zE;nbyL<6~&>~1Q!)6tSwxYY0F=pI{NG&#&OaG{cHOY;Ht#ZgB7>8Efk5BuC8!sr5t zicX#?)Xni<7V{6MFvMyP3|VDhSaRKWT(mi`B7r z?NakAA|gLt+ra(u@D^eOK+KZ_&l5t&!E()0tk0LVmVJ7%6qo&2AAp!A`K4q9folOI zS(H-Oc72TWo<7j4bc1Q((HV@vS_MUXZ*d(k%sQQt-sG`3PeLZ%6B_RzV;?ECt3iXL zDTh=Fqsj*3Xce~@C*W(VsITK)o!de72x=uK&Z6LyRGs@4aKZ|YiW%J62guf@AXSu0K!LV1l$a1aqlVc zQzGL7GR-!Ah{>P&(Z3A6{-IOWWrp4ItoLQY&nP(A>z=YtADh`!h|fm@iAY4{s~Nty zUbDbSA#gFSjKmE4=x>ej&R;%9#{u(3dz61rNG8>c*ovgS&Ccw~O8W3IoV2#6$~#dI zyRE_yU5!qG)0itZOkxOcanDP8#jP!N5||ZCSctOzw1n>~Rbye>Q~F{dZQ}xOJAh|7 zO4S~AE--rh-kCH}{f95ZfusMnvMPeEf8o_61 z!iVP{;Vxv^iZtt-nKP0GF+xf!MjS3QNqL za0?uDU|DJw8<~1w?5OW!hL>&Q3kz0L$L62nO4j83m2G#;(H=!hU)-38(y0zjB#QQ_ z32XG6m3=u(URz=7eaUQCs(Va0iY&nKq1q#n4r*HX*mqt51Qb8Xt9q9rbT778%F%w?$3^=R`njeY(A!z6pxYT%>2Ix0&@SP@52Lui zUJ$+9LUat=*m^X<6kTZK$1byYW#Dc;h@i zRBD!d(3^JJGvq|Kbe&?||r6Oqo!lJJGN$RrTRaM4-9A)?&Uz#B*}N=M$Ilu&MtRVNZXRC8m|VH z2bcCp+-!Q~dxeYuA*T~MlR3>OL${aiv#6(nF3aHq#~GA9e1PMI4-oJ%-4^_dnCAKo zFUdQx&-_WvdtcE`NwR(>Hs7}=R`Rmh>mLVR7l~EUxHzjpznsD^hJ{%wRJGoTMPc4d z_ZpO^7QC{$v4 zIUb@omy|O!e-~*nmBbbv#ksGDl*J7zPsSj*Ih*`v2jA5wcrEnXMR)D;n%*B_rJK0|5{K^J@klHCTR@a$wue|4cDq2=m_4a;Ad(N!TGy%C8HhXcg z{UT=c)os`7iwT5NMa&T^@}j_1W(Xj#0a3+gkHfHKR}`q@;ydO9Y7lo|>7I|6fhweF z?%eNW_OXzTQmSO-5J*Z~PM>tHhkc0uE(fZH#%xyA%Xq@6Da6d_;J~3I(#nJ-kf!K4 z0lOze|M0K9Uy6Uz`>oJ2HdNqz*%jHS(&UnKEFj2F;{r4SD&rY?3735NUezPyY>xK4ZxkC>)!$7UGuju>Sd ziP&!S5Djmi^W@H{ARR9=9MQ%!Maa>)@h(9(9~KWMtd7QeDIf+ZjT2#^&0(5%&V!}X z=3<&yeD*4sQry17vO6uaz;K=Oe&W7*o8n>B%@NI%$?0HoYF5iw^#*z4cTC*Oqp`Q( zapY`7+&$JX0{Ay~%4XXdkLmXI$@P%8acVaMrtW%IB}wwEyiO@Ip3gbAY#WV^6cYCkv?==M^7IcX z>he(GJDa1cMghV_acKX974gH5v-V{?HyoT(A1OtJNr?5(8oPi9(At$dnbCP^D)5Ji z0gv_4A`jonL1)|_UC!7qk@-EqY(D!)8$ir( zjm*0iA-NS&lYz0*?HE5w*mOS0^cd}1+}E->xMZV@C@FWFx$|VY2lUw}+WGVZ|UO?9c@@vD4QH-R4_a9nY7Y*Y|ZR9+4 zYueYm7(z;{y6!8kZ1!eJXd{o{A7Y$x z;0oZ{>Q-@nWWVd+SQ=s(*~b&hF!>MyO_P%y*SSDux0h)k zHujo7@P5KvpEUX* z#f`>q=ZFJc0UZ@Qwk4FGJ>P6^F(iQnPxhsFMQT+f`*V&I1uZe&-%LOZ64)ptU)JbO z?3zd+uK;O0Z>NuZp{!x*EhnFpRMOvN%0FCx=`-Q;Y8@(kn+P$cpT+b{UVO`>6LZK( z;4G~Wxl;4C1=r;^S6VSW_XP8;jp9HLdgpYP1mcZSl_;i7Ny4bYj@D$~!cwrcn!U;s zFcpZv<#Eo+yJDLsnxA=xGOlSD4lS3;V6FQnYzMnhnS&~UW{I}L zdhfi`rAGF@j`1)1ZDcq-d&7|3b)Nnr4*yeRtt*FC9f}~qLyyPV9P_DnY#a~;H|b8_3JLFuB^CV#h{=zrVP?F?-3!}TpgFtM2HC|F3fUBp;Dh@(<-;$mUT z&{55lW!^4zQ)j|4XnFTy=UC=7Fgq{o`MZ|yy1s!=91Gm|5*}uFA7x$~qnk_6M;d8E zBBJ0V^c_>An@pD`THk-vVn;9D0q~!PEq+1r#@_L)ss2D4WFJ{L%J$PWa zTwSNzj-rYa8nWea+`-9Bu1bV=5^tI7Ey}F57`4mB<9N|x;a>N;!?F(}0T*m*H#X%W z<`e@{SGmk)Su(*jCY9G&c_@i`iPa$*K|7-vR(vM|qGFl5>J#C*t5E)Zaf5r++r2Y2 z0>WGlZ|j1WslMppGRFQuYY#fLC&Up(HB9bebF3bGrc3MvHcg%yYi)cUtB|8^UJ1UX z#Y_BH%qdcRV6-52kab$0UWlfE<(ZjOB?5ZT+=|joDV}xr6V?W`a(BG!dTQ2CEx}~o zDtiW(7b7n4bg_(jKm7jjuGp^p_mNc_6nS7_jJC!P>dHlBtM*3#S?4c5LNzFPLTCCO zQYOjtv)b`)WVb1hs<1asG<9-RMs?cTY;<(euc%er7*ge@zt2O0N2^ z2>joBdB)%SXZ5-n{?HBQwIwbh3;4(ZHWgd>S!-1c%QI*DQC^$-(+8dRUHfW7 zPODYR)~*n7Cz42V4#GQn?nfC!ypIa&@uBzY#&Dy%`QjG)12C`KU^^Lujy){Ne{1I- zWlPfM{$?tnY+!2WS%|TagVIBXkl5SR-xu$!f0`qxxGW*F7dS(dgn0QOK+RO^ZK0F` z@YYQcIjk4Tij3^ccWy1m$_}=u(9*IlIc3RiZ1Q!Vi=Pc;-t0|~WJ#UfSoht10e1gb zP2A)nM?EP)pX((4B>HRaD2%?F+4mL!Qli5Rvl`N)+~ z75Bz#dqsVg62&qEQI6+CJqDKj^xn1I&|K*_PRk8rl={gSGrCa)`5}{^2tTJ<<)QeC zE%wnJ%48EA7l;Cok~xt@{)~nw2^a}x9&`=+(%{|1!}l;-zQp4ZKQ95nMAQ(zuWiGR zyyj!q{U!E|xVxak2e7!l_4U&=e(Nj&8fWzh%k~L}kRQl~kk``}OcPm=@v36j3Y1uf zbEUz|A{lSvrdYc2)LJsX5;=P@DbEq!(1?}OxBu+Qcm1pV5Fi0C|Mb)IpHTb5tmTir zk36{K_eVRh-+ zt}z-HP64ss}_9 zzX4g?TH(-c=F3I%2^9x<*o5gqM@y)_e$ zD3L2)r*%ZBC2VtG7GXcvbX^Y%XrmWYfHgLI%*3!;R5vrF7g}Y~j(vHN8?;Q*{>=WZ_&o!RY62V}2tmV)zx#r)p zSMG#GnP|-{Y#r7dYg8G`BqE(-i(!bgwz*1!NjQA#0NYiq6M-Y;jlXP?Uo<0*Th%UkE9QhX?6iwwF6F$q zD6v3=hqfn^_+I&EULC7nBl+mNl<#kE*W=&ZTHn(gh1^4oCDkU$h~;(wxIG9Jq@ckq2LhjGBz1lRjPu4HA<9={n$$m?yPH z!%#K?4Y8GyTBc2HEPURf$D%wR*GBw6_E9&gm&#*@d1^L0L6HJ28Du?AQl5HjvLg-V zd?6k@v{M3|M||jh%&W?S1wnLa9UT4G3HDCe<{sud1&d_3fLJFMkmzRxqoMbYheT<| zppTw0+{``k4`fcc&~by44~ngC zsZztPPa2kUJ9vl-=bWq<^l9Im=~NWz#UZs8z zX@0U}FTtJx95z%UyB$_3OrT6Rikqtw?1dZYb^ei~3l7-4Pt!Q=xPReINICdg?KN|e zVRw7#l&H^PWntY|ViI5{IeV4C7(*Bn09#?7ew5$!`M@5{l`#yYh^k{RF$r=GJ z>ro1^+M~Fm2O#`F9ptv=THTGeBgj!IqODDP0mAfX`X{(^DP8a)a0+*p+9{jOI@76c z3|Kdcd)xe3@NqiI<&;X_tcJUM^rbAd=pa?0B3N+*!0D?O+iu)MtbC#IS32cS4&@)! zz7#~n;J|t}f6zeXnm>YZ*Q=n|^B`YF=5 zo1sD;!x>uQ;>DV5*5BE68wH=c=&Wu7r>dPDK09#}b975}EXaI_VT%xMGd9)C?X^Qxh}gFcsj9 zz2aJ6Tohv^sGEvn8+`BpCOXL zcr(1?6-8RtT;USfX^|rql=hX%K~Qi}wzbm3sesukW>@W5bN(fzkR{jmaTkjr@5(*N zn~Q=7-IP;lv#7^QD|UB=ogLY8fuPNH6J=rAto77;cu4Cs*+hzIIhpVOkG;1Ji)(52 zMoEwmJOl_X!QBRThv4oI++BxYA%O%B?(QDkAwbZ<2iJ`a7HkGWh5-2{+g?5AJ@=md z+&}K~ttS~~_3G)>U0q$(U0wYvkeNsCe6vrQ;ib3J5|*#WkOc6li!>@x@v;exC zo~^1V-)rPQNYESLJ!Yfd%J_&m{>j#jW;;>qKTFELN<%>NM9K01l}1N-X&vKJWr%-{ zzG=R`*LZEBK?d~7$=V?dd$#8%Wm_^*?E~d$JR7eo&YDd%`KPZ>#IBI zVYSB0oBM4MWf31vvGLRTp(x>0U+M=@T_Bh`OaOXx4PBie@*O||V2|EP_@!pV;j#L$0P{W2-`~8` z^GXr?&51|S>E<*e-#)Elp@R9HI(X|Fn4jg38->{Dud%Rr z_<4#GvDeo%l%I{6s!yGjZ5;Nd`(*~^+DB~dM$Jwts#Mwb1qmby#(iq#V92vg*(6lI zz1Z(%gVV|gZ>HDA)`TWd>*+X96eo7+YCfQwQG#RX3V_c}_{cC_q{pQAa@Lf@x(b{8 zcQr=(dssa5p6=&(!PJH)a}64yJ+X*0LaC<+#3>-nw=K1oe7JmDLp%nQ3}w!`_P6EG zwP0YDiGI`*Kh8yKaFIql{YU{%BN}QC&B)0jXPg#|q|mo3TS0$IakffamIE#XCefO2 zzM}w(mV=TdIyLfn(r|!Vi=HQGZ5;QU$|vbNn&(8e(j{plklt2DJ^4F) zaoi@wofmQKB9~t4qua=+k-4{n9Zp$uQ*(dB-?$Y@+uu!v`hwEtIELCyze5{{igEaKE#vM&= zy+FXruqAZlR*roNGDqRG?>$_v?f;hqt;u!>I@U=?}oqkme78@ErHgg zK~&Em1PnWuCzs+e<+BS9&e?2In2l0m6;{exqzV{eX%Nmmjv@%z8EA^PpYzm(#u$`k zcryM$SJy^P58<5P;VvhBn{26+KSjhVgdS&;XRqI3^EKYVt0c7J)llIwp+BnJH|g-z zcv7u&#D)stXiUF6W4dhfUZDwn;}q%I2&TdV8=Ox*G)~&F1n!v~%k*7Pvot?QmN}0S zT|abQkj`-~W*YA?LEQ<(>rDm(lrqt`qhjvtAdC=%o8ikBAFv;jQDN!E0VP1fsnJ&p zb3RWiCcK#aOWc9a{fz1vgPX`~+8O9av{Ndk$W!_S9TN7`Sbtw1|G9zun@87Qv|Ju+ zM(ll0N>Uhsm5nE!nm{iE=99D|6ci$t0GL6<-_@eltS$%nMeFsXcFoy>tq(fJZNGxX zQia*FdpbJUtG{03_~#NZybos4L(I*Jw||P!k{im`rS_?fMu#)$1-aX=+6fxPY}-G8 zSal!`a9h>vFvUBSHZ68)@_bru0mM4!Yx@7$NOjTwhnDX?d*-qV?9LP{ZT2 z)r?Fim{3=q_rB8`5;+mK5UXz9dOqh3cpmW<&m&n8pq)gX?{CfY-<;t6tEL?=7h~8W z=q9+#htxs1wldZpj*rt$yRKF|rkMNpcV#dMx(OBcV&Y4w@(*jTq_C~G~4 zTm9{7QpqH@`uk{wg81K5UZfY=Ua#P%sgtiVE zO`k>8yI{)B0VMe0R%+3vvwXO;@Zh7EBlTU`n#l{v<*S6R z%BiDb`wl$*H;%44Nw;o$5*Pa4)Gv#vwk4Do@5r=|ZUTY&C9K2SB~Ge0mKDDp?4Vc> zEfX(*s0tXwYo;!+;-0bXlq>nwUeca;JHBZ#Bu~L-%}R;ZppDbm>fxx4{w;pRi)MAe zL8e)WUFpVJfW99jb#(AOpQxN73Pyc?tr{1df$MNt_O>-zAKwU|#LRB``QoL+WxHMC zGQ+k?y$`toDPF)E)giT~Tm0)~;~wKLYMvH3Bpe3l*5;7MvltE4vn7%?q7qc}F&nlB zd3ub1x=9k8{-GEDPIdfM@&MX9@ts2Gc5v@RVSE>7bu&vH$mg~}FXk4s^uKadV8Xd3UxuKf0%m0r-K`X;?A zJFKHtFws4dNb)smj^7(faH*6NeJbB@XNK80=6|9x>7r#;miS2n&kuoj?(n-9`5!Fi z|3#S~LXLbx9;4dfyvRFPLIm09iJwQDz}1Dj?+kode8)kB%c7TMS^ehMb$+RhF5738 z@+$5nByJtbq#6gKE^Lp|b^19bh=Biuq{W?H@vJL6F(H24Law7BX)>R%LB}ulrXduSL#HR_Z^&NN(8m@#8u#n|IJAK zrs~q@1&;{cC}QJ+R>Z@dzpj|JwMHoL2!~DRQDe+UiXmdQ@j#Z-01b$UySgV=DY5Ns zUAZFu1fjEIeFXUS0)j}4N$8(OriqN3dK$XqUn##oeS%A5pJRf&@U}wmg92YN7BEWh z)8N07`j5)|HrW0e*VW=+XQ+DW?XnGFZM(zOYuEdl;)64+b0sy=?NpxHh}=hn7KeLo zY;fKwapT0HG2=UR(^SaBm8tycHgtP~sm91r8r+l(O#cKxz|(-a|ES{s#{K^#r7Dc5 zwGIw8$Q4O7#TZA|8`Ht)JC=L?#!AQ1(TkQ2#ZqKFwF--S`+c3^(G-kgq1em{EZPpC zyF)IYEcIdaWZy|<0}bVKnNBb%9;Qlb#chrBB{6WC$gg!}L0glWLf8hqh_a46Iy~rd zt2`yp^N}q+4gs;+Ym2nmiaos5d0Hv{zoCBD#_?!x4!)@>eJ_LnZ?!9Cy5IYiA5(aw z&aU;hw%Dqu=8|C3;FkiK-UxBF;Sq1xGEk zwkM=l>``DDMbKV_hg5MPSe&v{KyS`uUj?y+Q7hab5PXwH&cBNad0gzYsDBm}6(f%y zpBg||N5X|i1W*68J^oje`-^n@@3Zr+SDLp6uxujzT>EBd;u@TE7NRWMCE!GHxyHcX zZ~Z#$eaw{HdX?&h&ZZ))4x*O=QDsztZ6uY+*gzgR1&LoPr5cK1mc*72U9&Q6Cemsy zI>zW88vH+VBGn_=RiDUX0dDFp0=fy-g6t&IWq`+nhgG^6$nuVkLh=IvRL;5?h-Hd~ zc^M#nwzp8cmFkd#vTPn080cUG-s@3*JAdhXUNSIfmV5bAn*Zb8+<$-j7g-km5bY6g zFX+Oi-BtUk30{J#5RfuV~0xUt8HaJHfmhs zCwzZWMTpvyt-G_~e>)9hOc%|nIq=;~y|<6r(XP*(C829l!+^P|l>Hx_O}+ zlTZ-!JIL4Xqgc&eONImXU{_J}@N?w-T8lI4vm%oy)-NK&kK6)Mj+q& znzau@2~7-sR8mOB?mXjq;Ij^*j9#0yzu#0rl-8m$iAqJ z7nAFz{<{AC@9LNoRZ&L{pR*42&4k+6@S#xjp;(47 z|9kVxKQ#KU()9l{R|MLe?d!E{ST9_fVvgLTwgFfZ9Q^!T02sLik~?Th|8DSYWM>w;cM?t^rZir>0d}tZH(A|;GJA~w>Opk1%U^~ z(QL}Uvfb|w>CJTVjU%+C9@$cV{%n;{<+cc3w%_T-e9$s|Z{&EbF<{|D?#C?l(U6cv zgF=e0=z(Tb=_a%VfnN~IQrC1BuZrkl$k z7ncgB$6{|AFxY0$36%>@SzA~xo&T7C{fk!kLp)DP>{DH`+sWLvg$?L8rVd@XxfS!n za5(Ul%d8#!c`{>k55D~e?T>%EhB&C84ILi8@y&6))?YO8zA!c(5)$d8d7$IRNT?c^ z;pFA8dxtT5#2*3KRBwv%&$3led}65HhS(JF_|p#d`xNSrn*A}OI)*JuFYwhd9e(Z5 z&2DXdaxCl(l|$d{XDY)wRS1inWVtl?msVCPLGpy*R%6q~pjvbGiq53jrnjshz|^`` z6g5;#WX5+;lhPE%5e^aK_7TIs;B?N&z>3t+ot{aU54H9Y(+#C1RJr(L(eO`-;rCme zy73;{nZV{^<*H+Whb#RIar}wl@j@o@QG`--sTN>DRqp@306hsvhy+|<3~Vd|!PdhT zcltPi;z`7&%S1Awv8JK#>xdhuD;8mI7tRREtQUClzGxD>{%`yaZb(y$7c zCO7NW1S0#MfW1*K)C?{x;!f3-p`k(t6xM0Cu>73-{R2-_bA(LBk1a7KKJFTJOp-n! ze-E5DR3)Z*hrp0huY;bYZu0uuF|u)09&x3>!WmuEIe`_I0>9Vpq2SVXRm*zfwjZ7{e09 zwV)?wi6t5993c2*2|n+_vsHL!81TC+yT4b0|+fAq}*Ty-%+mXnHHO} z=3>+rv3mVd#8mDdX!}pMb;P%W(BL;^!Or*Q$6sfZZ?rWRY?U*La~<^K=jmLT$vWvQ z3!CJK{|=-1cb?PjNj4wpbssbk$mnI~qtKsB-EanX4T~qdva?4r}8(Z=)1e1Qmo3jlrQ^u#`8TFD%jo+@d9jahF>PJUns~Nh4UK50 zd>2x^s^CyVx6pqiBuib*J0vvRucP<8^Ua!Nb=s(z=Im|3UNjY_Lw0OU6ycS1L85{! zCtaMviAX&oq1t~KlZddo<_ycnD^qFBN~d%Y<{azYCtdag)iDyBw1y4`c0N5bxrbu~Kk+fOU(qvO^-l@DsR{cL5mpAn)jIo?$c7a!f6 z-VW+nK4f& z<%t|;Oyb}24?8XrswyGj`yzXK=y0S}I;_;@p%s;x(9g$?!NRtAb$n`AaA%?_R*y5~ zCTskPrF_UUbHe24_X_hiWRt?IY$>q_z=R5x<;ExaY9AA4ML=)Eln=w33)>44ZS~xv z&jue4Y8w2lV);A_N9jDbPh0E0CDA=xjxuCh<`_W?i3%a z-mI<#fyl_pk?F>_UZ)T#^Yb1 zYrqhl*17H05H&B?tr)joj=!e%>=3H`KK=^$T#Qv`0jsg<+HK0pn6f-3Ya{mW-&AoA zYT~I9oDYk#Q8FXq5DW8sET@J1D(w)`*32Gyo%#i>pw1ERaKUDQKGS!bWVNu)Q$2K0 zGr+a1%rO=3h}36XKUIK*5;NTZ5yrTqVUAoELv$ja0z>uH@eDR8L874>NQ+)k$&k4I zONkmwd}~_-WP|lRxf04qgWU5=8w*`p2o6id|CC4mt$^?;z7?^ zR>VWlDj-U?1ACiH2Lfd;g6!AU9EB^ND*^OUUxX(0by zT`k(!pyFD0SRh)7PxHYnjuRp_TO*Ug9*G9)UG5j`B2{jekAh*iV~C(^J~dMrHVa>T zR$)%nL5^fT)*p;OTtpCnbbTkIjFw>RwP2+&gv^ZzXtX&&Y1kThyT0!xr0nJEU4WPP zOU>+gq}_)=uGLRy_A(rF zWM@(&+D}zEk>+%KzGf$Bx2FlC(lymis^V>Z+QO1`uWxkho0r`a(!Ah0csECs`9dyG z(+ZgXm^pS=r|>b8{hB0N3Stc2ZUP~uR#W_xYBN_t-pYI*lXT~sS$!f;4aGE3@MMmv zDs|zA548{vu&6_ z%9grlNMtj+<@HhUOYqB9?UI3`0lJi?HpQg);)90^@*EnlvOX|Yq~@niJtl0-3%jsccc&RtXcRKlP=3QDkM5SICYQr&S!Y@)oa zt*v5F7_OnUj$T4W`h^I>W>2^eQOLfn*$WwGjkSb`$$tk4=q(tj_C?|mc0|w+%oclB zRQIhbj98YlW5RGXX1k{LzP~aDYyyWC_1lao92fRgQzOEX&Y^2S0LP1<lq7WG*k{c1P1 zJ-1FCNO%f@zzs|V!}qmelzFK0U$gMQ)u%JmADgQ~$(A8xO*JI!pOBghw<{?%ZbyvG z1<1(M`AvbtN-mVw8a=7o%NCcXY*^NJdb&{r-XBD8B%YCCU+Cmb>6xeBr0v?)?_(>B zW7Bo?5Obni< znM!Xg$nN&kXd0DErJ__heOW?X;k;P1;+ob3VyN)ASHqEQBJ3IE9uVaITA-o>7{pPXw_li^>{( z);Q1uNz!&L(8PGeV24E(?@1vr;iz&X(mDdElAe832nj)Qr z_&%}FnXP+xj~GhH<_ZYSLgohNX{-EWPiyIK`s&}$!T-Jd{dx_BO`HQ)jbRdLfx~WP z!8viWB3Z`K4}6A?JkqDDZf1OTl~xH&tALLtpItbRR>8y8-~-Zana0?%hBv+q#%t6{ zE_R`ctMy>K<-BEb0#{zssyshyyO_gT?n=ty( zFQYQx4~(TOQ8TyEkkvZC3NU_Bc(D0uC<*1G{a2%)PAL>O&({oXA4Z@eierMWHIQmq z()!@)H3u!ZCLPeBMl90%H}*Hq;Pus})WsH0<)ieHSaM;4kVq1vZUid$)QEXnK+kK4 zsn__;WCNe6@i_Nt^k>B&Vhg8W3oRtWRWSM|B)^sJ3mx+;(Y#tyc-SyP70n@P2@(Oa z*~XnRWX?hs6z~H)D*>(~6hK<$i;F?6bxvr2tmOuuvZ3g znW&9u1jZ}Jsk565c$6ATl>vOgO^P#K?GMc)-)Dy#ls69=mcHBl3Z{YK^KDkI+ta8Uo=6e!Or40M{RsxVarmICuM4B4()-W@?4Xd<|;vh0n0qJ+TLCLQ5}hPBaq zRO8IeF8s$6!8TE+V1?ajqr;C%c316D$%bxDv=Q5-g?tZyZc#-cfUY<9!qAH8;La3(sS66+feVNnVdb5bD9j++art1VoUi(VCYa{{qtY>AzNZBn$z ztuY3cZkD4&O1u}`ycEvEE~mAxL=^83LWfD`!j~M)W^3$A$HU9{x%n_EKV;7gcz-?K zVb)vilJL7%{MxPbzZ z5esrnj9SJv-wK+6Vp30^<2`T{&5+~P`Fe)%{WY>}_L=gYq!JkfmDsEXRx zC`0}X^RPo$F>0>FUY@1n47xwT4S%H)tpYQhhB4#>+{ARmZZfl<(Wi5zlAO%Xp)*~| zYM*N$ggIPbb249ivrvMi7r`#Pd9*OohuK&o!#heBGpXd~nuVoXCj@`0 zGbcJ)0&fNRO*`CM&u*uJQ`S`Zng@$_pGE1)E07y_>S-?$DvNf)mAy(QZ2W>M=ua*? zx;hZ@)9y(?h+!93lRF)&_Tk{;fO0+&|Ly9=DRG1mlmITF4?SBDuei}#KIv&3&cm_H zZku;f&t=JQ(38-Z{qhBPlcy6VDE2}-t@O2%>Y>r!0gL?8de;2MG6pt=jjOenx2KJT z3pV1Lo24T*H-MUp8u3L`lvBpd+09ei-NMS|=ZzO1H8&R@;-QwfIH!TSrK63NHzHlh zg`1k2_xEHCPd6)V8*gJy4Ow}_6AL>V8Be+aDIk>q}Bg*5H@pN<7LR8n73-Ox2EA?l(yo0}u z^*=w*aPxBTM*P}XfQy4$ke>@t4_-bY4jz79E?#cxza{eVa&z$W^K)_YQzM>naPa|z zgm|g{ed^x|Fcn8!e>R=}&!(%`xY~K!Qw#8L$~!oF+jw%yJ0qwjYlDF6w)w4LzmuI) zTH4Lun2Uq!=WqGBgg68QgoOA7sRe`(q|&nSa`W-DvhhNY4)IuvQ`^VV`zKZ8-8@}> z{rE{EYHm(d3vW*ce`9V$CInS}ewzNI95wgvo%}mrU?@h;`r-2HC~`trc6pirvyA{= zZcQ#EX&EnCv8UR~n5bUJVltT8n5b6T%3`$0ytLuFCLBRiCNijLfwhtxp-9L`StZzy z|1Gl#AYS_iv$=ZPxOyXMErdwZ`mMbh2oB=Zu<%4YpceXBO+>oAkBcR>;BPltZa?{o znu}A_#@fN+SE3*?@pE(iMv9;P`!9)qM<2nTCH`Gk{Jj68Hvc8-%e%S$NPay{ZJ4>kX<9rF*vNZy5qn&;mR zBOM!m@Bi5#`boIos{cDD0k{PC{|zsRi~p@_1NB#3cSfEBT`Ld=IHRVH+nOsvTx??} zR6?CYW8R28ehDWbp6`gaWq!}rlN8?4VlyrxN7TU zZV6YvV=3rey0p!*2{$G&c21J*oZq>C+?UPNRC=U1H!pr`jthyNH&|c$u@`N2GMr_7 zWw5n>1<}7MsD$hUXgh23weHPbt_LL9EVpRfsTt|z+f%0$e_A zuHXf%k6`L7&)e@-24W0o_=0aX<3qZa?YZ%PM6Xp-QXP1+;Z6i3fiJw>6I^m`S~dLQ3@fdU>(M;e*xfKrD4M9gXa~1dWScDH9bVJ@s4MqFR&ar%1V7~3e zoCk31NFq=_OU7M$w+hslWC_2)}?Vt7|FsT{sU4KuLmI!>Op=Q4mb1|(y z_3a$6ufR3x{Z-iZ)qGr4r5bsii(ObuaUYM`$nzJLgBs##o#7kL_m30e0FZ zqiEU^sJboEHaO()Db;qiqZw%Ga{`9e-6RpTI_h!NY??wK!p7t9>n~wZr3go^@ zry<%mEc&~z*)EQF>9Jj_*spZ=OuTCCK9f4o(K399r((-QdZ^h?(6iBv8m%>S98|pia6Iy2jb;m7#z?Qg-Cz5W{Wz%% z1q4u!9621SbR4?cIdc8N>mM|@qo-50P?L$F-nL$;0g8K(Fpte)i+!dJ#SZDHEP{YmWjb{q^0?) z1|k$!BA=No=|PZYzH_gYXW~Be_!C90()^0aK_R(~7NvUVDV_{F_oW6$(i6Lb7h4SxaNaF?BYG z#e42QoGmoRIl4J~Aq(n|zmPOkGBAi8?-I*G-vu z6w2@rjhwjhy_uLFm&lIwZl%ynNkE`|=Cv%3ePsR2M&afiAdjG}8mlz%d+X z#3sSrS8L#GOCA^Z zJUJ`G7x-*QqSA%`aWU9GBKz$&@+YMT{1jiiLvXSgrzTshb|Yt5FtJ>>?Q^ttInwax z5)ei9pwqlVuwxd6%iI`^PEc}Vv%O)X!DprGLkk(^$M*PwyB#>~C_H%Rw*73MnN8jT zoSs_s?b5j|7+S=;E!1l$8V4C@rvkfwwCD?|4O-6U z*WvWc;VmQU2xt=!RL)oIogUL3R!(U8@Rv(cLg|L25=EL%=ggj!IISA9h05k(_E;D? z+Nv--i93;;=RLzKm+{q;tBs}IVJwnlL`D!?-*rZwBdOUz4jpHn^s zv5S!$hp3Cu6W#lp)2Qvze5hxv;3Cgh<8if`=m;f0ZmrPpab6(xL9ILWlIDW<|XQh{9WK_~j$2Iy3p5wDi2XZ-+g&_JpN^ZKzTSNte#da{Zakf!*Lr-O{mBTctB+DcFu~i?1C7X zCz~kIhK#(RBY)uNqY8<+%vyTw5k8X|4n}H;#~WGM$yGF;^s&;xR|>RIRPTDKD^5X0 zr#Rp9&3esVkdY0!(a3CqnAoWAfmC3!F0)co{wA7Ml2Wo`(2iicjSm@a3CSqi4Km zTk_tX+#RR7kZ`p^J4geCUinfQ=fdj4(1U)_OKBE;dSO&m+Wq$&@FUY(iDajV!Zt%E zMdd;QY`=S;hq7GRdA7>LT(wP`wM)r^+mDKG%CXCDjQ6((Y`+h9F^vY%$Yfb(*(+~U zjdajvsNsr-t*nG~?krMQ%c*xQJzz-y((w1(y9?L5Q>rU|y#7)Di@sb`CMQ3NN58f= z#yqC)P++nLK@Gv)HnM5f&<8lJc2H({r_IN_K>^ICn%v4Gcbo5CW9IOhydF>4ayEf;Wb>?^dJz?{8;QJ39XeYNdeQ=U6pT}my>7R--tnpX^>{LZkfI*(?-#rpMr~2+4Lz7Bz z*$=2GyGR_VruSL=`>se?WnLm3LR6+IS9);csInt{*2c>i7XmB_WCyqcw7Y$Ojb#`N z9vyobwY?COtx#t)w0}-6V(ORiHF~@uv3!cD!CR?7(Epg>!4{KuD)Oqiu&$zFf@-w= zfO7<7eG&Vdx$#K_&r`}WdR5S{l2AKgUd2PnmXv3CNiw(xW{?}5<0jflnMKjfZhe+f ziMKhD{qI{W>>h-@%6>Hc!&7MSL;gA9pk+Fz%3BQgd`>Nf1k&Qrw_zWl)WxO>XY^$b z&EhY~&csN*{Ya2^#!4s^vmj^rXo5DDG4&}r`@mQVTasou@BktA61|@laaj3Xj77`!bCZI zz963;%k*8(#q_&m^J9gR{2bM8duPHcOO2?Zz1AMI*3evs&z9%6^&;fo)!O&nvo8di zY3b6Q2;%!2wRmk#=ETwY)rYgR3mhG925VS?E?x^9ppe*wVZ0rE{UwaDy4;>C){FmK^g{ex}SSXsYy=1c8gh9WIcd8FFPAVlC?1`3qq;EIC||U6Q$b z1^*Eyjb!!s1HrF2tLD6K2E~C7NrgQjo(qez z|8bg`-aew#Q@G`U@wtG!s?*CC)0iT0(T3v7QF|g%cU*ixcZpKjB7VzRBFnA~nR@1t zn6tu|_d@T_$KE;Pb|pNcI;G$c5>ZUA;Ht&F*1grA=h?0wJ$|ByGm}8ry-0geT2J^ z*EVQMO4#v>IAdHhc5a|)7g^}3+STjVhI*FI1p76c2(|-;7WH0f34Rx01JQTbV4O)> zA``gBLPu%jy)E?YCy5?R;I%~Y%uEux|Q3g5V(MdS0Am(45|!!3EXN9x5SK4 zJxLl&A(&5sz9S__d*1n?j&Qc58oeRObdbcgb?4kXNuFJ{uM9ZByi+)mKqggLB|Cy0 zCcdD<6&k^wg#*feZ2KB;Y6q>lqS>9_zlW+NAhA)iY~_94o$~vTPS0TELr}{rXExL5 zxpX6ERDL+OHphMU63s}6y9na}iwXd_g^o-5U>cwQqIqc({rFwkd|s9FdX0WYdd^ri z<`$K{_&A!lv((`!4aF zLx0Kk3BL1Ad=qkngIP8L|LhzVfP1v2G*xV~T<8&6BA#{uyYcg073 z+HQJ12$SRi>DrRD#8;$;@5TXzB)D6!*-#k)P7xa!vLoAa?(4UzfqLh+CdY4*&0br& z53pMg40}=Vm(zvnK;svG;Pj%^3F(nx7Rz6Ko)3&K5td|{+Bkk#9QN2B&0P~B?$E-`!giYQldE=`H6m#>X^66rB7AP0Pmun8F{#u zUx~L3n)jSLN?REk4!M5&roYa)YhZ;R_deWAS{aK>S6z)K^u?o{u@~wudO7gLUJ%PZ zSvIjOJ=`nu*CHjgAYM<^ohw%1(LaHJXcd>cFPv}PBb$5Nj~}XL?U(T1@9*jCi1dG! zgVN?;!zYE2Cajb%v%|pO@do%EFm^AE1P#BfF732=M`u8R!DSrDQz4`R0b$QCXr|e) zM~Qfpqd&|PLkVZuVLzsj)^S#D|DE}T=cg6rA1np}yj*;If0ztpWpsWwKIuDHBMejA zT-@B8iZ%{*_TJQj0s@@U4&GiGHl8wWF79rwKTR}(e`hk}l(q47u(DC`v;h92?dlJ6 z)UW2jznLO_Ed4k|!jVGQAZ?JWLy-{Ik5wcYBs3Hh6jT&6R8+LbXo&xq=xAu@m{=GX zm>3vXcv!z)cvv{Nc(^!NPoEMHJbg+|Mn*2D6A~smvI{co!GkAAKRzSPA_{)= z;OF)86+bT&ROE+f50DwR}4;~^vK|w;L!RMCJM0;xS z+~YL?PeQ>X!g^`iKCPK8ULwnLI?u2RdKrLrVqt@oS3jRDV^UEg|1%wH?}6EEbcQ#N zF^EatCdlEFFAf7E&^F07hn1w2v+y!dZl07&!(+Wma~KtsLF_SV2Pn*%%3= zD>dikM%#%j(dE57rqz3Y_|!yZZw`km{j@hG98mLAK1>@!)`~GXcDRagr`PKX(rECT zymYV})+zx!4^&&!h`m%=01NBVL2x;NvujBwOxM7^Lb&nhy=k;Q40|IDt*-L%K9_MH`E+$pW;;bNjl4Lu+lRc7 zI)S4Og=7it+X1Tr6Gze{rztuko)2_}3~SYOFIWb($H6U%-ndhnqUOMUDn>I0VO}R; z1jg7Q&(h|d<1)t+!_R&$PZB<8@z;pcv@Acnt-Re_49tV8B3$9dF>efVT)ud}&xOdR zobGCv2_za-yUXYEb7lrSm;Rt^s(pHFW^pa%cu8(h{k6rbiRmu;fS7v$ZxQ%b~ zF1Z4Vg3aIxdMz_noiMxXQ~KuF%ko|8R=p^zCw>X9U1_&gH&d4+fimvbJz6xCo?Cq^pztTMac8))FM}fgHmj_Nd%w@r$ewN0!FR6+STXv-gmNN zP>#_Bo6Gaq8t_}kQ~l)F`jU`X6qArim9EW{g&r5M_lU8ssu2a19=PN~lZWzi-e-|J zeAX}XE8yR(TgNxE^fRd-DP_sIeGFi_nOeT+=wvomR@WqVNG(7WUIh=gfWO`i0flfce_AMY z7TuXL z%X81f3ea)7eEflQ?}6}Jr%Qcv-@JGyJNluDinVUv9O9OxkX2je#tM9KeCeB3Bt;V* znCdMH*k?%`?0TkDs(DH34m25|h|q#Lmze6HoM-ybalIY`7>eGCYgVw`p7$WF;^!J-fUfP!PgKj-efI$fkB--{ z%`{gZ;}Qh2tW(ip(hc`O#U>kR?3_ja&Gf1cw?f^3q&p(ab?$~Or6n{sVNac7xKbj^ zJ7w4R zAc3wZWrKfLZH~;Rl(8(>|K6%^U!56fH-4|vGMhEN*<<`Rr{v*!T4Ffzm$78g`Rw%MGFNPe*^1ga&3|NyFA7o-bQh`yef0+g1KhkCNaXS*^dwF zm-tJ_IQF)mki2_wMFLS5KWxprWY+@L>E1!QA$Ml0`g06qiHW?J_QP;ybTisbl>6Xd zS3`Ifv=3n4)n0p8%i%(GoEaP0+H$w#H5n`qz0N@dVfL=UHIIaVH$j>z zT0*1@`y`%aiD#QQ5^R^>^FSd}B4;2M;UlLy=A!_=LuE{-458+x&|DK=yt~Su2m{!9Lu?Mf>f)DxN9q znhq=jka|BIrmareT*DKrrwtn>!5s_D%@t%tXo{@KdkYw0O*uhgT}2Lj4L zgk5hhJILGR`3uU+SIZ|s*-qnB+4sN>n0Y!tf*NSX?#ju6gcyJsF)Gl^B*Kmd7hKx| zYq9U5n>VMr%NiP2QU_Nf<75Li*nc^3 z-D=zSI98IwM~_zo@##=}yZ&@zmmb1piRkXkg_=6EwPj-me6_FKYbF3p$JZ)6#m*0J zzWBokN>60o1R~^~&|}Dn@mLzA&q7zbo5k*(ut3)mSdi? zwFK-#@GCIwG~XFN@CCcXXpdEnMPqlEZz6;+d4VyLN2F{}wNlvsW$&o^ZT1hOtPmKL z+s)o6)$Oi%mN}?5WhL2nN>$rvr{r*^qNpj2G5UgX20EgDyJIjHmw9JXuXaP9F$p+wYK)oEC-K*RtwZkamUPA3Mn~s+Z zIlx*yQT|_c5|_n-UH3+I48E=grl=$MiEzJaf=U~f`+5((PV{w&$ql~Qg!{}oh?LNf zUh%LY&V0;av!IX*n54RZpab1)8AL>$wagr@c}B~@1R{I96hX;4z3T-NfMreO`rp>T zX1?ZHODq=WH=XQH_!i}}>2;pKSj@e@B<@R{%0U;9y z%kRp-!-B(mi@QA;@YX#c(d+&1-#btIOo6f4AI7ZWW{KB}WaETAmXmYEm@l)D;oSwm z-Z63I+io!5@!17KK`A_r*NTlM)i7U@l-OX@Jgyf>oM19#w|5d($P|992er%U>Lkjk zSaulJX^HE1t4@p2xUpUcA8|(&*;Us>jJ{M@ZZy!!rw-_)-Z)#wiQ0HL8l32OKJ+jn5kw7z!Irfq=`@Jzth&L^zKuf zHOJD92YcVPbaf_afN!h3s1C9+^0Fgc1gPsNBKmYly-_k<5~K4bjMrIVwy@ETy(Xd+ z!`V&U*=7O`O1Zb5DRkJ)I&^t<12oU5V2QaFjki6sq7$zQR%Co2h=`TmI*EQ!3nKpL z+G2CHEG+Y7Pr!j0_qzAC@Xqm@l=X<-{6QBiNBV|^%Zde0QpJdDL6Z-JG* zM}Pc!dYRXSibz8nK%|UNia~~rN|WRxnvi?37ONZk^|`~hT zXH^3$6&kO=#XWIX_EnqKjlP}7`6u^cZZ5b{7(`?jqofdLWhdKLy|kMiVO3e;(7usu zHJi{DE9{rCG_vqCm5!x9VTK{81??msu;_QEy3FsG&{o+8{mGx_&IFU6upcaGPH&m_)CI)IdYeP1k28^sPy&{%jS+eW-(JxwM4#D|6uV9+4v5fg93*C zM8OjFS3~K+-pZ(8yrahDg@Plgz>yd`f`+74mLn8vg*J=dRLP`qJhOUIKkXedV}1+? zqW1wKz`P4|Y1hW1y4d^jqej(HJGI9v@6_9)WxO98ixsX4{BlsnUm!*s^40|KiBXeP zF%QNTMla{gZ(7Csi|ell1_@VCLDNC5mBOSagn3%Zd~N{U#EE(?f{?qWgFru7vn0zadyeEc@xjEiM8aLNz8AUz|y?i;g7v*z4Q#!$} zu-#lgsjs!W36QHW8Vcx6=S%(Mv)#DjUldQ(E3!zv?I{YEgq zc)X_WtF10{o-^ETJrAK)q?jRu@_D(u;m!GIA=}_)tlsGjt19qZHV0!2cxn`i#zuCJ zqc}@qz}{bH?M-T@4IS0ZefS%O@+=<)pI`JbemsA+gHgc1H_{68)_`J}rQ4r1?w?dR zD!G`k27W70BcS~%P5rv4T`nQlZT1HX4n)tH@8>nFAKgmyM9mTt<7=C~x1Pq9lzBZb zy{PMcku!pqc}>ZI(E;>iuuhA@mqUte@lHE7k65Fn^cr;x#DdQ)2@K+8xe`^%jrzWd z5}=n+lRmFM7r2b@k19ymUF4qDo!DFrx{@=T-*o$B!AUW*`S$W2l{@otF*=YIM@9|z z1$cydc{W;CyRy;dgG7GD-CGt9cJ;X`hrhqG+b&}r0V-gjatM%&{Jc6HVT}j8J0SFl zt<66W>mn2&&QJS}U7yw*31w390P;ljG98 z@Ai=*&G+C@>%5`Vj_rDSXcfeL&@FNrt733oQt~KEyUx#QPY$W(<32cN0?b-91RYvL zhu-#_+c84g2;bN9#OyQawz(<*Jx+#0(*d70#+GK1y$%HP3Xbw#PQIzNk}t=5I!l|@ z^gI$Dgt}OJ2W=k!KSV$&7m-ya{17Efj?pG3ryXxo4z4ORz)9f+%=UmN`>rp`wdNu^ zh+b1lFU0l@Y{S5Vod2AwEj*lJgN~A zdXdx%GqhV|ce}Oa=5=ttf1S6hoHe6@wv)-&sf4ss-!Z!{BN%psl&Yf|weFSvR*KK8 zN|U@6Oi5q_UlkZb)zDMJ$+J!;q)cch>ra*z&-XOqE(2X2PqpfP60+=Y5=?mLoO#O^>90VC z(O}e3g_>YJ+J!nDfg?>CW`c@jP=ZgUo1LBryW7i8#*bR1@t{jU35i;}1R?RI4k`l`=VF`Ch8)Z3|Zx>*T%&SCat{Eu}NKkNr{qh)AM3C^_X7oD1}*?|P1(xJ~7D`Yl#Z^bk8mZ3I# zFAhb*y_*e>&c4gbhi5Zap{_eKA3Hp#_So=FBSz!stAY_nh{oE~Ckyw~Zdt89_^a!d z$o_L=2p`HA$7+ zp;Cr8ZwpPVI4BWnSaqK{X^9(c92r;~6qQv&r_JZ7Y{kJ*<&+aXh8iZ+%?*8_hO-K* zgZ5F>SD~r>x*8cX-5z9&B`6FJZAE?`J-!u3B9jSwS`o(lNs9^H@OytQ|O-` zhmGyFl=sDJoV~j-85ex#kC>Thj;V&?}{-Ob72a2c~r=V1^{N9$qcL>9HIa zka<*nQ1we>!lKd-7Ftr<097KRqdd#bM9tPa&XK!`F~k=3^ZxUWA5H}x;0|5wJii3W zaL$gH&xhljV#eh^MV2k?aHuF2mc}2ryvFZ2rRSu{v-f`5f7AQ~R1zw>1!(3NJtqjK zX~Fbn%@F6;>J2#*%Pj4qo@h;r`gXqsl$KHu1{I*2u|6FWqA9^7g*iCz@xu7pFOWxWWjx8ffq~W0?CUbGWh351H*n6HEcAnB4e9+2uT^n ztfWYJQnlE>w45?T1P!Hb#M2d*Ykf9d=U7kg@~g6@Jd1X`lN+zfxWFB!w#P~Iph%{Y z^3AC`*C*wRiAlzgBkgT^Mw^N>?g8$X2bK0m?-HI^3q@EHGng=XDhX>vxL-;o#F0J~ z^-SspCfMc!1Z2*G|X*!GyYJP7l^^vE#Ihnbyp3$jPbi7^W6M zzLJ3m6vN(`dX3p?H?dw7{c4tE-2TOQ^LznNaDHtAaf1XE@lxb|I2tME{XRD1ktYsr=6bQ`&GZHg7P&3-NSsU9t0pyr zXh3mjmKi-}Ub7vB$-HelTk~P!#R!N6YUtnex2c{))?f$+Uzdza2tnx8;pZ z^FXilxQunTXI{UT3W_?GtlF<)tvs2FdNilcKu|B1)sV&1*|1D|bWSuc)uO;rLr<^} zV7jK>^W$SXYvt5OW^=lF1sV=RVgc5OrTEYQNZ3-L}~+$rQ51PPkAsrS(y%(3dP;7PfcCRP;ho@B`$ zr*J>Z6fX$Pfx>erIjB)0*xWenKr?l&d~13uJKMIH#8q*P3U@~6O*a~2}9n{`&4m)fd0bHK5kj?&OZSksTl$cPA#aaLc zi-v35bH-qK2AO6m4)sg7d;NpUeH6PR4c?^gYZ170f9fUDI@qa$`V-Nlzip{y8nyTMCAS3()xMqE#bJ|tpL%+mI1HY?h zO?r+Vzn7>jVF%8$zsV?NtWYDoHSDPXD2wqIUFP%;^^Xoc+p+gDopJYUe9|!c{zXvO z$qA?^RKAH#W0}{VdI(WItkLdsjvL!q*C5)o!H=jdc*b9XxTLtR#83LwFZl(`P%$-u zhLQHRnvC(7*a{(p+P50?`Y&5hZN61qjHC0HzrC_MPipRnrgxF}uJua18QYZ5us7)c zkYzZgna!exjF@P7(&EBeam#e;=bg(j_jM*B?;=s&N~>fhIY45{(_C<%&@8j3cc z@edC}v#fU^ZKJDBlY02gj2AbLzqTw%E%^xzZl3>KrqeVm5#t(T8Dpp8Qa*1ZiHsym zS&y1do(T32VUIp?y0|i@3RAl8kxv}28w16dMl$Bd89T%Q`C#_F=ZYGcW%zMX(IB;Z zPN?GP#h-Hy6L-kqe0ZV4>J76xIF1?6vRf_Y=Om_kYJxR*VX%npt@diG;3P{qV-b|8 zL!x|sR{#{6z2@62tTEb>s>Q>lbnd<_nD;2tHK*=3o{VZKHK1^$nJB*Z@+!pyyM)V| zs!Y>PWiFo09AdQ$fHFP-y`oMG1HhMX%Xo=xX#0nwsQXva>|;(hp6^fycxb};t)hLw zKN~QXKUiet(L)_KhN19?X_9(h-)bWq_~!{z7P_(vmPquoh?a5OEM*9QcD zYv`y?pbbr>WbsdKRb{jG`deBbPd#``aAIX!JG$}ZG9TQI)JJKCQ+m>VDz;rRxht-n zJ1JZL;t^*GF9^kle9;aXTp)q^*Ht~8g2%;c?vAhdt-oxyJ>TtS67d!!Eb_{hud11j>LTA7c_=0EFR6&~!g z86`oQ^7%0)nOeO>WJ?0a-H-aBu&3GeN94aBk*t;c*qk6%)d{l8KW(LV|Cn&a&z5hJ zVgsXoO>6)5fTXl*L?>C1&kk9J z>L_K{qNI{Rj>W&||#`35EUa$6pcdQ^cI|IuyO{KK*ab z)8FpP@3#DRFZA2;#V6E;Ni|n26D=pItAzqBl9v}nY*OxgDL?q$TP3F08y*o2v~XyE zh6nDiu1Bg}JbL)oCH`B_^Ek~ILQ+>#$kp6CDvFRVygdQ!8ln=tcOj z*58qVKYG`{{x+-v@iW0liLa>4fO1siey}7;hO!PM*T1$kfF7KvGGZo!OBZCH*t|h- zA2Sz3r_UFyqh*ju9s?>4yC$h5{FqpBfO;vCP`r|P)?RZV%;t6V&nG|p_5S};HvhqI zFgk#V4b9#81(7*04as0sl}>xrdtX@Pkco_<4)qV5OMAqAZvp+2w{;zN!Eyo7bJ7R!_w0*pD^=fjE!$jgy7vgPhY7sy!cO#afAb zs(Ju(Mv96Ui$KZ&ti*`#_`&k}zJM062UWDDxq#7-i_n2$10y@KSb(e7Lb~|ak-eL? zK!4*tb(#P1)%@YR_kpeo)Ns2FX`hDj=4Te!QIy6J{^J^s(@zkcts(Zpm#5?zXHa$gZJf$+I08*&vTzO{{{D+(W4}?+;Pf_<6n3^jMC;QW_Xim1n zF}oL^cKAp6x_692c6A%NSZ|T9WFL!ol6Do}YyuUz+)TxxD;7cSMdL_T*uD?T+urqcCVmSxXINVz02t-rqqnqlR68l@$_)& zHuddrL`xnL><+eKZeMA@*9?lAPT53szV3dFB+s<94wU(R0Al6-VMgJ6CVFM+2g`U& zMEF10pg)krKxaB;o&gI7(Jp%ZU^%0#zyOjD13DzW?{3r3fqIz3A6)y#`u#s#T>~El zgf$U|n430_`I)VoO-h{|ZyDX}=U!eaCURzLztpZ=?z!X0NrP&362L%{8l#D*4ro-D z{3bzs?t|oo@vjNMRB?vzg5cR{VcSLzcHkRE0k5VI3}_2N;f4SVh?*JcF+irY*|CL@ z%cbz|3Ef}zg#w??XWAOMuj`I(&e{*b6;F;j>lND~hot+{iAh&QV&f#bVQ~ej`GMxpt2AkyFbZ9bLSnU-@HW64 zk>9h{Zp*Sa>E^N>vxhtrMf*OrUU_0Z1DU6B4&_+EDnBf#aAXuMpYn#+X;!V35|CdhLbuWFV^%e<6pd_k)F4OikHqSVa@x57hp#82m;0 z|K<-5-qBUGEL5_1+&83y&zQ!S;k35KoHpHM*hPJ1&3=x!IFM}!?y3KlzWJ696UisL z!VE+)exAhT`WZbNliyR-Z?zvujgN+KFD_^tf@O$szQh&fDnb&OL4cta-XS4&xBo9R z$w$6CPF{~OwPA`uhpIk7kh>2K7Oz%tl%3KmPPk9XE@9*6F7N(m5mF1X7-5u`YSTTV zQb`}g)dr#=UO^3AzN51`EQg%NJRWc)0J!_OwrfP9`)dcqn27LsmBfTXFf4I5-VL)e!D$i$$N3W^ErI3&+8T738GvU8Mk9}QM zQ(*w{+zu)!b2_!>yAOHwqK0VR&VNVz{+WLBxJdI=DsW|df?-tRa-^aRSgc!eJk*u;Pe& z)R8y)hL;J!=RY;(^gQk(=x#^sw>s1+MlCF?0f1SAiv}!az0`;lSMbS*toi7xv#1L8 zOV z*32Lpz~q@2X7t%XdyFYr{`+hB|JHss7C>5tHA11v(8p1*QagubtL@bJ<%oiEF&!`eb7SY8i)z~n zhJBnqcE?MNLWh&0O$?n?ZI-OyW-7hly(ya2{?t{#n9|B~glOu+ zFvddp)YMAdhooCmz|eAHQyX&2P$dBz4HK=7eKla(!tvys>&bamWoPdN;mj-e ztzlKKqZG6>&HzqIJM%bDzU>{n*vqThMk`$45F9VNp_^6zE#v(90*r0oepE|v)$$aOKQ6C^ zjQ3vbENdh3nYTB$oYdF3h&WZ>gA@Cjc_6u!P%l;+TYw|< z*8})NS6e^>4A4ee(*4C>(vDMYv|{*1$DM?Iv_k}Tf`*DR75ywPG{42lS&nVlUt15s zyz#FlU3m4TCDkn{uJN+KK|nb{X}mO3st)-;Ocx z+mI$Op>VYzstr)Vl8lZ5GT>VqddDxfgIGpRDWCB&XAjP20)ujK3P7Rd50-v&NTT65 zVy*Vmn@0x|h}|xqu3mz5ots=5*85Lw{yy)i=MnDk@CwYi#y+pJLtm$hGTki>K1(|E zW!hI7HVDmjx<$JqR^)Ta(V?Qc`R1;*%y+djfvH~jFGOJP7h;?y85la0%nBr@mlWG{ zas+ay%e%O_6J~RLW+^LX5oEC};qSi6IkMYjx(t{bIdXjngO;VYsh;4mi59e7>U2cD zvCmC)msHnz^Cp1Bf9|$*@>`W;9l#eS=F8#cEIO%#Qt>+5`2!S*)_vMCZT%%d$2G>1 zi?0pu9R-or_neT<-^8H1rFn00h3PqbBm_n0Mbj+*8hK4@rQIOg>2uMY?yo|ekIKfm zR{v$mQPf1sDPi+!Uzab^ zk$t(XLh#@!%;j}7wS9=@ihVDLV6lE^aqCGOBM`@OKasbIN??nU)I_Eg6h4f&Pz z)D(HIxjWR+QMD(#juV79&g`a@bqVhg0iy$9{FL!XgQq+_1k$tXVXoy*BRCa(7tu6$ zA^l1%l&peh3I_SmflNwT${=VP1GIIAxm@}8)E#i*hpT6t?Y)%X9H)CFzDT!E_XP?U zv+BOqj~6Tsu-lTKz9uoB9)i}m$Tw6FN%KoRm{}BZXAF)RDMeQWq2ObC!qRf}$J5Jx zPHpjB(YQ`UWT6_SG7BSX45~lhT?8MPlgZpZ^3BYqEH{A$r~{YEa)(SZ?kLFwQv&X%>v<#uiCZBvM zOT1bWfjd_>Zn3l7@9Xt6VoQFqCSxjYF;^-lan`- zm&aBNtdb=6IX)2!^mP|~syh@SX6hN%6P4seR`;h#nbSIpipAskbwOA@14DYwQi6eK3jIS_akd~v8t^1ROH89nySFE0-{5zpdJphb0v zlS>%HR}GlL)6y-Mt!%fZ&Xz+beZErLEN^U>-|Y+)|IIA?=}x!~ROTKE-tTlTVesRW zlSY0}^UkF@5%QH|Kfe@L`if&G0h#z~A9h&{xZ#;n#S;fn{HyMD6TDj2OfNLPboK7s zI;<|n$!2uznDFVINOQk8Uw&PuS}zdo0i;gYck1L`r1Ngdurc{xMi^ksAV?fGaAgXV zxoPC^`P7`vvR%ep5x2nbW&9^=1J7tp3l0gZ^>=Uhs8Af!qjOSR-qFR((N5hQqo{Gm zI@F?o;B`$)P~7a{*p2TaHWn7!xUO8qAU>7Z<*6NV0|e+;$jNIpvVgza*2Mb>T0OB<9 z7&+Uv(!!VA@l9ABCuyT|`^JJR21E_q_*kl6YdIY+&#?YHY}m~#(G$9gyHQb>bUC7OU4 z)e@cMPP-KcE-U@k+MX&Y>08Sk%r{Z`M*9N2yr+nH*>E|#(IX2@5dcx4F?KI#Rx zZqt`${#8##F&Kg46J74~Y-qw?rZyLnRZltTIc@J69K8c`GiUAPLHCqUi_UA)LFEG^ z6FEnS-TENF7|dM5uY^s?c%Sj+taJEARlX zvEFhm(4?L~!jsC_!gs>&SXkF_nq63Diq$~U`> zy-OQUESt^y>W1~EO)mB}S)Z0g1@reO$E7vYRftADo<4^y2%4PMrD1F1gQAK{hAAK! zAbf&@D{w-|bLZthxU$JJVI?r7@{5)}lWPMlB<=J$Bbvhc3;&QkQJ#2ln03>5NMu;E zZPA&ClW>t$f!$|6Sln839#3UYe1+j+0!E|qN$qIq6~m=++XGvMx~@U40SWBdDvifU z(9LVkuz73+fRu>*V9BAS0yPYtbbA^bt+Agw4*r}xha->8`B&>KDaqaEyK7ZZH0=Jz zfz0pyXc#kuUNEgU(WELoiRuFS<+I98*?s1yo8@_W9Z7BwOd9a1bNXhM7i|Y;^V-LBOLI3WCBmD&U3@WOUafZ%jfs^ zj&I&|d?%Kxj3%>@%1Pl_TRw?s>3Q|!s!;FDMm4)cJ30HxdquCvr{j9&y^dhosO6;D z?{q6xgbZstNHlBOs6K6%!`6@gSe=pb0kxF#&T#=sxRhLeGv;lkwSgE$I77ZX?Up1k zC%e*JL6D)L+>fS+prxV`i()JsoIg(;e3~C2CGo;x+P?4HL)(oMu7#Mk!!Xz|HzG$O zG-EjLgGw@k5~~75o0k&g8iFpwD<}+<9h?Zsm$RzNHJOm$@2JlU%pru>3Iv?c z5xS!_D!s?@!$BtB2OuVszxD5XsMXU7)Sfhc5cnt2k>?S;#R4G`omiv zm8CUBJ>>|>|6-h`=nDXUxia1=oY}xA`B+!N+1kX5*d#(OH27j_rq~?c#g3jnysbsw zwta|NwdaVxWTyb&pOIn}QNLVl3d3pvTkC_`(M=PCxz z4}k^{k`T;Lr}JcO4cLyN-+H3r=;9gHc8%j2wV#R>iL%rhrH!{DXkoNyQHD_Bpq-fk z&*3D(*eT2O@l*G=m8E|JxhYJ&sy_|nK8m8egg!%+R(tXX`i2@PuO95W@QADUYJWn) zSnFE~A!?SwceRUd)vTg`dIoO7>NvGbNci2+U47;kV0ZcKZfzZ}h}RA0EiJdaMOjwTR@tbO ze!GL;#^p_<-g~ul*~tWh{V$jW_&1@__unu~b^p)c!Co|IbI^PD3c7O1e8tJJu3@}} z*wXTBX3uGtEX2_w;xs|v;tJS6_anN0|;UvQy&Bmgg&)#4akvJ%P>S=~h`&)%7qe>&$>De0rpRMF|38+2Y*EM$A zMQ?wwTpggloE%G0eEX`;Whv(4l7Gn9E-G$n>t`o5AAAI{O3`uhwT90R`#l|Ai1TnO^ItWYahi zRd}(-!>$wWmLZ;VD}Ikl>D0~(&c#MK9)=WjTK+m#lL^R!-7lm*%9J= z8O-a{nYOU1T>}g-944~j;awRm_NH~gJnq8Ni_;!2)spD#q!i`l;H4?c=HlaDulrp@ zfCXEgzr2%v{5wZg$UrlY(Fe&?ha@S{?D*s$`0K5cYE`=`JYdFufc*XJdjvNSXLod+la)VAr!Q<4f>0AM1^8EpoZuU!*J-iJr+1 z?)+as=kxrBc|-APaQcUnh$_1|tXm+d%ZulOyAmrklK`S3CoKvDo_gNxp)5Uvw-zUe*NH|7$w1z!#9yCdSWHt4g9WsPKFzYc@tIejrjz-Ks*@wRt^l?+Z*JKA>p?(KbxIG;Ehsn3f$$N4GIZ5m3;?`!gO@aLi zuBMa}K~nEg`Pu+~x-9^X&2L=+92(suz;~q;Lf}JuIo$IU;nXXPt}$~xbOLo$d{Otg z!@}-+p$aE`h9%e9@xP7PJh0AKe@1qvH^3A?QW%B4TNK8^4pR@9zP?%<`h$hr!|x9Q z`)sBcWQRJ{XNCkuQOK*-+_V4KU^88tH(5V8@m)HH*Gxj=Qu}ieYF{o?OkR})-ruj% z3=a*!aJ9y1=A2RHRqf|^-M*fm@|&JU)32=kX;9xUFPdjCZf(T|H||__BvBx zs-2?suPIv%Z>A5Fh^1NYArT1G99XD!6pEiHft{k!-oKaBO}^ zUG8?l(|^3Vb>;Td=A;@YFQuF(R}7LkR#w`g5_wX9|?)&bV-S3(q&YBclb z5~7ZQqiTxn=$YS#UUXYqje{4E?eNsajpobR(X{m*D5LyB{2_)7k7@KQYsdga5a08@LCJVrMeVkLRtlg48_(9O^0j$X?$;#T-}t-$$g z0^^4Ie{ZPqKLz6d<(u|{C?bsMB2Qq%A$M&?WWb@~8vf~}?blkV{i%+HNm?I^!u>Pv zi(LW1&~yb<=?@mLy$M~Vbrbla|5nrdm&w4{ld<-=U7#Y(9KxqH$xp>*Wae{}hDX(v z_O1@shzXbP!+aGjFW*pLb6~A9qM$Le=GR->fhzG_Lqez+SdwBiF7Yv8A*Mk;zNVz* z^dw$hyt>3(h5Ua@53Fl(qH(~Aye7kUpDge`4joUj=Gr3y$S$K?Ji{rgAMLN#^(r?rMg2hy=s)5C#+&D-&QnRV@FQpN1weK~E6 zu~Lx7X#p9PJ^emIAlXPLhRRNVuv@kon=zhPeAZCuF8}3Fw3v>y>E{~)ES3)|E^S^S zk#Xt(bc56T%mK=IOr>uombWR5ERh*%F@=>%+)8Z79?lKi+kdblfoQD#Jij;hTsvEi zJj8*1L$76`W-F1afM*Z*u?X9@{|wf z&!Bvqvcdsf>Qm)B9pmY+R5$Pr$LzLhOMXC=mThwH&}J>jUakH|pXRT+hjJn?s)j67 zHz^Msz%*nU%bi(NHr-XeY|R~YCSSwawiY4u_-U65*)uOie`f$rBxit00DM?rJ{}?l zmIGLV1C{yuW8}z%Xm3H8Gc*V0H#IYEc2)n|`Qg8ilx+!7Sb(H_a}=pFisV6R9eNqt zWh^Tu{hIv_mql&gHK{$;!xSl|LpGWiy0d42gOW`!!wL`q8mvFl$z^hWh?CPg6Lv!t zg>%sg&-)^eW{B_MhK~AZl-GQ~3Z6c5+oU2<(;oU^xQOfT<3}DU zv>jqu26<1%bUYbZ32;&}(j%f5Z3oUjvf+1kJJa)dW+(W&jhF=k0kbr@YPYOH8s81i zC=koOw3sNgHCuPvUz~dKp@^QTlCUu3-==+0vd81m$TTITC$I%F9r11arJvqu+$zaA zPJc`3u*2BUYTK)98Wl43_ORGSl?|5*4ULte5!@H}d%^iGtqHfji!5WQhgvsR20QPJ zF?va}Ap5Zr*`=M4>Sn5J`n?V%s~PU+3d4Z7<9vKt6gv$@NmeYQ6M0;tJNlNta`L z@axo3iSc)Wx6gpCnAsev+MZVF^!?Ci!9&7Fx1t;%ON3F67aTT1tA z-1I7ee#TaluZ zG=J*a3U&-~{e(9M?RXIC#62r*jp}=d^I@j3u_wD4od3@D{Wt0TXO8vJs4u!y_fD-M z?$FsID|>G#xO|OmD%+*yuUcxUTA{#4I;7hpo>1=~8@GRzrMr~}o2wBINCyZ?%%&DH zy4dK~K8ytSL1{S}eg*5;2c&WThfjL=zx=d zUpe+Sn(*I|mVL{1weZrZ+zx#z50B~f>@`>2V&d#!;Y${jWV94xi<}7bEgc{sx%WtO zsex7V{F;&_X&*kaDzQ}dzcWj?HsYblm;*qL`rN@?W+^G@@U1O{6B;_rZ#vrp1Wx=M zc)wVm@O&P*P~c1Bqi~X;Sy_rzWP=aqZjI~)>TTU0RGK<*fj37Xa@`!39&-DScjTJ+x0G=CkFmIQyJ#{sE! z`o-qO$2OIF8t-je!)%?V=QK5Qrf%l&YwL)-mmQCKr?(pQ;WZG2=U6d4PHai&|6s97 zUY(k0_?l+>(C-j(e{iAR-)!T5lbT36QjhhBAfUhvQJR}5GkCf#5 zrLZ(RA#9OMZn0mIeMr)o^WR|as(Rz)=Y6+LNZT#ZwR9d$0^(bA`0cuu)>fJ1tIG!G z-51+OEQpW)VMOt(IQ>hIdTfC!jW>fd%e`VXA*?G=;{st*x+!_v;0Qk3)Ui^VSD7e< z1H`ExAs*ujIr6zVBASpNeQ>K|$Us@gR=^^|>3N|mLTH81+7nep23rp@?=(y{!ELr! z4@>P=p*qLwOAE}}JePR$CWj0W&Y!X^y+_q0Hl7I=tgpOQ0GL9LU*I+)T5~+9a0nu? zc7S6^GGx4imcen7il7VDW{@zheO*P&4fq<*ERIW_kMiF)$o8V@>?8*&2r3p23jc zoGD+v0$n%Am3LCz;t_0IrypM}@jh{6`kq`NKo4(Zz=5r-Wzj|QrF3G59p zo2}E}+n3Gw!;eFby?!u99}iil4?qWUSIwf86*eEfq%bprdEWXlc;Yun0p}(wHn>Kc zI;@Q*n!YJW$dXkttR-jcXS%ySg{n_HJ@$9(;Sm~~oKf(wri(PIq}NH~%ocv;zS1Sjbr@*tLPa|zT0`$A__NYfnexM_M zts}v%qW-wyK^wRGZZyBFhFgHM)$Kdzc`L|-J}^Ht4{%swEN8r?__D;2Jlm+|RB#Q+ z`o-&d;{BB4nK9-u{xQ}|(99TH{1%(s()J{wEfi(VK{bB;dHI2Co%)dh4+es7kZm`& zX?R!HVV;VDih!2=@Svs30g{z~;@?kKEw>p6V|q5evH8I=tB6<`?Af*%;kY&{48W_@ zaI?DP4Ygq}-v)~aPVi?#l#zzW+>^*VV4yLw3Drkey1L+Q!q*{B4PYSpsgr9i2c%ZI zv!N7mpkA*yW5nB9sw;tJua}U+ZDDbuPJzN8QAq5~rRrH}+xLsi%(l<(zj1l-{DE(8 z*X{g@qxPX`+2{<{2fkaTM`w0VtpYb`U?n5yL#y|uu~bCn7qAqgibs1%p(j{7STw*~ zs>kjb#DeX?LhKHMOllp;{u0$}ZXcd{hRaX6i~GPKW!sE%9S$$d{qn1_uV=Q-?rvvW1)}}M*IE0Yo_>K8$nYffdUQrY3oPJ z_Ug7D=rz5>bE(=_dm>I8!;{7G8}Q6vF)C)$v1u}F_rW-qW*hS9fy);}#4U3)j|EjY zt}IHA0=Gb4>CMHpSl5VN3iOt6(Cxv0Jn$leo#}mdG80M>1c_|T%B^5)dP`aqpHF!k zzA(OqUtHwsGIvZ0Y)fGfTZj|Ml|U8wu7es$haizkYB6?yoHuywSg7i19N#bU3H?nz z<)DKbjF&%HD%%Zzu$07Nf&K)2)W^)SRXJuFr_w)?>fk@)s(~gsZFxO;L*A?B)clkV z+E|>Xo&WN?a5|4*g7ZQk4KPw7baM-uLT91a^sILnGpVDzhU=$K$@Xztm(4uA;tGry zy-hTZqR?U|ua+FntZEnAwWh=^Q)XhlcFt7+i3|0ac0uN8Uk+FM*cm65VJnv>18tVJ zKXR>=-${KbC{o3D#w!&>cLJsJZ7F#bZoX)x3=f-VK;�OVpD!r9O)A9QT6lce_-s z{W?bsaA0hlz9jwguUCM6yCVDd5(n_sGd|IQQegA2Y=Fzo3JkJOAsyZ`n)28al=T1o zbN>9RVt3~mL~9}$<|*k)6Y8K`Id-|BW+->((|LJb3MU>40&3n6U+$tz7&G-^XL*yUBjyLI+b7LO-#(Zx zntM63M4$wW-if-whu^LbHrElcCuJ#)M|J%7kKCi&WBr&?>Gfii^^>{N zM*y?BPmro!=;{l$ZEgHYylynplK744LBlzN)rwx`|5_RQkID1bQD(d?Zq9TXaMxkN z#+ExY?C`D{5k56EkIh|Gi_qWBKXAh1vWw^gnMFb$VSI=t-Ks=Tfc5>1Xr7MUNmrlt z;5dEBq`|t+5p7U1C5<^m6K^%TFqT4kn4&gC>?~RNSdnu>pu8;~8(?Gk0QIc?ow7eQ z*{XDz&AE$A5w^K*P(okNP-S_tLqn%w+WatlDdc>A6VsRRlRNH12?Sf9vY@101UgY1 z{0xd@+&tqL${~f(<7`RB2MXJQ&sGn)S5ugA1AuI)dpZEs{wl;wfcQja{6sTF;*3H9 zJwU8qfldLzwJn_18KABd?CXWCqhf2abo8@x{2h zx*N^DvINDeI+krpU%n0L9hTd500#AcqzhKR)$V`uqx|n^`2Vw23H@nI{sTMsHw@Ro|CbEc!$lw(61SR0GQi(} zWr_y4RvhtNc5H2eWmxw>`;*A*C@UoxTq~l3Txif>65iGjr zJf4;M?nl!fw$qLmad|&~*OX@?dK~Mv77)VQt;3`4Y&loHyz>mNnw$>Qc(r=SMa1@; zneY8U_Vw@&vXkh+2rvNl3O9rq0CkAl=-zkAEq@J ze~9IAQL|kS{xSspUo$i6P84)g6byFg$k7kA4y7}>5+pmReNVJ?79-v*vyB&d< zy~lCJG1XxOTqOBRv%pE~_k>$3c1FnW#^y(QO_P+T#k4es{Fw@?)fqa{FmYX8!p0C}8>ncw@qN-4@>iRSmB$?aE5@mZg&HUVH zZ_>d844(~%lvnE?WN%S_knQBV|1W*=)EYld#E7nDQWGxAfvGarp0IuFN21nQZ$ly5 zKTgRxRtpt2#ejKk0t6Z4<$sXP$o?arD4q0@yp;Nf*N-<0Lcx1Bkg^xY7vIHcihoi# zmvn#xPFlhXq>AeCw`zWY{AgyyF+j=7rg&p4DL5cL!Sw4)2v1hF`~j`(p2I#f1B4utVl{~+TAWh(5I^&C-VZT%I*kYpxep(Kr6=2I5{l+Ecd z)=mIC32v(Np1lkJ(yYCmeEq+xzJH}3;{G6G{DbTPbVPZ8nA`f#Wq~86AW{neS0?uM zoS{(v4MlFPnhV9;np4<=0YF1eGf60MO_ij%>`GdI17rPgEX4mg^>rv7SQdc`m_SQd z-R{f0-)u7I^4}aia433WDE&{9l5b4}FpIOzI0>R{15lZWaxDKB)AHx${P{9&ynC8A zRE*${N1lv;1#5!Zp7?+Tns&$TPZp>7-8Et` z)(>EaC0PcWK=gnTdaZU8qW${&#pThXs556%L!rJ0%N=Ai943FRPzCS_2@niS-YKdW z63)ZX7;oQ}Fqu^3!Fs*`$26kCTF4 zfAdTu2?w~kp%qB`RBhl#cmAm@oM?*!k12~GuZgkH!o_}Ji@$btFqSzj_<01L!b)Ml zog2?EnVD~85Uc+vGUm#OQAnBQU#FmQJhpN?79f+h=?+BLh4rILq7A}SPTwZ2B3F?> zJGl$$Fijjsy$~msnnK5JI}23?tSh*J?;jD%{o>{z&Mjt5V=*f`b(y#*-27y zz5OmF4+(4OVMW^~OmX(b&Vx4ZQEQs6PV+Ih7+mfu+O~gr@$JvGu}aTjj8d9mGTZq+ z-K|MS%Nzz^4igjGvw4T-I&lfqn;d_03;w|m@2wJYkl8*!A51!UULJ}JdFgYXj+&zA zMZWi5ihV}_m%%y`+^!@p!Lhq@Kf)P58Um-RZK78UV-S@%i#RBf6wmak3nFT)fnCSavL zwPtc8^M(I~!`FtT;lx54zCMw(yNh=kdg^;xG^^<&YxnHx%4I1VRHj8m?b#WB1pb}9 zy0-wyLHcF%e?l2Fwsq@%SWx!3Jxmc)xLtkkmvi@*e_9#1PCzhi0ig6WUM@;1@_E(3 zQe5o88Czbl7xSl-mU};S^reWt>cAOtZaVoDQ&z20c17a0e^~RcccDBj*-~w{CnU~K z!)3nyRhfEU_PbdxJASZUkvb#Yh(*UhSyO~DTK&0 zt{IbM%&TaLheGZF@%7yOOq-8C-i zR)uO`oPRuq{4Ia)J4gH)zYqfSyINTso15~TjH5$-^o=w#Kgtj6#wnzK0~$&3%kDbf z(Znvkh67gp$N7qr#TBHmIyXcSt&it6Y&x#>S0NX==(g&dW=tHcRrno0uy)K6b#8{! zZD>X4c1+p9B595(>z7njuD=%H3pLf=LrMK4Zmc9R#T7xy)#D+x$pa%DvFCR%?c+k{ zZf^4J4J}qm&4~PYIzS*h%M_FdZ{DR6CCpkw!IwI>0eYm;RvK5a=`n_(OKnCFPF{qs zMYse`bGS8|qb^g3GX3OoN{tKWt_qGAvC-UWJR>E2*(a*(g?WJj)w6Arzbx3_5O50F zflVqthy$$y{&F?A^vSEw50ZD3BW1WNDU72htZzd$2J^idyKt)+#CfNM{6bez9l6*a zLmsqtETwUZJ0WvYZy~Bf(*5O_IA+|23Y8D0OKH9tBxBZl^jeosgp>#~u>$StJvWNE zgq9GK+LHY}hom4|ye4i_dD;Qx<1XxcEFIQ$akg=ZRV_EfdPu#Z;=zWX;34w}vw(Ro zrbk4bu{}|lv$|-Uxhl5FAy!@e{ES-dOyS@Faib+S+c8DVLR`u&kH2|OciAx$m#%pC z<71qJuZw=}&Y#Pcs@t4*2Dspy4m^%r83W9KVs!P7>smFq6kg9NR0bL4*~13>a5j0W zf)pY{C>{P_%6fKBjQ+;+v6+iHnv6BqWCaJCbaxg#OTB8bf12`f~RTVIBCJYP0G8-X1yN^pcG@PyU*4lVpp-ysl zNd`R6lK-6k%cPR}CQojsrMo_IS6JUM0G$4b-xmd15=r*cg0d<0-mU2?hH2JLZ`t42 zMeR$T)KLub-rUsxE0JyIK4hr_6(Hqdyal1pc7B>ROiKFO>?Z7YvF(Z0IQ?p*}FeXlEK~tw<;R)R+^)SM?@?1aw+6-xx*6lm-0QC|qd+Kq*taX?q zib1{9p`(qgu}_0gKGtrvJo763bHqy?gYR31-yxw*QIe4PJ_dg)C4`;)PW}cJRxR;$+`nCUhn>L z)<1Ag=Lv|OtY3sN4bLBx`*LT!=4*D zrk-%VH=+Wf+x|>>W%uRuY{56`Qz>O*AHBn9WE;g({DprkG;2Dh*SG29N{{k0TwR0w7vj*5-sKY}S)FrzJd-Bxbzivr zH;(6DROMft*?;{*Gz``OhsTn%5_D&UM75|7^wQ-F_xmAW^kG;{Y;mfXzkg`J3wL(F zYfQ~H?j!I|RHqN>|^?<>u)6c0S$wU(x{|$+0fukWqU_>8JUu5Mr|Gi&OTqi=#dd z!=d}@Sf!d*W!RESv_eDjuV)dxgz*Z~73YQYu)Fzh^2-t`?ov3MVCQZ{&Tj;(EkP0} zfQ|$si#?BAfd&Z&iQYjPLH8C7!*>9B@98}X_+WGM*CK9)vv9w}^B>UW)SS5Vx#J4bj z3653gQdXN-mDHggG{N2~uG&+yijcO-1gN2INEjhdhdtaoC+p5XR0S}54ZU|L>_OVR zq6f?sN~45ZlSnw4kA>cl)TJnUP# zc2Y!r>cy9*Wn%J#rz%NXRf*DTTx_?|c{QioM7_>(QI$@<&Ifwm*Z4~7!&~ez6W?{6 zK+mq(5)Mj7^@*~ZQu^(MNLIz<7Z(&*IACelVYlYK+C__0EgAM+D@Q$`x zWs&9Xfy@7>|f&hx~ zj1@?urse@Zp-vga)fObb6}hh~9(!W+aOk#h!@b4imVxV~#Kn75js@GMOr==33+j1G z$Lwv;Em-wc8^c&F_OXy5?&?c-CCkwYnL^Efkn!$xof3o@y&X@$-qJ1a7M> z!|Xm0&0-fC&!?l3Es@XG)o!mxo&-!JyS8vbWk*=2jsY4%y3vS?I=AAcwW|JAAL`+oCa zU#F?$W`}62f+HTOj1HYCZ_EXEJwzg{avtpjJG?~#6F3qQaA}?B z8q3U%{`?8UdN)aD-H%~{E=>mgQ|ev{A&M$Pj7nj}RnNkT-O@S+Gg_ zmCa!*{cNv7=i@lHB5`nn+PJU4_emhVNJ&I!@JO8j)ADL$}1bYSh{A#`V|v z@7F~7?+?)B%j9SJ{pHyHJ;@)m%11O;yO5l4T1TmT0=03KNS}&E;kqmNQ^uj^5E&Ch7oE<<-Oow!+-te}gy2JXTY_M+o!S%F_sL@$k)AEQoFFAN(ff`%O{FFGu)c6gD7=6;g*VC; zbkr4UH^0?mUE?HlBHC;jFt5 zJN}4&)>lBVuE538GGIIDK|2f-+5aLb>TKT6jSpE_MzaIoR1gmKBZxB#Z{KnXJ%pRy zws-f;XjElr!SV>QC*5H!ZmjB;Rs8WOP6T&*#KZU_CLn?D@u&Omgb8I@HrwCO&8leo zvwO+K{_&l!DlE-53qpI)0ruo`6 zjI5t8n#LUzr%&1GzD9$CW8 z^e(7Dy|UJCmCXY9YJn-MF%`txmKxT^_u*Lj%n@_fp`c63k13a&lmGPd{>N(ZTV=wW z@{MP}J)#Iyo%x5Jq9Yjr@oPK~zI(}EGoSjE}c0+rFWt*{4VlY8SZ;01g5%AqfabZ%Rp};~8BQ$opGQd-l$VvfI)P z%>*O(xiLi_f&n2Wpx_DPc*?!aQNvd8)>cdFt~o8FIT9?42w&ks*WXWb?jj?{4KF2qa46O*qjt7@isyo{5hvd4@Bzd@mhZdcm)YH`)a6Q^ zjoZ21j?q`;DjREJX&r)L<}mS2vaRkVA0;le&D4syM%8jDMNB!QOofY? z&LtqHBHW@e9k_9(=f9RtZ?E0< zP(0B&~n@>Yb*$v{DD0{i?0RSV+Y=d{wfu{LsH@YkBA&g(9+a%Sr zN_3*(`d%)%8LGv>_AtpEOKUuNVliHBXIXN>K1aV!JtkdDJ)ZbFot@;R%h{fCDC%5DF9pO4>vy8z$He%@KKv@P9S zW$8Q@Y1`JCXkY@`VXI(!^(tqmC%sz>=Vh*5Xq@RLm`3fIRho{qWq{f|V&7w{qQYFx z;_7F=S(2=lR5iqN*+~1G&82fT(i7l>(Zumbq~}1F@dPultKrnUIUKJ5lD$!x@l&76 zn|1DG>KD?1FZNoV*~Uv28*gYQ6;MPLT%x!{M&^CtuZ+!q(!q)+4B$?OgI`6@%K0OA zsqCgTLX)g=a2%7X|n3JU#q%2i&41@}FG7f4!^te@6IED8~OEH_KyB z6CW^c`ZLeRia*$N%hx~Ji5xX6ruEz?`kWNIzmC7%?T)Wf7?{+&iMBqcUCv#<#10hJtGGXAZfKIr8xl~^7@3?4z*02+Gm%; z2h^QDwCQn#dV)LL{4=ggaSCb43fh%1|`bq zkJ9N0Gg(=osrJxQv9DkV0kWg*=W*^tw8~pAA1rs^W>EQ-?X#IOl=tLNuNDLQn{QXW z*7V+w#bFiXfwtI&_fEn)YfQhGETeWB)gP2J->tEBG1s0gW>o-2$-U{soL099<5hRA z`G9(IhjL`}+Ew(4wHxY8x zEt;G}B#vF)q7l#DF>zce4oPpcAE!SuN#AaX)GLGouBuwYPFNFYF zn5aFhO-=ovQnmL=X?SlSkbmZ@!@;H-6Nq}{_?~lc)hdbWOR7TXO6JyQ$&duS3{BjA zW!2+q_7-uk<);aj{1bg+1t@>&_I}ws0q+Z1qrhLXtUaz!4ARhtrI7&!B!vt28%XUs?O4BPnn}M{)&Ih?EyL_AAuAI4n(j znz&4C!tIxxSTn-}3hjnS^q{ElzID)@-JrhscE-dc1g0t!mbXG(L*_G-0`!Tj=+ zpwo^ZDNf=i;^=f>GvR6Aw&SIylf;&W+=XGYDUA1I1oJiWUqMf_a8lUu(bW;|Koo{q zfMBJGa%-j%aptR(n9DbwWdAxPUCkDhSlO)n3YrL%yU3zEHct~CKu*&^jTtGNKdYbq$}AZCGT0oFp+{#K23(TQ9#=~*s`Sb5 z1{ERR;L2Mxg(a|cBQRFT^Vh&N8osdIdb>`t01)uk`e5Xn@E6@^n*|swFr&=r79{-C zv?)TljRJfyQZ+d=IN@*airngV>d6dNn-dKfbyS}P*Lg3?nHCH#jXiRBS%{FEO#EzbKfQLNKxN$BfFUPg%KxJ8XV*#9f4i`nPl^jp3lrN zS>Tx(F&}&LIxq$|1sOVmQ&ziwGbx_pB;JMmCIYlj zCZbAQA~wzXgcdso+ctDe&u3B3vlq+o6za2I@*-7!g7F30?JNvnH?xUpay7TMPcGin z0(<73Ig0R7GnLdXonB7_$}anvISFh=c=qT9aoD-nES`^(9avbOXpy+A&7*^#lE6C( znWicwKGn2Mqh6YA#``aeDSJB7F7y%EIX(xo9NO^!JjH?23WAP!)h#4*tuhugp+vRn z??B0L25Lp-1#RbvGb^ktl3bAJIKu@=_(6@4Hty_OU^REbT3fibi&WvI2tuzMn<3P75OFj70CrMwV{t7w%}&s<}yU*O&9p~3>~92E42-}*bTKW zpBinmm1z>g2+j6z0_VnvyaRw)yFOkzID67ZlUuBYjy-Qe_ZX4(2bn{fTx)Ng01zS| zAF1?SxYzCqJz^JLW(WN!d70E-cgchD;wV z@5_=pE=!qMvms`HpJPrzU=Ohiw^-Pk5JAw?BT7m{s_CBIPt&Y_F7Q}p?I;5aN^(cE z`(Vuk99A}-)U#V!shFg*@>ss0)I5&RK?Sw}^wY#R{u42T=_n$F5Ei6GC(4n%#`j!} zY+Gj~Q;V=n2&*Jq7}O&qXtjjz@9=zm_rwEWUz#)~_VyA0_E4;U;u_*%c{;bp81 zHnE14RGV&DXl^)sRik@qRmXKac7!oVgLmTl#%cZHdBAi-2xdrWrZ!OvJx~w+c?w+{0+;a=cEL42)%^P zO*J1zR~&x$ScG6Rr5ZlAp_jGM-s#lp5HDtTzMK4@POB0C$USwLgn)+qr~QSUf138c z?@(!WtccRv>C>Bj{pG~WLpBtQ@OK2^da$ZVfiqrSu9!WSMcuYWOTR}S`qGpD5;_Wa zK?0IB0CWK^U-NB)B@r1;uOkQ(V1YLUIYVaHZRx@E(6?vf}c!BSx*{y79pA zo7Q<7g&|3IEcD^_iW%;>u+9baX+SGZqTT9#vpZB8&{B`y3)hV8aiz$+JToffU>V1u`>MU z+DhAMW5FbB-1)Byu5>OvV%M*pe=7Gn-H>rhRWBI{l7u)Mj7a30xAw#xb;&;x!-UbQ zGdRXE;K=kgEIuBDxdOMZR|_2d9kL&tL}N@wOXf<42X;fK%1S~RV^tM&{3B$f{eFeNQP? zMqYukp~A9J81)AUBc{u>c>;q17d^WyQhnTbX;Hy#rvQO3`#i*xS$N4+99dv4Tq2#^ zBWaRfs4wifY8&be^o)Tb<5Yr_R(eP7`H48|sCxt_^{_Dj-6LL~Qbh^{iw#0Cn?$ri zx-7CBlA$wG+B(IL=<(b3ClwDin07|4wTUUOvwJ-jc622a5(0eeE3M3Sk3CJcDdH;4 zuacKvNz+svil2E=7UI5G%Y%WAK-mcJa*&{_VMp#Pa@ffl8mE6~6Zg68bShJsz(I(s zBAgD-W%rkZFV!5MygOEBY?Z&na^(q=eOxeq9HAJmiI*W6b{C_qxSg}L#;5rUyG2}t zk+*nQ%USeYsOhxAq^-v>64>-kY>=ZBZXm0v4Y*(lLm;SKEjvvVg>f3ziwW03O-);;dc5sSWv-b&`;?+>u)mU#XPn0d3sy%U2sQ$z*s ze8Y?Ht~VD4auysO#N$@6TuH=Hs89Kp^h|AWfPL42mAYn&AFQUs@=t=> zi;w}>A-HHjmQPeukg(*L$F+pBq*>06ZkfKf9>83n!?==3VJE*;Tt2_I2 z^p*Pk67ajxEr-K6t`s++p3U1wkfs}tm`5@aUU6k}hjymZ?%z?T>mENFPU`ca*9)W~ z_*i3t{6_Ot&giiL)p=ejHyak5NmiOpdou>lCM}jKW0C{;^NT$XHr^eZ)+IGr5llP$ za=bO8#PU3Gd$K)zrh|Jt!h^RSc=nt#F)@EHlaMaGp@ip3cUKVdp^{u4Ds&bWaDVI(;TD04;DJ>@%28R(d~;);aTJkl5SZ!HcxFUdecI%s(-8mV@=rq?ECnhn zi-!{pbb3c~{?ssjY<*X) zCHJ9sqy6LZsM7}xDF<^L3!^_%0~-nBgb2VpyRDgh*1Tu{5|Wr%_@;h)0Hi^8d42DD zRbJafbnI3SKB~pf7n+*X>*FraRIJ@0pE@}BmAYv@M2ZHtn$1GYBaYzn>hd0>&ppK} zuT3B3xzO5Wb8+OwYHde62std})A_36E)(qFN(v^{6Z<+_rh=={F|Ni%lRiAjYWr@O zqay4_T2#nt!S|n^ER2vHf?4F~J14%d8znz!#K(O^AB82&e@4XHy^<2Mi24<;0)6Py zsW~o8vT^$SVnUF~xwsX@I>p}8GsB^+G3-`)BuCcgiOzL`w?31=$%i@z5kd(H)^Ovi z>oLd_+4NSOhoo#z^%6k=kJQ91HQ|DUXiBDMC zW?o>=Cfp6oY&d;nVh5JBtB`uUpzpP8Y4N$F zNgTKI`6!@FKI#^KQn(v6nSnF1_Eio63_ziC1rFiGJOZ8UG#)?JKdp(~+QBipS}{0c zVdxADDS~h>aJGTfP;m9*N4u0>RO&l+VnBg69jTW;Q*C1gO zQiChc7VW;Y#`++=8ALE;hGj!@53fNYQi;{J0)^Aq$WMZT4%C!a>QqFj z=gOv6D~14q@UbWlq2!lDlugY-Myk->k(vhGqd0T^DscsL>trH;?)C2lU|*@h!FX8K zdXHM*X5i14FRhQ<*%}%%gRElh@-<>h-jj3^LF~I80CJ6I`@|VVJUVZ#%-H+^<%nvg zr!`26jsH=jwvs%Z|7^oO@CceDdsaYLRnIY)l!I1%^x`N!vr$m*QhSqLG`;uZ2L7Gv zLP8p%m~e#Mdx94NK@Kf^`?(J_hv5197wC$jST*=OZ2RKES7ruDovxTTrc>N1FvGRc z=SW<~f@^DzR*wnSAy=}lA%um~DwIT{9XxTTS4bdF!W0k1!m<%Cdx5&-TG@6l_ni|h z_lrOCBP*j>$#GI!JeE0VKSSJpoGSpa#}J?ui_R0DE0=aX*}>7?FkdH zq%WAeSt^K$mP%EpYhxgB>(`x|D-UFuJ`&;pG9s{yZX=v{oTz00%55)hv()W}x|_|N zHpsqY9q;f3TxF?@qXv%DWs6xI?->RAX`9FwmT$K-j@c;f>##-U^EE%GAq7Bcck$Fz ztOUK`Tl8soz8f~0uCS%*#a=z?F;zhd4S>!<6-ff*#^{w7c2ZU?PS7(6XiiODU)&0UX6>e{Bz9tSB?<>eF@U6AJ>-u)BV#$24 z01V#xao3&fXi`hL(R^Fnw{((8^PKYM zu{8IcDEp}o44mz8QM*65t|-f)+75`(GFSK6zTD=};Jo2(EBvYwUD>+lM!M$ZECxe&ai8)!$K(Sd}q&VlbD7*gNsvHNp|o z&3$V9m9|@Mw-*?+YJW6+((&pFuUT+AEuizh$w=gC<}6LHEvl@(6)%?y6#hAvrV|;&Xma;rx1&w=(rWnpW}Ji`&LNGUjD@m8a+KTLN02!~@0^YsG=0Q+r8Di-ho>*` zx=00DC`0Ui4tG57vLy_mwIsBum>S*DFeMu_PW{dEyCw)D>%H7rz69vlFX?oSY)ULvILTrNe#14tX+kt6P8H|dtIFKgj+kI;Vya+!i*#W7 zm3mMwv-yBnNnD@k;YVbysNt94vl-YM=YeL$^8%Pu3l-e`TSea2Ba%FMwRoPGa}!as zfs>mGTqG+OrdN};D5yFiTP#9Lhr{&ar>d;9p%-J>gC(9%DQvPMkW8@Bf<_R} z;O6rtkh%j4b$vyoP~TUN#lUsx_yQn-mMU8DGMn}Q-h3y8C{XG&PTQD}6Ap3zB3q}| zX6abh;&XNlxgc0d2qSFxRBTUJg23PhP06p{h0DJhHbbFrTzyo#5!ArKky}k*BN$b? z7-rCP>rVNYWd#m^uQ)KJSirRM<<~=J%lq@bD|f9VVH1EVU1?2%htu(HB}8|R|E{%o zch^U?nS9zs)H?eX<>EmY6KIOqUSk4LRdRo{qKE~HW0(ZEK)91LhbfHRud{%TPgAV0 z7L$av{I`I#M5okghv;6=;#xw60BjaLJFGym~{ zJ*Lb9L6SX0br6~l&6<<9Z%>|;nTOA!-cgU(Ii;pwKpB=VFnU@5mhiM}r=vUzu}ets za^?un!Bzvnm?w(D-;*_+G(#VmCQF@{4?w*HZoj$lFm0SdUQ@z|950u%*bV8QPdX1< z<*PsS&kPkkZCSmJMLS@Qk{Za+sSq)9?$Ad~)v$?<@H8-UdRceqfuSTdcdwalGL^Om zZ{QUfqQPL=N?hdvbWD6#Z^%(DKcNzmFOx?(_4T2LQ}E5#Od+T`e@ESEVpq}D@6s|@ z#=Vp0d6(kqCmoF9S2*H8wBrPJTY0gb>_@&FFhX=I6O?^Q+vk!C$-^gd}QBv~~+%asUnpNGHjY zP35-{j7{7!b<5%RKl^65=lpVHvZFs&qtt9&QNtbd(hPTdyn`lLfp@ce?&A8j1L384 z#Vr*(cSC(8M3^$o+AdJRH-aVtzF%S7>>~a~d4lp8jf=nvRn6O@_**|NE(Ut9A#1-# z-~z_(tr(#cM0}t^ionTk@3=MHciY(DTao^Lg&n#g>2SfJ@_gD7`XR|G_~j@;fU{^p z>#YgWJ8=4>GHt8^fJ|FvGWcozI+K#Z!!n6^wN?Li8w9XDTaWRQpq-32D!Q zt0_&>Xqsv`$8V)m*9Z{F!eT2ki0g=HA6#};dIc`fpX29U%GDmKdh6BK(6co$!WJao0nx<_s9l${Qvg9M*AG2xca7m-Kl#e}FEF#!$KLJJElL&e8nf4@|a1UjgOnBURu9zVzNy1H=K-|s^Zkq zE;#?X;yxK}Jf#HBOxzgAjlI&VH;MS@SMB`m60Q2nq(+q658R1!9zDrpRvm$?l4Bwa6_5G3!@!l5+3(m$x5E~MRnF?kkG;~1W?B^Wmw%pu5$1=s$ zY*Np$@`bitKX(SPHqhq{xkEuFtw+yrQ{_s$*yB|v?(;p;_5%zab+pp}ypA`7d@YUP zX1{v8n9{0K#qYl%blMHQ4Y^%PLe%8vOQNM{Vk2w2oIc#LD(5K-E56mh6YH&Xj^2Bb zU-@RR^&L5$;Wtg+Ywe=h`3IwGu(glLQVw#aVzN@+;90}A_&^jJ!FBH2C&9`7K6R9A zalyv8Dx;_iaXinbMvJVbP-#Wg1keq_#&nlDqfnn(_1r)Yket z5g~j=8w4_XbCM@D;dzRuEf!?wE>nE(PZ5g$)C0sT#NgEUXVJ%->gih)E_4AWG`HHs zjEtBzUirsvn!2N(mvZ(Lxa7OwgH|&OaHx|7im)-BhOdPx&oQ-w-WU_DIYea80 zM$w5m(|l_uO9EX;7|cioNQJ*4>fx?G#c9~LP7Zhdy&dD88UB2#>&)r`U`P#f7}A<_ z8vI{I4-rewiy!xQHFu(N)s~cLlHV$R<%^G(+^~F9p}eb1M%61YX`#_}y&QnTdhxPy zD{EJXQ(PinjseqEc*p**NoC{H?6PRSw z=m^)FQL9i0*8NOfURF^gH$X;4f3(QG_(5C6_Bn-1WuG>)bvlrGW4F`&s{M#8Q65qp zFF6&LV-{7xq^i!KGP3io-J9okI;^`;`+Eg>HEx2%PTidpg=tb0o|Z1i zLEj3=QL4vNhgyQ;&roY?U|Z^sZpInG*VZ@J+4IIIor--8?XC}XB%gb5Y8wm(4@8vY zx*`*9Y5YPBupTR998ZvJc9>pwAObpfbJPw`uh}4Mn_6}09(T5KCXrZuhk)MQ|6IWF zXQZS)+3X|FWwn{XEfdA^)r1E(74MSUPzp<0$7%FONBdWWoI7op0`~Gs;cbFGnfzMZ zz2kKe+Ng_;RhJk*lNnoz+b5RUCssp>0|RAxW{dA1PqN%P3(*f$5W4Q2G{E{J=}8^# z{l#F({mkBroQf=Q&)ya%N(B*uQymmpl#j>?{3%eM+TZD1y&if;P5Ih{enP&?eGUZm zLuWQ4>xqUps;lWgR1UGxOIk*Wmm~U&BubepcV1a(N7EEwKor<`+G6(p=&)9s0t(`G z=94S@O58%nqvW6QKL*QTh!_YKEWNO*Id>4)?B~o?dE;{^Q_gIf`d3kPBbSLa)no`sCKx@$u9XSZ7rShrZ z#dZ5H5-%@dR*{*3tw8Tv^-1^a`Q#}x@0sf_2039#kYAUrVp5XH1rVMW z0D`9!f)D8rG63KkZ?&~wO*#a;9Y4^4!qBPIBK{wb;K>~y1@K>_dO{dia$sqYW8r|U zHDz(7%iNQ>xc1bBor0js#cd%X zRc-7yjch~;vHu*m{STu29qEW@DBq5}^ZTiqCJQS=Jf^EH|61H-Xr;O7f@msJ7wQ&W z|AFb{8;90H>7sjls`!?qfX&g{Bib$^&(ld(Gh(9}a)x6vl@~5Oe!-ihHZ7~kLx?>f zG>}Vd4xov`2wh0+N4vOaNvsw{HsgnW=HC<2XOGC=vnnLU#>EH8yDug7l~&;gxo%sbjzDTf%6Eck@gZ-TqYB=+)L z$#%2%6AHGyb~-*U`AYk|zZ_7RaL4i~cjZs^>kUh|>;vV>jGgzPxBN3h5#qkgqD!&@a=Gi$$QXNSr8*M!RI zZC?GoZa>++^H=1k%+W@_x!99Y4tKX2ud5EL@VEPSE2wFdFM%K3X#pE%yBgNlpqe#dxEzN^(oj*q|X!n&Yk^W#cO zfWl?=@nU5{5Sm%gDP`4s>t~C9+FVvT#~a#$*0k=X|BJo%4r?-7--WRkL_|OZ1f(k+ z>CK2p@4bUa4OP0KMMXhCL_m7)z4scWNloaHt_VRuKoU@cA%06|&)zfY?3vl;x6k*T z>pOqGxgdGhT2Hy3vYs0ytaKVR%he!00|bH#Wc^`aB(}pc(W(@(pumoZhT|_1vcHfx zHM`f*i^t`1RFX~(@3(tVXUvpGbe_<{$lV)w7mByodwh^j*D51Vs_83J| zlFmHqoNnD%EjhB))wFJiW4MNkUUXZk-XrT7U)e$ch zxt(I=U{>${B%A0zEhCQn_gs_GKiN5JQC#GJM14Gj+rOg}N=X`^+&7(C)1~*R(#v@3 zhRkI1vKUArs1rJH@hRJp{Jj`wrJYOMWjqs8N~eWwj41laXNz?G50v?&{E^zTk|8_z zFKi2g1^Kwxgj|?jJ8& z#|dwbF6b9rc(i@yKc!mwGmVv6UTOwS5I)9eR>JrMP0Q=UnLczYNu}@NG)<;TTAYc+ zv@HTp?eC>2tNHYbE(vcuV8rm2XX7cglC`jGb!pM2my%49Rm_y~g~BF`RGfx&s|s11 z-hs*Me~rFe9897WB`U#%LGt1P=zcCtNhjX&A&Rv*ftwM@8hBA&Uypx^>vy}}=f8)) z%bQ*VhN_MUC7W3(Vu>M+#U?KY&v|g;R5&S^0v(^-I`VG4jS4!9@9&|sr^V!73BQtz zp__KT@`XfsWcy`_D!D(D^tl@i4}WN#OMrg9q{~hFWAEpsUi$zeQ@m8{*9a!#bevrt zj>;~KW8(;?XvmwDu)9J4{ro8j!*G882)=k8A(i4|ISs*XZbDH_`zTn$E|%?})QC!8 zW={6knK|o-#h1N58$t9FZTr3Zushs}^RC3KGxkp)vOHleT^KSGy-}VWlh4;1?jh&Z zS`S4JDka$#9WQ(r*dmxV^DC>)o1JupP#`qul3Ex6K_0j$70cn z<2*us^t9Ne_*I|Ci*=XH_+G3IIt)K)iCjZYCm#@r&l5c3sHj4zt!l5t@fHcDH;KDi z*&JD1@9%J>(%4y(>7~#ejoLV@0r|JgRDUFu-)Y=HW!Iu*S?{u=29wzo-avN3`ZPXn zUOS4-edr1P9?#g%|#-^@ah$O*w}(l38;RUM|Zys4`y;MeMKM;>I|6` zpY}Q^${jzp3;b#LPu>{$Rm{nNFSQti$S? zzCc#km3j3AN}uR5t=N}_=#vu|}AX?0dr z3SeBiyPZ+?@u&ETPt-@BO%1FdvyNzN8>ppTdoZU+D4b}iUiKc(L#r2N8Lv3!yq%~& zMvQfJu``Oh#LhTZ;<-@blPe_%<{JDj_>;}Q;!nQm&{sCXbUb_6|0q%CH`DXyyo+G# zQZ|Y1bO4?qK!pZ#3f?=05`iOW9d=V7iSfp!2E0vPv-d}4rh-1#N1xG9>mk};>;rI9 z`0v!#@NpBJiRPg3YL6zz zQJ}XVb}`O_wQi*8Q+G^`+=|*KG#DIZM-7Cu6}WLx570R|k-o(eW6=vCJPp(W1Ix_n z3CT2HvBdAep8{OKf66~|5m5PNO|LoUUA0khF+EUuqli>W8)QTRKMK7671#wo%{K5g-n4r2up;TAz>q-2o;kEFEKcx=AL+& zb!2AFy@N4#KZQ(nL~_FB`_i0h{L1+pH0NJX-by_c8^=yGx(S9aO@DZ538ppv86DBL zh5e|w3Gn5U2&`-RXInFPXEw_1*B$?UC&U$!w_APs=aV>q<{H6;R?x+jyOhbTo>?_IoP&T-lQa_Gd|qu#?5Zx$!nA^ zXqd#vAHZDOytpT;|3eMo2g{&`FU~_g3;>TDs*twu{VDwV_MrPBZ>anWiEzd+7%EpX zvtq6(;i+T)q1~sty17-5Kh-Yna;>$VI_H4In58o|&>ZhLIA+CFS$OzX9tX4?udKE`*>l$%x>k-_SAma2;DnZTQC zBNI=zddQdA!B2~Gcr7O01hY@6>*E7%t43!T7axPh@O^~!vlu^_$nbE`s)BZ zvtn=jUhIFA#IG&{WOW{Me6<)~eMs*g`;dPrDYDI|lF*1_s4i2&XO{K&&$fKYm$AMuUyL79df%4-w=5iDu$ibkp^)(se1($a0~0?}?;v^{Zi1 zFj876J?@Mrmr`Q@1L9c*uX?O)wrF(5<2M6Eg*)eRU+EIJVDFanGsU&@#Y8`a6Fm%w zB}4F@X2#v(l+Oe8^T`J3DQVZ?1IgEG#`Lj6h9!>2gN{seDQz3Qm@YT{c=%uM{ZIM) z(NO+1F^DI6n8HLx0(jT7aVPUL@DcjD!^kk3xK*2*EkjdTeK8!o5NKdZ1l|uvhpSf_ z+3*VR=@)cl(5*Z)t=XXHIcr%u3=nvY?16ih6)mg3*27v?S=>?}9J$ z{XJ&nls80oPWzjLc$`APYRY;nWz>Nf8tpKyvGfy57Pz4KtxiB;*f|C7Z4?|${9f*U zt5Frz11s5(crT%erBn9n&KJixc!zJ@$=snE)26X+eU2p0!e^r@R{5uidKL~uk73v$ zOAMX)8G-KX662J-Ia>%B$-*R>@L<_YVSua5 z%mF9K0~CfKF%NIEzJLw*7^hzB424o@b*&|LMW%0zY29OBbFBP`OimVog%jNcK@;}J zrSjbmh#khX)ZZ@h*9}Mo#zhvs7!UTQog6Qbh+)Kc00QS_;$*zV)rxc?cs8(1#`|-&c}7ZBVS^yb_6BPIR@vMVevP z>&m<$C~98Tay;640{!SLn!95Ydt@>i@`#1@(OZ1AHQWr#7+c%ws5Y_rZmoEQm!WU> z5(CXueX{3MBIa(F1AvPnth*mw$IAf8Sfi*ftf=*O0X7kIJ5;lUC6T&1_VhebCAv!}T<{ zb>X8tA;T;%+E8YO-*{x)CW=`1RE@$=}4UX<$%6sx;BYQ&FT2Z;1EO4C;e#< z9Rf#!oX%nf4OFC-6pqcrf(0knpd|B|Xn{KHhuH{p1xwk++d!3R?@LR$r`U53Va z%;q+-9^vmcy>x-01_^GN_G?IiSw@)8(p0_X?4&j#i=OB`l1R6H9?x4v&P zbrR-wzOXNyTEc+un_EW-z>)IsMIr&bkvW=}jT`;JX%5~ex`T!%SyKR51JHy7C(wlO zYLnn4H3_B&bAKUOa9_UlJvQpIz=iIP$0g#sZpO>q9dpjFs$UQf+-YyZzNdP$nYQ&a zTk(V#pteNaG^C?jKOs!!b*1`Bq-B;V9W?glBf9Y5CseNr8THF77Q4_-?Z#~+0+6i? z`EGgBb6qy2bAvwT>qM2KdAblHo-TZ!>sW4iZ>IW)INgJF;6y&E;8@M|z?j7i|6UCH zU6+juVb|;G>+a6qllpj?z!G)%&VKJ_6Z45g>(N?AQbq~9w%Yz_y);QlGIi1h*V49X zo&d4Di!Ap2TUlwWcGm}!|9+11`;&i?j&G+oQB#x}%$4rr&6SG#cxoy};dZuxffs#y zJV;O!;{L5cut>5QWx~!C@)k;(Kr5#P(Df8)yie}4O{#z1LW$tLH?ebKts+54-!l)a z>)hEolT!Lmh=>j{PtQsTD5~BPR8UWLr>`E1+S*2kF4mkkt}Vznep78H5RP*kq{U?W zosITSzEYq2QiG?YscU@3*#Gn(>A|QQr@5LRID`#(f)ZEYjTuMUE_A%tAgwDOJ_)(c z`@`v8`cCa>!Gz8UT(|vNkLK9^%V2oxU-1(hxOhSq?ebpr24b78jwi(z*n{CE4SYBR3&p(+{=sU3XwVNg%f0UE*=mj+NYW3%oO`!f zVp%(pBQQN{G{>CtjQUgTn;Ru1R}@I$7uKC}CzzX?aPm>b4pZtOheK_0-;VmOP)D?c zsp1r$8|6eRdy?eucj zpt(~nSgw?ex@SiN#wg#3Q{l%C3SZ5*v_&<|m@LNK^f-4hfD>m7Xq!Fjv1S{YK05)D zdNqwn_T@RLt$uzQ#al_PbHQ0#Tlm-Wiv!sM3^wB}qa|xE^l-W&!z-J4d&*|QPR{gu zH_puYD))08-;H>!x`zKXSsO?+{z6g?9Dx>oW#)%PqQu}xJ6MSMh^H%0NA9Yqh=93@ zcRbXUtCLOZ{0*-6EKjmiBOl8n{$~U&id@>^V;$ubPkjup)EKJ)5BK4bCz_G>_iTm; z7K@+2?994ATLht#77NpBNzHy?$yS$t0yEYhASsdTe&%J{5NCa7Cy)UhU|Z5L=p{4= z7D{1BYyEZlXQ$u^HIB}_x($tvu;KNkXmzc}WHfU!9}+ClN!yH4j!p0(bg}s$2%4|qCeKvU-GuAPg_u8cE8|5 zEzjnC?f3bUU*#K!#WUXl)ZrK(A#b%psawswX({6B*S?T=N?$u&I=Uc)1S3{K zJ6DKI(+D@5vy5AqF84K{vM0+c@s8*CTpZoA@n*y|@-_9eByTeHqJ&q8K~?A`OR!L5 zqTGSLq5~ClPJ=r41nu?>e<9%kaj{Uk0N!MLM5uq3@!CW!-If>q+_)T&^ES=CiLf{>H3+kglc3IvYmyF%Clj7de9)oMrL+*<#K(inuy(|;^qn<# z1(D#=nwhd1yWjZL_~HFVVv6O;Tue+2TawF%LGvU8XFxQii^Nlk?me}W#_=tAXXE|w z{flZ>KAQ%(tMvD)+)ui0EpXSH^3Edmwt@8XA9o5vjz{u+TEeRk1?S+M%N~uAn5f%z(gp!gPo?QG_J_U8Wzu6lA>!E2BbW;C4kXKdcDrEZKj# z!#{eKd&Ibd4ZSfSf>SREOSyAX^x#NrJGcaILNs23Bu%wN>@PEH%i-i!D$q7DEg`fzBhGX5OmMw7MyYF^2mU$mV$ybr7s#x-I z5-u@&EY@tCu#!fB%ewgMJLUL0Amb9llXf2Hr9Lp*(^>FtNyKh&j5O{id-^LP=s{te zd`1GzV6oiHuuT)SrZYp`=bDw?)}P8e>VVD_L{81eAueSSaz+8|r%AdN&c7JC;|FH^ zvNg7}_{i{-#D!k8VYEM0l!b`i?#hi=7V&|q7kNe-MXSWu9r+fliPk4hSsz-9d2Y>*_Xry2NS!}gvktR{r z>QewsQyXQgsi`01_9D|Ju5Qe6wGXfKlre?T+H{>PsKF2l0Fh?`>k01J)TYzBI09+pM$|oo;qW`q-B257Z z08q5Y%drp)?IuxO0aO;<)Jb|H{uY6RB%ZTi;Z zO=4ya${+JHME#4zRKz2jw%aVQ!L?}cp>rFBtCdFx5E62lQ`4XU&pCALe0vN{;9$G`mI zU&uPzMMIWFg+Ig?V=;H%-l?0w&Iee0N&OBTJ$_4h~WTgq1OF+GBe9)b}!faVIvJJSnEKQ$r-a1;`LU?J=OM79}l;kMBXdq6vt?N ztFuOg7WZeZuczw#@*61zM0Xr0DU72CFO1=xkJaFRR%u&=7E3PhHbIy38_8L)l`;Cm6q;6)kA^#{T%_iU*;u zN{keowyQ2zaq&%FpvIqX`L>aNN)G>3ly1KOoH-!T%^m@|SHN}Fg3w!QUkl(8X`lf9 z=nQ;I+#?g@dwv(=rh%EpDL{??460l8gNd(F@TZXYq|SO~oX*6KBq_Z-X6fML_p_!F zO0VS?vO*;JuUk}Q_y?OiO%6z*%V0QDLN~aKWTjV!U9Wwdelokj!Tnmjfp+N9#mea} zwo`{F(o`MCZkTC*#8LK0q=kAE!IB_C_PgD=&bg*WGgGpTUX*46dP%ddI7>T6qpP;( zCBVl2?@-@65N4@K=#s^t8mWHsgXN%cM+3#efdJ@}C?)IaAFvj=($ay_aL4(2x4hvULplxBP0#JS3DiU3K+ zMy5#Tj|=x2F9VRd8CQl7Lhw&(PLxWGlg9}e@|Bf1)l5q&x@TGF_N_ zDMH%R(1h817sq$h_~_D&o#WGoi*|bJfGhyZbfE^v zk-{U@m<4ljHb9;^Q|LW4CW^`>C575B9Wd>tnNQ~OCt*C#3Vtr01M8Dg6JUaxI`~oB zY*x-H$8oa+Rh*kT_BvYlw#%8*DZ59G+pGJM3tZT*A%hkNmIN0IiLTIKT=crG*pe@1{$)RVuG5=?+WIw_ldUN;baR(Hha*7Pqw{0C!-ukGtL!9eF()l11Mbb8e(_Ym0L!)@zD0h3} z(s87M$k7bW{he_)1d{=IC>nJ8jL*S$Wg9{2Q}7Yd4qWe2K&7Y7$D@fxO@76WD#?l* zc43N&9=;4upInkW{{d*hccN(b@T)hg9pus`4A#|FHW*<>-kqMn>(@NC;_7{tcK?B! zv32&Jno%Z@WAp{q)eLTkD)gS;l(JamTkZN8v;n{0!;u2vtqliftbi%!bVLDYEI{g6 z-DLNGx0JVZxW7n)lv-H(ML5@i2SY?77}^jmGz}=#V5P!pl1dGaKa9#(RGUcpG=}Ag zv){xvzqjf+J}P1B(zq`fdlu^rHHS(S-n5PiZ(L|E>CV3~h6H`3;sSUrya%AHoZL}6 zB2zoMbEvs4!sX_P^IVmSq8w%3jDJ@1z8ymJVA#dF4%(NpCC*KtHSSEeYqSSzclAYL z!@@w{)xX6<1Lt@B78l5$dR{xqvU;W&IH*~!;GYIO ze$x83v;3gz&3mxt7E0WaB{mT&)3w(TbGH#=7UN;xJC%R@ zR7wX<5*tJ?!j$u^>3=*+(Q$BrK5&wcmuJNBREff&&A@mA_xV9eq9c-@s8e+(u)8gO0f=%;Ceu+|knD?AwuKqk?o%*TDs?O5+&yL_zF} zC50IO>Y@l8rLcA0aoP)ZZOskpe9FWPsF|c!o(j(PucXw|CE?2#;8+h3XIp6yWIW2@ zNFX3DbOzJ6?xBbryq*yJfCb^{0pgT5Vv9zwQzU&;7qV8qMzg~^y|)~`C%0#q_k*SR z*P8x*Z32V9poppHku^Z~YmiV3J``7#Y zOYQzj!^kq2Wl2s{cNKvTKtY{^R}C0Yaw)V=UZ8bv#)}i~CXPe=Y5E1X2Xow&A6%;2 z$t?3O|6ks&zt^-3whR5+?GiagNI=(=9>6K|H?`_czs_$iqdnZ4?BK%yds;HMCgK8Z8zFlp1o%81% z=WaGeqSZK59;$F&Gvp{16u2G%h{ax+@!`CWOmJ_P3&8yJu${Y4?~#cF$X^_6!SSZRO)>blF-BIIX8RMrOM2X!fu599se#?>-yQUJ|hm%LY)3RwE-nrO%X@cWsF();5w0sw| z_uS%NsU$1C!xVn4j7!8!3XIy-0gg#<|7p) z_O@q4K$~mfq~|=oi;s}nBucPs20@-33M5Rw3+m2)&CDP#IVP2r<7MrqBAz!mawV3O zhLapOX)j<=yHig45!vyh_xiR&0NRQ`&26d(?Tt9T=<@CU)Eg^9CTndLV~YY46XIsC z8wv9Gk-7lQ#JboyM5=byxppZSW-K>FrU z_-=QAAdZ8OXv3=^Y8Uk+>BGhXY5(W$s<;Vg8BK0IRLfSWq0jL6NxkW5aL0z%)qfrnpn#f6tIVC7 zQ?PH6E=$f%q?J@uO>6euyqxoRW9-6Ku7sT#$Qvpo?J!=#3u>Xa*9CUM`-Nrc@B24G z#Ho*cqMiz)i^@O(OCW292gorF#JK{XvJg*HpQR)6&we2p6I{-O z|31fa(ykG*Ob)?nM1F>v1s32Me&#-a1f_7`g8JqYDA&?l8ioi6-cm~p{o!dRYSETv?Aty~v%OI<(uV#z~yH z8+OQ|hATDX@AvXrLQ)V*r+($9rMqIeFp<7#3fYPR2t6Y`-Q0E@T)Y%7cJMV}o=vc< zBV)$x;{{2mJ0BM?i8!>hC@}tN7-?O{q~$o^&Z(RcioT`_Ot%=s8jq~0AIbAw{+HAI zsbyKF=j&sDSD~dc^Bv~Y;x-jVg!hy`nAA4ZCmjygjFUZ&I_jc)W9Pt0bNq0P8pN=X zurX1aP?IRMei*GBsS^F+Y9;AfC4YkAL(!<4@)yq&s&jtRB>WiH!I^N%& zMhMXjB(@51u6tEqGhCx~gDq?GxG)R6q^Ikr4j(|zi%NR{MBw)@R6GqZNTl%MY+~nQ zY9U-D+B_Ta?yjs)hq||B{kha%6+T2u)BJ(a_^LF2KQHeNhBNRXBS?FF0Z5HUiQ*WF z^Lw0F1kT>!&S_VDL<6no^sjzySeySDO-``3m=K(u$311lGkaO2#_wHMNFzl80TzqG z^z)G8l$>0$qNdI39Z6vsF3c!2ut?EF zr}vW!8^JVCQ+x@VSHv4e7p0_?)kwcVmucG3~onN*GCo0<+8r2*Q_&6^G@pL_~+Z%e{C>E z5n>Gfa)WU<9GCP8ppN$8fNk(z1)&~DeDx$_BI+On_t4i={%R?Edakd-roI)Doe4&p zPJ7!f0%tc|-n(w142dIZSDme(8_hM=eyJR6mU)msy%baOZ(i>oa`6|sGhnVn444imwo2ZAv`?-)ukibYa%DG`F($=DFa8)F<-kM}-#U zIK5G@A6&~X0uX9uZftzbt~pKew!yjz7w5&Y>FQGgT=WJih~*u+9^O3(-?aOo9JJfz zr=j=$VLJQ()OPu6*D&uVp;8ISS!UtVR;HDsWr^4CkrqG9in%L9*7cIsIn4?zu$nMc z-PSPVnSRO9nfY1Otl1p2{zhl_H3}9h(tzAo>U)o2)tcj8YRY=?S}kclR-#WqajY~ihdqpSi?qzxAx@2Is zg{IdO%el)k8*5T!JKQLXyH|awZCEMpQ4hEEAI>+XUHXHa{c3@US2NUd3UB{ z=ORZe3+u_6ML9P=$7IKXGk#F*+voMzB7(BXZHRjmQ*Fi${9Xj;Bcytp{5s% zCBgtPTHtl;-cD{>?s0s#qAavd>cqQ}0mS(t5t(Pv;!zg zVYx@h*ShbGw=_>(%qXK=L}48kRv8B^IBQEg<3_IJI@83lbTuelFtCWZbRqJ@x&deG@NJu!_Zy7PC?P6iSZUY z5enCnxT{#Jp-(J!2iNv&ZXP`1^}4emm%nIX6I=EXdBrFa8$5Y-z-?83ypy5r>*QybSneMS!nsEu)%B3#)zWmGbYl1h}l@vmAruqdMYrqU>n=b+@knGX|Z zv`R!Ocskj%g{btX@23U>PmS$U^mTl20-Ym6k&znC(&?cQe4|!^Z@dfVobnKu!jX` zFN}P+ebbO7$!}zX<(hIibF9#!?<>Y#%1tES$GOxmBuW-ITi3tyNaw(aivVNRn;wT- zWm(J_n?Zf8B%vRw=G+Zc?;Jj=?0(=ioqo`9_*)Hx7-HVnXO9g&n27kRPYFeEBrDOI z`GI_u9zB)$sUtHDoG7B3ACB0ST2h`-3ne*eIcEK+f_~q^A$Kr&8V=+!!qJ(!=8Ek@ z+2h=9!HuCPYAR~G2ML7U_CY_a2hqL5T!3C@fJ-#fLP?v@p4F;xPMcq&v8x8s7m4o= zI61!Vi4tJdluV>yZKd?TPNFENz(g-;8TtDS|8D>Pk9xF+tT2=BW~L>Mj;wI=M6WlR zaVzrDm0BMcPn)j||E;aYc?W^9TOq0wI{;3L@)!{8c=kqX=bBGc%LwNe6CXLv#%6AJ znksZ4%qK4qvSRw{Aqh!c=%t;OiMgv1zC1nKOTpc08faGO6!*Ktl>+xWCljYG3*w_} zmL=!e9Yp3-n^gD&dJ1>ALXRG=iBj3$HF;J-)+2PTq>!&*l<6%nLOAk`{fMRZm z2tdwObX)Yo@qLA!T+RpAdYaV09MHPQgcn^p@9w<~dv^q{7L|;`ccPGMYaxxT(csSW z3KCXstl)6`c=aU?%ZKox^guUw+xF(JGMIKD`1?nGec(I9a`A6+R`r|GQYOxP{NP^l zp}KU4JDO>+g!8`Bt@**PCB%7Lbf8<@l!c(f8Q+vRhhC?VnT%IerwmV0@R_r)%x`nd zr&w&ZN)n?Hn1O%yq>m=Lap0;(Uh_KB_RTE_8e+oUXBXyjmJ}WBU9Sh|k$@4dtLs1A!f`On0oULD}69t*P93NyEbW<&}x;iaY zwT?%xPQdgR3~k49qErjcaS^U=Zly>KK{z4HJW_~U*<2{?_L-&t1fTpqFS-Yqt_af3 z!Ld)TLw+ym|3}%8pSvQ~^!UK9s64$2mzL;(S@-2a@XBv^k$NSubLWYWR(0kOZ2Xq} zzXecwA3_PKr`(er>0trAV)naW69oW7(h{(=BfKe(ZK?Fb3orF24>ELPFp8KRj-)9X`a+Kj=M$gC)Okj}*NB0v zKF^1burlK0UH=PDunwZffosD;lm1|koWw+rn#V1`NhHuUaDeQO> zk|DGaM3lbp{tJl?J@FXUZqHC?bN6}>(SGp1@Prb9QOXX3kTazLC!flj?#!NkUi3<8 zvMk-T8bxULWwFO?G%Jw*!glmqo_Qqr77E{? zc0gu~kXWu%2(7EHQYa|Oo{>3UQv8-FgSS-X>*(!+ctK(btE~DG7=o>yK0FHsqp$Ay*j-xZpnO^gb(TQUB#_{r=c24U+wnmyWg+mnt!Y6-hA9X8mX* z9!oo0f#Y#EbE4hN{_8J^5)ZV7Cb@u&hq&3Iz0hYxr&nCI-CVav*tl&ID7#3gQ98ek zgn!MekrI}$aiy5eely4ZDTVx9`43`U4T%j*0t3;fjDus3C1V}?jO(Amc|0}v9Olwj z=*}&k-G2sir&q9t#NqhRm4A>FgJX(He>&$#cox%7mJwi{WY$^kvlA*U!P=He_y&HAgz2XfG9|0Oss0v^lE&87o!2HXX|0tK z=YTDpL48{K0 z*?HjsM^}!bgMrZ(iZxx7&#%G0y{LyPDBxXKBl;4WIJE<#4 zFuF)+8FKYUESepWMO<|CoIb}>cEY;KH>#VuA^MPx}K3JLYGflw@?UKE9Sq&KW{AFRA};DrJ9c5C6P8^=&i$c=!L`{A#2chOQi`Qqa<0 zHnk>_rZeIRvsDx6-bIQ}VCKeUg~vwYxWjn~*UFw*Z}ym5w%{0}x6OP%0Onu^Z#UCH zC$8&ID#nnwZE3k%pTDld%G%a&G8R^Gu!yYa>PnjjBP;A93Q&5+kW+L%`!( zp^N=|gLKXP^4RsxVbd9yk@LkZs7DWr(jhbpi*iA`Y%N6J$J)!&z@q9I1y=Hx;s!U4 zJ-2A3Nw^CWliOAeXJk}Re{A>)dh7=ICL7b)hr;vB7BN?xaTbX+YPQRRr`YleZAxZe zCZGEhkVxk!NOpGHBW&hb@S}I^z0n6Q+NI@!5;RtJ%(a)S)9&A<_pK{`b&-7O`lrU(B}TT6{3FEvNi;k@SInh><`Ax`Ge>?BEdLd3Dl zD(~;Ks>?&z2$L9BA(;NACXO3=xcFqDAC#xW@^(AV(krILX2{8u=EaX+U)LIf(-_=M zk0q4V(@K$glsOo9@7hM4ÃZVo&;a_GXAmj3Q#Nah+C*k?-U7$H<*X0_+dM+sd} zsieTnLmAcfyzj=ODq^qo_4JG%+^q9D@l7+tLB@t7#eZullG|(+Z*XmojnfzA~697M$2V85py{O_G(yj$f*=;v5Ms zz<;>h!0{rSOQv8N%EL>kp3|gjV#_Asj(UqOc`23Fl&+ne?+$CUgknfh0x&^2v@1_r zzE|O7hlg-QDMgNE=Sh>%_6EOStln>Q{3W?jWmDV8M#(y#^{9*8qa1^wBB3XhZ&#}h zUsGu4k+O(OgDGiE<2VeX^zK@Ve4x5PrYIEBCgbHe>t~9yEp%Ikv%$g*N4U$T=#R)% zhY9ALV-hk+92#VtV2f(wdql0Jd}4!^ex+@Mhj;MNlxykf`Y~QTt0W4wY)|dJoZAP2 zzM4grwp&>X6+_o8S2s9}4t{s|aB=Cxn3>?WnspCKS-YnifeY-JR?M{6-UEgmsEF3 zIe(3!;no0#iiCp}i`+YX^o`>4f~!=N?=JmvTde8;Udb}RR_e`8GmM8=i3w1?Ev;+4 z!wlDW6xwrAdF4 zw0b|Ksa5p4F2#1I?vlOpoadawt9SQ)>aqzqaVwGZu^RJx7;N)HVuVs};5Qxq;XHx# zwUdkDeT*m8*B3@wI((LvA57*ZA;nmsGE~RsniNUsy?a{PjBy`hON7bX83k7ZU7BlZ z5+>h@o|FNrpi-XjI@`{+U`-h+KZGr5k0MocJac`=`hdE<_I)h(v;?BGMu;*w!ImJ=nW@b4!O|S@5rp%2%^6 z*)KM-(aY}Bvuu#Y!oM!LiVWYigyMHJ*RPB3A1>{#%~};nYK_gV%0hmKcDHX7<0NGg zD)(m}vBp7C;r!b?fw+Ejo5?Vt($%|9nCbMOn~xuDrxrCmDxX#tE3GD2_{$;D5Ri3^ zO@s!VbZw1>bmx@W;;jzacpvY(_lgP%S<`}!z(i;)v4Iax^b8L&q&q~*zbfY9K6<=D z`l#41t?hRE#x}jyJqCSK(P>h#7#=I->Sp^I`ZLIj$i%nT?CS+HlOQE3OrP5L_2e!J zHoT3``>a^gETRLQt5KfUY}8ZAoc>&|$6aLb>^$Ai&h|H&`@hcvSJB|Zmm}=@b>4~? zi^MQg7CNF`?OYGm?;oei;S;c8Qj(PSyV<6PhFo^_5+qL`R61s_5R>_vgS1TQFYg!2 z&ts0f$<^wvMYi)4C&fP_$36Uo1V$@S%|mwO0+~FX0c(+(jXRTCG;eyXj$*|(6_e1F zbI8!Be`PoKp=Jv~(zXWIayAeS=bY}0xtgRoemL`d=5zK}(+LfC30q5v&q&QLn zr_dT5*=1*s0bjsy5S4SaU63ghrifLXKjD@kY%6AwMYP0M@d>lalRWzT=@x!kM z2XgI?_{Ytx8haJrZ)yrYV%e*2M172&^8iI;GS+YI@U3i?u5_O7MWg0 zbA8A-nJ1$`UFxA0V0Ezf1e!ulD1Ekjqi9e^<36QM6r(WFCW6nXdZ>n;87}DPqrd4t z;jP_4+hrfQl9*;>qyr;ARa^KR9zPdYhkNi~F+F?3GgU*i!BJXAxy4xJ;IktnQSU3P zZ#;Vx`-{SS!?b^BiGGn*JMjsCtqB`an?*3Ieg>oZab%3Eqnnq9TObc)u43t`4pnS^ zp5J1A-)?zgnP+dA|9GxUrs)eN(h~-bUNjFcUddK!=WiaSiThUBquMf z@?ui0{8gW%8=AxnHz-FQYw4Mk-O;1Mn*+13mPwx_S9X|N%X4^ciEGKMntPphT(`~g zdGliKltr7lJ2zA!k&>UuIq}h-4Y1oyJ_#)q7aRwfm1H|OO`A&Gr)N30JY5t%I2_IF zT?$F`N5K8q)9nh4XB_zXxWX^mJyyzIF}KgD$dv9rW~QU+7+K6S!@iIkK*4-=QR1<1 zsr&L+&063!zk-_=uOY;#?K^c3=!`{d;pU8Gj;6uhgi$IhzbndXy}}G_vv(=ATY5VX zy+u-gnCxnMks7&MLtkB6RnrM)7nT&J?1WVF`=7Sh6YAt!N|zZ&lgblI%wL-D#k&$S zSK}Ai818%c-q*Sf)KJ+qX(_l#$k@Y?>pPUu zNn6>+*BkRZO6_oY8*>3S&B*&CA`^tnj{DS%)tV^W)w8>EK;i{ueKAL_Sznml)giA| zh}p1{SE~3x9PJ~DnW^UeIWIVajSFXJ&(r_8T|?^O(YdJx1Mv7H-?^EULAMgstPj1N z1{5RRR)#adnL_#M2_b3z=@>>%Ff++BistE)#t0nk0m)@%ocuQ5mt}Ar<#mix&LBYs@xKaftf>C zG?pO(WF9e8HPGSbRVwvVSu~DUyJOd)LHDu>Uw<+Lg9e97q_|xn4 zO-y$vBuR*m7k|^lf`4u`y4r$HCDyaLIzmOdGvZv7@35ZI`NwXQi+t?q_!pZ3_|okL zHXJ$HHAhWbrq6y#TD7w+nXzL3Ha2-Tj5 zu(Hp+-6`&MQe5Oy$kTU18p3ZaqMKX_)(R)7#KB>ff?KYV>bgwRS3-ne zt^^vzqjq^yB6DG~+vs?!3Fmj=%(D5jvL@}B8XFU8L0ju}5`K*R&5S6GOOgGMA{siS zvD7!Cv(|Nj;YwFZ9FA$|Azbom-k9E-G!)*9DJegZW*#!QdHeoq)&`z1Aj~3tT*;we z14-WAtA5@GWivTHrN$Z;?m>l~2tU{QfL^70gpUmqRZb$wb+i1CcT$4k%N33h~quU5_UGL!DKXsc|! zvR+<2$+8sWr3kowcU0yFCJEqrpcjsgfFzO22twY{MJbg*VH=yLYIL zv5oA!Z0RjL&S<=lO`m$rz6Lh?k?->{nJlKZq_kvIhM7(;WBeeksU zh?0a}`*fmsD57;S+WpOo*Uvz5eu^PEYZlLO4QE?k;9#zh_5rKr@L4ydN%}<&*;7%E z3y+S%uvvBEYf%BVj6BBEMCcUkwA>X{4IaTcSzE&PryX0&(L47v1@0>F2d7C_v`?V1 z%bK`}az>2*6Csf){X`;7hd{YT@rjHEh%rP|MJl;+f5XJUK8 zNl7U$bG|9&L3$D*!Mn`~%ZNFgo+C$(j`vJ9F?`%g)smI8Vf5y*4E5RYqw(uvE_n%g z)!@P}8|*CZNKd@)nHc(edpbsb6SWre$hh6xw`f9_SavSNpvC1g=JXg0h6!DHa!aY_ zh|!}P&PiM=yo8RGUiUnW5~QlXwWiY33Q1i%lL{;(Mrss1d*L|5*i-xn_7PEc>@0R!ozUJOk41d)hSoIH9uu2XDdd^H9-!||og{f{ z|6w8O9xR<7geTBmZn5zHVeOrxD|`Mm;iQvvY}@MK#I~JuY}>Xvwr$&XI<}p3Y}BkxZ?z_KiyQB*M^3ayFM6sX@5OyCUA`8`nI}wUmOh_K-5Nc#?srm+bI8Vf zmfF}cN0oZttPPcHOfI17C`YEEk&s;9r=S_MOm^>=bEA!oy*OnB%``>Z*q|uGB10nd zhM_{W&C?M$^-=$yUt_;@zeC=)8#Bzqn;~yz1}R_c}V!fbjV{Zw~uY?I*I`H~5#YdHYlRd|mvxq%+sue#g^%FZ;zr z7|`^R-S8=w%?CBp*}|7!x|2PsR#<9zNnY@-)~T?=SG()veKf(pWV@7U5q!?5ZBAnD z**_)-*WR~z;op&DXTdi#r?S7UUKm$`QELTVx0w4H^y@cFl=YxHX;b4J8g;p@ONY8= zpNHW%$eZf^uqFA?n*8qRT2?la*SvM{i`J89^+9PPddK=yo^HQ=uQQAD^rlpN*=l!s zo|p$q=e9c@gicd}QQ_T`s|!(k6+c@V8d32TBwZ!n6Lzn*$Nw%6mR$Wx<)HiyI3Bs@ z@y=fK0t7GIm0w@Ka=igAcfsg4c&!#P# zUB62=&KNfr@kkF3lhO#D+^&eLFQ%LD-4s8OpL}Iy2>1jNc~_aoS9?c2a`!Gql<^Mk z8;g#@J@{|`!P2GjneHyE`l!CQnf*ib@>qQQZZNkwb8mKieL9rfN#AyPgg>M3k^NGA zlm=h%?lJe$`Gaz5;fZH~Qu|5k>z&2dr_Y5q-$&RdsN3znyWC6XgsJ$_O9sE~_@Vr6 zmuuiFoK9zJzcP7yhuoEZ?DC4eYeZwgUvOJ*nw-7rRmHx*Pp9AUE_A}H%j%qDWy4pf zJa)WucS^jMUG4_{-M|A#e)CP^qLr~P zGAc{^&FQn}5nw+tNO$pNt6m!TL>3ulY|^j=ds-Bm5_MnZ#9O>72>xrbM2rhijlIBQ z0#RF)R*UU(=h(oNs=lCGiTPB!85ddSw(oNVp6~L3^o}~T{c?CzT`u3i#jPV~gJnPX zp&7~K#wv?tfnJZ-3{Lg!+eSxLu2K^KNXE0}#jui63hN8w2b7=u`N+NZ7QMR!hz9e3 zBrl#edU2@Cp9j9^-ndOO)9>{eBhPu8@S_#>Z=`PbiwGIZ?jM-#Hra~zyhH_L*?57P zUY|6XsAltPI8;jWR(O{%v*qfcP@UM&2Z^(;hER>P8 zp@XBnk)9Ro*O86BIV=M`0Ug2D0T&lQRaW2J$iVUINZg8nfPwC-LY{{QAZKr5pkU;v z0gw|C`Kr+~F%r-M2)jCpDmdyn8hy1ERiG!Jr(^tAa~_`m(KEyU=$W3D@vEJJnY)n& zBOL=P0RsyIixxnHLX1OGQ&ve%Q&UV@R8y0lnTADEQ_tQ}Q&UJtijMxPDXpWKF(p9J z&DIDYD<=+6HnMjxv#}=l>JK1jZ(}R}C9cL_IsZeif6GP8T#XF>*9tis2Q$a7t2O8u z*l5}5+1c6I2$<;TX;~RqzQ!W>52dX1th5ZQZ0zg|1WfD_Lm? z>+?U9|7(tanN`Zj+QiY6fRzCtVrJ=RWDgLr{IZ*nk%5h&(Z5vtH^T!21Z-S2=xFI! zzOH4VW2a?hWoKt$BVc7`)&j^IIoLSa8yGnd{NpisfP$00<6pao*w|bB^YzzC1oQxD zJx6;pSB<}Jr6c&~TkEgw2pIml@H0(3rKV4z}gSoZ%irkK8l z`j_&8Hr9?t){b9N|5Zx8^11Oq0S?Lq7{gqc9AaCY>2^J4Uf9E~DUML5vVoX2J5@G7n-?C0_=(rl^Vj7H(}P!0_7 zEQtbc^))0*M2R;YLd#673>RDonCNci4xET_#SDzd*0X+xvas2~DChc8GZzd+Eo3TC z#ybrMH!a1?MPjHQ*^4O&&>yY8QR##N5XXonT3#3uW~c{o$MFJjlZ2@I;sAoF;q$5v zVGt2}2UJ8*#)yUd^@J{I#UlIp>tOXW7ckme^ljKg4%0d{;ihop%Q1o$3KM!U8lX74 zo%(Wo9xJ8jC^wo56~rtcXZ>70+L7{@YeAjy;_?Bj3je-So+ReO(6F<4iegWrI+ zK9T<=L#ucrM&)^)+8HlM8YOM$F;mLDAfj33fpV10rYS2=dWD3Np6H|hSGV|kslE*q zB1@7D*j~MwhNTUMQM9Iy)mP}_dENYsC7!CbiS6Bi_5L-(PyGa4T=|c6K-V0}W?2?C zCfB7;Z*j@D<@nkT8TJgp*^s}&=cM{RE%RXHhJ+Qz8sth&p5e+5Bwuca7HllFX`7T7 zP=#a7$>#QWZ+-N6Pn$ttz>=2@KTgcU=;H`AQ+mtRJ<>BqBC>4lo7#OGGPk5EWQ*kJ zC{3KcQB7j06DstS*|+A}+|`vYI-Pc1Db{+=ef(s#=$5NhdMAuMaz7JXP@2f4ZN3xU zmApUfDu3ycVIsBwN>NwN=CX`xpM58I?X*<;ApCr=jxxD+$6=RcS2B^A3;x zIFA&jfLBU?3xV8T7edp%<{UO}`s^%(7wjlBx~=s}58_#r2O-BO@sa;70nf&xlV5g!v3mNW{0nbA|?K^#=*LHRsBDt??0%nv)#XDZ{`bS(Ieak-j~I z{qn2df!BKgzGr<`vO9#6E}n8IQTP{B7_fE>1y9VkU;P+i#q_qLT?io7Dzx95t?9zH zo|eXZDWLe(9CpQuTg7WGJ z;T(V=%8(LY^&z3fP@e6UBQ=qX03k@wBP>udxDis@&HN$ zh6*MRHbvphPtvePymG=QI#zdTu41HgXu($lzhAY%^JF;yH`>j5hRf;9(<})t8pT~N z#T|4#ohaSyB5K(#xX>x7-O^#)Zr)6Quebpwd=&i|^{!d1%!3ZAVY%kK&yxL#n_k)e z(BfitOM5LW4MI0Ij(h^ztaedKyb~ZSV*hY}Uv1N!$eM*lzOkV=RP~O{$3U@;tDffJ zn1Lv*VXNFeu44mNyZ3nhpr>;V4@v6Y$-zwdo}x7gOG_tZ8_Vw8=;^}&bn}+@EyfQK z#IJwKfCIqeK`q%Z?zHffRRw%rw4R&VGQE=qKV<9%F`>bhzEC{!2Dx&g!2kwsUz2R4 zOJUgG))(%FD6R)}4{vH2#BpmK!Y~42pn3epZoumD__BuK!L&WOhiVF+>u}lrAh$5# zxPpi4^J>)?M$=NBv{eR+nv*J16r<+I6pNZlMQKFD$lmU(balx|QXBB&_xGH|TcW<9 zPuwx?hkQleW`|ATq7X@gebV{d= z%pSJ!9mN!y8g;bfTG=(0onEROeAjlX3)+5if3jpOHEYPakaY&p)yThJW$H|IwGL#@ zG{_l_P4o=~^vkTzOY_m1kf1Vk0iv}woxP{2;C1=sp!5Vl5J1koHmEPP_WARF_P;Ug zmLQL}J9qa$-TfI@tfYSJb5v?wtXjTY7=_0eOKVbVvfdx1w^~`vH^HR*ExUb~qtNQD zUAqLhtiHVjUYBlNOPrh43$9wAjRH>=hu2*<5u6+~!j;LJUf)`4T(WRX^R4Fv24{2p z*}>*J<+DFWkmQjAZfXULA6dbWRK|;>2vNoEe&d)eCn~VlD(Qh7RL5R#ymBsElC<== zjEIi4)Go{Jqk+q-+#c}VR!ip8c-KvR9^wa#b~t?p*OO-L;rD$Ac?^&+jvwyH3=a%F zt%^77lJ%^Qo9#B^qeYbHA=s2k)o+g5Q&?sgw`i6$94d!K>j+;^_3;VNjFG3uK1k$l zV)+}p5%;}T@;#CM5wHFqK9@KIlkRr+Tck7ynrxGLeBMK2s&OFY>Mm5OZ^Ic2c-Fk( z=FqjW$k}NdW$N)#`xoDpW)ZTSY1hkX8YuNr=8E4-iK*_E(X&U7Hd7Rn)%DqQj#HF{ z_m??oY=bb0mEc4SD;qF*cTewxPrJ_PC|wdyQ6~uD0*%@o+B`N{NJ2_@Qr+1TvJOb9 zbM=Dv?z@e7z0o_??7jYQ@GFY^itk%#jfiOHXc<=|iPU^=TyuW!r6McYc5Yn>dfg@1 zyIi*F^hd|y9NK8k^Xt#@Kh*0kbspMqddcw)!lZ;63TXt1(B#zIoU|@=&Jma;uXwr~ zlUBnmGwbHU`=MhH#|EtA(YeQ2$+H?OZ<&;E&A8tK6TNI@BtD)|R_$+P+uRN`Cth?3sq?BoSRNq(Hpvi9ia8nWt?Pxu?7zjW~%!8UO zYt+9fQ;_4I@=VEJI!0GT6ggsByJ-D>J?97f#I6YPd>u&<0oJ-H zRENlxlengdC8{5wXnLOIVYF(7i+PXc-{psG)r`TKmlA2!EI1UC3n@ubV<|0nCmc%# zF}S5J<4d_}^ARsq%DNuF@((aJloBr(d;nHitR7#^n#ThgHa~ z?V8edSH?+>jA$X7YwwmznaIV5b`9YDGPMv7gT6ua29KEla7C&+>s6YU#9JxdQ*W;_ zIdk{iR*Irm3r0b?(!KCVA1|H{hdVz+x=!j277Flpc{SIF=FZoT8jI< z;(BgqBJ4h3DGENR>7FDutnLc@HCUKQFxtX8UgF+!;4KN2@Spmwv)8X)i>}`TxUE5Z zfJV+9dtiPx?A86C=qWT2426lcgNhiU1x~>X1Ziv8-zu1JY~z#DPn=?>xVD&T8iK0K zVm3d#ir*DfDU(@?%kqGd!ryUWzYFfDa;4JbJpb&axJhe1SU%!l`k^Xr*9IY*8?@s{ zLWtQivz_w3S-{g)P$!3vi3{6fd9%+w`b$Q&+Ef*UyKI!)YZumJf~v4AAOw233BmUp z9Dr8I{~$SG2-5QAp&0V}E?PurcMhh@WBH8`mO(+=Hcvu$#UUgePO%!c??n^kSi+y=ove-WTzRhOp^F2&02I~^5f`CjCxn@FBa-*a7v zi-S9Hzn(06X~(kU7H1tMtQU7Go2t&V(sBG6x`0d4yn}E|uL|blyU+ zj99W-cvp5zrVNxj?_5Q_>WSH9<275RJiQbXM8N0fC!IcrY60*!epo0KmRn^SyNJKi z5H9BfA`rE-mf+yY4Pf4>Tky-;#8y$mQC7pKPW63gm6SbE=$I=Hd*||MO>FB&D9|=M zRGbvZoQegM!jL=&p$sq<}NeZMQQEf~t|lElR`f!b{G z$l#GP6Yu~y4u3ee4woXbS2VicXzqS!!>&|qSNi!(-4^Q4JfqZ|E4D@wd1PYo4#sC! z9M{C;BC={Ygvmu)`zMTU%pey?4t|3qx!*-xc!5y{h0`e#&YcU)az!1QUcs6jZpGhDVW4`;S!=n0(khw zDAg93pb9`eCB>3S$2*kQ)3ymV0#1e!qD zlSf)tFCv+1sleKU$6LQzJ_H@%Y#BSw^l5D(58?52eiQW z+oU zenINPO!`G?HHOv2zKv5UY+7|^olZyx&EyAKckuH)y#Q}-izY4Ywn0w3fSOFXJg4Jb z*6%u_-^ADFks$CXF^Ga8IxNB7P1niwa-q;ydrrD;;VVh;!6DZL;hU7RyyW;S5jmNl zEYR^WCdwMfp3d!c;YNG;W?1BW54xG2v8UmUab){ul^>VqlV|(5G|%b0WrtH2 zo#LK4b&Gd%G;^ZFe{dfF=~$2?UzdMZkF9A++zUM=%)xnJxw;rv^zLkxu^D@VMQrWXzad8(m%HEigXoZa}X~7Brn_;88upb7i5@ovmfO_N8;Tsq6CB>IcAW-jD*-@vkc> zyxfGttgF*wQ|I@MAnDIi&?<1j<@=&$TY6Le>^HU;;C)V}^u8$S{mEE-NEzZ?NZ~6& z@{Q?aeEMWiHBv|7>eep9H}Txfc;zIHp@y+;Qf{A>mTQ|08lT_%*Z1+vtRVhIZn=hQ zU_&U_()M|>^;Flo(e&RoSt5j1&3d?CUJ$TIout$8!B(Z9cUOA!|M%f)7o?NnExt-q1lT- ztJk+ctefbvC$Vg9BKK&3f*RR^=so_g?OQx+JfGK1AOvQ=n*S1L{fkciJF@xjKr8z{ zDCIwx`TsxA`VScVe+{($1Bm_yp!M(hzl-z#5@`KjXCeOk=;~i%{3mpkiR}wk{@;Q4 z|BSA#*{_Mgc}i)1PrZn~v3*kR?m6RFztiYEY`@k&>a5<`twqT|VZtm!_Ms zkMEJj7K(tdv9a-WV^!bj)?xL>(xZK<$M0-1q4_WD4V0H-^Q2U2n_`kaiHPA1jQFlm z?7Pz;i5FDjFg=IAqSGO!(TcC*W>okmvJmaOL6*Z6Cy6Gld6_74T~e!-l0=R#ijl>l zjA93ZtQH|wZQ%XHS}H-_o(_cEnTGHkv%-2+2`1*=hEx{4+VZiw4|R35Bt{Rlz)2}X2&f>Opi)^!AOWSeG9o6nzRR5~AgtwQ=NC@9agh+BpZ-Ad(_*3CTA(kKNz}<>QLW}g zgB4jc+T)r%4zzrOPCbLEle3%o^ecDp%r4@MrS(QxHRFhpp+aY-SuW74*B`uNmF2}U z6TL!&WfXT-LyJXDuRuV+38h!1(iB{5*x+0K`mvkMMum&@cl6>Wn{~g$mQX$IwqxI4RoE9*yGL?C)czqS<9OW>zIMzcqnM5&`n>^+dS>HRjXYhdpU->qzDjPXgBkY)@ zW!UREkcL!w(>oa5U&Qx(F5ih}YX_@*<@wI9-Owksk=7XbRDN3m3~8T2Sq*ADIF23y zluc3GR$uMRgE!jckr~3wm|;qt5==Zu44aS}=R(2iSqgNU;2)mP!&Q=tGEVj5ga%DpbhPQa*tlsY{(B1Ckl zRE!Pp^>#GL(5r4weXh8^pQFI0y;Zj>MdOQ3g8bXy%A0e=0a}qs(w&pxVINGf#^w4f z{V2Lh%1Oe^`PDRnwDLJNNJ66%bfGAWivyHM`FROiJ%e?;lWcGjLc`m*B3w}hweRfM zgO65J?b@A|q(ezqG6iigPve^niE+e! zWo;L;iA2qy)=^iCZE(VNeD$rqe_T ze@oc@Fh%xGN>R3RW(CtOfIErL?}s;^b!il*3H*%~Gz1>!`0m~q)Cd&X3qkJAi@CN} z2dYi*s?%Wzv~&xe?qzhMnv&wPsWMzB1pNIYIA|1AEdSd2N?yr{(=LCd@}nDL0UjT3 z3lJDnFHYl833b=L74vhc@xk!EUC+jww=-J%6Uy>^d}815it zXnkmH8!3a|8Kt?b>i>bS#l@h+HrX#&rq-6Ov;Qf(?z*BIk0I?aj+LY`*P;GOBV>tss#@A{HuQ00 zBb?YOr=m^$2Ys8d1{uJu7eOw22*eJd{F~LBFcmJNiBl#&$tF`IvNg{r=v3&!oM=T_ z%X-l|cGyh5-!hI_`qxJ;Z_G4zT8*daTKpVe7`yfRI7?(Z>r@n*XUxgA*kt;YAg}TY zkpVGB5YT9+vciXv3SElekawI88Jc zX)hIHU-_Da)xHAC>F(0#1D+m38f7Y)-NT969sO~?`tYnL~bV{up47J2FA zU5>l0RD88$54ug3k`&Dj{Y}I2J#s5fKaK0(N*5UBEtPqs%!%s8z%21b=I@WqDgqPC zZMyHW`TB%(RSZ~nKO=lX876v&MX^mye;&-u7>XTl*Xsn(xKCTv>p9hs(eoi(mb=P6 zYIL@>VtAlBge&jdg^O=E^nH#?UIw$b?0 z1_pZxzd&lrx?69d$gFqRomS2oe^%YmyPh%O2|pXOvKQ}p+#){DY$-!i9{-*)R%3oYz8Vo)uOH~Nv`yI5< z%^5WBzg=LN&FTg+!&_h^#M7JHBTj$4KvdNlyXd{X^Z{0)t$kDQqc6ZhUs1p3=cmil z;uCTxOGCt*(jej`8wUs2s_#IYff+S!AalU~Ap6K$8NJY#@Ew~)4) z(fS0pQ?c5xHGis~);yZzlRaKmbF#89SvM*DYRexgyrV?8QQGSaV0Xw!(onL6 zU98Q7;D6~@m#w)!QzgMcJ_0$H&Zh4R8=nA^(2P@^2w(Bss~Wus9_^j#F8d~Qtoge+ z9^o=un5BT(Jo9x?9+xwca|iP=2b0NGP~$dmU`_mW1IbPH=JxI@tsKFmVG9jDonZz_vXyP8)h6 z`SQYFmZ5h3#sintkRYe*dN9<^SO)4zzC{Q54W{5r{^QiV>zHN zw#HY?q8=hVS}g>LN#m2yQ4w8J8^c(}h74})l05b5F4doVgI;dTz~$kM$AOgyd?ss0M9UA$MDu}0+Sy!? ztd$+e8(4P;vh;i_vKoX-{bK9)50bSD6OSpF02$FusjerX>#5J1q#j(QfhVk=Rl0u+-ifeal*)B>&%jQIdE}wGP^?gIVOzj zsY%|2f72W+th?`*)s}|aTj5)EYE@AxUuYB z8aoQhovio|vMfmSXime5AG_1LB9f50)3f!1{meMJ!(7nO(O&P680Dm%XC8O@iMHX{ z4;MK)s4JeUUz`h*NS)^aj9m3;5Zt;anm9F5YN0>}FZdczm3}g4RG#u=_O}WN!~-#0-XeL5 zoFNo}Q^an%>}l=3XEV_8QA@Ib>3`d6jJd&eW%o^ap!J$*>-JE4GXgpoz@U`8kTZ_y8YGrjG{oy*nzP zoXW|wyvp;o`!r%4!mB!k)mrWiqn%AInV+{i;Xpc$ywYE=D8p^`U|E2KqnG_R`RY$q z&*3G{KRb1MHpinwf^9^NNUwd{3tMb>9Ub4BDt~DlaciLNR)}dk>?-_eH*ORzNQo|o zY|8QL5T`J^qDe=EQ{Er^BI^VOhrm@hAM;xmf+PNU`~F$*9rRcKR&Hk4`dg9u%bM9R z;XP{Xga6%D`Yy!9?Ulam8`QlQ^>m54o_^SU#4{mmmr72JI9{AyI#ZLZ-qfxcOZH)* z;M?n95q0rOwwssliMo9;T!?OKm!O&!Sh)RdS7^xOcXyB0wE{2qH>zSHe>DByrA(ld(;gqkDD5n)HPK}PP6w3{9bnl_sGpXz!)BgHnKXFDK z$|r~pSq=NL*7e}}Q^VjSFfz3>;dMxF%5r5+<8ja`%f-6_fauVkbIecW{xF^5a$Q^= z*YWPRq_OX->Xtr_GEdj*7Pq4RJh4R>z&&HMrF!R>&2k#JpuV=6YPTC}L+aUyDMS|# z!Ceq8h~({w=#a@6pM#=OPwB}c5@12rp9iU`Th)m@qmjc#jro->yYXs|qsKR^RguVZ#onHjs7DWu z#Gt6xabx{EM@F`8qsz=uMv9((6;8jy_PwnG)B7xbfNbujqWDd!5li+fosn`q6c)(N zclLI|2CV~wF#_mHCsF_0ZwHlo{T-DV7**;OAExgXSUAL+pvgMA>;+-x3WV(1oEJn< zZDW~dRdspKGT;1JbA`YZ#L;Sv%rTYW7YZ~vK^jKQ?Zp%k{Ymr1cYkUrE zQqTGcPr~#QO5)pix?@_hAQl?bX^WJycZLi8M2u_>{NfDaY$5fwW-X_z>xb+rpy;!Y zMP31xE{dDk4>jPOnP3VpqeeqjOz+nl;~wl8;eU>*aL4PMbJB$65A%~g#YDnVsCtUNmU-{UOj>7@* z;0A=UVx=;Z$PH6a&A)E^>XIZoI0xq}TErJDp{Tb9i4TO~(N{i{CX!wcL^{I~JVPgR z#?x{f^G=*@{g2?f*(Q8?TznF8Sdi$)z0DZ&#W!Yj4Nfe)11Im~WKz#r8H*#gj^^D{ zVvF59wIx}Z>O09pAP@7yEt2w3J~l(ClM{e0k|CWH3Id-mkb;iYR(QMsn&4Te`}eFU z4h)kg{(Sm^?A*9CbwE_!5an{)#0O7$HB>`FDxJu6&%z|$i0&-ctrd50l5Ir7Mds^! z#I2dPjv0n0sc^$J^?Tp_eq4PDYn94GZ zzUj{#CImpKwGr16^S8W-0$9Gxby(yqgKKHgrCQWqTgcRh=zmzC(X z#ln~lxuCFl91At^r(uTVUR@<}oEmu!69Pc58I zcMoaNxfikWep60}M!b+nc+>RL&T3!?z&q4M_OqTqZ$qZQ!J>5pQ2$jeqjzPS2Ly1^;1O`ojGNO%y-swYD7UCnWXK|iyG z6?r5Kk@-R}De_|;JE8>oZ3&IA@aEu}WFizfO9aKjIRbFLtrvlbS*_{`NswYPcZLWI z^9fm{71v6y>es?zz^IvoDy}v95OpEc6TkcCtaWJ3;DHXvAA#n+x}FIHU1Q8Tn{q6R z0FTH$2X|=M;cvvu3Dxur-KudGuQio;rH3+YOFV|05VwvsuW>@TjG2HU;~FV}jf08K^53+^B_OueaVSDF27X$@ zbjc8Az|K>nf@5P~GJ%dYk^vn(x3E*XS@E89U{>n*V1$MgU=nGEWDguE3htcdahuGztCKb()MKO41$`D*4{tKTs%6gcjo5l)-5yVQ+ z73tmLhE-Med7oh5Hx78NiRfN~I#d0?>V|N?ehFd>M0R-pD|(bzw`5ifdRP;6bISM&|wOZF!5faH)*ANe@sOIRtO2XWXm zPokY#%d9n4nM!G>C+y0m_g?h8vA5HdAbBN&h9}EQH`9=V;KAIG3-_49m?ySYC_P($ z1Gwlf#>k&~OI}^h{7ZJ}!%9&w9UIs1ob?I@biaK!wX6>Z#`*aj=KC@q46QXF%z0Nf z-4_9Dw+5G?GUk{MMp4VC>V!3*aMbn#)p=4Z%FyDs#K@uKVp*7cHRdRiu+uc^+N?sF z>BJj=6*P@L3>FM0&9e%vwK|SOQX@%5Z19`nj8amo0=Cd+ z;Jjpe8mCkO8JGWo?>B|PiF|j`q|^#ay=3Xe7bHTF9PuA;)lyoPRNn_(zuSWTe51m_ z*}p}l$p0Z+tKDfQ%LBo6U7gou4BHqDz z2vte8t6CtnFhxDo6kiu99k3#`q@W*cIH{+hONZp$nW80ypZR!o<>0izS0o7Lht0k? z+B0H}DX7{e5eZGLQ<-pM2Ak&*d(GicEu!@!V}}y9f?ZRZkCY`R*G;6rM(QI#N0v$p z3$t#$?-k=~(`Xz_VKH0*-l6*QFifwkr!!Yc5LXf-D{y{*z4g`hPy%+0&3O@5GIDaR z4~*4shTAgJC;IJB671rJmeC|UIzrQ4wt6|i;!t`~D+HlaIqETXrsq0n3hhwZAP)h4 zKwkL`zr1y40ed)a(9bpM?Gn_v;u+>UNHS+?hIINSYln_S>PuTE*7Qv?^m^sb{xszf zkxj0qo^ID%g<3wh&UD%PG^ln-xeD&$5;GS)CBN8TLg_W3Qnh>dWjBTZdBbzTML#aq zw^bJYe6RPlBI@v*m1|2J@Y|Y@pSa+%iyL1TL@`yG{OE7r#k4~*q5VCpEow_kp|ub8 zs?HfL0^H9B?A#k_aCVuDG66)!-lpaDd5eGQ3E7!#HShP}#I^5dB74N$*^=iIxmXGP z7OU3+eKSDRohcj#dNM8JC;3wKjr|3mDdZYW26*qF9t?sc$t+0Sq;#(eD79X)pK8Pv zCK~4X39{a;s7Ch}rp%y4v&`*Ql~#g2h=n>p7zA7T_8CXGMDNq!GA*s9wZbf5lugeJxNtQB^!WqLpa#z*p>)|LE7@^tu%I2)ihUrkCx|9G4KTDdY;#|CE^}9b zPTscMhGkfigJ;G`tt{}=P5klht<-k@cWH~vG4a&$RdM30RTuU7pSEM8nSd-nS!=z& zQDaT`;AW%#TY|hR;97n!&i|OJ^SK(po8@ggbO0a{pAo-tPpuA9L5)~M>QVzzXTsj0 z_Wtc49(L(UQUlzvc)^t&^kVLLk_=_1H|*qqDQxlwa}N+7C2+>uH$=`Fl58v6^M zX;Dw!@O$)($8@Y8!Q)yJT~z?tx<+vBrt4ihrPm7u#dK9&NiuWIHPND4;6aHqK*SwjF$uH z;&j|%ct`@glf~Vn(-~l(!sG?%x7S`^+Sz*VHXPXb^|;P=&VuZFDNE*Bg688)a_v7bl zu;GtQ8SkJGDG&Hh;S)cV>4=A_Mz?=r9~=%fHD|SA*<`dIB8qUh@WMIR{q%DR5*{4d z=8pJc)Y>*Pc3qzD-JMjrB3AZ6_jvI%wzhvrZ2DyQ!lG;xC~s>S?%E6LS+e*;6mi?V<(UoW6`zJ$WJY(pj{viMsMeeoWtQF+8NFd%GYG@3_st!z zh-2lp9-%kgh-)s-&LqaIcwVg9c9t9@G9I_z@19aRR1?zRYntsFb0o3%sfr+CN9RC#e)8Fxa1DMbu~Qro%$== zqi?hpPQ5b*CrBR=MZFUU7ZGHEeiOJ84t>TnQK5Y zRF!$riOY<^xc&H4K63bz&np`yULl9I^jF56GM>_0ib@(|FpOw_khs5yXeUm!CwxegevO4sObf)%Su>yC!? zP9XPxn#Xk2jP?jn%_Q!Pn)r)Fi4vdK1X&e-A4coDjz6mEcEP)^ zPtK!SNQ0*A<3>YD9G;(_UnM4gUjRd0#e$)n?Lu(m{=k5d6&TY`_K{G!g6udgzs5!N zm`?FL#@VvI{H-C#19BMu4U_d=0ya86IZA+w9g8IIn#4di9=C=rYbxMRU!M_Q5FH!I z9byN6nOQMIe(1zP#P}K#j}q+fx`oyrfaC_ZAY&j4ge{!TIFz5a&3X61HXXn zMa{Ghab)ywxcsT5kiVz3(?NawiGoG3a81g^?TL!cKV#nASVDEOgvE$!;gBTnco~pG zpF9I9*w}wn{{b>H=e?ADi7)$Y(5X=s{Z@XC@aLu~nG&-@k%3n+%n8z%Y*{BbLyyyw9e%iacXuq9O*sE>8uKsauI*BD-ugB)2?Il)f-j?D=iPn zCgz79U`&m+Loq<@cf=Nqkn)|nwQg4o0+~+(Y@HV z$5R!6Z^MEsV8Db0mNwgxMr{~L=oIkFmPd)WB=ddjBH!+ts-bgiW{6jxn;$qg7)?5A z-TWTmHV@BFX=S_(UCFBm<`h>&y0mtD7V3-`1@wZ*l4f-DbGoq)n_Gv7Si>uL*chLR zZQJOOHb*RjRFY29iGd`~!CiU1G4Mblb9JSpo=ne(Xj?@m0(sJ`V9q(0jkdBPCMX6i zrvn-DRUQU&|ClgveFmL^UOu--+gp2N^*4L>gC-tz({h5U&$D_O|K^nBhT2}-08e$e zD2R{3dBSh@PN<6(49ZFcyoQ`wBDX8)oc1qfCkD4)7;`6;@2Ia*S3$|N}Tm|o``MO zy70Z+n$4MryZMOviWC_Y_&92a*N#xJL4@T)*bzD7=(d`;4s%E{JxD0muslG?rEKR2 zSev=#BI3?R<4+f(^TvY|gBlMUhh{)&Kg$#C7hetww!s<1qclJJ z-tPhOgXv;C^9w*PNO6VVijJfvL6Ks-Osz~$!0%Y;~U1 z@Tyekrtibpyv(4CbZTsc(dF4XY`^kb0BI17^e$b{O7=dLaOCURA_?G^8mp@a$>=gG zHvSnmZNj4ZS)#5}vStHpundEPhb2-|{Qq(GmO+&@&AK4Y#@*fB-L-LdcN%xs#@*fB z-QC^Y-Jx-Zh6aZBo$t(@8*?J=#EDox_KsMqDl0Q9E2=WX+7> ziV1De}$V-R4gT2S1mp(+Gc9@09* z@y>~eea~=nlZ>}-ZGOHeOAezd$}@=%Qgm?j6yqK83jT0g#&ytsCm^gN zgGv(91MIO20>M7~KnK5rgrY^A1mah}W(majgo}R=@B<>{XYKxgu`B7z+AHNBnnSs5 z8sf0YhRDNQV~#`Mjxk!xGhXgcu%reGht2#`gGjf7pG6ch0DGywW(Rc5>ECSZ5Fcb1 zvHP0pwpf-fMx3BQNl-ztKlC;HK%7}fDIV5e%0hrl#|yCp9w=<*XW${=9M}M(ix8on zO0zsiPF4eQg0~~K+Fi5cUE{>>CnLd|kpWN;TO-uClNeaG=ig20u$qGT1>ED!@+wZ~ z_hi1j_OlKUX=A5*&+K2CA!f4y!}yBjco3H=!CSC{o)O0zhI*Ury}kUvb)%CvEkMYj z5nV9?_ca7}Qvw`=Eh2r_{SavAcU~O7B@0?56anG&p05?U*kOw6Ew71NoXP>go1hrw zb~#;sMHaxp;-_0!IWzZ6Qsn$0><@Kv8gPo6H(ro5G?Pl-u6xz3X??3p@FQ!tm`{}7 z>$=?5B{UO>lBlID4-;vQxQvURTG+1Ql`g&`BOK_WYYD=Q#=iBTqYg$~!gkL`p$Ob{ zwY0K#bHF&<2v%iQ`VG-F?RMJ}M%SVfqi@<-x+}$g`)-jFDA*Fc<7!K}SWBfud-jYU zA~byl#|B5$B+1Vng~>V~1&EL4v5VwZaDq5aT)hPi(Q>|E;6A)<|7<~}Q?)S6*RK-s zPspmQL@`dIhHKYv_g378`i~W%Y!wmMn5b3y3xcDw`D|qr!p6-)7y`wyWo+bd{?k$* z)3TMMOE!AouT|MneT`NJj#ODb8dcQ~NGi}ZmcbjUuGlStd7*KI1)+U)JtWwPj;_5Q zEK{yE;ltLlyZqrzV_b8kAXsQc_N7+n;Ot<%SDNN9td)x0w66rMwtwdJrfv|?cRT7j|5P>C zCg{8s@_B|7EvXZ_9b^<{)F?wb6KH24E7?#}M2vTza{SB4Yv#JaAJ}+^ZX)-tf2>-! z!Hx)i^AUl|VGxkCT+{TZ(plWjnwpdl$lzEN&|lME5A*px)3u(Qq(Ip%+2ZigJyBCcCM~oxzUQ?h?iHe#D39}I z$W@E5cgh8ssH2Kzy$|A@EyIeyYUKO|Gxk+vZ2`5V7=s@qW14{^yfsPG#g`+dBc?5d z8nTv~9kqeO(Y?7fq0660tQgSnB zVDBR2yc{!?V&woHsw49XA ze^EW3_0-pze}$$__6SgQ^z95{ov1anWfaJ#UNX%0YPr5kyIATl zcYD467$jIDipKqKa~A&#n&baLr(pRP1>*mNPVq0&!+#Y`{|~__SpEYS@n1k8|Nj5~ zowE1`XyrfP6wKd0{{v3(KZC6J&S|g`a{QZz`L3`2U*IodHY|iJ{}z$|kISzARpfm7cKZ06#t8d0^jv@Kv#;z?(`~74rSid}>v(w{gL8)r=V-dt>_Q;cvvJN5!*ZV2< zC+=fssoYYEr3D$5(ekavPVMJ zP9uWo%{U$(4Xen+-VKSAoN0*Gbz{+5dcB7#0M*>{KAvpBQ}UrmGU76Y#2f=p1U7+g zD{~+IU|vik5g`>RX(6na4>up#7ZmiQYSp= zdF@13iB_}}h~S2%%Rb_Kty~LSb{k}xrif;q6ua76r&uT*D-KiXCv22Bs0|EOj8%HzjpAe6D2VaONpR?VLvuDAz zkKb2J2hh*i`^6w`ll#I_+GtuA(J?3=$BcrrhpP%Sq0Wea=LB137FJ5l?1>_E6&(Q{ z)FQp+7;nFnnww_&OTfBJsJw2(P0B*e%w-~Q(Ovl@u!jHO12>P6dP#an#WrlV_qe83 zkypbGNPuXJBKHsZ!zZ$l^RwzgLNn@c$Rt)7&#$K?EtQ`^*AoQO|CsdHVDJHFL5%=? zmyM`{1`81W;(#8u$pTj3adQSx9yb!rI3grDlyK45kEIhKq>Mj7{>e*oSZrdXgRZy- zG@eFy&$I$lB)~eLdl3#@gG1VyfaX!lxd0A9y$@N7Z!uEu;t8#?#G3f%l!s$Ta~d~^ zmJU>820=s_*T*_TtbRN&$08$1G~-djr6~&8GYAsL{c^2EF|-XW2fj8TBD^r8rm5=L z!!^B5#6gesYCq33#XUa(LKK%_Eo`M;MI3;iT>m>q58wr1G2(+8g#l%LkG&hgG%9~A zs^HDubLfMjsJmRTB^bhr3!MN%BZJ1XG6u%&`L38dA`*flZGez;rk&N$+St824uhGR z?YWMJcXE0Bx z@PM9$vKI*kaC&3QO@iw?A)o2BY0}?aQ{?U>k#GNLUiI^Xko7N;H}V?^J0I!;y`(&n zs_^7r=|Rdf&4(uwsgx{+ytA9Izz;o#rBSM#cds~ki|YG{3!hChh@kD8# zpHNo{20>ZFEN7nc%hycd3@M>1sKNt@p7OFidMS_ZB(YtCeO_c@Tmt8YWJLsS;P`6+ z?1(z+4PQC|UyCeAaSY2sGPcOb0Iwu1N{(y-IFP`z09JxRo;S{v&4V#B83nn& ztbZ7%+$)hs@WDLiO>z*iK@r{Lt?m)&d@pTCpiFGguVru}nGy%+%vYFL@_IYV8>Z3U zKJ+1O-g*yax9>clktsGKh_GyWTp6;ZleRl4 z+Y4e=Lul%uJ>iU)<^a(pTGhDSM^x&m3Fb)7P^RPAnw{@f3e7l{?mPR5mQIygyQ9eQy6h_ChpC1Xweg6*ArK!zeA#>WLgh=AkIXaf*gt~a|j1k z*7Y%}8Dy?UdiE96H#j00kS$9&Bik)8>eslh+VnQ$M?>$=Lj?Q#uHkLo%5A(vax1z| zVD7%|M6sx5ouNQMlMJv=PCC)+20h5N6+=3+KY+<|35)I*HF$jM_JxBIDv|bKc)a(f z9e3w3#5S;`Wj#!pX`~eHn>T(%;~|GDux^%JA z%4qQ93#ZWBmaK3c&_Q>bhCq|S3QwSW-Xau599RBEn6xk^j4h7|@euyAV+NZ6Wv+g0B zauUsUj_4TZT@MqD-?i&mZh-QotUj?g52LYN1|WQFBQ*?et`c2@;2dn`mLuER9WS~~ zTpLRS0`>GR#JjAQ4@=UpG6>l4~|jB%k5yFwpLKjLl#~txVo7|q!!dM z!n`#VjICe=uUO$|pbBcrILXV6OnXxC^eEXgYYUlj;TXkU; z|9!u-{)IgMr3l41uAVaW*KgAwJ{r0_U1D16Nii|JPbU18RmIUA{&cWxY0TeD60kla z11_rLM|-a)#V*8q@PicO8q_na0#VawxNfrA3(jl>H?!V>exuB=5pr8tmSD(Xu|x8=VIQvu{F>D{7d#14@cdoHmqy6aTKf zqNAAM&x#g;zcJ)6Rzkps;G4W;w3zT*or@N$AqBj#ItNTEK%_fx^m^GqKo_0mf9I{EMoY(QO&8T+24n^m@D5fJDQ z)-w>&Yu95pqoRn5Q%qHDMo%Y(Q+hB^V)}%hM5!dJQ@CT&7 zE5a2HNS=v@nzf5FO5V;TNl`?wF~jmng{GhZXwzCGxQnLPS&DxPdjy%D|Jfc{zmwF5 z;a(vO$b$8OZwbZEjgCUaAZ((%(yZXBd-C<;V{XR|f@HTHFhx%HEoMr}+qmL<^21c) zyzpB0We&%TBZOh_+*WvQ18X$wSVb-KB`edO)%v^W+^j86s~N49;FqA;D1{JzBjDd9 z!9y~I3i9VZJiXg@P=QR63c}~~Ja_T+DzO8w^rHL($Rr?M-t5_Q~o8FJ0nS>vPLk+_&@kk%DAH>{YUm zF0r|O-H}181hIy7h~hu{H^`NI+W(dF4H93vRw#{uOn23L>h!&c%aM8_v>v8U-m$f1 zaRm8yA!u?cfOkBC3I{2pfm6k`Jj#j!VlfCP-c_S`?`6U&Nf63&d=6Yz(YBazqp zftct=fL8`$$&==XE+Hh_hH7Io;SDE4Dt5zFjw(s{;!{)bVgwx!Uyc~inDBHzL=b)( z@n{*syGB8GhMT!T2^fi639T|t{9RZ=RnTH%w zbI=yy=-n${g-YO?#kG(7>KoUXU(D4=WwOG8HJQ9uNEx`eF=r z1oqtHGA7lzN1n530Z1rYNVgeX(!qr``{IKrJ6^^pvy2k4kN8m2> zg2mI!FR)1De{O}Tr((M8s>gP3bJ`~M5zwpH7s&|1+XnF`x1njfdGt`06J`|{z zCKFCIak_i*m72O*LG&CgRZ~^4%NcM*@!iwlPwVD^&4xHQmVlA`QPCt|uYo;lOXO57 zj8ys45UVUdZ+*UmB@JnQ0)21~DT}@4pAz~ke;IpQR(CM$d*k5hC&pwnI&Z<9lp<3# zB3^4K%n-!qG5*f$)7{+tJv2rWsbP8%-dH}EX0(U>b^SvGN(mpqjD7DY--_scWS?-5 zkmk@tiTJKEz7Ky^xbITy43i8j<0gf?H2cuk3#xu|N-l^>_HBR)woUrV0nZk~UlEFu z=WR?zA^i`ku|&??K&!a0J6|OkfIBST9Uquq{?#@K5-Q&yu?+)YcN+7~87mRZ7#9a; zE^7GL;^b~2Yw2gg$~bF=Woi0&(1YG}m*IS?ms_=$w06*za?>xq6Ezlg#XgR6VVE zwWG2e3O)On4VxV81LXcbA5k*sEyxOopHv9>;y5f*;4U{KH#2|A=#sC5!UX0)28PZY z%8kpIDtXIxf6SbFNJ{bMb}#K?gD|W($n0nl_gv#^FS8Zuo?4}dllow$Era#ouvnCl zud$C2*VuJarY(3m7FMT|Ll>&V= zRalqI_lk2#ZcsNI{nWB5MXeONwxMxUvq1{ixe516U!%=&Wbz}2xK(h+@c4setCDD< zD+$CVQej%To#&dZgjPnTQAC>8Dp*BIOTzmYbPn&{+@toUTHR zwWpu05Q|LBRv~L@FHRNoOHs<9($b^?%C)mPozz`| zPvj@`MuD*f7-X?$DoQ3XF~=m`5L31noNJh#%6|D3A>Q$8va%zIXw zC5r3!P0b10P^twHMuFXd9<{-SX=rqsP7a>Q7|BD!0uXvx`2X(Yo^{NbX0{4VZz~1`HF(CswwU&6dXH zPeM#DajI#I&97Oa(8|Bey@#99zgOBy4#pO%4p`Eyb|oHdF$Le6G!*VPt7BSRmPUb>J3{q4wgY1RfG!kt zcl|&cYQ#mV5mFC&n}LjI_36b;Km%w{k9P+IxUk4J0@ZtSi{xG^9F)V!KJ5==f*_7u zwfhqW1m#snjRWJ}u@Z$7C(o0B<2$IF)gH|7ih9pktS$C#_`)AsfJkF;8XK!y0EhT4~>3gc2^w%uI$p9!o%vKtf64Z|H zKEe2<8-~}wL|@PM*pRWMJT)=kgw)b#~_EoAU(L<=IqsVh(Or^=VMylGR1uxmPIo z*eN~cgd_|-JT8;bKja>=WnaG3qr&m3!J?5@{CxiP`-)kn8XL&5x>YTv>cj0)vT+$ zAM`bgF*?4Y+_GUoe9!1w5duX9yv2~9LF2#cUurBch5at*F^hy88&R>&!7RyX^M5#o z?Df{#_TsUEL60JQ!3zg*#{<6j1Tf1-I@LAs)b8oS%z$;nYmsAf`9k|`*kJvRk9d!! zpzLz%$n_JuWssS$tl%m+Nl85~v58}&3CR1{d7+shI>Nr{_2@cna1@K-zP> zuXD_??7FN%(3sQ2*8v41$co`ARNQi(J*QhdMWX7{1(aQkY_U8qKC|@)i%gFch*B0~ z?s)B+2sL33`0X|es*ge0?jUTsM;=?$DII3w4Do=WZj4o(cm#ZJX=l2!+Xp$=Q-q_@ z)rOU*?LsM&^iNWyu1VN;YsyNz)a@TAJ;~FeW=ceO;ywu89SuJd)GvIrE%^VATfZRj zlv?v=Tu_bg&?g1emh1kV^mkZsD($N(RggLFp7Lz}UGl&uo6744RjXw@qi=zz4{yWJ zWltu`d;r{P^cO#Yqn2;+n@AO!eHPt=qEvD{4#ith&xvLL8!-f(SA{ze?^UrLnfgWP zUW)9?K@O?BzskT@M~G0fpX)A;)yAoG5VK9Q#Etyub{I`ba4f(K;pul-T2Ou|9oEUu ze(1ac$+nF}9?o9l3&U8E_*^4t`BWOQ64OS*0~z-sSHfJ(a*ba#>YYOovf(@M+R#X^ zK;`m6KRS>k8{TExC%Ck@@rq7JTq}lGvzTC9Y*W02VImko=p-%@_a%#ip{DocuR5YS zlY3?v1cI@tZ`yg`{y4yHZ*%F9s)JB4kEnD8%PLQIj1dg{UK4w^V4+ZFoNaPpb*t?O zo~C1P;KZM-@y_xh-ccWNpF5$)T`_E&Kin_Kl;EPwG*RRf8DaY=Y^7zT#l3@kGY|-o96Y$FHrseL;cZ7CGx_i#eZ|GmZAe$;viVxq`tJ zmSu}~iXN-hmXpW*qbW_~NJ~VtaKTeGdNbRXdGQf0>|nDvr{p=apLnU67I#p`*R8-O z!=MLe2HqhRE4KqWpje*>Tt#X2*KdTWubN7jD{ubxT%6~JY|oL1l?5)Y*5yI-I^xcl zkp@_0begNmwv(ZbZf-%XYIT~qR;(+RC3rOAQjWkNwaWo25IAm!Qf`HiRH^rhw}Qej z>B$eoAf`g`MKAp6W6HZ9Lvw2QSc;=xiJ~C#8g?k_^FVJBa{?J|lDhXt7AV0lg|Th2 zra8`~9nN`G+|j`t?Ro0canr?$(|N-~PG42!`oS>+Tl$x~0)>P>P)pFq3U)q^f?92-S4Qzx@3 zVY~JJEibq|K^xplxVC3v4Cyv!^hz9y{HQ^SbW9P|N^@#2Zs#-mNgLi|zo3d6y<;xc zSZS^~{61EdgE1@Bm+rOC#gA!yj-P6?whA4?3p0*s zrVqP%xcrT-4g$3w2fp9~v8 zZE$o7^mPiXZc;GAT%;x2B0c#~HxiJL==S2t5YOXLz zRX>U8Z#TQ~0&Y!5(dp3^Tl3CV>zSv*iBWZB7hnyYn*ln6aem(CRQG%+uLP{vNBLwP z52*1L7$JqhbK32=rZ8t|nMNp#1-K+1i4M=sLadbdr~!8QZ7VBr?pJj0;X_t~*V_fF&Ejk;IKvlO@4OC#iL%~GJw=cvg3AT$ zZt^+p6x$aO)k-F!PFrq$txd3RTA-t`O{P5qB~9|SM4Ke@bihPk-2el9GOk_xw|V3L z4^lr9BPS;l+kbO|i$UGO=y?lIR=dkLX6yi z9FIOHq~&6Q0#*(j^?Slyu#&HEg4ULJjKLkS3qL1c|?+d>837*W#ycjTYQC8}XvXG^{OVF-D-}J|@U|GB5roO zClBBkul-QkEeq+^Nyn)>2EX)wzTuNc4jMGHl36cWG_Sb5{Je^eP%^-5`5TS+yWC<7 zkra8|;FkbB3`pPR&|wUGzQX0TP>;h0cZDc=9w6Z5H%q^k?M5#E$tAXbQw}^GLV$p1q)KZ< zP^nyBZadyyCR5@=U#{01iMlZH;bCCJ8E6YhNEm8mOCU_ES^PDWeRi0IN|)bW=cOYe z?APJ{5&BSAx4W{lREGHC;;KGs&lFHz-YK#Y8r|*A#NzmPi=^(h(4Fhspc6m@)F)*; zR0sP0u;bu3KD}-?z&M$gjT6w)yc~I;tn>N{crG>;yykFpWJES|deHy?bXz%Bp#~$o z;^pzqs@KcU5|&CtN>(gk>Nt3GG!Ogp}FL@v5N3{ojI+ZUl{m`k0JkdjhUQ=g5($l`4uNsr6OB1V@ELW+0C68`DQjR^uo z*i>E3njwG;|PWK29AYx4UwFJ{PEEE?e1RTCe-d{Zo(fhJb ze7=|NgMTvqc6nz)J=+1cb>&;gNITk)vR69vIz3%}{#6nY5$xk$;$G=Mj)azDuxA@w zO7gYdS@D+{MEQ0jUm^yI)kK;7JCK{L4m7glkBe*Ff7Ju-TqQt&iHU>Pi1lk{*X?SO z!Or&~dfi%YjQg}==57A3ru9yip%W^DlHvkiD_I zMHYb@zsgRET!yg7XmJmYlkJ@x{7ehG2%@e8@NP}QIJr2go*L4I_pIu{K$)jv7EK*S zu)cl^#qt2~d{}ax>+ZItBUHtrv&G)*OulJ+I73fQck{A_s;;K8w#zB3%p`P--0HHZ zb=5o$ApRMpo75Dc^ zMJ5#lc(B0sCg({~An~YZ#ovzqBh|RD@S6mgY>OWb*icYczBV^2Ee(yYBRRfhGf?KA z4p0*3Uq}B2&VU_^+=j77pDZWhO(QabvNO3|q%q3KrKBc#CX^ch{00{al)&%6Q7d-c zWMyl!qf&{S`K~P$D-{|om(4d$B1(RINR+b7l2AvI@v%(@%7A~jji@3XQ6_7pv&H;! z*$*4CI@9{?(2#DCTH7n!b{FpJd`su2fq`w->JZ;%Wqg{@0*NmQQ{pAin41qzCBo%$ z<|HO&=BMXpXV;^TO&C97)=Y9CCK=w`{Q?@)+nqxP`8G-#CKIgygI#sydk7qkwzd}f zeONZm0Eup>40UWw%sO(fsgv{6HUPo=r#`20hrJl_t$^u6LDhv7T^6M($- z<@ZeifWakF>RSU}iJQgJMe@XT^CmbsPe!3Zgj1>_M%mHCIt2N-@-~wLvV5!S` zz~A~&;I*_%4J(#FaBL6|t*9$A+XaXSUas|!P?&v5-GF@ei0#7m?Jz|O*4UTYj}GCr zM0xyb?)OV$w~X|a-5}rMJ1}BLLn0jUpo9q<7RgoWULJlsyiSk%@5sqibx(}%H}cIb z)S$rfRR}h!GdWUkIeS8GD}le82H;&?DV0x+4?hiduVaSL83uCVFn0*GQzndq>A?sG%@GeoeKFMWi2*2IxPnoS&T3)!Cb=zXhh_KIh@_;vUpgDn+s}l-}{2)R;G@pHE}Wr zB>IN{X&N#9V$7AG~IsW6A2RPr;H*HyfLp8kF$AA8UKF4)VevX}a!UtYCnn$@{2z?Au zS6OW=AhNEdnWee!5UOSGypDJc8BcQGaGxP(xiGbo;oF)TqC~0T6_vq~l3beW_X4?i zo}XvjX(>HUqcc&1p@}coWNT}Q5;Fx8)PpDIu)sQM-|PNK3K(Pp3WMW&d{x!dAeJUp zx{y%6!HR!V$KMIGK`{m}a{%I0z6!P5j}O#ttJM}dA|2bg*E4f@b&P1SU2J6PuxL2A z^^TcnnZkwQ1lPe6`T(M?o8Rz9U^UEaFZU%Sq@}pyQy_Gvb_u1W9GFLrAiHjEg{am4 z%6oL4>aF`5)zjjUvE?_ z+5ZP-gKCxmI-Qn`YW)?bQNLCVlig?MABDyg3=}LC6@~lG#FL~>Ojuf39sa4`vvCzG z;mYZY+_Naj>In{PA%cB=QKoEiskuHVDJgF}DEaB;hJYQ-#LDR>*3t7%I|yLw*d5i? z#c0m@lBBfc7ttCGd5af#U{LX!jPWY#gj>8z_aY z*9+#bRg@2)*dSm|s7I$a@@R68-`fB^_xf8kU0qbW(QY0$3AvfwG|eP*w8Hqv79zj5 zY1---g7I{V{(R6cR;EVx*F9kD-)|srzyVx{)$Gi?4c)RJZ{IW7h%rrf0lt=TW))Jy zYOF@oN_-F2G8sAAJe$Z*+39*}Jw3&udE;!`o5Aipv|osIVh-b9+bK~WzB1{+yBRAm z3UF(ct5BL8`fmvy+R3c`J>)TziV&t3A;GW_8RK(U*zsqbl)Q|Svbn;W$q^&d)&HI( zbrMfc7lN<>ZT_?;Z`kpGDBQ`3mWI0eR=1)>tl3=tA5`jQ05x{kQu*WQ1|`H?!D42y zkPw9mQHFm$4iKilxxBo*xB~xh4JYMtkXI5mvXX{z_iF5z=yza?mSX79;0(&}DzpKZ z-OFvFNc!CD&Nk(hR!-)erTzK@`Hv=u_#fe6UAo1SEUVR&wUq9S?Qg3yuo-uj=KAtj z=xD1Bjf{Ld-Q+9h^HlmwL^zW(c_t^;(m21zAEw9aeVe(-?KpM4{pss_cXx-~ti_R> zlF>0`{=?gP>qv-I16Sni>i4$+I%D8@ydb|n8rxE&QVmF3tL5nU8{9hMKvI482WDGE z(lDNE2b))CX{RgP6^mP>PkS>*z|A%#87e>&IvD;tMyK;YfdJjt$nvai9NB{OiArU{p$ASx&P-#Lqw=jR6^B2{njj6y; z@)3P@HV;qtT2>~3xRe<4X7g00am355ExtADnj45OIAtwCpi%!LnPTP;NXU753;KWj z76>dH7G}0L4xxiUTr5pREAo{LQ$`w%o`v~D{9)O0nOMcYh3F`UvwdP{YjkvVmcihB z85p9UOV`54l^h}2`Jdp{4`!F4of@j-K=EPkkyQ((w>e1CZ;;tG+&>TX>N*WKe~^^V z669ig#74infr1wfUu1zDeYGb!*&iX(zTlY_2P+;o} zsNc3kAA(}Vexilp+3mJ^{dnP6SY6x!b-UToeb2rK-8Y<*mywa#$Yf2mZ-N8P>OUD7 zS+_%|Gsx>Di2lC~j9yD>W71HX+^l;wz5cn94E-ZgV{2)Sk&}_}Du6G5kF$s88WktC z+rQ4AcWIj^)j6ApQS;(!H7ysL$8XTkiQ*0Aa0gXut)(uOfPd;#8XNmt@34wA$eJ3O zULJ}&T?um8SHI6a6~DS#OCj5YMBTwS9WsBdsi^5N^YcHiPfT1Sn{d2@e=gtO8`#e2 zEX=b9EgTrRHI!aI1C=s|;T&CDly-6pk|gr^wmU=pM`}dsVrF*opY8J#k3JTF0B`n4 zb5=YC?cJ89$&nFg80a=!`ow-IRKWRnK#H3sU>YDHaS*1yro@o|$r(NO+U80HaSpMs zKIpwnwNOyjME9#NDe#Ka*6m)iE>yj#d^y}ne7NRm zrGDwUVG~q$GB(z>qDeg!&v*Y`AzE13gziEfPI(wy`~p(VPNhNF!O{>w3j6DL_&ye! zbBjsIa86(e7v|T|DB^ztg5MwR&7R)=cJXGU&h|{nz(~_1;b9-1<2TcsPaM-*;eff1 zgP3t@X>C+q2dib`5}e-s4FCa&ghA7A%W6>lL(f*#L{#Rb;&N)4PoHmXQO>Jo7%2L^ z_iLd^E~x=Q_EbGjc7G!P|3`>my=3UJCv=)jX&W1Ziu82J2zdY^1mG1!Resh_O7=Ft z2Fl?*64ol#h0F+w(rff2B;@uMpiL^ZzTjC?aw2SX(RC-TYF=KcKk3RQP4=M~6hCI7 z!1Jan6LQM^nxya=8w+9oS2IyPI02Zp{Zh!}s92a?he3W+}+~4SaXfQnDIc zKT3kq*1orc1a`Z+@$u{6KcIPzs$eWp13Szj!%`%-p^jQkPQt>*?h}wZ7zWpem&b>b z#U2V0%Dn~p*BTY;R4g4Ec)d>xq_s>%&f48(EpZBZuK)hm$aa{S_<`Z@E0#HNcV_y_ z4#>U5=Le2)GJIcj?J&mYy->oV{0ph+C3r>8VwT3_#E z(U2ie9v&ZRlgUFud1lWx*M<(BcsP7$^DXcbl$Vz(6O}5QYq1rv79o@*mG>ya)nXT* z;2;%kIjb-b(EwNj7;lI?yFcHWYoVV{SvlCcalMcTFnVo2N|J(x!Wg;CVJyNWdFf-1 zC(Ah1fD=?-2;3DY;c@Rn%xhzvGzqRA98eb%-G&P;ajUB0F;}l#pU)V0xbu?7Nn?NQ zw$+~v7>M}!Yogcebm1LG{H||sH#fVoi_~fLkhXL@A6mmimwz)eqmXcC=9C~#AXU=j z%uHlRPPhxoIJZm}C5J~Qvc9V4^mf$o-ucY>4&t9XJ9t7y;uKN5tk!m}?CCPUdpxY> zgZo3i@8TgE>~Nf%o+>9MHZU`fj*h;Vm{jk+av_`kx|&FCER_RR^>dVphz$1j4U@nn zz!){N=ZB(5Pj;GyZ}`eN3A`g0lRTEZyOVu-sx2u+z(Di*CSBNe$V&%+4~%8UOMAo) z{$mf$K0{V)*b)m#Q`PH~X}h^G=^ArLQfO&HK78B-3)X85VG-T!7pPDKZ4&G6D}2J9 zC*7e4A0Qyd4Reo8kDc4wG3FHTi7wwq2A-DCCg?N~qCy3Uqcq`Q$oS%Awbg8xC+NBq z)&LlPxNq*57(-jR_w!_nipAeSzFj>#dW>>Wk-O%$>Dlp6UHLsVDUHcFS@ zVth;X6V3hOt-0aBalZaAwMIu5lfd&aZ1N=N5>Vo)WHqBE(qHl{?LjZcMIVE6iAz^) z{kgmhK3qypF(^DZW>S%w`ogi@s(!VF?k#9O@kespWDwC5Mou9buR_UwX;heHfrj zv)G`9boY7k|E{gM?_thysu1%z-_9Ys88U-ege^Ib7d2d6Ylxdo4dii3BxPjXTmC%Z zP|M6nYg<@WEm8;*HL{N+6~#uzRnpV){Mcv?*4LJX{Eq&rdF?-_3KUG~>GySPcMF9m z0pBDy1lNue4#3jG5cVJ=# z_uM-=`cTA=g$>es{^S|L(cRsR7bTxtWyGYZoQU$TMun}M z8Wj+4jSJ((K!W_$*d-tWp*F(Gy}n(az4*RWEj^du?8eU4Ix0G1od`Zfs;K{9Z!InD zZDZrJr^mDk{U6bRx(*VNk-2Eu&B1>JL+=48c-tHF`S7$<+z(Y}wn~V`b~rUF{1ujl z0kJSMv^YMBvaqns;K0s*9lN!cyb~h8u9mNcSoBTC{_WS2Ja5xZiUaW}uetN{osxj* zQZ6Y)GgWc>-MBy=a^)tpQ z6DBO-uR-$2t7}dJ@Q46mdubs(Uo|m^?Nq1@!W=;m0#WIzm46(BX8LT&*{^alt~+`J ztGHn&ix64f!iA8(a&f-fCqp~gIB^j>8vV58V`f!9kn^?jmu-8p*wcCYubb&j27SV* z0>3^!taQQ{!F2)wSJ*60)(uUUul`Jeo$YU?>N$VV?N&Q0bZ>K50bkm6q=P7j#-ob=vH09fue`cCLIyj-7a%AOJDsm)w4U$tKFr`@bQ=VxPqVmGY=csQw1Y5GH1MkXXYtA3WWmP^tWfoN%9zjAvQZT@K zNt_>%2_$zzFc1_^0|Xe<7%}mfTwIG5sy2?j2r!0%!VtKHf&!w8LJC+CsR=MhxD(Ta zMuD09FqLQ|a{_tY&9cUJ z-E&ez2`T6p-;WUg&zo_Ok@--B?RtDlr^x4(mE_~rT`qH*e*b=~@~tCXsGpRqG-)3D zIxv9;@3Oq;YO;ZM7P~`+B4NHZvSuZ@F_pV$xXDt(wKXMs;cDhGV3E~Y3+z@a03o8p z?qB1(Is}*k!H|Nz+nu3e+6!g;+q$bI&R zMFDZ_TD^Su#s-Wau;;;SQs$Op4-eVNQ@wh4uFJ{E$)EjIQ|o6GC17llmx2}(;(4{v9j_+o3fyx0OLL!HU=jQ4lm_%)4VW~16u^1gvk!{Y5 z8VRN!q79h>*HtB&nP>t81IcuCnxhn`@K!b-w>dvbS=u`w&6e z^?%Gt3uX@ifG$SZSp8#J8vTLXt6?06+a@xqTcpIvVVia&H}&gxLp7!3eBzi89?pow z_~mYHArcoU>SThHm+*L?ni50pbj>D-Z6#}IvbmZw5e59MIrJ< ziIeJ@GKuh}m2-;P`O61iyF)2C`Q`=>h%v|*WH{Kz zz3q#NUA#IeioBxgq{c52&Zti*b!*V2|A`(E@Ky9YJXx~1av)lYIy);s6o`0xcx6kST1r7dGAJJ)0Xavo_F-Gh8TmE@0}LbH zUYF|!Ke2j!9vj^`Dq5f zr0DsWzPcjhPQWR={d9U5O26l6)a|6t%R_;)T)Qkril~KgFp}u{%KZE5R}K&>7+M$6 z$b>*HcazW9nUE^}8;mwtoE|eq95u;Gl6hxkV{$76k{a+~gY!e7+w^c?8x#eMe5dMI zlrEAWDJO@Q`{P#mhei$$?KD1)9O)AheDcqdxw-1K?rsD9_8VDg`DaS>b=}~(m`2a@ z9+w|>M<|oidW^n&fY`WrpmKT7%KB3rbuwE!qaO+T(DV=N{r#Hh;|{P(hq$COw*r;Y z$<6GKW^LnDZGG+EHFGf(IM#o+OG-A+ZIr!Rc$R^f6pj>b4a1|LAa~dA;lK0KC&m0J z;rd{DSA0;oq%dpFiV@xhkyDVOEksbo#>^f808Ms)WKB=c#A$K0G+_ANSfU~)4> zdb($1;K@iyCBR1Jema``@(%}j`Sg4nmmL3H$*xx7{W{2cYo{xFe(>zl+;KEe2JflO z0<7x&I=5!+XXR$`OF?c*ZbEW((&vW5?<6u;BW_5809O=UYQY`LWGf=f$NQ!{;~ zKCo5@6BjKrH{G|sTx;6(Z!~fc2(PO+^BR5|{&#fQ(` zsIU;>uaMpwJl;H=7xh4!NHW*t2s#Y#Ll9I|9{c4L4#NGqtH2?hnQ6HhOcbv}p2k=L z19~4=Ga%XboOGOJ#;#)*7n^1QXueXGL|ICjy1ct+fmM|FeE4ubaQrCcdV zTNY^7e#4NT|A$4mqtIGo5(?^|ua}jTjVV}M+thY0=puB>1hU0MlWOIbtN&{l$g=0V zhrD_VuvOmBI{NP+n@8)0xzWQP<>VBghtnH6zo+od6L3jlmzEaMU2rmpd+x&cxO_dl{v1eBij(rN zFz)UpfSyF@6e=H9^>=%+?O3bPKx#_9LSs2vnxM^@^6+)HwYOPwR%mKAH8mOA>1p#= zt3AaFQFIK-Mm2oC^e5ZBIa*DFUzCc8g6Jw!6AIu%eZydHknqODB_;pIt~hv8qnc$J9*2#w+QBBqW89?ou1RLnCk!K!KuZm&C*wzQ|Ur)m=Gn zQD+@?|7hR_kwcMci|i6RAP>w}_86AK5CH`iX$LQO!x+u5(YSJy44 z%sqHKt9ifgeIve;ke3wqem>~;oScXm8WJPV&LGQB3WbFScicY_xW@Po{k2|MeGEK6 z1`($4npJ4DG>s1r19=2lL~*6;iW>)VIT`_p&Vqs8f^xTOcH{&RlH8%aUK1q36^UB8 zxn<~VgUjeq*?B@-#Thc>)5HOrzm3i852xHT)|TgblhkzeXFQ(y2{Fqc*WArIIhI!< zzrFaWiEr}q!i8fJM{H4xWSyk|1z^t zW}V^vA&vwo6irJ{)23g2Z9d)Y^#xXX^rz>BaFZoRbyS+0o>N2)B?ulz+0LxfWUn?j z*9G2*n-JPwFWL|@n!`6UKW&^_ZZbN$M~`h?Wnp2zCrGMtTi+NjZvuZLEf-napq!91 zy^Z{inv&DIKpxPT7~g^p9wA=vsm`W^H<4(@lvk|sL$J+6`zZ?>0}R3cMd23yeWGoB z9RT_mrV9XhFG!qH@6+AMYJDvQt$XbjV&TQcF2~Cn#%;ITe|#gev8My@2hQ19@ySh1 z&6ONh%}}+-_-Hi^6S^xjooak$rl_mR#y&)ybaHNrm93ro%suGp;NWNkRj< zl>GD*Tl*ibX9ex8t?u>;v-SB;+~c7J54R*y(9lpd^p*IYNZ4R9?d3>Fhz{(Ri+}!f z#@z9GnA`J?lKu95`;2VmaUH_(2uk40mABaRYTBTtf(s+yxsi-F_tsg_nBohQXnffh zF-H2mPfU3Du=n`fkB6;o4gUE_Hatw*zi$_pSgdbPKz@Ww+!o>F!FaBHCXx&jDa+9B zZu_Vd^7`6Y%7lk-CVVsrz_OH-U#7v&tAJdCJoQrYY-eqzpP2_wRZaw1Hg@oMG9yoo z)(o*X9>*88l!|1OonIJiOdZD~z|pBC{Pgz+D|FJc)Iv2@4{{Sm1T~%hSX`el?pWXA z!-qozr8+qcD8OJd>OcCnu?OEe5ZBIto+b!iSo<9^E+Iwd9x)YQfIds7h%z5#K23BC zY0k(Lg+b+r!xPC&hOAs$lbeg(3^rIc=KWCC{hD6JNm%b5y1KaiaX$*TvDV(kFf4!y z!2=P+pyqrNKn}H2Q*8jT1TgoVW2C2+#*|mGEGvy9JT78+^U)5~!@!7!PM<8SBh4#VC8dTI=%!8)13WE4MS!!aj>5u> zClvhickt=)?$gkqxvfjrIpDC_63gsx00Uf~F$X_KGBOh5R7pueFmM&XF7<=%i}MIS z#L8l9bCr>p*s1x@yrk6R-@jl%Vig|c?WZ5(X;5-yqNRaQ# z%QQ`mRp%oj`c}h{4jNOq?N~(LKT5+=^Mf?Jl=l+VRC({LBjXd|N{zZTy-yunRFk_f zXBLxJ*OrRCiqe^Q|9U(D?WBE9Pd(Xm7sHWATiU5EmlyT$CxZ*B^ctz_^)9P(W=}J{P;bP6Rt#uL;VnE z>FGnh@H$`hvsRp4m)KOwC8eYiAR{5(Uc0+H@h(UI7CiPxgODf=NuFcq6rZ~Ba)ms{Z8rf;rR=4CGz8kkz>6q zCJYY*;8-UkpNWaVtdltf4Gl8i0il+GVPMuUIw2ngc34zQr01SgK|vAf4O+QUlarH^ zLqj8hmPpy(K;`=V>dXy+p#yvel-p#~6($@^$~Us?bRP2Kr&=M^kf>;Tf8U1urvFOxty3EoeeJ8@pJ39A^#^py$x)O5c?F(&2k(cte~Xe_rUL{N%NhVX(k3nU}K2aPVf4zE_6X!Hm-I6Qo{N*hA41+I_6Cv zMuGX@yN;lczfc#sxX0Ir zV>VDX9l}5yp9V+~Crw=StO5_-+o`MF;&5S;fici-Lx(!spISTmIs@d*`Z_c+lk}8J zE_+|Qd3oOwJ4Wj)k;JLe_yqW^2T=yy9L(BcHNgv(kH42!frpXmhgVSGK>r5QLqSEHgSbS6fxr-B$J<8`tbxg8Zc<#^^%^}A8GOF=UO%t+?mb^jkR|DL z4PyW$1WpAbC8jvZXQv$vb!Fx6S*|q9G=yioftRz}s9u3=Q5LNCx3bcbxA!+q8FF9I zC&AZ)m_O@0U%^VNPfaXRS$5^fs;o-`#xHsPym_2L&m5-DbH5#O39g#@1QQ{4A6iOFOJ6%X z<1!!m5rJG~cA>lr+X4s))xkT-2S`|Y7V7z(G*gMm>`oU;=g|D1imY&g$crvNEeETO zpDfPtsZmw?VOg*PsSz@YH+t|$kSbW!$5(@sQ-PC*2Mj7S=kf|PR1}QlSn;r!G0f{r zhm9@_FU9FxXVSi;8^TayJgeREJ zkI&~mLv0ZIYEJ+j*o=j?wl?4&T!F4C+v3uH1l7l6dBNEw5`qviCRAOTQBO&2@x1#{ z*!7O30{sYfW8PO^cV;hoeIoN^-Tb+}^xC&`^`=oUE+uYy~A50c6Yodd4}4 zqp(pF_eW&H!GN_oLk*@L##y?T6lnJqWS4N=$eQ?3%9%=w=((eDaS0=VKxYS!xt$Hj zBjm?wNGhRcBYaaUiuI4VbO#wuM_d1v3eD$fbK)((U)((`VJSE`SkZC=(qf+iw`H9= z8yC8ol9GSZM<419{eSt_{ZAubToS@Ix7{)eBMj3-DBMFAG7Zr{WufTWN}wvw>o@d< zD&&(HQphrXV2Mc1+DyH{ekC0W<;W>tYNVX=avT;7)`bIUM3jQHDRB3Ykq+Z7aW3DL&!hRg_7INy*bw z$dQOGdNfzBogF{2mNV>RtgK{YBrOt2H(sAgs+Zi9@BqxR9{L9^z|>_5edEy$7rWZ8vH)NS8=1JNC!Bth-1vtL_jZnjjUp=9qF_5=g&C}``8Ul!RahNugy03nbAJ_VR#$}k)M9dLA) z&rj&@4$Y*JW`9_0Fc`eJ-|uaeI;%@5scD4X+@i$jr0^=jUzS+uebkIViGRQOzn554 zj1(!0v=};=iI*7ZPj39A%rrzyRRtxstIQuij@!vm2_sJrj}?j+6)h}6Ds{jW@a+j)}00McsiEsRm_h#Ca@IQZE z&aexWOCp7%A%@bKBq_(nH{@4S?_HQ{uezl)-E3he)ur~7N6=bVIf!#{bjx3Lc}Q#3 z+muH7eqxzHiw+OO4gL`pj1=PFM7#!TiW@&Liaa(`sug%(UDz?(PBKf0FB`n{>CD5EHt#wi}LFzSs*1;2Sye9|rWUwOyXU z%kFTxmW!*)f)9Wn)2U4w6ik1xrs>*svKCU?vtSYX7Sj$f#V#6ox``iy{$-2=!wcZOPy-W&v$6< zrymj_*0>VE%*MvX$`3!MRv`3sf85KvdG!Wuc_pAc#6^!FOcw<1kKM zy3Aae?)^toD~$in4mP|)WM1&Pkg;qb01=)ti5N2`ZfHL)R)(H3X43o$1)6lBqkYs% zlo5KP`19{lab$b}HD$ld(Fd*G%Z0L}_{c%KR==9YXp?2|Ws)1cv(dzE)ASExK+ydg!*@){~+qCtVs8KgYekW(n6*-H8BQ@>VQ1dZi)cr=XV;Yylq&y zyf&Xt4v)zZ#*v7kJc!=pX6)qcjB*rLmCco^ zruHM9saYFQkr?BFw&Fr<}xl?wu<7) z&M+^c7`Sq`OJG1v@>wmgJ%0K)@)Cwb#GG~~deW_-p(Ha)waQVWRzbOX#hQjjUy5;K zhIL>*P_y(8w@U*ADU=kk2>R~^uVVX6UVP~I%uGn?yZyB5y1J?8==i6n*9{$KW8{i2 zbvWFq;AppYk>A8O280(+POUvXMMa_<0tV{ZR*sfNdTQBOIhXV? zq;4m~+uShecAy&AhvwR$H3YmsetLSkI)Yh9h-u%c`bItUt3>qR^shTTCuTTBkh-$E zva#*BxY#cjw4CTLyiljiGI12#|l0CI*E>MGSU=DGCCJ4v`dagTs0i0eHKns)|Dji0SeGqy=+l6GbV5 zB2M@cN2*SH@>LaQMSMo&t(xEG{$9CX&lhXoQ}6wIw)Q-qheKG=!aCebKAWjy`?t!D zsqNzQG&4r;U9U6p^UK$}1_rIRPw=eyi4n#~)&O!xrEz4p-@8yi*$$<4RXa0wabyT08%MaGt}u#rdG7-5myTje&R4r6BQ!3`18 z}=pYrUoVp8*Ob^ zY-|rnA?YLt!ACp8`r3Xov#)|+{t2Si+<&IWV^s1eNDlBW+S@?(VR+-7Y}PckMp69> zL<^6IwF?BPd1y z_I~CLsM>&Mi|Cy3rS|(NV~}nZ!2P!DQC;5MT~gs?3k>;XF0Bc?_CIWb`s2^Xe%rKE z($Imqzd4Czv`;~sI;SkLmE=`v60%1Db>|kWLsf6=n9vRulg{l>=4OZXtvRDaq*R#T^d*gXsF3cR7#uM#-@Hf z*8_t^A0cAUh>UX5`h9jfHIpM>mBcmjhJ+UIY*3M5II;SSxFtx@F`O)anImok4QdL1 zy{gf=5tVrlBFhEEd@SqYkQ^Rx&&pM-yjYUghv8gWnTbo=*ey4`Jh`3{Y`^#YFv{u?(BMPF~FKCV~-eZf?_3NkSPd;j=MwVqDPneOHg^5J~-lhnJR>o3r z^h2h-@2#q*<#IiVtag?GFm-D3LM3{qDCh|YCn{8iAu7CMhMEy!`M4FjST(m`)IO3& zHia5leX#~4g$;|L$veMVz~wV_EG`_Z^RJ8Hh6t39B=*GLH8xgG%E)=dYm*wJW@#F? zc}-yS8fe*jJ})l^g$kQ{nP-sH*j!*%YJ)Ye0}(}Dc(A^prl5fF;|EY7_D2#|cM?^ZXt{&hA(AlRdDv{ZX zK?gdJ%PA=dvY~x}xOfe;JQ)HMo0t@#O$W1#t#y~;CqoRaOC8SAGX@7qc zS#SPpt>z+Y(T4Ny`LeDM^wH841>KTa2j}nKvWkrE&(CHvMWH~=#PojXG9=Pl5Gbkx z>bMx$(rZR`9~l)Ez{D(zp34mI0Rsj)I36CJrjL%Lc~t?{ex5)W!;VXk^s|!$VUTbU zS^Dzt``blUZnq1oW+dJ)4l+FFGlfXZq1mBU{gia9QcY!DTe!Q?-=o5hEiOkzMR^D& zM5t~gtU=<8Vs)=`j?I7nG@i@?ysxqr#X7rfsHQ|zO8=z~Ak(`4LJ$=1cp4pP-i^r( zWSOAWbNi+ANmIJ{EXUrjTV7}f|8nQj{iP)0^-y>iKQXdBWBs3`dX>%y%oSd52kLff zzP|ZFz9$uxn_{?Nlk}%sTg{)#68M@VQ{BRzIAJ?aD!H2FwVQ30d%MF%+g~07m>w;C z!J%+}lG6kG9EG@X8+iCJu_Ty-(-)VJQDHUTf>vyV>^yXRxKezG!Xs5ChE6{hGEZv% ziN1z#t8E{Ix}%DA)pfl5>2#%Kax7h}cDOM#2B4rM?GNT$<}72-5hO;fG0#Wb9>?Jo zX!fWI2n*nRT<`Mtd!D!eR*0w%Gf*=~!XxBEJ9Qxmz~?nV6G z+?KZ-ipi-2BMH6>Z$?m8&t<&5Lqa0p-qHagu<_2y&aW}*EYu1~*$WUX+?^mi%zvU< z>2?+=)rNg>>l7gIBeQzR6R?c1l!huQT36tr!X#KW_KFYsl79%KoZ5`{2k_+NG_f=<{LDxV<)5aeTZILIa}Hg2$_Xk$)M$ zn=MH9c`*)`g{v48;Af{_N3n*<%)+v}v{d}O_EX$coTRu(M+B6YaO#vaqkG1nQu*GQ zMEkzLYW+{W=i>-D)s%Yxn7mV4>0_ZgvmmLDSinN=Mf+D{?WWq*r%1{E^FmVrg@}k; zT-fd@7u~NLEvSz_Y(iMQ@+HsCYt?v&1kUQ5Cr21!Ah!JWKhB3F=1yvA0$Xz&?6?=) ziWXKC@#}?r%j|7z78}vpL~y}=#uY9!QY$IZ#c+lVJ*&KMj)5gRVgHaDT}(Y+R-RX|Hq{=GOjCuXHl@5CMs8-R2{xw2 zU)TZOtgrX}jwjytp6oT4 zevBZ0WY%qc3L$=MyPRKS)*Zb{hQ?!bi4XDWXcuXln`F?ZNivf%(s&=_bP^H0@&|ex z>=q1cb?mc3J|`=K$EcwT{51cX+y4wJNI>=r22^p`Ny{nZn+LqN6@HM04dGqx2kN=-wMV&q#)WTz<-NA+;uXnlLpK zd|`vJ<`3`Da%e?>*FjbTKP6_)y$J-7yUZ=ScK`#$zPryoQmQ8HFBX4VtA|d z7kvvH!vX@+sV5g>J&C|IRSfLcEuNjbe`%{q|8J<1JKdhyh=jU^EpCS=wbwH>$pZu5 zPE_`yuadsv3q6pu?k|d9uMuH%*rt7|h7rPqzG3!F#S+My3c~^rZodZI*a<{gLCMt^ z$bF^HI3YP$OK3=0!=Nd;ll%LfNLUt9(^KGF2pk+v;P%fdDkvo6refq~D4XT$#F<&0 zR2vklmm+KjA58)ddd9<#=O=7y9B-{14JR6Kh6=%o_3YWnW)$U=va$vahnJf9AK>6%l3DR$EsAM&anh60c6YoPW~>j? zGrrFk7MR0>(TJn;61g5cjBW})-#J)Q7ff+d>i?*3a8Jh_Ho1i&Hf&DOv^%eE@|3m> zRy4$AmWmt?J>SEJ8ckmTS5{jW@DD^C@)}FC`S|Pez0YC(!FviBB%er7bYZ4%$JbAL zMGl7r;@jN9!^2>pLy@{f19>^(-r=_sG=fnn3Lae;MKn6tds0|m`Mx3QIb|Uc^$Y6| zTNQ`(=B+=3I&Fy;_22|LJHxibp{4Z;9n=yle15?X)^JU3m1^uV`F8e??!I~R4LBid z7YsQD6-(#7T@LWpMQbL&?Ud4NZ%;0Av(sB{<-8C{1Q}-AqJe<{AybS^lSG4zUcb@w z;~*p)V$_%%NItpq=CD?7+q584zdn(QNymm9CxO*!m#W@W)k9~A9Kz?9j@2+H&^Eis z+U00yDu(CcmA=Roo!62+Ks$W$IVW?aXG&q|1vB^Dom~_#1VIzS*j^6mIuEa2t<@W3 z3)pgYBnR~#ULXqOTOt&;ck@y5O4f662HsDW#$e62w&vP8LN5fAT`h*k>_Q{?X)ZJ-qLgj%O`$>d+Uig)ELA5-dG+PnF0wpTXK285z`9izG!2yLjs%1YUp_>G5OIMbm>ul25%|N&!ERp_+gXL zaFQ_g6}8knuCwHUfF|GGq?E5}XxzQr?izeYDM`Of&=jiBj*t@&F=@%9YVk4dU0%rj zRC-e*pRSC@&Uvq{&G|F9Kvb17Ks~3?H2JsE8M?d!dqQTu5)zb&A&S_{rt9iMX83049I{XH=39+EQURnsURsoH-i zisS_&otGD@q+~{Is(g}MYT!)DyWD%wDBPu91vwGo))yTyGwqDm`P`^ev7CZkU0!uM zY}q0$YM4crG+Urqm^Ky8cpDjof=#bDwoDTYjMVvOSRDK8Fio;WFkw&%D8%}{nkjzL z(*&~MuY0Xc*wf7pD>*C9B9H@=(DHUYp)v48isj}RrJC%Qtu_QT6f*D&S>%~Yu zZU@CY9d=>4&d2GwoT7>Tf=2xZbOGXd-99fKh_uLywmKcGoT9^DNQ^H#vKLkjO_grB)7(Zd_c?)itiw6@i~!o0|#QSx`_= z{}1oLTS31L=bg867*Ir?ntu)lrG&E2Fr3cd&SWMQZRt))u-~)@z23AEe@;sw}&n5wRy4lg| zHyfXeh#2cK$|Y>&*qUo?MRI@A`Y)hF-}DJMF5_u2>b3dn)vA~Mwt3`?(Qst`If9x* zEnlgQhd?R#Pv~iyzhaUt;c%mC)%|<>nkaWClYq|hNOqU}CW&iOPIN*9)g_WVOiWlD zaoC{21-2qcSQv4%3-h+Zwp+c5+MzKd>QQ{FX{VOKWrD^N8Pfd8I@j4_ndVDvQA99h zvL~e{<&>+xXa4#1* z{NCU9$ygo}eAwG2H+*1R#R|O)Kf)SSsF*FbGOlbaVjjV#SehIKW|XoQMKXi&8ogl< z1!CvyOkbs~zrQ__*y|b{_l2NT6;c!&0ug|jDZaV5?RU06_&f3UYbTE$*WYw)MKLc# z00J@ru`uSzX}bNwn#l_CF0~@Q5Ftw2@rL_s5*g5$4^+6X((5nhCLaqHTWC`aFR0L8 z&CAOpBf>rpk0u{Zae25SB_oBTWj659FV96%WzAVLnL^`D?Vh^tlHJ3rge+t)H(Hdw z1K8^;KoEazO1#%JSFEL5U9v;0V4%W=spEpr5F8>&A*rkLzPjLxK~zl;JBBD>{?R8o z*y5uoaY3k*f5i&C{+#<>ZxfQcw-M9UhZr)_?_i?jrJTdhPR`l(`8z#%=e+gng;o>c zAdZxQw=4CM8Oql!@#;{$DzES$q-SS=HFIj|0T&8X3_EOF@ZZrc9`5`t_(VF+ z;;7(pG&%p^S266mqkCQLSe9SB5z6&L%M%R@`h6fBgI9Q5G52t&hiFbY1V@glH0pgW z6zlt}mGBS~jhnm;T{88S%|C|(e3>}7im|V+MxmP{2K2nnN5)MX=Lf66+Z^Um>6=Mr zsO5|Dhs6)n8=jV4cE3!HULNTD|>s>!jATCUvn?)iI2Z;Z+z{Jbm%t4 zgMb_%UecnHijJoVDPLEYscE_@S~?*Cci0eYVs<~j68sRkP?g&`Q#7DY>ESuxA}GGa zNUmzt{`R@18NWk2kdo`fUv$Eg(@RC3v{e<7Eo0x(oBt~uO$dqvly^Ar#t#j* zAydKf?T#Xl(^-ZOvP4_7Mk@jFV|Bhh9RV~WNVTxVlel_i^BNxu&fS|U)}KFr)KZcM(K51rxqZGp_CkhNQhq1z`1(W=T9}*X zlJm_JEThEw#K`o6t2r=RS3=|!tkA|%;>5--?DU%t;>W@icEHVS4~o`Ow_=5MF;_r` zLS-mpXNNz$$YVn1ppynijR|LRN1jrEfF-7+3h;13k$2XXLc)XL0Q;v7#YV^a@$=yk zVG$AFZ~z@{d~B|@y}cDqd&g;*dO)Ln+7j|yj-Y~vbFpab#6-nRi`(t< z((=eieec^vgAIT#ZFJPu$R9HPJwYiaifbI~At5fFN8i8Va7>|i7a6nX_2}*pXt)B) zTMOOo``HL&4lFpwJ+`|_SFGYBmsT5`m>7{}AVmuScHX$)BFZ_z#xPs6vPL|!p$9go zH8e|!(aW~<=Lj_St$gAo`Ughfy`^F8L3AU?+gg;ic2-_Msp#dOwWN=vxtF-O@fl^) z)@i=K$IY$);b8C9f=R2Q9yAf>ivOb1%lgQERIOecI}_eeC!6K>Q=Y%5{*Y&G?otZc zYUO0#^>uM;tD^bZ`qfEzI3%9tdya;fwhVuhu4Rp;)18waDU~2e!bK)=>f#1IH0~u} z5cCUltGFoe^1Acxs{I){QkSm}kr~@jZ6|^mkn;|SjXyXfMB0!ozv<0oDKJkLL>wSY z%n|P>>clzaCYeEDHv_Bo7O)CV3`0ik&IQ+8yhuMfS~y4oF$fGGkfJQe#(nL|1J7Tc zr%G&0Udz=}boeq9dOIoK zt~_}(D#sQG!z&l(6G~&P|FpWk%p|jCp$I!uc0rne2D0I-Qn5J$o{6etr&& z#CRSz?p}k)1AgSGZ%~j>!E^mir;S-zaZidsaogYt*J!Ycqy~OK#WtRW?ie;sGCLy! z-oun`!vwZ4zTj80-3v7yPxRuVnJN2VA{xBR5w& zRgYaAtWy>EtS|w_CS_4_k6;xobgt&iU>C@(X}x(8#l|FD2pfq47)pDPLU9y3+&M%z z7>@He>WC%dU$SQ86hKzOZtVh|PgPaeo@|(m*kn_e>G}DwN+YW$XS6?;b??r`fc4#6vcLRA2!OhkxjGm&^b=9F!_O9LX>UzUO^b#sD&}Y{BqHmf-I}*S^Z@z zQ<~+uc#O{Bc@=;#SwwC~Mz2yg0YS4?EIP}mH_OLgA@EV7|K?M$6)J5C>&EWoU-P7z zfJv1B-u|>R-q1@y;g@T;z7@aQ*}B&FjAphztv!Xrn$9Ke058bq=pRoUH=a04+M^9w z3BRmj@u4Twq5!BJxTuZIf_sUWIsJ}~_CAx^bS)(0XVx>yJPOEEDoC#Z8*V|mB^>mM zJ4`gykXTr@lk-+zZ9f4i7*!m4NNmN$AUKXNR(hr1@OIQ7)6*sQ0^NDn&VfSr6!f@z z2uQ=PO@B%iPjiwofQe1sX?*wX!%l+i+!YAR`d0P?!}wpsjBH__t?p`NKAga`YcCoa z7MNANc9M?QK$e%IyPBirA&PJF-*5MaGRZS^*|>FXjodKyu;3tc!-pfLC)U zB7W9Q25A-Mrp=HVmQO*FiqDzXqz1fa|1V%4`I=@@jn#Y$y6NvG7y0IwVSY=1n3z~R zJhaHdsovbzbKJ1C^Z)ay0n@v`GWh#m;_=akb zZphyrXD}caP})eLoJmExFpyNa@IE=1^X%A|7?=ca_mxeW_S^Uh`;_qyg*vgIeB9@R zp|utOT}o|+RURB1&KQiUdP;2}9I{_s$jy^As@6jcnz1Dz?7H#k2M`ECO2r1_tfFLX`FNd@E}ap3s2;3z;(_5>kC&U@s?ihtKnMMU=EEt%jsOZ-$OkJOI~{SSzMq0um5KBJMgBDV`lluPN-{l7g5U!n3bZ=L=;P` zqbREyL~b2h08t8ijS`7Oc>joXwf;4kG$v^&NvZ$7F;@fJ+5Ww~>5PUYIIE`Y!v;u# z;i&3<`a^XI?Zu@0^lrNPc6PfV;lJ7KmS$*kF?9$d)OB|~tW-_^ELXV6hFF-E{uq)) zQ+--SOH4*KlOK`x+TslSw^HBRaGxHiTz zy?^F4hfM5iYx8}3ebf`)d8JkVwAcR)tWaXnQC9}Fo066DMn)6iAcjq}tg+^lQ}TG( zx$1f5bW1xm71}z;qQb(PTX_7BYcFzVW8pwrF5zdU150Lh zzXNXZQ59#C5fq$*hNR8MSyqo_onXLD}sHW&eMK_MV=AUjNyAuE2}3I6}dI7+2Rriy34y@(8sr(wU@87@^V9a3KBZ&>p^3 z!-?c*WCf;-?e_^@MO&?y%WutzdpbKwX(hZwPY=MLlT9GxmNvW1h>MS!aprXvk{^a! zh6h)*5tmgE(h+3%?Vz5Bs?YEk5A#oYfwY?8H-6$Vl$qVp{^2-t&z$X=#=%m6*)e7K zJztwW5g{%v44y;5d$nzx%z~)8wzb4)Mwyd(53type`{-(ac^XLLV0^ad;K>$5>v!+ z+7TA8%93%-A3wx)l2|l$kz#Zm za|Pwt`m+y_APb;Kg=`=#agqmhj%HSn#sb(5bV}`w-2WmGp%{#*g>bFFLXC}rP}tGI zxfd=(F_~skrOioW8}#g8obMVk2BHUL%;8SvG^#lo8g5ZDRZH7+uD06jUfb$#=B~Et zVZZ*o!a;h!(xoxnoL{>ASTuwKek%yhACRvxFwvRkf5EjX%G>M9(;cIBZzcRcJ%@)K z->!-W?E!~S9b1@8H2?1JQl?UqP?98K;Lu%2$OP2tlwEqyoWgy z6X7qhhps%d5zs>x@hzJ4>}^sv?bwsp3#`1@rjaknW8i5Jv$QQx$IsPsMCH}`rv zvm(?p5>vl%9BTpD+74s#HVkC@Sgq@)kbr_R?4fYMXQLkvkUcS@b*`JPCw?z|SC=K!*UjxV5CF zWss7TmY0!%4y*uT_cvdOGykP864w>DQ}WiEUCZT3@cEqz{4^?Ml?OdQ7|mHEAuj%H z34S2Mg8vJ|b$Uxw?zk-k5q1T@FAu2QG6UP&O3OW9nKgI^-4B2e4oeG%v|~k*YFL=( zKDO(Sz!F8MPgAPvB+0@xoo5pM%Fb8Zzu}!GFBhReF;)^mar5#D(^VeVzUq%pB*g)? z`MS-w`+Nx$-*tPTHe2{&iCr4k-eNLz(HS#LPsbffAo>sQeS-cc0x)Z0Q3W6^%`ZPx zAw0kR@i%5rxv*P8MI1qDL_`S=Cpm+N#y+rci_9Si$SN~z#PEm`3t{i z12Ul{oGqRGYFK?&u2-+WQm4L>6k2*Tjl0Ury90Lwpl3jVRmGU00q`bJN0=yy!x%@a zadgqpsKpC4QK-rx83tN%ShbH)(A1QoiC$n}U_~x*_}6%nla$LgHH*e7eitjt7gFB= z59eGx&+q(H#^CGa;3m-2gGHkUHQNgGX;Gu9R)~i=AZU!lpx7;t8T8tkmj{Il-tXj; zl(q6Q2>M4i$A2@kGg~LlEZJWy>ZUR>Tu*y_C#O@`uW@l%TCo^~$w%xq+CBG_0=$G| zg;+>>4zei2Doc>%qIRW-FmW;WiKt65mq(7_sh~gxU{JY1{c;NSS=a=2Q1SU43yTv? zFKY+BV2+gv%hAHn6i1liT(v4JbkH5`S!U>efAfy}deppcQ$Nmw!mkRoZkSZ<)jS>8 z|K74H{C)c_-mWZ^qE1gkM;Bv&UkUqoZ2x$GaptY(F4}B)n!EawC!gDAMu`2zASOqv z;?SPse$|^6>`~0$@yw4!5fF)}nal4CMvCzG2u(-#?R^$QlyMb)9w-!GmwWlk9ykq} z;>GBBE;z_5bRq}8j=e1_{EB_TAqTP#&NwdCHoR}jp?=e!>=CRaZv;~O*{JZ!UR>b@ zCJ=`3Yzg9uQL`s!-2?FOyWtIpVV2R#8O_b;jXBE_q_6SF*qTQ`tx`dZxMaN%>d(zZ z`QsuifR%}giHXj(kc={AG9{e>?I$@s-Dao%_(%>NonX3bj@T?FwqDT?*_*GgR^7|= zlfC2F=X)1SVP&Ipp!e&e?b<-Spa(wg=*YE-hRz22(@=>muvK3y)I0QtA6BKlh6Enh zPYG4aOjZVpR5Y1_u5faMQiR7jXH;lG#x(AWmR;q0<((%@VX{?0Vh}dKI4KxO?TSrd zolHa~L7fkm3LBY`kZa6=3j)iDt)JZL67}cpG*=`_y!J(nRhLthrn`<`)!M z8b`E|V4@b-=EC-eFC-VjHbG9Qh&$1R*w$Un#Py0(3@ZaL_f@IS6z4FeRc?}w3G@Lo z==1MQyBmCCAJNkEp`QfniyHJ5^1riBF~Ji1Se|6k0_{ph%BUv z-=b!b66Q%!Hd5FhfA#E4qW6kcuk)VTo;YxDtW}+%FMmBz;$swbS zu)>g*C6T;YUFwl1vcdl!06Ia%zMp)G2VjU1IXlZJ#8`qpkjoS#%|PE!xvBV=plshh ztJSLY@PtC}4=}m9sq}i|45}2r$J)$3#U$Rv$kG!GySYSg4dMBf}%Q zx;iKIEdQ@wHzAZ17sL7QE#3pZDbOTWxT+>}50OM#x_95}`1Y68G$BkGojY?TIy|DZ zWUr@(r%bK@Voys>_V&^O`Nu>=%H>KvPQ=EK;hF2 z2d%(%)48b`hWoQ4AJD=E42snzc^?^g`Fh@F!^}ii zLL)o)+5X8r$>(UC5H8i!G+ez3k=B-$nu{0fE?v6(%<8XR{idPe%H=w6hKGhGJ7t@u z31Mnz^vRRV+|{2sE@-2%R)p;o(MckvP)s)=K??u_OI%_EKnu+XuMs+VYrHO;zW}SQ z-Ma1LZ(2Z*bNNJE^pdR1tOX8yk=)f^BJpF;ESg)wp{_BmxKCrz95bI{#|Z%@2tWdg zW?-g(`GTZbf~4c3qoG9{C@+W2RSISG@#F0s9jVDF!Dfs7AAWRnG%qK|*I+#QuOmT$ zCI?4s?S~%@_V*vEtaNa~65^KKx^ZK2Px3i>-GtEI)&>I#r}>)Q-1~D|L0y zIvVQhYcF2x7<*yTHBAUpMoo?1gjzzrsH#${)CjR+kwoXCgAsrr5l&D-5-oJXGFGix zlbcIYD1Bu*lrgL-_}e;)qyYxR66Yn6k$P_*fC|__Z!l8G)*o05%?^;y++ui&1a{k? z=N0J32tfxoJFm#_hit9PQ>B%u&4DI=|9~N@^^+Yt3_5+=_1pC|m(^0mmuJ7U=h=3r z*)`P2TPo%yxJV3Y%&6pT^yU>Bcp0u97XSUbO8R^ICiNhnr*T5~UUYO6>V0Hn1l+;F zL71)9=dZtb{`?Df>$%_E+%__Q%V$wVMhGz-q>@@KtG=5Z)+#b-6ArH8m(8@E=uGFT1v)q5`PE(Fuc!Hh&O& z{_GhKca5vkO{?)bb@F6%RE$_G1@iaQcmWSUrTZFv!$TKi1W5q~2p9%OEWr>I!Ud#C zs!WQE)yP#MlEamfw^w-Ccwqe)8-P<}d?&{JPfOIlTJ1f@q!cK`jy|&YmU;X#Q~SqQ zcXzc%drK?iV;!`v$<5Pid^AdRNpbObo}U(d%v&zEA;DJTFIK3+e1jqqmMG2o+??G0 zM=!36$<;(c7#<$3C@;5|&3A6yg2$3QdvHQHYtH-t|G-B*J)D=3NAwDi1 zI>YG5aCT;JIXUEI=L`)E{rI&9j~rcr zV{H@$cyJxz-85Fo3KcYc3kd)ZZX^7@&}l6qLO7;WMAaj~Zd%|>2)!wC9t;I6Dq_(* z#Ns3pGVB>v*g<1J1*bXT`xKMQ(f26~HfdvUZ-n^&LkJ>~ghm_?0w^DvKa}A;0|hJ#%+va3=V3<0RaKE1Q)=@00u=7D556pAULP((NG2r zAAsTm|GL$6?z`u{I(1*)bAD%m<;JA-{k{KZd*I|iU%$)cmtnQMv|K4yd~Y-sUAWL- zG*143PPV3#5IQ?Lk`mWte6i()sb6>+6&Xdhv9q;j96E5o+;O~3#YIISK_NPwe&4R$ zqa!2tnwlt*vbME{>_L&*+MBl~E5cbqm^m8g?~hm-9i)fZ6W=>DG!z{f zwIzMCQm$N|xNfnZf7^oxgxHbNkim-spm-fWa>UM_DMWCI)CCHVzpo#Uk1$vqHdi1N zU`&-WL+{FR#1R1oz`*pB(!Iq5rEn5S{ys3$mPoV+qIqT?FaYo|73xsA>1%oVXugdqt6n; z%+QYLs8$=Bc7oe2WQFqumDM*UzBlj4u}p^jAj)Y49zabkH@3 zO-hgl^Y+gY!VJ-#ox8M@&TM~?nidgZfEzf$y}dm>KBTs}sWCM?C#O@Zh40oSa<*2o7VB*qI-#rsZmBd$jYnQ3y`(6;AV@*6-E= z56sKd&;Sp%k#o0_4vS?L%RQJ{R}mqyh*f^D-ut~AYR+!$hlySvuJ)pB2Y%23AvgzV zhI{%Ur0Vj@KWoJBgqG>`fflPQzGMy+?!~bK`MStpts(Hxfv-kK--XIEO>Ypw(BMF5 zNSH+Gbob63n4UcTFSxLc4Gk|$?7hc{kNcjyz54+83r?RN9vm#cbO}BQB#Qrh zTVP%t92iimHE=7tf0__A4G#@fRaBIhmR&9@|J5j~zFr*~9Ae90%@V>)P?NIbd(6=B23q5G>k|lw<~* zZ;yUW^FwHRwE!D)*14WMapITY2L@Z}?*35QFx)>dWyd)sdV>(2J$d5lCR>@X(maii zi}%rg*4NVmcG3CsR&<*yRaN7@Cw!?vtJOUp9x_j%3nNKttV3yusgJn$-Gzxn6}vq4 zub+HMx3L>{SXtWw7IOJQ&sjp40jf9NjR*)#;&V3m=u0k@zRj)R?73KlVjY(quGigt zMG?FA<%WtyiDK-kNYI6Fx)IJd0&bWgYLQG~%R^Oq9+wbY=9TphGLH@?IwSe!7~LWwPh&AekQ08^sl` zNzZx90HLzP+swI1yLzTQQPWNTNiF)y7ual(pj zef2Gb5E&UmYN(Ot&&|`&kT8`>J=Ot`-imHpa`C?ti7GuM&C-f)@4&QoV2v{vEXebm zSwfgGx_QT#oSJs!#*LxzN&x6tcTeQvfKv-CS~w0F)|S~jc6K~@T2Xh$N2M<1GMX{o zJ&f0kaa#qfy>eOZ{)4z!g1asApbu?2!Bi18m0?pFlD#qb)O--dA? z;oKk*&OmX099N!O^<^f9D}Vw4c!}ep06%{)E}T2t-``(T zQ`7eF5rlf}%2f}AM|?~SsU}bV`_%LvzixKgRmRM8~K#)bQZoZ^rOH_RBpes_Ty9 z>(sWUok=>Tw&T>|13})q_qF$)d-ut{cQ4DnSx`VXpz;t{z&0W%4;6}niweZI35LX) zq%mU}J56UQbs8m^Y879>7pTZvKm`O@dw@tXF(m%65$W)qJ9o}KbIzIH{N|qT@AGTC zaf4w*i~Iw}vko3c;8TR~!l1B8nM93ZT8Mwxkfx%KCpO+PEMOow3L8sP-j2rz6l{S@NCb}OgCI%_pM z&&du?ZnM0!Miaks`QxFX^XJcBx_AM07u}tA)6$(YZNPV>HZpG6oyJ?wGm}4(zC{R+ zzbfT^dcA=S(?dHw^PkfY-8cWK*vCIm$A)!vc7EqM?;koecY)7_{DQXD);+s+^AH@z z`9+%w2M2~Pb=}#sLa{`tQ1$ipK4V~zCt>w(Z)1b>U`nZ5NCB#>Y8Ld`V2e7*&a-ZVm3#b5+#z+9WR9N@O5FrPk=I78}HXj5O!+xaE24)`zj-yDKVL9qTGB9S+y+Zfa_R?}<*EXY||% z0V*>i)7!@{COU?PU`dHd0|NtRPoIg5jG8@X?t%RWzHaE zE^RI>QmZtfGR6D_KJb_3BG_ohx-@1fpal|%EYcpO)3Fc?$S17<1sIH=pc_Ci2(iju zDzm&&Y{JzNluIPX#%HZ^Vkn;=D7j1_k;!NR&^XNv51o@^@LC(bAeHrU>SRW(Or`j3 zg351|-fpt|Dud;$Kz{KTN2&gs^X#z+aK(ZOihXjZ~lqmqWDL9FKDj;}5$!>;h zB9SWqixkjYlEDcoD#ET+YvBMo0|Wzz2Vevkpi87CCnv2~30sP_Yt|&JNDzsnQML%R zT9cfRs8Xt{s}8y!J$mo(VT;i;(W&yho+lxks5!1us9}o%d$%uYYvD}kmXevjnGGxc zr3)9w+ja2Gx8`~KXtcWMB{6g6F34V!1^Ygm&Hl;Jqt$O!L3@Wi;Mp^0JQCslz55pV z2Tc*eBvNUq%WAg7J6==Cl{}Q-lOR4W-eR`;1_ZlGO3$1=4R3nh`rJ%s1}q8)y#Ih1 zBLsp!+%LetkwGZ0%#q^0%eF8{O9k`Iv5}?~OGu2u`9L9-MHwtYg2CjXmvh2j%{9!) z4*zA5=@&U{n9W@D`Z|+NZ`Bz#IJ3OX;u)o;X?ZHeiio0hc@9V1!w0=Rce_kR(}BJF zCOTD~*VBYBrh)#iK?t&Ue_!wSA2m2I@X=pBEXd1y_w8zk8F1$4ks~IvRi)NK#{2x_ zNe_mdwkl;&K=2eHOd@^skAKGUiFQ8s20&W@dUF^R&_;mT89G#NAXJ)T$7>SPSJ!X@ zAAn&DXr_?MG{(~OLP{VDl>t_tE&we6y+8B*6$0#rY%!5)+eml$LHSE>2BJGa1c`qZY$8E92rPI!~V0 z^CW~(4Gj(sJ$f`c9(ec=y2lT0X>Pi`qikE@#!a7}It4%P-|N{}P@q<8HClErV9t+^RuCr*%xP}!z}f-gQh2}#my zG>OHb@X8OH0ATnS8OlhK6A~06nOCBETDg@|YW~SKOWv~}2?borm0M^xSu3|~kG3z? zJECMUX048`+J9i8Gv#@WM+jpV91gBsxzcd`dSCAk5W=YL_wilXJ4N2!fJGLn&JDl>_i;f&x;# zR;$!7Jo4r2@Zr*5W_fjkQAE(QSF@a*HjZl;7=j`cJedd*Lf{g`bhxI(Jj@0oZwM8^ zHo1>M$SF5*Ds8V6=w(WU1Q$4L(RPdN{e#t988atY;)D!Q${muj??MKqngwWF5 z>~J`Og1NA`_xJZFCnb6N`opd(go_+I`tbyY`b|v{!V4?6+nv2W_YWjliSm+J-M?#V z;pBsX!FaRvDlhOBK$|G!4u!%&J&i#UXrqyOJh+7jxsC_lBzcVhZK04WG^(d*FG}zs z!#^e1TX^7xKy5!kRTLEPz)A)hTv0URPe0_Qh`t;mdh0g#@Dg>9WK{u zr&F#_LSqSI!}x%ScQ_zaeDFW^%RQ*6JB#B{skOT8Ke|)h^)-mRh1{Fm+;;*A2_Xr2 z-Ft(O@Cbyc2vVM|542LA;SCnBtFzm6tE-ONT5a7H>(1=jT8Fh-YpZRQw$chdfT~5i z>tpu>X5Eh4-SH0~Oef#raelvZ;Qr?1{?7TFeS1GVeJb0P4KLOE$KLgI3ddLzNeFMg zeh_$Ug?lB~AD{UMKAtGCBs4tI?nu3U?b>)h{c+?cgvqS4XU-UnrqwG}Ch@lN&0CdlQ>9WXVq#-rRn_I?27RL2m0h$ZAF6=_BOVR~goMO2ikXf4 z%m$8%jB>(O z;TQROBt;`?7XT+jTlp!?+>?#gsUBmz!z!o}=jLfeFC}^ZPql7C*~SNV@A^80W32xs zgx$Y?W&YwowI+UWU;yrTWK?u5}KmD`2yCV2HQyr_p5tSad(&0UG%iTzr_70So{nTqTiFFz6?|#vp+_&k*x* zdL2#P#>GGgWi)Yf#QQalvLs2xs`7Jkpbnr?oMznw=H|eqA#Y5M`NB^KlUcP@)h3g<_fqeXt}dg&c=gH^*nDmO{#9;w zb!An&Mk^StPMclH#HQIDyp@C7y|$nV;S&FcdSpP%rv7Hh?j!-a4<$1m3AFe3&zSalYP53$6T|%_c@EpIvSa z*ve0D=0`aJh$1>gO@vDqr>aEloM>Cplr@SbPu)L9kmjePM+o88&6{Sk)t=(`$M@gE zvb}99l-8^L{a_z_^|j#8@V||Wgg%#_5GJioefn8z40}@=^(}#RBay|c^XlsxH)7Hb z65}Xpo!LZ@RB7qjESLLL8R8`b0{ttET_7dL8SD%#xj>>iijE?we3shI;QJYwg`y%z zx`iPQFtP%ML|{N5C=|nSZ_}3(Bm?8akK%!s5M)H6J~7R1M!L*mm#SX}3GlQqvN22AX0tahjm?xL+3P z7P&Qgv#GJhgW;sW+c*mc7hJ2+LMf2pUP6#5*d?oV&#p59=pwcpDgX!m+%%T=AFhD#mXC-Qds8s;(tZM7iFPLZ6TBC$G zIYvEQ;SHPC%1cd3@iN4y)ftkxMKn^Bv*z}9!@kbn*y|BO_;z3*P90xTTr#@cux^7P z(RgEU5bU=P9}Wr*egD1VK0E6V138pWN3_}S(1t*sA!TV zDH^^piWW<8i4=!Pfn^mMO*|$eQ7Mi{WDlvpTOz<985vGO-GE4-0*y8yIcfQ>mtR)I zs$ybQx&(bjS~|nX!M!Fo&+T%#oX(o+8eb=GTr^>XaOlA6K}$nMDQjq803Q15Z7+?w zR=8K_bb3D_`1ODE-K|@BIjbEhjxT$9_U+ljb3#!;vCEnD&DXt0-g;Z3Qfm!{Co7Fp zJB5HYjfySs=9xH?e zwDD0|)~HP^ey$*!v*)H~UpRN(*ZCV4JwgaT{;P9xDVq7}V$Zp=XG6my>pgX&E>CT3 zSopG2Cr|qBu>X;t5GJQQo`#mN;8QsAB86dbnzmPVS8o4Jqd4j|fqX@x=PAiy0=Z0} zU!coYQbZ?>e=`CJmq_#~f%K55nWBPldMAtjjwQY%&|XTCO_SLS)kmOLNVJ2(BQe~W z;p90UTEAMYp&2<82ff~~zO-~?cA%j|000mq4OF03$FD6e1_PF*#ib<$1v%~&8jV)q zg~Nvqtt}~q*So2{;b>QvGc7$U!?|VC=AjW7{)uVA2;sAH=ftQK5_@NOxS_5tBsAju zC!fInmA*dsuu>fkKOy+_zcg_DdTfj`EFuagmQxzJr(Uo;@xuUMjCO9JD~=(UX_bkh4x7lMr+M8+$GZ#H zuVo5sLT+aFyT^Tc4#1 zho^lgKOsy)LG@psk@<0C*liN&!NqQ+GS8XC8;q+6*+oJEgzyE4_K`?GfezzH6XU(f zbrk+NExkq{g9K8}5OWE-lqUN~^ixW59TyK1;!a9dD<^Ns#6!6FLt2XCL~^n{L92tl zPtpud($R=CIXR`FrWTdTpfbQrkr1CC!3dQ~tyHQ!Rn;u3a3rUwHQFSehtZ(AqSBx@ zK%4mE-hB+s?&$0k_@w26&75cq3J!hm*fC#caqKl=gaC!YVRs-XR`2n|sWnEE<;Kv^ zjT<*AD=HQ&4s5KiAJ?Zpru>93Id$;BfhECVdAWJN-LWGw8ZjA7PG_bfMhQ21l`9)X zF`Ab1X3N|{-Hb+l&K3(Irz8@bAQ_5g|HXcp2Sssbal93GH|6dcH>uhryKYE4QLgFf z>EqYkGXo4W4976TOwRy{Tm})u3k2~%Fho%S0l_jFv&8a1;~F=_S~sp*;;Q9E0A1O* z3J6{xC@2PuGPtinJd-W|z!_?bZ~dz4^{=ltZ|c4Gc`BnHDMJWm1jIcdg!j_)B1`0% zhG!Y20iH4+kD<`<3;LS=8q=`cr?!X(g*}|m3kF3NkqSh^jIA?cIfkK$p2F;D*-JBj z9n4~BYKL$pKRy zDLm7I_cWSAl2Q{X*#$)CIramF-@#FBj09NX8YA9z3*p-k6~%M)w6qCHSMiJs!$J2a z>Bo~o?~;)WjILl%4Cm>of|)`W5D2_Lij_i6$S9~YQIS#Klzx5h?%hXLYe#$gxBK>0 zRaE?N?;fD_ z$Jy0=`LbnqA@~D8gM=_Jb?CrBNYx9~=R<=--Eji`Ah|+G(JU-Dr5LgMgN+Epk*w5( zm@_jWBqS90pHlENjYzff%>t8asG5-?u`G}*$@7jWfmo3+BGW*s6nK!z0ebzKr&KU} zxlsfLV1V`JcT;t2u-~L;-^Y-!Mr4V%GYswqPoHU)q>M$^F4R1FV1;$%-xp6DD7$^5 z=}~ugOnf|+;VZI4q4}*C&wYDJ?H$Gb*H01x=vGTh>+RdU_MNu2oh2oe11X^f31MI= zE3aU!)A+lz1n}SrGrh6M?`%e9>9JbBMn!nA_ZZouQnVpQ#p1l;^yTOYy7g#TS+B#VPMjF;H0jHd9bFF}&QDAlKhdSUhl>B7 zWJ^K--S6mtto-qQXYXlMMdi-zC9StwZ1Cw<(I6oVOf@w&Lbfhmw3tQW6rlCda=Zd& z0XS;4z9fxg6nZ=wK=&sZKfeGVf7qviB6$9tC4t!HV0VkRNC=2yNS2o{`WIoDppCHbl!HP^+u6T9GTH%Al0C$QiSeDh^ zd>iJUf9(C+7sbV;TXtT%SljFKthkxO=c}vl%v2%E~9pOy~Kg-Id^Q?*G>RIuTN=^pH+MbuR z)A&}7cBY@h11KhN#0k=33Hc;_nO+x2GOWM9PObKl5Y*)0kcIP;0TurG)$UCjHadXx?_ODTqGYJ_53o@6ND}ng| z3=~Q)8u7k9zL3bz%mqC*R0EpDO{&rJb*=^#D`5};3pr2L{BoTcTieG2`KDoO%?`bG zu4!0)Pfss0JBMkMV@XP-_-(e)Avfq3zy^7J`~WsQo^SA;VmN>N3>Bus($&~Xe=ObS zO@H$5d%gjO|8=6=QO9Ge{joef7D|4;{?h@~CHqA`NeJhvD&t~jkR(0IarE-cPnvHw z>jMotwr#uL*=a97U(g^S*k0$)SH}hg#dxVc(gmKWsC@Pkbk&yTW-#tQ?5b&Onx4GC zUvFHf*G=Wn_lUTHL@>A_asnY541SO$0Rygc#4-+zXE=`KG%UA^qpCS#v4S1NaNaE2 z&_f8#!UwQ3;!G;Pz=i;9t@}?DwR4SEIckI21-god_r(=h(s#$@bD@- zbSOS9KIWsC=!hsMXV;w7*-bZY1Q`W4cf8^9W$?ND^l7<5IXx;`2r}hm=h!E>uTwTA zgza0l`1%F7xJky2cdGpEyU#Xkh>D0jf9~7>DV~FbV0&4+x;AaxXcjDH!MyzAOl#M} zCzm=qIvTIvxYyne_MO(&e;z*Utx^XI7GAE9Q4E7PPOg9{fMGa=!sFS|fJP!7a8kq( zh{NIk62O;1bz{U81`J7G2q2zK=Li3SP5 z#(H42MjB1kQkQ!S-XwFI#j__Xt6;yO?y^O&q|8ecri5{Rx>OI3V%$A}lC3luiZxz` z5K+^E2Un4Fy%HUiQ_%5ik+hwXe4!AN+L7iKbLcovj%BzQp1r^lmk=?JLt2)-(nAQB z`xe+8G+z7+Ay70zEI%zG5;y?{3XYd+)EcP_EV;MJn~>2xo65NfvmzrR{j|P`bLNhl zFsWqQ_G3qmK=C06vX>A*-BxQ;W7EI^;Qp1(2muO3a$?fEquyV+BCDdjT)``W4 z>y;i}jvtOWe)Q;T{rW3vkPvLIvcrcp-s+gB=mp6setrQ9Q&YRUt+1b$n`<(f(oz=q zXnYI?Q_`HdK58u|S86m`8nL99;BsHBFC@D}>*40-0h#`U5Ez^xeLY5`>)%M%5C7O8 zF{zxxG|==N)(S%lo@awK&vC*#sXAhcW*7iKzA$2yarA;ftVqCeO+)g9Aw5sQ@$Gc| zgjns!Or!I3U3_#*a@;HwqQkQMv2|u_z2#Ra-Wv;wp8UGbw)P)dTJ09x*Q^%^0dhQP zUXrt`WbfCda0+;^Fja)5`r6u{AQRNy%eA%kbhl4~gkXEcT7v&0bG?S7%}DyC>%^Sg zHL$t2v~=p^|FU1+!A%``92bze<7SfF&0We21dbX4uCGXM%adeTk}cVmE%`|XJNQg5 zHUh#f^dy$<+z++sb?5bgx)b(8n5Ay3U6TBhm9WoKN5*;>R* zP~#3N8bIh-%oP(pf|6p9uS6^m$a<=puEBQ@bNyvPI7Rru6#Kq>ju6$)PZsi)0#lS1 zu(vMex|nO9~eS1(=GD9SaEwg_h7TYu6sVB*|V|UG46qx!iylW zl@%3_A2<7HbtjG=hxWB=)_^Ier>4Hsx4)%Y3BmO$K7O2G`LUzN)YsLer=%)XsHVEA zsi_I@U|wGS!nyNds`#?}kpl-J`$c(p`vTbq2AB}0mY@(7C-~a*?`K%sPq%sYH6R|# zi(McDkq(nW6$>L|*Dj3clx*po6e`Dn2XJi(-b4tWj1B3QXpyNPTWUK;dT&;^B3A;U z_%PGlsfhhcY(Vu z&?|&cUt1d(XilD(+)|x9X$q84FEU>{v%t26z znM;JVQqobN<0-DeY$@gSqPT}Ds}TbaTt{rCi1t&y3Nht~DU-X%*sxPn_QE0qXNMOF z;f119VdoKpj2n~iT~QII6jG*4nVt2`fS3UzhmSaX@DPl)`O(9|f(3E0aU0gHZ+`U1 z-KlkvTt*1nwrqhdqO_z0V(n69Q3^4>GNa0*7Lb)Aw_vc3I(n`U?Er-Y(DHXRwJi`*oOL zQRF+|ZkueWeU`2LtnjuuPG$gdcpV;crGHEbd;cpN{FEZc>JuyN+ORox*vJ7f1E*(X z+`HEZ-*V~nxv}FDQ!_G7A3y2tyt+=W5JE#;U0`6)n9&I>)fWk2>!wZJd-ghd_^^8h z`m$OH!DZU`{iczGF6MlPX|<6QpT!&iz!G1ty!^Rvcg^|@UY_1N5nG3u+lT=w_>pku z2?uNd=6;c|+tjL?h`ou~+emJy#%vlzp9$1NXbDv|3e0Y*jG^Q-<>2le#B5N5$%{o) zs6{ek6l#QhRCxt+$s+a;-pmGqj13|cHL$Vx__9gk5?3x;%Ch{-=`-#%HtyTI_uAE~ z&p+qk&p%heoO$xZeHnGNTt^6-WkNV~VE?bL5yGH>gM-Y$t%UH0(!-{v__$FflX=9@ zVWL_K4397wF?4u@3wcTiJoyL!0I=b?#e=iw%D&$=-|iD>R1@*j6l+UMyBx{;%K#s%|KDeZ zx3s*WGhh8CsCr_Th zA**BOu7w2)8ygyu5|h$X)1m#^`55);WnZQGUlofQzbCx z2xlRrql!+X_zqz!5!3Qa4VD8az!kR;dw?nplxC`NqM9&>0vLE;I7t8o{DuTdL|!BW z*jn^@1I9Eg%(i{&4|{g+&P+>}>=C=R@A&cL9|-v0D3=jJ(}Vl4BzYA&^WW0D<>eZU z_S~5>(7w8;=%3wstXQ_}oxc4o)k+91)1tyco7D!vw<}h%oKmClgL#+`pAZ?*_r|qr zBS*&C!mNN049hdT%4`aDb`EO$|{jbSMh!ouiH4 zFJ4)4@aX-Y8r_{mH|rHb0NpMtGn<3qZ^}qbgY}*`Zv2Jw=i_4II(6xumy_eZk-nr> zLU4`l)-@!A1edWMRbbyLX0cYdfw^+bmU#ABID1xdO4?fQ-t|srZ}InSYSk@pd1riG zW(KB2Af_D2_)(1*=yC_Kw+OpPEu6sIDa^%F z+(#gc;KTrdCojd35 zM7mQhBLvWzRjYb_;SmwhcUo#X#$@OA?OQf&7DX+m6s}yl^se9jHfkjVmubnO#RH;a z9^AWU4YT>{^m*CYQISzg7cUt;WGKuN0ggNbA{uh(Lvoqi6Yt1cB~Nv=uDAPK$Mg>C;M@8v%3@NPQ6YDEn2&3b#(t| zKb=9N)geq1;>VmREp^KX->6nXaG8GCwI@lf&S!llnFIO<1)UX`2E@!`yhf*Ij!77I zfb+idj0d2A%iqN8)o08Aag_`PScR#E(5?cVAY!0}+dvVBjaHLTOy-Jcom!Zw#seui zNtIQY*-!WjgadMD6nwW5l_#buL;+dDZIZANyI`rj8WX-5mA42Hp=%}r!X=!Qcuvm)k-OEhRIGvWBb>{47YsBM( zFd$Xe)!%&Zui}I3=y(ifafo2RY#|f1etpQI5Fsrlgg}c}g+l4#>c&f))~{Y8levgl7~mgp zzo^K1Xp3dxf`z12kzsRVqwgHQ=BB2h4e1!th!MiGXV0oCE6Jdxx%ujqD;a5NX(vzp z>+03fg9k6h31Q$WD=jT9E^cdmHi)&QB_(TDuOR?$&>Q5gZXTW{gm{sN^Y`-;u|Vmj zBw$YnL8Er}@$x1N;N&DDiTq=P07cNjOa9h&(}W%7i3#TS!?nNMV(s!)D)_?ldj=qu zV74MbFG85mD$4kFw8uo^jOlzD@Acj?HLG?$@YUX~Ghe%K`R$bo!IxG-5QNF% zqGN_MoMTLV2%)*5K_Za_2L-?0Q~ghk6N2qJd^mB5{hw0o?N->^M}4-by}f;)J3Q}r z@%fUammTb?SYaaKc#QKA^B7Pi-1ltIWmSOQ$8-BJTM21G2nASl8#9HF@kD4EhgS3O z5v0pOCpkd60#F5rS%8rkViCjuSdDOk1BV%aDS#yq2_gP~L#$M0)agl)5mF*X%&{ya zgh0xOtE=2-Fe;TQ;^eU;B0Qq^nPX$Nsg$a{yLTIOhV`GWBj`JH5glV1F+#}7JoD~* z?_a!dVOZbt6E#i)6}-}7 ztu`US#3^LcmZ}8_UOmp9ITPbNr&2MP=!DgBE!`jRR!rsfO^>7@q2SZ6`+bcA)y-5Yk4r4!&-#$u&Czc$z;N9$b|A} zc6S8;-6jHf28R_8&4WPa;;hpfp_K{{E#l;IMf9f496(|Y5+g;%krvISn5|!I?S1Cj znwkYc!2xpv<^=>kC@vnlh>kst7$Mxhd)Hw$_1EYv!}^w=sBuEDUCBvFGMRJ5!-vgH zO@aOaM-meUu}Nfj$^EQ&gLW;o>r!^~AImujlkN{yRNpQmd zeenQdP7D#mzgw*nB$)pj74)(<$nQ?CH{;9_z`qlreZ|6f^KT0_Hpks=VYBT z`ucbW`X?PcY_V8|D4b(beF%Z*acynbvQWKFpP8OKe8P^@`cFvr>DmW@RwH}^GxdOa*i8tf z*2zpZ&sIYEF`#34&aodM*aN%~gQt+H0Q3_;>kx9}tbPD_5esKIQeCt_3Xb0#zblt6 zd3t!ZG&c`x7{{(gj1VmC?MoJiFe3I3GwjqFtxl)!sU9+Cm*dIDhV|_>YMc;kS4DX_ zkNNE}+rBw^RHM@T``Wd}`g+@!f18t|b=Qem=;Q6<;v$zwoy8oKN@Ro?Sb#XlcP2aH zUe5hqcz^-M4*m-JK&7LfVseC5u*=I(KsOt zNSCf%-=BQ^^ttT$AIporE+V8KduBnY&5i#Fq&V~(9UmZ&PB>G1t2|FdUX4yr1W;U~w3R!k0^j6(=BeSG$vf!-zOH zxgsY!yZ48X%gd#2FGg?P{LPUh+i~dsRU<|SMCY=yPNkmAOwZ`78)rxX$;>#Nekv_B zC8enF&Tzlo7L60afK>P7Ny@Qf$C8teBqplW?m_cD(rNYoynMO;%}o*%?=P@~Zb8FAoTf0atjhylwXnju;HBS`SFVH!>i)d==_3lhwNoo2xf^8oOm z8xjO@ChPL}ersb*6SsOenp{_gM>N#c_5P5?#)gCY_C;^noS&CBLQx*Y>Q4y&VZTg( zqPWsHK1O%rZk5^ERJl1SF%gNVfY8%D-Thwo+{4T;&cPhq5Wy=Bl7JELz+Lb_29d)n zY8f>tD<-D0>Uu26n$Mf)54&@Er9`Z zN!~1>U}I?TS}f=!Lo_(qpce)jXsneAwYoI+82Z>oWBVW}W7r^yQ9`DIArKxQv^d4` z&M-th?fC9Fh|gppPD2zEOgYLuO*3M#gnoFYcxVJ*1OP@e5JSE*#%%*(`9ieU1WXKE zl9TH;k;~Q9TD8t(ij;_Dn>TH8zQ)h4yh#Xsgwj6rGVddV!N_h)F`LmsyHZmVpOBc9 zk?GdocJqhC_=NZ7q`o`bESD<*WVI^QgrLc>QPF&X2*Y`R;X9Ts1zEm`q~TAo0mV?H zBLE#CUdvQF2%*U6u%C+u#ZEkUeDAPAtv)%nvAV`}BDVYYS6T}65qhJ+Xr7(Y-qz;5 zBo9gTO9o+1z=9efw;6C-0k_F%_0AGs1G(iaQAhVd0yjKpa#nvmgkvdYJ`1dl?>gG+ z0QW3Id=GJSp9YA}Vc|rNiZ7Lhxe-E-@jN0C6`=hGGJJF-QOFhQaMkIPCtW9U_s$)o z-f&?5e&jb>ts;T2y1Lq1IUb7gG9i3(`0yNa3S0gA|mt%0>`}Rr&6iEyLj;_4%JmYLKuWT+PW<|GODq$ z@%;Jo@-W5m@^aVSo44;3#>X~?1X`h^;$L;v{T9HzMq}4$tPyZ8|Jc~Cp{{Fyo5Rv$ zC`QA;vkcxsV>QmMIvP99@-8_Ep%!xg21zkwaH>z;CwMg7-+_XpAhJhzKmxu{G$Sb) z5d}t&B9VA)Non`Jd#)3?-PI+Rg_R#YhWzgCZknV{pFZu?L=Q1}lMwz>S&2>-ue~u= zD3a)OdbEL{y=C0^fc(5XbUJ&|zdjK82w@OH>nJ8V=ED!Sv^2M*r==qTwA<~j0r|_9 zXJlj*uE>{4WH`b5V&~4Kxl0fapv6l=RxFlAXml74u>*sQqMa~6!VrTrz^t11PB_8< zLMlK=z=nd?XDUbiNyl;l#3AAYnq^1lMvO!Z&|j_Zu@S2bFD%t_O573Rc?-gZzKtag z8(wG#ogN;Ml5q6Ukp~a@Ol05Qy)ucc$KSC4DM8l0V@KMYRPSYZNXm^6&~wjQw%pHu z{O6y3*45eB8+NX$DqN-3XmmHPUw=v??IRx{3{HD??M|I-PD@Sw_RJZ#u0sb8N|h?T zM6#KMwG?&}V5b0)3h5Yya++3GL^xXdWdno(sPwl2ESCYVQcMN|ngF-n+1?vd=e#!1 z+-4d>rem4OE`)&7Ob~EbDB3q%@Bs0ER4Nk)M7@xJ9F{MXO66#?K{aS{C|+7z>^aR7 z<>iVnMQ-+Dy-vR%Yk?a=JZthAA>3+fV_AVpHKp3+DA9TA*7lD+_VW+;cO1?x$@k!FLmLV4e#DsA+Mse1fIGdwJFVIksfh2q}TZ=vpFEBh`WE{0Lf(jFI z1$+{{RTT5;A}wc~3Dx(J`G!&X22QC*>F;q$-8tcrl^RFq>VI8e;H);i@JEd%%G6j} z=Qho*jt;XqB{42O)tn*`OTVkG_I9F&nA`{f^A`0p4Lqe{WDhnV2Pepk^tot9G8&AY)4X^0?uK>i%`<0Z&!2zw z$`!Atd1%RNgmB{6v5A4ftBXtA+B@6ZB@$Wk)Z_RvP2r?ULJ@!h z48sL)Br78ZiE5)$em6994c@Bxil3-#pEW3p6ihsr}92BUU0R za1{FIii{(Q43Fym8k57EkT9~)_=m;1$3jA>@s(`dbA?X2*ckb`1mFL~F3)NH_ris& zc^Na3r)}G^#oH+!dU7KKbP7;}tJ2caJtNEZ?H~8ngcHY)dv}2S%SQ;mP#4di!y!vg z3QP=~q~i-tS5!<*d^>S!vQnjvkBauApc=xf5bvO|N@8-lLT-qRTH`GJHGS)T9pJVC z?h+QfOfmBqfB*ooK|R0@2*?W%ul+Hy3EhVw!XTVX!%!L`B%qx|A6)|}#=_E_6xL2* zrx+ds88=8EDaRH0Xy@Sz#XJ%sFhCVYE>|Eth>uGc;FSMeS-HM+byDKnf8Dq5iRN~w z$!mo0^_O4y`Hw$-tlYCJIeEHTt+m^1PjmnSk&h65q3m{h$*Q6;Z;lJWh)F>qix*|@ z+4bkx$SAZDV`AdM!W5_-F+iA1j{addPak36GlExgb*LzFHbqWIQqVYXtbNG%r>e%L zr~nBmw5UnQRf$8i5=tukU0wudwX@2jYp?ZJ^oOl6jWve{%usUHIs%3l8-HUlAXIRV zL>SXPyYH9nJ&=-~Zm|@!UAyt5Gd!f^MhG^WEpuLmSR%Xb zQF-(8RzQYdUt8<-G4`WpkPtd<-?`Fcz0rQ_|8lpova;a7;Lib3^FQ*-1SqN_jpJik z>Z-N1rLin4Ng0n=f(*>OH}l@>e*GRZ2+R!UFvAQWia`;vTttYdkgNydeQ|q~rc#=C z#YE%ACgm_r)7{hW`~RAyHqx?5 zVUsjk58|v^fVqv7G=kun0uPoAc3*%MISQv#MsZYRixE}Of)F685cN-iwZp`XC%^zr z2m#n|RAjDV`qqnP6^P0Z&?mqHh)5Bk!1`Yhf;~r4MO4dDWr(_p6)u=pv>=4%oPc<^ zU+8`PfDHf`;8WwSh2%5B&SB0MU%ttDAJ)}PoicTLyq2R_1 zCBrRUyrflqO;wdBsuLF^zSCcSLv>9;D8F;3q_`N|-S(j#K6n6$m*My=n>ItR6H=_#F{vv@jK`6_>l^#dNPY6lN2k!~6iuH#w0$1vwaUcky;S~sIsCWbbKiMgUgpLJ$w4p z`E%!XZr@=v811D_=_x6fE?hiy;>5}0$1h*J^rxJ+w^LUmgwtp9qeSOKd%GwFy=(U# z8%L{q@E|-gY8k7z%Ta~k;EYpDaPY3|+-M%(XJz2jVDZ0*Op%obZdo0_3|lJy00Y1P z=vayA`2rrqky!+nSux;`Ht@6*QB8zUf~Y*A9D;?#0tTWe#EM9f*JEC^!hjMHEl>6k zbp9HbyuH+M?G0xd{ol&6=^*Bdqj1qR)%em&Li*tR`%Tu4(n zalyi{&$ziguC^>deO+QUwzwuG>G(Vz|4cW5iD~VE5OH#F@0L$I43&hQo-$L!dI}e8!xD!;MDHCY z1=b!BSInO$igH9l$C4LMVc1ek-$9h6jHsi8nI`gQ1!XlSFXyOSk;%i#Bmpu9VG)rd z-jk3qu5L~o&!X0ZV96wy7X%etRRRM79h_v2a+(*M0|SGuUcL;c@yyASuU+!@J9mx^ z3~DMzPmh@yJJXg8>|E+FLU{J{Y1o8`kT*X5WEfm6HChiyAS3M@1`i#fWRcF(3%=9QMl!=`&T z+jp9|`$}V&d21QjyG3 z59Q=Q9L&y!$hDRsjvPLmYpu-A%sO-G_lj0xIJb1ywv|4&@xo&2G*f5QlSf59c0gs>#})32ybSXYwj_ zH9}aIvBqo;{JqgqPM9<2n@yW-8Lg%wR*I> zyDnjV!uenGy}f)85hsUDI+vf{a+!}GJ$l;kq}?63aqZgT#KeTTbKj#qy%%*DAv88N zzSHsl6m>O106e28#gDspU%YU^%fmA}GxIG5sjaCgwW_GF(A&d1V$$T~)vMuEVgTO- zXHisXwH`bcc_Q{(Vvwv0=(aY%$>fS<2-BEjcnjonJdxvp|M2aZY+}@+C}oF*Y9$$F zNwP_jZkD8)Te*LC2(W&@hIEr8)zl6iK%|)=dZd_M;)gU-|5zWTezeng?eM>Qbzk8x zS!3?E(2p0K-9wChmgs>xhK0HZO$%GLc*(=M2hEFo*7)?nAO*Ou@3gjm#^Hu=cg&-LUs_Ql#}1abQwp*t5u@G z6u|$7{Bn{)xp(gDdza<4?_K4wJk|I_B(kVE zsUp4+0NKjEkP?2Z|!3y@bP*GzW(JIFBJ@=10 zcYbr{-kE#v_wyr@2p&W5c0qCBZZH50IO`(;Izzw%Il)UX1ayD|RLXf~p%R^HLxSBz z@tVNL5x)J_QJ%?Dp)CR%P55DmI0>IF@KHQZVZtIFP7g}XdLpBvMn}?e4w4Zlas@J* zEtUw&$UrHy!zyo))V6ylbB=*4q38j@y(~ft-3~C zMhK1HHpazxc6{)`&DNGX+s~Xj6(1W{TvYs6-`+`eH$nh4T)9%b^7mN_7XlFnu)yv!^a9t& zxL|RjX^7V>b?Y&M8ND*dsh7G82!KH;JU`p&!-Jgvgr~r(PUsl*R=20Sw5! zg%0^vsTWs2d8D8%Q17eMuWtoe9FwG5bujgaU{^LT`VWz|T z4?Lm||No$S2_b9I;=#Wex*#+2#y|g^s6yorZ?A(Gy}{T}fB6ZG^e%NbLI8o`U1|L8 zddmYF-{M6}=J@*^kcdwzq!(G)ksz_x<2iVu%$t+{I?vT{6d3wdM6DoPkVZ#Lp?*C1 zB_f$hbQw{n?uLZRnA$F|BM>q2m3oMm@TbI11WHnXLyw5(`lkyfKSf3Et`efy_dT}B8kEiD33 z5~X_AN4FEz>T^|7CMUo6>p`oE-+ZiZ@1(jRA%LzoH#a_DW&t`~`E{_-G~6G~ehMH# z#weh={;jot`S2r^N+W3_gi$WH&1xe=WePSak#ay=O;zuJdQh9D-KW7YHQZ z*Z;Fwj-M}eyp$epv6%g**?KRv_jw~cJU;SLO>M_C>uc*mqMf2yT2WSb&uA`d!4LVN zJL_IT03O)$=_e*rD9tFgZQ1(XmaUM`u^fJX+qQ>TV~=du9ZD8#{md=*Kh<`t4%2f4Jdh(^ zf-gYit*6=x^jyL-e-{*A5XD!dty@G-R*{2P-iRpZ%}IDE=fi{cg4#-8kSsqpFLQQ= zl#(kDuVAr6MtjrJzSy^qQE#eAniLyPh|*{X zjvgI}c_nOJgngSDGiYd>v1dV>_2;@o%Uz-su6|igPOTg^!$KQXfclyJ0Q{2Zq|2&vM|jv**>%o ztsXrnM*Y`qf7-k2)5&Qma~I6tzV;oz_+U}7OH?>Ylx>bme)UOz{-2QUC4_eE`|LBb z*{szXbUK4tqy6~LAGWr(+_$%DSFhI8)ZD&oXgCQ;U1@1Uef^^ycDE&j8#iw@HDB*! z@WOfX451Md1abvY?Ep~5GwYOG3`VsI$)z^q?qCC`*COfyqG~Y(08owY0ssi!PFOJ_ z2273-c^#teyxUF@Ko@nGdQZje64*;fwpCymOy&|)&e0nL4r8J~w+W%Wfa#ALJ8t~g zu~J$N=Zx(FYoLkuHlOOGF)K0OWAz?xO7A%96t zPL6bs9-EjnHgQZ{ZSC&eyLB3Ed1>im{Si-*x{?q;P2V*Z=I3)9wpeX4ikUQVQe{O& zXCACMckXr$(%M>6U0r(OL}kTEs6WEtb!S3oymqaVk>{(c9Tw|^aj(*H4z^^7Da;cU z%_2TPBAcI?Im??401tbUCrS0|v1PnZ51A#8qUlUl9Q>x~Y(lL)F!8#i8W{{Bb$X>DztJ166}ulP-ynsU>( z6u(VPnKpE|KSe7fQfj!Lcxv+0OLeu6YS^h$Q@R@=G&eUFF3Zgb3me0e+`Rnj_shtx zH#fVSuGqM^Nr_4CE9f(rscIvEBElJYzJzD0Z<%{FraloAhj`}f-KO8}`_Ooir!xp1 z$dMTY9~I=yofkO#s(9d5Cu(5va+DA3KRmE?1LqaGye0cJNiDTm9;Y@(fd*ArIpYLT^JZZvY z_=5ld5*Z_)Z>h7-QpX67Q5guInGBh2 zlZ0q6jR(xKh}Mh_aO;s))oZy6EdGBXgq{U1abk$vpq97|{WEOv#$Ny-h3y&fspIvczIV6 zAQvjgkpx7MksARIyRB68Ys?z6OvG?rS{j96&Rr6FEXbr_i6tO(&>mo7)b z@NKfi6{@8lik4SB8auz@K`D#P4G#}jsZ0Z84BvtCno&s408I## zAV9w%c%bpfNcurqApu(=xCF!R;hb8O^)tqPJURz-OELB@x`sZ|3N!QhT0%eo1L&&9 zs|evaC%}kIYR?x4X>=$$DjFP8=n%(>#*|3y0UmI5v>=v9wFyjCbY%44;DB1K7TbyH zYHGm4e*fKft5&Y=@9V9psp;(*3uVpFCXx_Zo;=ySapPxyp6&gmcX{c9#Dv7zbG~qO z)v3~u$?dGQT{*earq7hg=7BN-$kyM_pUQv|wyJeja{8=`o&ezcZcf7z!~W;3I4l#&Ae3BB;FCe)d41N!g0Ie)fos;8*Ey zO9+$FqsC&{)g=@)MfT_S1qs#S?~$Kh77qj0l+|xob4tp zQ>l0e?0-6D z074HA1rpi3jIK_kuH+U zeZ75}8XEM^xHmr^a7A2nw6C}CxCd3UwFx5xfNcW4Kq9j*R$U(nHq_SvdHU0gS^M|y zGdZfN@*yY}8;8q9ZgvhhXxzSeYa{`%!2$=z(BP1vS2M)<#6G3=UDaD#TKIYZ@cK&%dX;`%X!T zR4M~PFxEx}+Z-dNZgH8s&wa)g7mSU9$m}qOsc}wP1vXCHMM~OftCszX@J}Q zyxp(!X_==P6GCoQR#HMj?=T+d4N&NjZw`Y`Yos{8WC4Le3Wb{<-7718rdZRD9XVo{ zaP`WSITqxLKm0iUnudl#Lqb2B`4!`p`_nQ&uGy%^= zaC;29hqD`z7eh1%R5LoiTI*1qt^qJ+34vSb4ix&jfuOjpt%!%b3<kIM^ zP+hzBQoKoB9}*nAGdsuRsQY*Ck}1^F1*i1g|QkG>uIT4hydr6@fB-_~1d0MIPf00q4Y^fIy{OX=P0Z zuL~#mVE6g@_=2hnb}`CAQLXfwAOwVqLi24%d2W+;Dk)x4g3g=iEBiQIF?F3YDNR92 zm;Wx&mFgxUFnro37p;Q7JtP1;NRxk*ET6r`VN#SmX{F7lA)+biiszShD5q?5bXO?z zk}{6$IndM9J@Sl|rDd7hl5bzV(b?YqE?@uqZN`K!V$Zs|eyM!e(4YqGp}|3IXbAic znw%$L>o&lOmF4B2{j{@FXk!}@5n-5+o|+1d20y88yw0j>tE)G}uaA$3jgN~n_KorD z7dkr8n5>27LPS)s%2W2QOV&C=m26}|Z7{Qg$GlE(kK*u;IQK5b zZbG2PqhY|%f$jtKZ$eocaCABkkHS%)__f14palbr9fYIjac(`zRu5wXK#nW|$sl-O z{!Ih|8I=wUBSy0Uj$(vZEambr7&rp13#{po*Ama0#XptYx}}gS0Vmkl*xA~Not>Q5 ztXd5!Td_(t0k8T4vk4=F5d&qSS)(a0D}C18J<#8;5B3i?UNTKn6OQv^*T(9*1t(8g zT2U`wxHyu~q*mL{Td;VsXMbPc__rP!%FE7K8x>`;bbzX|bcrV&;+Pd7^!N3G-Nfbc zygipvDAe`w@#Bvf9Q5>D>f-DY8x<9{VkMi4aCs<(3wSWb5ea5RI8G09;Mr=sFE%v~ z!yp9Z;gBugV#R#6Oo&0~C$S4jIZCZqryFQ`mh$5i`IJN#pzLqM27n5FyVZr1sUYPl zejmSpZ4Kpron3XW?9`saGrUFGc!pwHq$6#ygb0vt-n{wUzv&OqOcDav)WxbB;F5>C z$KE~pT~D8${QL2-K??w?HI)$+9@*N`0=g;5DS$RhZr%jg-`mrpx>|%_gh(WAZ*6_e zg>|&GwY9XEbTG-P&=r=}G_xXvC#|jVLCcdEwCw~I9vsr%+4)j}rt_($y~8k4^`!Lx z!fWI(01wmw5yx1~DAa&JHLu14y@Rwu9JWGmF^1p4I8DPC0K8f|734)=A`IKhM>Gfv zOb%R)vI1}vR1q%>MNo`mLGX;FX91C(}*c!0%p@D%w zAK#S3q`i50F0OJrk)*DsIxaRYI52SH-uQ=N6HEx59qq@D9gAHXqf{y@%F2qbU%ysV zG)6wq^!0J^)^z6ivuDAn zAP8gwp&h>nU2BFGX9dvF1CE<~*o zi%iSVBYx+eoOi!`_nz;4_ucRNeLAgHB$C8L$8@x}_x1F^JODp~=`+nXrKLm5;Wz;E zRjW?-OQ|C#GmB+y_ev20l+F1C60u~_ym>Nrj|!s~hgrJzW)1v(qUr1D4G#_X^YM*~ zjg?4Txje$jNu*LJ`E365Na<^Ns&V-$>tHFt`T0@<2oDTx5$^iucn=_807lFVk{~(i z=VJgY-xRuofu{T%bpGI%g>S|a(5)#5jIi?m?7?%%||CH$_kr7d_tr#h{7tLQ_VfAVeffw?<6+6DeM0Q~&MrMvHtTdt7cBf3@NNwh@#=xmOrXap+>fEkC~iH04gv#c0O3Ik zu+=n}%%|fi&;+nL0;_)33_?O9z?3vO6{plRSvPYUZZS&cZl0g3e=sfH_SF8XO9h7d4vTz`)Uu`O7Y&NeD*| z9Wpg%5d;++6wKw~nu>FUd-wh6FH@whZuPabLoD2qUtncxZ_cus2MlpHCW)9zs5afoTifkpeZcMQs-C22en92&km(i5Zcu$adeiQx!ZX2Io__i}2lz za%9tQ!+;@1sshPZkDuq1lCb8|4;9^Q9rL21bF;Id^9S}8jt^&$Tm_QiiDW4HQKZ1< ztpIXY!QVzT++#wF31RQ<-P24=!$QKejg3p2Uj=uMGF>QsH_}8x5qpow6?UII{AeSut z?vXf$u^^*H2>0*ZgKfnOQ`ky;URPTyV1(sA{@B*q>hB-$-ZYZ~fB$BPwRi8_$;rz0 z@$tnl9tsl&LE5o7u)k2NeKw_~cXf7-l#{OlA=KQsQFP>JXGe!&3Y$K79DTjL#+ERj z&+(m;@L@uIeSQ4W_{fOJe(i$?5n&P0;Sr3$fr}IP03jHtp5Ov|rSqE^iZ{{}wq8yE zB~XdbPVPhsnMuoLBinpzeMABlJ~dQo66Z1LBhO(GLb?LU{52r)TDEeEnn_uc{IIw0 z;e$T8Tv51ZFLb~8R)a@`KN3#9740%H+!4u?BblC)mDH)jMMDG)=A4dZ-PseT8?M)k z{E@u0GKdg5+S;V9?okm@hUxNUE8s9Z!{E;}H9u2QGMsI@ySnb^yYJ}r-9r+^ua>b* z2wxTMcXV`4TC>jI*MCjo8W_d6*!l7HcI$1e6MW}fsku4Ger3{zy_jtiz+ge&26*Dx z-isA-nb6@7`KZL#0<4_ko*=ntLh>TXX*8<)aC#Fi2UsEv-s4m56m^Z{G!WPoz_|)I z#)JTkRdhcO@Bk$QqzsS}K-v+c6+yF!7efM%2Sg$lfzW|rgd|DB{soYXNGxGlS|2@p z_{n?SsJ?#ra#ee4TSjVXWN6sNq~w;CmLcNSSbqxw#s`bzmMx8^8No2W)iE!lMhHj0 z`)-yQ3$~vR2M|{%a40)-#_*S)-}~^*Rh&3p?Cj+7VcZh2M4G)RYmoi5Ygf4(-m#)%7c}Qj78i3kxJ>3T zH*l_0A`J}+Rw-0Mfspr@5FQO4Npj!-fKAvcoyX;=kv!Fe6uEtXgzGE-7Dk6#8_<_$2wv#j@88s4#p&$W~2_l+q$6D-lJD9BDr7Cs*q`9+70aZ!<(IqOVm>F?v{vof{`A#TCK+?*WPH~qV$)YH>@|JUElC&?`&AOyJf zAce#F&LXHUDL$Ou0@z8KcZ9*~3DlSn)B}V7Glm^OSrar|1pq0hNB}4fcmW~60Z3XX zbaZrZ;^F+HHEZ46WOi(9e$2e^(6B32Rf7U@@9y2|>(%-f(dM3=yOb(*Q$qvnTLS|E zKi|H6WY6U#l~E&v!w0`LHD{gI^rvEY1Wyw}RAiK^+pq}X|1D#i5GpDv9fb~6moK-s zwZ=rnWTa=nRJkE}qweLP%sZCE?u|)r>j>bu2{a(@K@rh)wXu*iv#-K zUB)&c#70H!-0?+UPtWv{|%gW8kdt!OFr*}bQbcwZ@mc*_P z)b4X6rw(vnP4B|l$qd*<;mrWm5?BF4OeUxUG~PsD)z7j)Ex?)qww3}@aY{r|b4K-UWp1QC#^ zfP%pG%mfROtgI!hg5Tee%-%MBP$9C0s0YfJ80|k41X?w++P3#9-pQF z!XMaNu?^mNNG?AO3JBbmojp=B^h#;e2m#mJITlu-ixy)W!+=;c_gqBY7 z9?{Wv_=S6{zP#$&7@M;9#qAX=xaPmE}1!h!C*ya=(~7W$Nlx z5ly#lE&X`unuyhXJw4~j%ki%6q4Rv^Ic{_`wh3Y9_U(4IvaOpp2lyG5g@=D%d2x+U z_~Y#9RaUc>&6u9IKmW03`@^nqz27CvnYU=7o+dCDVB-hG$H34)5uONbhk{l>)C2M? zF z;re^HsHnE4MkEwhl$CY1x2JB{fVY(Vefvjr^kHgj6GCfCE8aT77KbJ!CQ2pJ(lg&i zuK9Gq0{yjX*AnC76|Rbw=H|!FU+?}Z$lxCt8J(G$`ekmekIsXl;l|9&oEQO>${G;yf5uB%2M^6(1)^`=>937pePMbdMgIQMAB*Ep+oMVY|-a6px`u}Qd69QI! zU0u$uoGX_vn{RL4Zr;3O*XJo|XUZ!Ed2Y}7Vg;Yv%L=x!RKAc3Vc|3yEo5LYi&O}1 zWvE7)z$G3-LIWVPgy1sZ-Y`{p+=-1QKt?iPGKFLiork>J{rq5tge!o19U2Y%ZV}6m z6+%oJ4JMx!n%IQsSrj4w`U+7P56tAS2MGlI+)L+aVMU0<5`63E;8=O#0*-54^1AKWTg@@*G}ReX0C zL@`b@YJ^}E-f2@(F#AoJI&Ic$3z8?8V`0Ux!oq?Ad@>#J(TAb2O$dF)CD3)cq@*Cf z;O^bK=AMHG@}t-M?NC9%psLr=-tM=+S88J;wXv1j+Rd14sdQ69#9CNd^OFKGxbTvO`cT;uU3!ju%S3?p)pXmgLk-)>%n zSNBit9Z$RL+q=(0z)xa<0l{=3h-3jS(0@l%8Q_&e-gk&s(_isx0h!Ly7a_OK)KhJ$ z@&_1!C5>hQlxM2)HT{q3XrdDF!WdA5_?R&;V3b0xH-lEO@G9WqpK7Lw9G0F*Av|0e z;x|*)*zF|4w}*(Njf6lyNeFxakVtIoWe)CYZGfM@Vcz_>=$QE*`GhZ75*!%h1G?dPjEhnAV1u{3N6Hkuk6C1OeG*|VdP^f9PWB?MF5$;-_R@DI@GJn#zj_cIji zKk$;^LEktagdufrC}8gF=m;yQ$UpS7`ykL^wvQ~x+jo(VZ|K6HtaTeK z{pAxrmi{I|@phu}tv!}qqebeh)=w?_h@m6hD$c{EpO8VF88-g?F#yt0cuRG zdO&LgMh3WUqJ<_vMk6qpLNbKTLtYb2Tp!@s3W%+UnnW>44BgcKYj|}U>_nE*TBq&&?_0YNN_V3#}+NmEy8f8Lw$u&+00ayQuvhr=4 zHl3@exPR~7(++KS+SWuwxVb6ze7RepP+}T4`04>bJOOwb58exN{&lA2?IaavgBq7? z&L-`|wMv1Aq4;2SfWzcZhk+goZ3)ZDoeVVS5i1wRdz3bhz;S5B4f;2%s7Vz`zYa-lU1@C&6G)!z6%l zqJZI(G@3!9TtwY@>Xl}i$Uy=}gvt@GuK#I`>BUx>XrPIoC~`go7!POy0)yT!*kF*$UE@Cc%uL;n zh4~@s(@4Y!5i63}J2^Nw1_mtTQ~zPVT%e*l&N#lL7-O|IX`7z3p*?MFc?j(8y}Ntw z+?l((EU>%CV_{!Bl-F_+iv&fnR)`RE4KE)oDhk?j(zNO^N+mwxqtp~_V>G^^!6Zgd z9?Q!t;ERXo;_2+J5DX$#4R8$qGiT4-@6LR8X76kK2lZQna$r&!C`pmj=McW&9{m|(Bfdi>*Kdv@=xxpZk1R{t5&I3bK# zWv0KGWn6B`e5;(`)(g3Ro%Xx5lm&P0SRHRWZ{7^lXd_6z#ZCnd`>uiH0hX$Ug8HG^ z05`^6jEMOajpPu03WXMXd~jo&u!$k-``+Tm&!F%fmY&Ztn^`K01tJDf4gdg%q>+$D zTpGo&OdCx$FywVWB(czoMij*;d&CY31OhV4}@@~r6qRetkUho;`gj(VtO`6T+C);?&fvoP47?tc~!x2FYfg z*IbpV@ruRq*4opf)P@@b2zEHpC$oVJxF=KA~cAWPGb zWu?JE!4!amfRagN{(kbPsF@+bs?W}zbxfz#Vk!H_&dcM#{~c(Y5C-b|KnM+H^Xu{P z%QCax%E+v`aA9838w=;oE8Vf9zrlYGALbFHHcW$92{m9BQzGa?+uR=r7@KqM8vnx^6u=_VBEydwRfH$};%6W8fDg#&$Gcb}&ZQJe~NFTAaG8Uhb+!bydC zh84S{t1yk+i?7z=%v1B!K0vIH$@RK$ZKx(aExoI&%Ql%iy;Wz=UpRH{=GAMXw&KsO z9)%FPIy$bkxA#96c@KhuP-jybgQT5X{D{NtraV0 z#L=Q!D6lZ(ejzVD;k9Guzc}#WzYl$Mw5VibtV*>33NK+E=ri<_Nc0u5J=P4edmO>> zD-4;%GA=Xohm|rAcNM5!45GU1ECjy zfe>b}FoprN#77}l2q{2{z^uq9UtfQM6dWr5XLo03UQSNh;Zr}T6{PctzFCIlZchl07gVa1O*1| zE-R}%ckbZ+{YNS)<|ZYR0!l#9^Z+~nB8F7?PBDhPn5_m%pI74*oRV;C%@cCe^bFag zJk5(MG$NS<@@NSkOp8}I73qiNfd>%6I^FM+0;eufeM1+k9ps>$k`X#7SM4%8K+FTb zSghi#*KlWnzywrE`hS$rQ%b`8C&=pVo4X>meGJ5W2fMbKlNRNuGD| z#0f{Fxv?>OnX$dCb)?4l0F50&XtA^|3=iMz?s~w(&A_13=PHNkaQE&bX728}oe&YZ zUEqza*}@R-L*Yz?a*4r^uL?>ERk?|!Yk;W5Zt-<}gm4N8tD#`9zdK4BAj(*Z3y=pu z+gbWLL(~q!gDOZ?vO@EqIR0&B$gcr$0}y{=8BYNF4;Gv-88J;z&<@B0P((3kI{t)+ z_NF1O`^bDDE5XMRE0&6AMko-SIDQPz?(@pZ&?E zl^r*4IwE^_?b`hAyCXb|ff_r6aQE)r?eD!892k^iG)Cyd^Kx<@e(mbD{vVrV|Gp1A zytv*xf=37_011J}5O0W&k58PP7#ytPlR^e>OQ{bAJFnEb#wmH>{xpLmTrh2>+{P!x;m(T!xY0`QnR39RUS1r+Db{mJ^goG~o~-!j ziT{Cd4X=k2Lha?t3WYK=BlFnNkNy1Q1*-~h@Qd^3F?yQo>qmB^_i5}9f~n*$?|8V~ zfTRVI|M7HRW?XI^0s%fOFOQ9hRjakJ(J?K}O~YpX03^G|rGHBq{!;?Yz5Ogub4TkuVCdCXC02Cb=DHcoc;e;gHnZ2kLYhCzGx?A=td5j*mwswx~b zW-XnQIA@hy!VLTGPo4GjsYwl7OpXQw_a?CT*;vs>NWdD+<^!K#3O8AWT>^;)gN zCeO{ur9|}n2Dq|D_Y(e4tn!zsA-K*d z(m$WCnYdazu~6q0Bj?VLdMm_bTeijMqW}%_GnW14^+2cf8cw#tLocuUs`@GaALDwE z9!>~{%F9I*xLkD+*LRkdYSdcm?c3ECFG?kn2D5o&M|z*e4k2vYwP&@vdxwa3RmA(7 z>mN33+Vnur!?%N^KPD!>k-WRCOe&S-X6Fo>xvH*y7EOOicv@(Fi`}g2?Ou;74Gh_6 zf3$fzu>B4OgMpwa^=Fd#F5Io^PXjx!8V7)35MTCxg35RhYl0d?b<{Grj!RuoN^GK*&xfr#Y_ z9&uG*I21$yH6YifXND*atgv3_%C~;?>eoH}n(p`L-}m!I9OeKFD;UIsM!>#hk1B}2 zIFKhUBNXCMhz}ki1U88@2v$J;Wr~{v_#6Ov1Qi+_LenC##D~i#O;M(tjLcts=$4z6 zb>i4D56^ov}E zlU^P|xYN<0)o5{D#rc0OUD5<8+i%|Q(s#<>f}4OZU5k- z*>br8A47adArjb5D+z@{5s#$s$qf2!AfRkL_I?Do6kr3c(r0QegYDZTTjQFh!l;1V ze6oe{2=%4i2)?iAjg1;szHNZb<)4@k9z-7VR+^d@;`?r5(BQQAYGGJ~%=8VaSqbt_ zl;Xqt4{0?_fI?xtay2nE>D_S2`w@Qm`T6*t%6ZTI6V>C{>+yt8dib9hGA~ruIGXr$ zuUx+JVokMJEV0&45W=AL?)yAW>XQ_#O;6j<)!jYfm^+=FdY%4GM<>p!Dk~L=z}}vo zQD;6|T^$n}A0IN;g2<*}wvOHWI^fJhFogy!U}QGHrhfx+)*|W`B=U#w8^~?7yZ`X1 zu2P&6G#}%^HGA_WVDF`g#Rw`O^u%BQ5U>YM+)2bCZTSO8z@47}^Z}oouQ%f2qYP5% z!WOuDvNbC8f_Y&wUzt*=;&O>IRaGTlmMG-%^0G3go!)M1ONfn=`ugFoq<@__`SdSk zLJu@vgb=+nI&x8@qms9_v;-;wPkmjMm65qDdV&xJb-JQ5EHrFIT-+jK#EIj_zi(KC zAv`B1*O4=7&CcyRMxD8@uW#?3JqvXUg<>CE!?`>XUkLzhR6yC!EJ2ZW<_CP}=`Wrg z6~I6%2b#Jj;dK?sBD>h0aV|G>$rGq*ZBzi&); zSC>MrDEa%B1O5F)7K=e|7#J86@iw)!l~$cKdP^D!uewKVy@BGKhP)UdETg#Ak?;U} zJtl|0(u4;9t@Oj9s2W*@)Q|uFnCq?MzfOZ2m9w6*^fUlSqaFOBs zpBacDkiaIPw~sIWRVtNflo~G%&!E#S2n`d7#0L)?SQ53wXb8s-AtpL{_}+0fqW~%} z=Y<4^7>yD4yPS)_gq~RAMF_jL?*KqSzrJvH_be(byx!dW-M~OzPHwO^B*JJ+h);Nt z%W%?5MF{*tK=rx0f&X_w{|>uza&QIC$>8yLT^~KTjj!(bCc}P2Ss!!EDW{ z)n06#NbC)f00*C>$RTV`5|9)MW+nzswP@c=RZYuK51O^zqo-K3?sKKcjqnQco|dnj zo}`?TseWkUKQwYRGuNx8+GlXd*33v&KBR>ljZ2QkHBIG`HDtV*aoG^a7<7mB7Y8%J z5eB17=3jp5l+!kw>#VB9I#z;+wNCbS5LvwE^QU?x6M9sSCj_jvn>WolIe*@{)6wi` zZ{L`aIX5J<<;IN{wZ;+jQW3%!b@kiJrpTyerX^c9Z|Uvn8RK1zR%=vD+)_j}4>R|A zic^hvxN}52z|ju8Fdn=QkQ~8k$ZfG>z=$Iqyiq~(FF@`Mz&=9rEkf!n;1nZ*hBgdP zA?OYeg6M0Cca!3rfjnP`H~@)+V*Fw9py1&8gR77{Kn@_yCjggEvU#MR%N<3$KqrzT?g8dYwKi zBl86>W2jy#LKwTg*tc)q+|cE*aph%YW4xumzaLYXRO*XQCH7WC3u%F9=xHS>1U&&) z=`*=d%gVMnxJ$lvQh}CbvBlfzYQ98>egoKQAD*v>hiE7kPES#>@|Y(i1Un==;DH$_ zLkPjlOifjZX~CNtR5%~(v}j#d`KNrkuIpB3dsEAmvo)>tjl;Kj@GoCX4<=b#w5)tB zD=i??Vt%F%{&m-=gy7K7k3L+_YpSbz?vK^uo`+sVLcpr4tvyp!b-%lN?6+)8PnXFR zA!5-*$ZY`Z;kMo|lJDalZ1k5PHxR-PDYOp}?ZfVM3QQVTDDE{ocwkPr3b`G?3qys8 zhU*bHM)P$D2GOtua9RPo6>vgn=mF4TLpVf2g0z(#9s&Gh{vwe?ES5;TeFOrrP$cH? z2!_#SXQZ!)PgwtlltmH0J5*fkv`5TKMuRaqDfz$m?j0#T0x48`?b@?^?ytATkr2MK zzuckC`r6vMaecY3C?O2=_g9@hU0+)__FI~pn;4~1%P<(!d3?;>5T8)Mw~;-?rEG-2 zhlEV@>sZAkY|3VhOCIxHyyCq$g=?y6S{~ybCPQunZ;tq#RFzkVlpQRc6|Z3BKLs8* z#iOM0lPvYeQ30-*Y8yr1z|c~tnq`t%)^12lT9=fOx;%C5%5{l1>KdH3d9viIcT^Os zNX;r@Sh1qK+zCQ>mKsh7WhYOXvval-ZQ5K|RG43YW0S>VZx-g8&D%cvV^f2592e}+ zD@q7IRNXzj`bCSwQp} zDb1!x0gXJ!**e)I?9|w=3$!c?_TRXos`c%X4vMNm{cP_s!yky8a^g+XA$;8N*%MS!PAxPQ``q_!t5 zL%8Q3pA960oXjj6Hcv-Tt0`3&-U`Ze(l;2&iTdgAcUH_ z`gL2k+bT6lB=LmCYknRZAfa!h@w-qUhj2X26f$Hh{W@Bpq3TS}{w)t_yv_s(?Ev4v zP|4FkGi7{npB;^Q(IAKrWDJ}E!M7qZS_JsUJBKlqCx8%8Bn1URs`Sj+-8*)2d3+8B zmq?`h3JW!AjY_G$R#~}vRbpd({gUwTBSl3kR>Y$Pwmg1Pur5C>Ipy5hv*>42Q^VfE zJvI044e`EzbY?L^Fc^B>T;-@xN`{vwUZ~8#XX{j&|%y<-}mwpO-5zGbCi0e4TdO*K&c>jy)Yl z1}#{Zm?{E{Rhkl8r!&GymKH0}eWrl-Z{9T+3}q$fHfH4=-Fraa(TUc3rL^pJ<+W$6 zZ9~2BADay%go?5{8`t3VoY(?cZld35E0VIAV9w2X} zc!wEs5)ersya9h;-ipz%MCgwFo?=@D_@9 z2$FJ!i9&3^7<&ntv7s28VrbK~gA=5Si|ZU8AH`6mbL#6Jo<4C>?xv8tD~9m1{ZI&@{b}3U z`p(Y5lRG;)YHMpBKWaXG@>J3O{qb>eGzGXg@sbT*5dy`C#-^x|5F=5x%F%w|Cxy16 zDSoAO$kwuEhzUpA-^RFOxk&mopCqeC zrD#4)P<^@B4MEGeMb2uu^5FghMZ3TKy$WD!WLSbTmgI^B!QiFghMwM?Tee{{Ni120 zEpu@4P(H3}w9KIlLwk)7dV6~_GcwTUbE&)>l>~ih=wtVv8xR>8h2r+MHa|cA1z}-^ zehC(~4}uV$>GiQ8q4ATgS52{wogdWM-7`q?&Fb5+vGI!|B18NG(xl>ONBiRhrybyr z0sdfs%D>T3P7{sa0fY>~uK=^3A=_wtU|KEUUxb1Pd*&WQY+&>n$X_Y~jsOyXVNph( zN->cl*aq-Unzx0aCeWg(fZ+g!1V%70JGygdAQA{^$cR)*C1I=_NvT9SmBpRy<&8Mg z#aWiLCJ8O=$iaio5~+{ZY*a#`8gorTVwYZDeXAM?M^|SDO892mw&>^>3!TdFGm8;I zb5oN{CJzh@dfL|7*J$YJIa_kt!QS!Wg$wBC`O@!EeNlG)!h4;`AoigU!rwM;5sM{p z(J@-Jws6<3K{5@!y*Zg#VL`!PM@ISi&JFONCm?C9N<-2NiKw6-Bmf~~c}hm7s*y%x zdD@W~8p7HBizV{kBq+x(m5<5O+02rNrjP<>`!N|BqjqP!A3{jejEz-{PEqxZKPpW# zB26%nN6~LrYFhPsLV9u=O6; z2F)K7Om+Dc3!TdFv)2gW$EQz`>;}#c9GJOx&mI<=_rsk#DBiT8K%>?5bU(M~T>IGv zK?wi4P?msmYx(S2oSn+&o~o!cxBsZ`YG`eJ-qYKc`J}ZiF*Gz0=fvWiY#zIcZGD+H z<$osaek&OCIRc5tG=BvH#!*bP2sE2uU|?Du;GbrM3vD5aYfOZ29}+GM{3`{CSJE$_ zndy)@EFw#xz>@)Nis4blM(U?7q=2{QYKO zadB6de%CiU_7xWXpunX~e8bM>8_S?QS`x*44v9j0c(qvqz0 zXV3a)q79dbonoSvqEq(s^($DvezFb6&fa1AbO~f^e*y^&Zy40x=BH@ zkvT?Muqx1vU*ZP%fDynyEq6DH?>lFMKF`pMK!3VkXQYKbY%o6&mSh^A6N04+7A)-Q z)Hl>T%u7nYSaNR9=54nst{P0&W#&Wwv1;Z<*`=X4qG4eJ2|=aO=(HZs z_4?N`Hy3Q2Xg&FIMTI%mdCctT?zZ%FU)e955N_YP6&4a==i*i<;I`8^65@N1pU&ef zT^z9?CwIB6Xe*Z!;^K1U#w~OHuA)Qn999R#YXtmyla8ZT{tfv5p^d>?*#NObBaPpI zggb;2Xr_=MTWNfdWW*B9uM$k}@{ib3JXCS?Q-2h*XvW4w3-kasFgs3=p0hlI`~&y` z%7)F`y=(X0UAtKvoCcyLix#0;=J=5#{(k;%+@j}QU16bNGG|$2_+miAin5CqJfGoZ zmLr6cl1bK6wr|@uuuCpipj*WpyEr))9XRk_XEKQW@(BUqw3nwBAtdv2b2HP^U1hF_ zshpggbF;HuWo`j}bCvFjq{PG*27@_&uD8#H($bqZZU}`Wf_9PuG%(V;SS*IjwElzu z3GkOiGNUxV5^Jj@6_%~F%G23+N|>pnP+~VG{gq5x*b%Yh>pn?PS*2%)#Pch$;OlmE|tnFmF2<#C+2R(5l2 zYOT%gdi)`ABZmXbboX@k>+b1cn1SJ#lQ}p9WaK^sS3r$!i5x~nK~B#yN0m`TLEWTk zH@k5$7!%Bv|0s$Jr6LTcfS^Et5pdrCK`c; zIl0-x_V(bQ5Q=76o0^`O24iF6<74ASe*8o&uX#-fBTqlv^wQB-{G{ThveNGwQOkI4HQ}|1b=6VrkTBd zqCm&*1ZX-#{fT6f7z$!W;|tr@k3N444;leFNb-&efiw9$01#7|;rY8H4R;|vPJ&TX z;1iTwp@fGWjth^A@FNU{OXcM?)z#}m4KuCNts7Nw(J_(h!;Tz0I5IqJsqp;bWHCZ$ zYiVHxu7Lq-`+6Qfs~s8n=IaVOhc$b5?}1m)m-!fRrJ~}M&g2D}=Y&vq_pVgxeW>VQ z!iEH#q&+>oG-}`EO`EdzW^uWQAvDxzG?o+>dwF^p@7IrYfz0snv7<53F$jjE7zvXR z5(3OnJ`RYfNi9IY(#>*c#Ls*=Pq(N@Z<{QK@W5cy@~z6r{9vI1c_3eB#)C|ig9N<0 zU14*?4-&$n5bEU7kEdNFxLuq${z-U5Wcv0U#(VcI7@uF9o+AXPf&TuDi5pkjJ2COK2&k`fZb;t~*6R81n? z0O=>Wdr@8hLG%LT5rtMc*k>mtjdd9s9!^c)mCr|7C;52Yk8b_~uZL~g08&e13Wl}? z%qfQGqGl(GiM}lWxhTLx8JeYNd}bJ!!VIHF5}crD0qgGWArgr_J-s(3B*INeEmsHx zY;MkhwA7t)xiU67Mj=;TC_87}a4g7TgaCD=;OQiEH7pWmm4j`?WR%M5NSVm#9_ z(|+jfvtle3^mIBxfNE@LNJ&bTNIdKvInanX&Ro6T@6?GC@YeU=edp@x_Q~;&hKGl& zG-b2qGa)=NnG%zdeF?n4nX}VXXiu|;XtIgIt|QJLsMUUQ#Xp@KTL|7~E=~pc1!G-0 zy1UZ~ijoNa1Mtk>n*h=XX0dxCKso`kQ$Q^ROp1VNqtM2=;lacZ-wXH=ERD?p4@L`3 z0D2S^Xo0{Lf}+q>6crhD?dny6pd=D^c-ZOd?d^b!q|x~*Ri!0o1_uVM8;%88k`RW5 zp8V^}Fa7lXkQnV8IChR4iV`HoCsbG6gm-sr+pbb+Zr{4~s;BV+&2vKd+qv^x1XIc6 zaNQWz6~*u`E?juj-5nPbCvo>!7rd^~*jRSvtX!t(>ga&&O@jk)PYwzS<~Z|&tVpF) z@qbDP1Wl7PjnOFf z{vWR_&@I`f@b;F8+$8_F@VRMl@Y5s5c^E%0FYi%Xhqa@zpy>%=LhY?Boo#JnpW>K? zhO7w&6PnM20JV9`)=I7;^zIIT)>2p~LrHL4r`6qWZtd;u+qPq;&`YKX4NEZ?qGdiY z@f$X~xz%#)p}E)3+1%?Wv<)D&G$yC%Re(7oAiAkpaN_>x^BbntGy}AiLA~kOApxFE zAP`7Bq+EpW%tJGGrg?aHt#;t(G+LP5c{#bKi%)M#NQAFHJT$c4u>Rj)eP!)9EXXWN z62gRfA3bVsZEbI988VqBDy@607}Go_ggZ4gKHfeMy!ix>N<9(i_cRN`mz$jf?^cvw z(kN9bg>q|hijQ2ORHz^-*gHBE<>o7tYK%|vaWExq`80`9ghcrEKF#Yz`q%eqp8545 z{fZO?>rCSAu5adOpi`Tf;bl!f?`0Yn=zhQ3S4z3shDsOjRoSF@+GMM24k&E3!aG&f zQBl#~R#d=uEGsS5_-d>jhXsvmCPHX!X(>K=YJb+g1KBwT3yL825A^q2Wrovgo)f~R zP03d{b`K~7`gjLGuF?E*fbIuGN!j_D>e~iGcxd8AUzzMQmvhnCeh2FE5zc!I&?Ydu z#ZMHY`N?PoLoFql6ozV}kjB}F0l*-%)52-=GDqdnYZlnrLWJNE|G{Ox2vnI9ft*(Wnn@X92ls&aicIlKWBgT z{;aHXrDx%0Vu5kzsCiBZwYP5zg(5D>kBy4nxM2gwg%=VW@?l1%Mx$+i@BprAkiWlL zr3SPBCqQg;44@du5_L5-p@uL7!zX7e;L|jpVq8c}E`IH>pG~38CSPZEYDfZwy0`Xe z2$4|01IxE5Z4T>Snhg$UUxOu2XOla(_ORJe|KI*q#!{>cPVfRfMREE!!rk5uq}NA; zJu#UwGcw@&{c>hArI8jn2M%tf#4)Vbj^$wb=3Hi;c(X&{uUu!#*%P8 zEu*vmN54m*4@o48r5s5H%*+Om3UGqLNs6KcB_~hbt*Hsn=^l1=KJM!3XlomL>#FP5 z1N{6~**W@nc|Yvvn096QdVBBQy#r0;XNb*$<|a!LLixpu2#VV~Z~_7X;noos6DM*L zuYB*%QIS!7Jw2~_^ts5AgwWmfu%@QA{bly3k>TOOynK>kv>KgQ>@E_CF`T$}=gzM5 zjO|;uirpk3K|uljfn0lN-!z3L~Se&L%U}up2x={B{hH6cu z_u{-sW_?({B3aJ50Nh>pMz)z2%*^nA5JH}Id9wUByM12_35(!wp6;Fhv0o-YQCw*p zAI-XJ)MVY=sx7S~>#{1Kz(9A;^z`fQ8D?OZ0R)CQVE{$BQ6eIOfXXRnL_i{tm@T(F zSd2y!t7MgwE1>28l3gOorJ!=jWdt=`g8K%P;D9olz+|hIf4%B@-S2z--n?)6`+i@t z%-D|F;uqLjb1Nk=IXWT={@BijroQ_F?v4cQ9vYXlUlVG^bZdpM(&yb=%6-##tE|wbj)ybie>| zv8c%1ef=sU<8R8#XM1&Bq&Gt7Xm8&g73I0X)8E(cW?kKXF>_=jG$hn;jVa9jI6=}p z{+5712n>57qL>JpmYi}Z?*kK4bBxUq+K5E9MZpoz)VtJ$^BS-lY*x*^qkT2RRU63Xlzn#)%Ns{!nbon24 zO9%oeq>7NpojbN}OG`|u`S#l1KRjH2?FRhCUtBCPbg_OTeBBBk>xi8ZPx`eZ?cm_x zKTe;GPKXZ(++0~!F@tWJ)2pRL2oHOEog~g`SM|`~fR^Rqk3L$t+Nk2|GQIC{3h9Rs z&R+OD&CIk7W8TA=sYZsMe0ElsSuik$25*bowVN*#i)?H+sa!7MXe&TlIY=gt5=vwQ zPE?Vs9>Su6gYMyi4|xPk*}6bNBhU~UV=vOFx7VpZeYz6XQn!g^BpQsc}mB zeqCmrEI8q4=Kw3F@X#=or&nl1Ga?&|o=1h0~yuM4P9Za`#2{XdJz5Au&&ie49hrkyIoNjK3M_Wq^Jb~2Yl-Srftz)@TQY@Fs z#`-12#hZQoX4YAAdYSY_2`t$Pd}@zO)L~;?o0pT$`uc_Tgu)gn;>Zm1T!(Sa9rnTZEXVq ze&VM}a43SHmZr%Wc`8P(@{ci+rSZa#TMT55`-TVo{tk zq7*Jnm1<}qTk%?kf|35?sHSPKVYHvCTH$ABCgN&n0q!$X`gVX=E|dK_&H(}p@cnffOSGBL<^;G( zH$dP^U~44M2tM^M5^EsPOp3#t5MnGc_e>{3}?LOz}O?SR7-Bx)X%)n<}{VG2mz|#`0;lPR_@=oudB0jpuc};V4%0> zL1}R@g0Va|c=q@8{ert4mwpIgWMm{KH!o=2x)7;!Z+t>ePmh+tgU3JAH8l^t_C6b-PBYFVxJqNi- zVmyjMInT=wpt-VeKvF!C=1L{b<_KnOE#&en1y({x1%RZjtZme4H`swgp&{Fo_a<7P zEHiWT+p@B2m6e~KK6C5l&B=!eBV<8-zKu}i;qKw4c7vQSdybpaMCpwXhK7cqXHgb@ z_44J1y}g421B3njo$c*OdlUa;u=0Fi;jFH%qx3@vPe*wvz}GLpFF>hKl@^ywHfU{W z>F(&9_@tt|+`-=A&h6WeA3h8Z3tMHpHvh;G&kY+=#6AY| z&l;QLV1VG#i&LeVoIAz<8P8E6Te&z*iVOb3ez^xlapZA)P1M9nEbHq2(8Nn!-HCt? z*qzzOOi$0UEFdfk3%kSYE-!gYLgprOXX={_AS@$w>-MqVu04T>g-tBzSIsqj(8Z_~A!=y?x`gY!mu_Jx>To2v)^5 zwsYoMphKdSHE*tkr8N&HY9Cwq~OOis3Q5-2JHJM zsbDaW%Qpk;8pU^~s5vA&L=$ZQ8?ku{i5;Ygw+Tp7@M~Jw1aLix9f2Y(MWMFlB%BYR z5I`DG79<@7!Ci{q4)D*R5GSEghg+wC#q_7b2_ISU2EHW7(AR-F%4^+A-6V}Idb#@M|YHVom(t7&(_&Cd) z!%dh=DI5CDG-gs3J1yN~VW@m+3S)qTo$A>!u3{@vu*hZx!hp;dxcpX+UrY$Y z16h9#``S9^aQ`Jo@(%J8QckXcQl_M;Z0+s-n8Zv_wn`#3J5$a=FWg2M;a@3R!54QM8R# z;~BSRtrZXd5e%LM5CF4f^4HVZ>8T7~E?p2Jo4G+bH$(>QB~w!wPLAfyR24!98Ur)q z-QQW~J~LKfaPa4CJ>5unfFhGcJitjWA&mG(-R0x(WiYWv98 zl_x(gDm`+nrJ<>_{lWMx#DspOMkE9ys;jH36>ZS8?Yr4+umz&y`QE?KA1glW|FIChsC)32iV3YwZtz*w5n1B366^88&At0ehyG5r-KY$Exm zDbaEXyhifdNc>AE6aXk3l3T0@xPX8-@eB_JLP$`yQhOVTEhZ4*e37T8w@S&NNls5o z3keF&&dAJ4&qzy6IeF~(I8FP6j4l%sf)Ulxe!uh4BjfgI8bbK2__K&*;e*$IFW8@z zkvX(h!?HO!IfK`XQdM&DWPe}3mzQ^-e}Gz}4Gs*3v={)Ig5va)6g8_B2#L_(V2Q*| zL{LNVfD{0Mi}Y1w#{<4ReUoZ>qVo0a9+p8)vm)f2eC_NQSFsf-aI&3-9@wFtma3YO zq%D*}dLyA|LhSL5(%wt)~{*}_cS=E2EkIa%zK?d+uO9&c=62wO3Yfm!R_IJxRU zE1#5MY&<}(&SWQN4G=;GJ86s3(U(za*h5))iLr5^0m111t*^e>TwC|F|H+fSz9+^$ zEly}$^#dUQiXJ_1zEG{!wzsv8*pFVXufB1kwx;IhkZ5XbdOt3Xi}AlbcTRUpS9eQ? zY}Y8mHUS|ZJ?Q9I9ka?!;T9F~-j&MAVYMBPIPfQPcqZ zeHsV>#0h9dz!S8voxo7-9TG2qBAkT67l46?q;LU20K+vpzeuI_=qJmtiSY?MjK4WK zd2r!MOG-Q#4-KnH+q$*4=kfT>{Dh3C@g{`)J$n~g^G44eR$Sixf`Sp7Kkn|nSzC)z z$I$F?>*mec8`nty!^0z*8gHZAa=)$3D5EtEAs~Ho_8hBXe7(F;j_r9oe6yVXqI5-M zVG}gd%|E0hJ@{TpaM-?4}@h&S2k2 zRK69X;A9Rsb*}oq?3Zg)Q&$$pqsz|Jam|NLKa3U~r(G%qgW)DOx#ym96P^hXAv}`s zkn#`_gz$zU;#g4;K}4;Hq_qWFaHym8RjpbpD&uon9x|gS@)G2Yib@d)fjN+=#R45z zB|_KYzxJ1V?#<5KXWjGPzisfoop1@@TZ?9<%2~Sht*1y=F7OZw#Af|}y-UIRNXhe% z3N0*Ws4Q8r5{`%T+WCV10$W;R-(O?9TecW8#<#MG$;ex}})m<#H!JoI>S>fm;5ALR)L=uy>z4>uPR)^5XKysWZuJqemp?5lu7!qIyuruZFy~fiC}l@j%z> zTLCv7(SMUd#6`rnhL9Km z4{(qIU~gxqb0qBN=H`}c4mZGW)k<$4KVLtETv1z7)7ACNP3aP}I&jkm@t7hm6%^=o z=chDY(l;Sg+$leP^yn|U$5+RWRouNh?9Zz!E8X1Oy}i7>JiU|yzJWtI5Hw<}ti<@P z?K1m=d3odVwppKM8XB=*0U=;rzjn2>q~wQ=j$!W(c9qVKjuqZMmKN6W8{!MEU&DBy zLjnMp;m<6EC_V%{$hEV4ZI_(2Q$9UaJ|ktIW$#hWN_G~Q3mH=|XRSS^1wG=yu6`OZ z_O>xxX@d!2+J}laH#!IaJ>T7mmEpotyG#!;%lRY=7kKu>*MF8#tPr#59$h$#N%*-zj9@J{J zqsdlw^Ja)jr3wrR9xg&v!P2F+CZ^n1KnPYTMM%aO~(&f4^0_ zN)zHY2B|_OcDJWAO6i*rex;s1e!MLy$!M_&pQ1nByZ7*+FF(u7`YbCu`_oKDXy*Bz zXI@V3KJ`a8ii$^*O{38yC2YYT>PXgsahKroUkxFQNq87h1*sIy3T&7>k^&k79P5jALi^r{sQ4z_c(u z(UEetKAC;MFbI>0f~8in0)=_mIepK2Bno@hP6yU*B}=U^@Rroyu9)ckp3c+&H5zS?W|Y}BZit`%&H|&wZ0sAu29Hk> zq`-cX5dv06=d*u)U-X_TG*Cd+L+-sX5<(py+5oYhq32O(DTKw4Y#o9Jx||pZ?jjz5 zXdXaCkk^0-f#z;NU@<@>Knz5e5x*S}RgmY1kP(IG;oJth5TF?o4p$PBE!N%gG21>9j;%!8WA3`a)mE0?aJlL%iY~Oo;>O6?Oo~X8y~-MqBna=qtYZN z1gu97+Tvp47PFTmZccdG-i}v#G@56fKbTus#;sY4ZKS8WyH`7^vGZP474EPL4UI;! z4pRbu;PfOU1gz$!ru?&Kt;G@}V-swf!x93`qe+H&W1Z~v?al_#_A^qPacl1H*O&+P zC}$-*Qz9Y5rDv~`vG%&?5f66B-;9-U%@M`sy|LAql|DczX$pf-Y0i!eu~IfW+V*#e zviZx>;z+QyP`Vix70eobdPD{Rh!~ z`?eH38q`)-k15Zkf&!cYhf?V}BwXCxJuED(I9yUTIS8Svx39jXwcuZc*KYj#d3SeR zb4!Rs+(ep;2@fy^v;d+E@?;1aP-qPU8~cG_7(DnMk+n3p0TA~A_c|m~MKt~uUU>|N zY$z}Wm>ZB7!ZL>UlrqJO{tPss(9n@YfboJcGv$)_8`z(-A3?_3(>pLAFef|PWC@YD zdCNDaPD99S*|ZrWfv(bhYPH954=*q8#Dv7>UC$2O@2bqskON|ARrKr2G}$- z-Oqb_wdUrQu`#h@O48NU6&@Du;NavkjCHuWxh=Pm+Hy&1@(==spYl6*&YjDztf;`3 z>Wqwav9UE(RTdUjqyYRJ4>*ABV>3CC{kiJbB8l)w?Z#q z@FI3A=BOk}2l>}WkIP);uZLN&)JoP~`P?AsB4@MTCpi2e#@1fx*iv&}S7v?9y^!b# zs~|U*m0ks>zwPZC74J=H!Zp5xfQ65CYDx-bct2miM-LvPr=^i&Cxl*Y&wolwZxol@ zDEWDm6cyv6(1}y)<2P-RJH;rK2lBpXdej~!l{WI&VXg_c58j7SU&)k7X@2E^Zl^f!>-G5{6LG`AN1j~p6k?pg#)`ddFT1t^`N z$^h>OBM?DEzU(u63i9~CM!Iyh-zq$+6o2;}t|crq?5(-;Q@L54obbVbNUfhwA z;_vU@tJUi2?dQtOND>jIPivhR38A}N6W|w+w(#vT_Pe)vCb*$L7I2aY zQ~%&lGbd%7fNTWBXdq#r(Gz8m=LDevg`yDMNOSKGtPg#-8F1A?Ktq@)q}wQ>mL?i$ zUK=2~C}M{YET9nh(N8!48q-Jz41;|}0HB4rrPM|$6q%C(nul*A(b+R+a77s8F68Im zDJ%Og`(+*!#g)hLk;J2F-O689VsI_jsN68!Jw4s;y`CFpI0xo}fT)11fC>g-RYX^_ zx+=)w0YVOs-71aifv}c_SSm@pvMg>aS67J|?)yeTL16bGiU>t^IpH# z@BOCV@AvuX)ZVvl{`X;fI}TjoVd33fUE}3svqr985h3*H%Ee1!iBuw$^GP~xb)0D@ z|2*?EYg>nLCxol{`C_qj{hGA|C3IG}L`O#9{VJ8Ne_9*wL{Qc?_88q6fEWfbLWr+` zhO?qPChyeqb__|@83xN3PeibQMZZbZVp`~BhEW28Co@~%n^fJz1dY93wruILNA2x+ zM+JEoD(@6+TpiC_&hgSQLU?~*;BNl&>Nq;ttdZ-N2%)FD>wlQ^23b#!@iUhc6)gx{ zFmu*N48s=Py=T%@k2N9Gx3t_TuV{YoU|?f$WAg?dO|g@8oseJ05|zD)p1cOB1i}e1 zU_OVc0ci{&lmVh1kQ+EKlR+v7{|)In_T_j`{A|J#kj@p+bqsL{3BC|9*8p`LP!Sw- z7(fU(^9zRH0XAHid~ERZ)f-5fv2(yNn85)vI4GpDva+GRzHe*ME26_f|B$*piQ^jS4y~09?M23l7ERwjpc~CSCmJlCdS_pbiD?mU15D5S_j+>0L{~p8Oe4S>PnxGZR z#GI|rHb^=60n-|WlXL?nkoPug@T)C$=-=J0wsdE{Ig)+9wz0Fd4R7qine(%>VqT1F zLI~3O`?fYVjg6Dd8YK-Sgoo`9BbP^FvZVw<0nG>mPtC!R@8hFsZftz5`D<&ct12tI zx*nT$*`q-SB^8y2b53TR&aG}}7}$R7=SOhRsl>agw^p7e7B%Zv1 zCBEdC=?u~$!or{0UW6jEevKMSLdd^zMJ{(48a=FR?Dp^7_gYhRcXgGQl{PmunRbPvLrIVnQ)b)aCV9PBsw%?@xlcQ@m|{7+TvEP_V@G0?eqQTcXo7O4!5y) zlFH<+Dpxm^8_mF{N0mX4fxlnjK4qtFc9c6OfWO`7!`rEUA!TP6EMq(op9M6W9;Y?| z9_-ThV!+F~EK+sTW4%bNvvs)p#QtY?>Zh;wnYLKQTOpn0#T8t*jCbAfpuO_;&$U07 zn$W#wjhBWJLQYoJTq_%Vz%?4JTJ7C$`1=i864u(>JdUgFm&pm?QAfwRwdiCB7g-beQwt&bo&MZ}y#f=me;2;oJJ zu3!nA!SS^cydu#TKS}iUfLPCg87y*x=tn51doq0ZQGJlmM;BW;Rt;eROI?68<^=Lp zcpwn60CL`5UIaz+2|-5sp714M4t&bfOYPy|b@!JCgd5lY<7e;-4GA^q4BwtR*WdoP0|#8)+;Ndq zT2i7?s%~Dte&M_Cgf#n2ZtnSW=M-}1jP&$%YvP;;0Rwu^rbH|iK0B&_VjVT|NofXN zreSKbj`osI{)^^$`cBsI()3feX@oK{XDhS{R#{*?NF5m-r0OQ^&|9QvU*46hb@Y;| zX&KO|CZs(Fm5E8XSIgU@vxpFH+3;smxzwz;M1u(7eoG5Z7Wi~G)YXme!iJa02_fgy z=`Wld?z6;0hRC$D$=LtbzVuIz%YpfKUodPHi1*1Qn^el zu40HLmMAlV3i?xg4NKgHlomlt2E}r)3X;YQFTn!=SqI1#hW{@_6A)d60*ngH43QyX zZ5YH3-~kpO9C`YGQZ(bNaPjt1D_vY2od~&1;mD^jFbHTiDIqaCD@(7@E}S1&TvTkl z-#R-x5A59+ziL%-Vq$B{TS1Rz4bqqq!r9#1IaW4%cBhT%TSh386GC%S6FyUUXM4fm z`@B5FiHq*u?d!tXKP)@~-*k0#1ua;l)#}_;ZYfF0CypK4m7KCTC>Q`lkTeUp=NQkx zr-VQPN44yooxQnu0?x&|^e+HFhGE7ZJrN0j0DQQ{i-++bd1QF7LqBDk7IEB+m7ct` zm(Mm~kKw&|H7+tHu2j66q?xdNKyRimZ*?vad*gOe;`U9OHhrCygMXUj z?=Z`>8bk=q4Gk_z*VvfY@mdJm~ML0K~V5{vOheEO8Jq)&Lzv!ghw=&Jf!;)@}eH&KR&tz} zeiCzpOfMm9-mp=x(WyPvfdTV1J{nRW#AS;}B#|pz61Qx@pFz;VMePqB81K@rPo2ia z(aOju2)RQC4;nwvtTEG=5yGjQf6SR{Rgj-Qs&5&g-k1=Y8XH<$T86flmz5zTx>s0; z@9|YEk=`x5-Pd*GtHTsU=bbKU}^%=z)8s zWsy4FZmYS;cGdxIZWUEk{au*Z;xKQBHeyyR<6!7IjzG2ds~yCLoWXhzO|Q-hlNWXbN!wJMt7h!$zG>K=E~2?dUI`Vq%yuL(pEo4hSkWIb~d*@`9LP z2>RC6kf4yj1q)onl1O#ry{@id0b436og78UU&RaU9nznEedrZwiAh#g)`x0p9vPO2 z7B0w`sOXTOU>HnNe4I=Eq6dVvujriw*{nDzN65Ezv!9VIpEfd07b#}PxKd0&u!;9q zNs&T@!5Lmq4}OsZg98jZGiVFh@);RkztMP@g^Bn5SpDc;FHboiS?gg4$fe&67iE5x z+|klDris{IT8VsahZHH1V&$80B27Z_m@c1UVi;x!kkfT_P;SdMmEP~^nZ$t^XRLwoDt|W4J%8p*aelsyEk?`fvCcwoPmjOK|M0iXL*T)BYOFVzV!W9! zLx5yz*1%kO;oSN0evrwSUXs;8|^Ymi$ePm!Y3SzxSn#;US<^z7M*4 z5AQo*rWPZ)17UgepTb3FYL1R|BDR-SAmM!YzR(b+#|kxxQ-<&|GSUz>m6TZ9q91%X z*OrO#`1T|v7;La0RVsZtZnaP1XrC}vL%4SJs*jw$)ovL4nSY(}_x9=T?luw+w>Y_{ z+)w2n92^=Q9ka`ZaR;|;2I%)BJ6a+s%r8hvOq9qKIWD4`xb5iO;0A%U5m*WZW^<`! zJfdkp0IrS*YXoQ$foch?1)%K!t>goWqW19dYXAlqbfS?!PZKOQNpS|NkAcP=-blCk zu9<1+EH++RQc{qU!?3hTj*mZi{J7jp-lWrohA!mb&>(n9S}{C*b97{1uyx0_osR8^ zZ6^~a6Wg|DVofr!Z95a&PA0Z(`|IC(>wEwAT77TbI(5#eU3>2%le_gyT@0$-kvuVE z(URqnQovWBL$fXG4zg-A7lt|POq+U_KQ>|o4~d^>9Dk(6lj&0+yiRMYv2Ko_tUOC_ zU&njnKw+=Hq2cxrs55j!DKRJ;!p1?tK|>lx_VUH=c6uLyvANl$%Y=uE`q$>Dg`+A} zXbWr4F^oye64t!Jj0K0>U_2mY|HeJ?==$8CG?nN~k`=6HRNJ7fzyHP=qLhpWzEJu_ zBXhtOj6|C`16nGcF@Rj@MkBF*aWYBjaRDR?Cd^5bTSqFxq#?RHHVD19H7fl|%s(+U zkqI)KL@2r`lR;@S*!*?>mG&nT5U?ogN7h@m|lymUkglN*<4a{`&S7Gf*?Fb4KSRQ0ovNtCKDup+~Ac30Z!`){x4XW`@P} zW36kgVJ$S~ijs%1U_Pk#Uk%JL3g#3b5ke+=w=NH$ON8b=NeWn==2zs&S>qtZYVuWt z%BjFsj_`F%GuET7r3(C%c4pbF1M)ZYuE*AMvvGOwR0S5rs3<~2e#Lfw5Orlu=(*o6 z*JVDe@MrhDtIC0+X6rYOx*4n#tNabfF0W``Td`UtD#q2Mnm~Phdwo~)S1AjVKcsa} zM%)na^lc{!*c7Z*BB7+h?KAN3KVwSzcIrxfb~<2uGfOTN4cX5I$<#J^aL@!}eGd6J zGtK@jy$TCU0t$+bWZ~g#3BO5)#DPSPfQs}*2@gCfwP}uMw8ZFFc5($vY2Qp@meRED zUIYSEBBedr`8)oLuBo;Rvp;a99WjsmgRb|$Z<0~v`Wy2yd3VjTt2j^}hbS zuVzcqcrr<4l~rj?K<`QFu99{GvyWipD2;_s zYDeZRep*X?Kbbv+@VIZ0&x)kg2g|mAttAW$2+9e4Jf0nNIRgKHn9361!|Nyf4HpV% zU0PUJe5FV5jRUdzEKE(^HadSaS=1m6p*cn*sXXn+q1ya9>VrZf)IbH9CYL80Z$Tbg zBIXh0s$&kLE4xE_#qT$QyA~27Hd6&^s1oAD4*(v}6z%q$egM49w%--N+vV%4Q7ii~ zrKID|kMO`nR`PA*scELhK^VC{Ve?3TKRuH>Po=J!U>DOAdfv=UM&uacBtA(|eU%%c z!8T?MS%M*~Uuf0d&E>4cvH++oSBC2?3eDu`%CkdCySTVq(VP~9Mm@NLw>MZ!H=jMA z$t^6(BQkM4s5HsbndI9>$MC@t-B^IsAE~vg9|OXlJjYNMNB{VGAL1+N(&XYkm;Bsa z$|W|9%^)@%-_l|Z5K!B0r`fkrDE7m(tHxx~LBs0aMYsc4G&rJN9^Bji!W?z|xUlHU z&*aBAGK|y=`xENe(%^Cvu);i_TUt;5yTzpZosC^Rp`0RFQNf%L&Gh04bc6Ho>YNqk zs$%FAEg)yw?)zK+7v-1b`XH1D>JZ%$YlK`Yg;qKZ-8Qzk%)j&r?y=v7_l2dD3w1c13s%*tJG#4@opJzu;I)$vQ6313}@t-I!Tz@ZN_GCYu zv?k#7=sNd}ev&Y0LJWK7M^AGX;hbAwIgBeqC^;!rp@{D3ccTG9vi>cDg997g1LQ79?gh9ZlX;&feRT=TnjCeDVQ*G_$OCGipr&SV9w8=!?=v5x{ddoEu;wo030%Tf%g6!g?4)YsdfY)}M<**= zUG}bSaeGS`-?Oyv5RVS7yVn(|24npxRePtW9)RArUs}3TO|D?HvyVbsC@xtbQ?Zc1 zCff6jkG-MB<0^(w3DmpL)YJgky|(as8U{F$J^lS&k;6}WDsFEd*qtu?i)%3M2XBtoRJr&dV6xGOoon z(<61kj*eeHjmSGE1oi19`c5SG%44jcLw56LI#u@l{@h0IEZ4BK1*NKpYXS%eI@-ED z8bt64ZGcI^y@tP)w!uI?EP?*C#P6^tyuur2=^EgtIa-Axso=Pxl@ih+wKzSSkPraq zX-sStYfN~`-6CfLApmP@$qd<)=8nY-!HALKXANZEAmz1**he&TTi-<-lDZH73tVbllRfIRHsF+`; zY*j>5OoI~6ltHcMtZ{p%{{?S^nubTq5YKNx=3(@{$`uzHlxbgza?=i%;B3U?V0Kq> zvX87Tgwl#YCShCJ+K0>fsaZMEzc8ib{Cwz!lzoF*;`$FN{4_!TXyJUYBbdnk_=zy) zBHtNLE8>#IZG(9lzd5GvWcNc zgl(0IMIza8&zlL&dU(Jt6W4KlVXON=(iAuMr%kX)$>XYsP!4t}GD>e_UHvD%sCa|< z6qTB=;&G~@i`u?t(Ok}UTX;w?>ThEH!i5t&Y{%&wgArh!%9yNY87g8LI?b|ee<0Y| zLy*RapiUVUeNji)vRj8WX>X7k%#O&>8%$gbCmtl8@S5OTF zRMl(|7fwMFHN-6~2`BLbl)WNA1tg>akZ}gHLEB%DpD6eI1`|&*DjBsB658uJj4Nd! z0k=_fFjrWblC!IXCI<7x`LlZ9*q@&m;Z@N8DpmrOA@ib45ushUvHPt@5mVLT z)`ROX?X2oQ>&d;}zXjcLxttk*nLK`{4a}hy7UoSqsz$dc*ceKbX(&~odLfVn1**uf zN3gn7%0v<4F5tYTj7Ee-p>mjDsj@DEkU~Z18=EiXy*W<)oeXVJh8W|Eq7l}UO2^rI zuXYi!apDX9r?6Vc#EPGSS3vW-py;?DM5LRnM_4l5Xq)5ZcNLFAPFhz&(P6dy7WP^Zr%0dt-FE8)z-mXAL+1(2iBMQmoW*vs`rvJx!o!{G0zb^dU7D%Dg z{%jzZTB+H79J}>zqH=aqv}RXqd%Rf^u&s!{^w;zsVV+#YyK^swa)3}2jJs$p;1w1- z7IGOUk6ud+0kjH4p@c+V3C$5$en8pI<4d2B;!lf3#fTmA1t#C|BcLL%-hmT`^90vI z3Rh}C#T!tQ3KQWIf|OMyj7bZ%vlMgUK_iJy@dYXyQr7nNIuBu37()_M@7~@DZb=Or zUMUL>!OHqNL*WA$!%C$URZTio(49uT?;($|t~wtylgA#RsM4f%8uYA}8k%wxYz4iE zb!JgDb0lQ?z$f(zj!S*fNVJ5#7m~*_fzmSiSO`ystc9u1URRIBrb|Ulv|!labsx|v zP=Fc*Ra!I{-~|wcgp@!HE~T;e3eQq0LL~-oGb!U8d@ZK4Y$DolGCi}ofLCq&0j5X% zi>mKhjhN|DW-csn?PoHn;>%Rp#Lgk&6R1gi^u&|=ZG6y+Vled;v(O3J84n+xyH~$^ z@_X8;b6y!o!|fzLjdG`cz7x_Tdi~2eY{Vo}%*m_i4WP+)fq(=O9`(C4w6xYdz13Y++)HD5OPfPI_a`8gHxX=M~!Q>}qLgedpWP+Xj;$h|-vds=OLAgk_|& zw)FvIrhYn^FR@H(XMT_kkG0@)5fu|_=i@WjwG9|y#_S77PhSiPO%4x{xx6eaX(IpNV18Yk(+GxdXnHL_GErbyALwQ1JP1q7kyhk{ zf~pMo67B6ID-Y=<0r2C2M6G=dgmc0=FoE2VW%l?LCgA`;EvsmtG6j0;9qrHpl0i=4 zJZ0sb;}eBvqWqz%nzB}EN-XC-O0&aOTw#JfG7T$i24`;X{qUwM!R&WYU@rT=-<3~6L85)4nhYq&phgNlAujiK z{VpnVg`a{~Bsk2A8k(T_OpD^;JbFSV!Cz0l!rqaYy9dp9pEWq8+LYuKLc1b=Rylvj zM1}+!9+uE)g~KFR2JtsY&RFEVPG0;xW}=XB7+R z$M5F+v|JHE2sXd1nA8I1uK)G^M!~(`Gi+8GP@LbllGHxZ`<>zuv<6Rm=t#Hh+{*9gIoez z5pnXNu|8+`wxlVi*0$Uo5u@dHEVOF zQ5!E;w`Ts~>FLM8w~2lA5jA))ti<>%31Dwt6(i%Zl3x7fMabiC+oQkV>SCiMZMOXQ zU%Wum)hBiT#*z=sNy?^2F{o?wJD~%&m!Hm0&X528*s8tipS!b|E-ER)I(=vxpg^R4 znP&VEYDl=d-@Q_0gk>W?s3TPGi&JVF`R!cU0hn=xN!hgWZ4M=&7lOcb9z5w}E7@|r zV~`^S{u%`}GEO=;6?Q_>QpDh5jsf|u83BFG8fX!6em6Eh9U4}3LuP&}yB{K2YYO5G z<#i!foq;4cs(=43%UM%06%`deXQrjoCm?9C`tWY`U8!1KZETD_BP?-Gw<)Zpi7fBl ziVXcm5M6O_YZG;GX*?Cs&!h5l8=B%Wb`84pl^mr5;lvAb9Wo6j(2~<2z#sQkQXpsv z4BXAz(l#s*IZU1}=xJ`5C=?a!cp}VF4v6ciEbKP7%!RlE0}zad!9<#|h@%QO-ab9H z{7rD#`cIC+eXG$Zh}paCM+*qI@H^;wd%9@g{~Ws>iNayhdT^UC<+Y#-xfbIk7I=NS zSWB)lapVPa%NO;WS;Qv5$Uw!x!A14Fzec+^p{k5&@qL)h7Wn>$p7G<>WH>~*RFC*{ zY*2tdx$UnJF>$uB2a2GXPL*YuiZ4K~^yLR86+v|QMmW60DVKBNfC}0`YRl<sDC; zP8zl86|}!vXGmTXx*R{zCrRcp?PZnR3o+>me~x4NJ9q#6tF?&jpkWlKP(Hwj{z4J1 zjy^CJFC{LHn2w^%{^E8RA}XxY<$fS*@9wUwe7!jrn~jW=RaWUcxAas`#lwk#{%h7S zPR0ym*J*8Mr(W?5q{p!4;w&_z+h%zl8C3w@qp?TMac--gG-W1(Giq0I!^h!{eSMi` z^U57EqiB*8XF5NF(PJ%*e=^Ho+lY;C!y4N?ZP)^+6B$Vq^M_EtgrkZ6$qfOAPlgUD zqD)t)3VFN?CcifR#J@cs*grUn#q1y>?kWCVm6E&`NNylLK69cZ z)wh#a;NN_2uc5+qN(z(qN7968n|Wji$c5SMerAU;L-ik>+ecPgdIpz+ZujT;iYrf; zq8e;YB2nDi$Q9>55@=!$H1DG5QCUw~!R5 z&ShQ%es+=nf;EgY%2PY#?#%O*!d3Rj6C9>T_JgGZNY0QnV0cG?HZ7qcadEUU(b0t7 zY=#?}YzbDGb)uf@YZc1R%vz1yY|cf}G*jNp2zp@3e|zkmRz5$Noyw;zL`6&A9@kRU zS(85x^{11O-|L@a&{Mr4NPTw_BmtY1#+OYA)c?lZ4`7 zys^N8?3&|Uaq<^FOS)gvPV4<|J>yE!o89JfB&-`iC0-xw<+zSkbZtaoS24I82XcoX4icp+-ZC1@?W7mzRe@i+St3qFYQg zM_0ABzCtmr_wAx{7&#Pj=6W5rnvv}&m5+UjXw;QaE;e@~K<5}7A^9xT3n_$Jc z7Ac@VXuNik*!#8*K~f+kB!#>qz@u*Az+GO zAdTF*HB{LRUrz7;aR>s*g8wOV0m6&XP{CnmP?jVO@wPFwEUrbXqGYbHiV_`ZFA2by zC|~oCKF=Ii$|?IXds5O3DTz5+cLa+OqVFK+Q zT@3}6YdA^j$gc>B`k0}(;DSET&=rT%KUh4|>ew)b4ZWtTH>&L~OO&z+(|+ohqyP;C zYpSuMXp>q}B1v&0I*MAYa%(j6t~JX!^&X!e{zGe zX{vmIEsc!pNTH)qH9nyrAJJ*(Z~wu;?`(H!X0m>B*swtT-6k5>$B#f~yvErE?39>RCX&2zj!cW5Z3FQKsGNR~12Oe_=)< zw&>l!pPkiFJnJcUf#hIT+Q`Dz2tI{1RSkxS3_JpYpr-_NJv}|-EYP#kh#}@RbPABN zaGRJ7Z6EA~Iu`C;b9hX%mngn}0GWs+%2yGP7oR^yNB5gVAad!sBiKto z0EOA0l}_p$p7zaSDD&QqE7(1BIgCaLnikj*EY65{BQH=2vHy*RpQgxZhXd6LNMT7~ zYKR*Af(|_n4X$kJhn|N!a!0v%aCmT3g{3NQ+^5ThVlV4F2BHLBe_7jGcevJ_f`qO* zI|cqPum^9C?X5ogoz74xAmLA9uHXT$h+gJmpsTw(ZoY}EtZY;ZwnExza-Fl8i;0QK zT%XnD`Zx&X|M+MyADoYFBG3Z&*dZu*_ex$Z(we57bAqpQ@yL6mOxR@Cq1{m-ZU7%n zJ`Uror3}@Axh&PP;6zF&Qa0oaj<|zx1v45Rr~h>%J5puo|Q|q`(fSQ3+p?0QHeIK6h6S&}mjEuFdrY_{$>&2+C9jbjH9Di3v}9MspE% zOir!SoJ;73WX?%MC)5b9zymgEC{TJj}Dg&NrcVONX(FO-nVAaQ?7dKqDGa}`n4Bd$M(6kRL4 zu@0Oix0;-m!T#tSx5G7vdxiqFOGxmIOavSlYzvv98`}r5k*=F{Kf7w^Ux5CC_)yG( z$4O1;tWc{?Hhq2;)8R3Z3UZ19PS@9?kp&)nyhMD1ijrtzh+uc0?N82!o8M0bO)hX4eGFHSN17=c3jJaF}bC_~G zR2Ai@fO25FfTZ76JO#iMrf|%`;4pIvKbxB|sDux&DhO>*5Tfz`P3M-CSGt~0@Z!b1 zw*w6w1=Cr{UCT4xY%a6Az*U4JYk6wM#x9@R>V@>5{*O;qJ zGc$zSUtrSzQ9`os#WC|shvr>NcIRI2sq)<)Bd#iN!lai1pBDH^2%lj)6XY$dFOnZs zS^5)rN@^uM6C@d?RUU-xIydlLV*)0;-j>-jC+)K4{_bu#n=~KS_c<~m@;^vVfbG~AmF;40; zIO(Tuwx^|GgSDq!#~jg@9*+45YZ5x2=SEr8a#u{`hNPMS~Zm z`Vx%zt%M@JVZ~pd=VwlzK5Hoq)ZzBHR+8FD9ZhtDUM%GR;a_5USt|-^IlKhas}UJD zHfyhZpkXab&D2TJewb$$YMS3mls(Sw6by_G#9sn!tdtCS_{G; zM_@e4=2lWO4N^tl_864$NG(HREfc{v&?=p2)Gu{oJ?XSiv}jE{s_dy-yFaA(B=oKw z+=U+8>fEx0<^3s2RziVM=Z1MQ@fk>vc$1=~l}7-ta0s;H7s z6VF$|WnkM8da1WR#j)F80oefF(UiU5622Y>dfr?+6xJ!h%> z8ym{XO@98=Z{)j$`7weC8iGEX-&Uc4Rv;C*ZhvxXC(}ZvU-{Y$uz^yTlW5SI!h~U@ zNF|gs)aF*yMeuvCNUr?#^T93drL-Qj6fb2q5E5D_@-7gW8-0}vU`$bd)D%J(p?DSG z+&GU}G*35a_!*cxPZjTF$_`v*YxOTt>1PBJs&e!Wf)Mzk{U4zq#}z*bi6VUm=@5Mb z{hv8%n(7{I?lBAB-;C3N_s$*1e-T8IZucRC*P+GLNnmhSQSkvzgshOU=YX*kN0iVK z80e?%QBgQPlzcaILDjJ{R0_a=pfK1zVjPArlYo%0BX05&iZV29P+_&bOBUP+`FemT zq@B-?44yH|L3CVov11Sq^uccH^5@p}<#+czNs~Pr))INnVEtrCCQ2E61pcln2ADyL zac6;)l%Lg8>otP#m>09Va7?oP{>3GhLuQs1VO}y9+U>9;8S-{tP|QiLy_BNBMOrEf zp$!`CJB`R5dehH8c@2WsIUV242rh=7y-0tJPfz+SP&OKi+_MK?US6(eW@a5sd-abRt_UgcQ=|UjOR9-N145CuQ=#ysR`hRhZm9 zUU5kwRcXN%n8Sj^7L5@4ileHt%rgpI-?<<;k}&#Nl35)KQs zw+0@<6%9L}CnYqOL*sr}^-Z@_(4x?i^*Ag+MzS>)w}UB`UtdazByd%TOcsW70wcsXAw7z#YH@Z=Ej_ikb(9fW_HLKY=fo7*{zsU z@!du`TQoB>ODl=gVEFZ&Jvi!UijNmGGhfsA?_WO@)X?J3pa2iucJGDCb`*>Rq`ZoT zhK@=GRzahkcV%MP+G(NI4#HoEna#Yyi9~FH#D$hrg*HT@2P!^``bPqSs(d}^DYB7= z7YgM!eE%ekP>T>aBjDeloaYpFEAN`3G*=3(5sBlAoqIx+J*N3X8jaCFDREd##mp?R z9_D1?Q;q{#3BKvA)KgaWY&(hjCDIScMbfRZ(tZ#p^x@@r14#BarUFV-GS4T3Dx{uh za}Po7!K}+9*Y(D51d-c>#5~~QD3{O-FnHLu(yO&*^Hk?>O-%U)6O|$Ev@(^5i z+ESWG=`~*2tV11lm+Q2fU6``E4Hxw!!>vLcAI`@0*_ee$pv2<=0YxPxnHM&&{N5fW zwU#Y20qG@HCARA9)#%bUs!P!L#gEeKJ*g$XjT+=qU-Pr5Enur+iOUw!Ddm!XUzF*G!-XpQ9oc8*e0`&SMDT0K?G%3p-LSMpWq=Syx#J5Tf4Y5Q5AY`{ntBb#f8NlV}Q*mh!VZEDi?A z+Qy1iGrCS4slP33EHNiDJc5U#>(HHM)ncid((YA_?oF7GpshV||2#fxFKYVQDmDx> zQA0}=m^fvjJ#sZnCq_3|Tu3+;gelAn*Z}9v0BV#C*h?V!uHG!%00&@~KpX}r{SGPu zme7!pG(kaXQ^vAXPl_w8?BcLauGmKhmku^|C(oa6g%2dm$OesE^lOz+jmA9D#lTEP z9*KEEgj~sjZ_UVr4kTLvgo4JG8(~49#qd0{wa@mF7LV@1g%SzI-!F!5Jc?6c+Wi}@ zc7+l5@Sy>&8ZRX;i|CTk$&fbWt+4Uvi&sTpMt-?EoA}VG_V;<5(;B#+dhgaL{S8ov zS+JAt6(Z9nrHfPZrBVy`12M0&Nxen-h-HbARd5G1cL{TMhkSVSM8(Y2Qd)a`v-emh z;K)pc3x4=rPVRKmkIf_Y3vfHNz4wW&w7EBm6rM8$Bnf5`X(L7~hk>34?Bl@^MCTHq z*pRxRpfVEvp*SGohV-rkkkKNKi^Lyt1Nw17C+9o-s$cu4Bs#a2xs%lDE z3}6l~?`KUIxBM)!e`(iQzT)-u7gL4Q1Xv#Twefwu<+WE-G!#@c+*$En z2h|Bwq0J-aRTPkWyG0}PX9eA7iG&Jr21^QK%WAO-y90>g92LY)dx@R{9>@7hBN&jj zo!s>FHmWwts#dFLJp{N#r33H(T>5{YM*t~kcBDm7F~vLnWPGqFpJ5LcyU-=Xc-?MaL*8YmiifUyC}kJ>FnJl9FUMwH6LN z9QHeh*X(rSgkAp|XIHnv-m}@9MX0itV_yXGa+lj##ZYMT;-X?vL4kLw{MAFvT6lk` z;V;L4((+ke)TJvj9Bb!Mja zm{yg@XAMLkIxO@}%uIB>zy4#-Tj#R&cwi-`qoeb=70u^3fx1{TPRxWFBKk-y$6tR4 zZCMMx9S~-Si}XS26GTb3Sp{?GIkx>0m)xM-C#=TLk4pnpi7VV8hGLoiEjV}#OOV*u z_ncc{B3hd~^M$vntGs?Lb8b_>NAw>0&P_*W^_P<1Nz%v8cI}Ks==GLd%8UJFNQojZ zOG{lIkl8~_MYPWXL9kt2cl||Uu)*T)$p3xtguOGq&?Hp(V>iS&x(|VTHqTso+e-P9 zuE0h~iim%W#Wm!&S~SS8_0!a?TNi@4AC1d07SZnuF^fu`AQO`@b^9(wYe9IQ7_XOv z?)rU#Urr-^gk`f_c+(rgL9g~nUQACyy~Y*|CM?80G;sCx=f|6$uNq~G{mNT8frNx? zk#%80MM4yTM8g-aSNYw>$$as6;pWicp}x!-n3ymL8^TclZy&-BI*6pjS3$O5I%qzs zBu`f5o*qAy^24Khz@S*5ki|759(_KrU-;KgvUCaW!0K5l0I6QPwcGMDi2COk-v>7q za@9rWS43F>Qp{F>&9U4}HXBc#%;Brid<+Zf7tO&F1sMVSqn%fYS0&^=Bye6rSQpUM zg(}G0AfuM-s4Q<@Z}WNM^B~EHP@Ipm-U*Q6oPl-iJOb6>a-;*zA8G#%OzPm;?7+Hbr$~s zE|(-S(+$Pbju_z-9I$Y#TCyYR+DK|&|syhO3Yf~h{^}F3KB+L#519Ir;J7X8k zMo(>bj+|p&m0X69w8}87JC`hu8k?I}|B@ji6P#GVYTK|Ydq60MqEUo)Bi$pOj(k&R zuI#eteTO>6-0E^0f3K)JaZ~$esz#H|?`=vP0kY+#Q|H7N)w8>(CgN+h+pPB+U$p-T z{dB%cBq%6iyu3T#+rawLRlPut9s_+@|4~~5bFc(SuA&v}h-QaV{3yg#Te%Fj( z^YLEZ?~5?VCRp}R@lV$#Xi#ePs=96qX(rJP$RDLw2{%*#F>artqfpkg!c9$SW*Ggx zr2YC?h93}f=(VCskka34#3z8%Ys%KtjQb--vT#?g-_pCKU}z$p$Qswd*1kS^z1;^V z>+E^?FNnCz>4OMHiHd>oc4QRb;_lLduUHKpovW-4I!T~b2WXxbD>E67P|2=0>}j%U zffWWLs@0#nC70F~!;Fw+11izCkL@8C1%U;U;I!$+$eG}Rg0ii;tme@uleNatf4vXj zU(M6d1-`M7)i;^Ic-MQ*?#d+OPcGf5fJBOoLl|v9#WYHgJ-8WJ=9q?Pu{>mBTo39LHj+am5OjlFK3AnsaPn()(l^A&yv@#;_g8(dnO+l}3r_ zS!;ZV64lg^nZx^+ryi+WX-Re{IvJkUKe3$T^eXMKT_p6Ai{34W{az7nPn>Qgj#5Xz z+F#f^Fhkz?yvL|Qdm&LsGX(OQ4Z@2j@3_RnSX@;hi*fwF=gSV=#+8W}?V@w@v#TFL>?i8=N60jsoqnWf9v zSAU)!f5QmjWH<9|L0WY?1r+5(uy-Mb7s^D^5E`S15S1AwXjh*nAK%EJo*Nu0_- zN73j6IaO7Ut7R}PZXWCU6QWRM$Bsa#Ysv4kopUNeNn1iFBz1Nmx6t!kW~5%Q%f$EX zoe;Z`$qWAQE@lc$b}#34A)$_dEq5_bef`U;8Z?akc3GG~VbQ*2dsr!CqT4$nITK8P zqjY+53f^sVgT*P#K5p=iY#EgIe!rjRQ;_|>vhN2~ex?{!x7>pF zpbeDo>`nk-V|vn!y6SGG>1M}l$2XCV*4uEaSp}q+W(H8*XYw9LRm920KiNmeL-RW6 z3n@_|&VT_~Iu=k#zy;@z8|X!+VHK#ntXRT^YsSPNxw#=dJP71ps*8y^`+{$6kk3Gi zl~z}$arzy4p*-P3)0@=3>Am@51!cv7<_G+=9--ybT>WyRdvjQ|c!EAgc9Q_(zwRAL z9}Ks@mYH#KORL!d%$&df@y<`VQrRV}BF~SR;3Rj(c0TblA=I~@Gm+MwGm+g!JIs!e z=a+`FMpzz9lI51AH8#_I$0Jdn=S5q?%SS~TYN4gkff7;n(vrv(;Q#lHEY3+Tv32Vm z1R4fRZ+>O7HynK-!zmHp8Iq3tfkY@Tk{R{qh3L~yGVr}QnNq2T8kCU4cYO^=CL!3p zo66=%NKS7vKA9^A^^0-BgTqJ0uOkV-LHBs5ZYTH2$uZu5N8PhdpO4X+?sE4(W9HN8 zF8TR=3Hebu>fu1o=WUUGlpT^H{8r|6RZA^rryk>=d8H+c~mE2UQsI#8UL=PVzx=YnO1C~wiKG1d zehWH*o%h~iN8w0DmLM{cUg0G*S1*EX=#n=Z9z^eXH}){I zvwIR0H1wZX=6anA6`M0DlGf-6)HY(bA<1DC`^X$>C|TJ)=r*r_{q%5j`G|k;D!(7@ zK8>i*pra^kqy=bLrvnj2 zRP!qftIO}TA6DWY&u z0aBlb{rG)#x6U4ckGJP`&+<>yv*4u=sV)$nbm9wfR*?UaEMWdYoVtjV^)2J=#6G21sxj7{O&+DvunE4sh z4w9+S?Y7V8cHI%xcURPvU*CZUXp#eAkj6`Y@cMjCPT)LtNu#V+Tf%vQy1hVq~g)6!rIPe6do8RhM@kSDSP z!qk7Mm(i2BxvyG;l99RZHn9YQL9YpXx_tSAtRAJL!u05(XHczeJ`WCx@@XlH^JKxc z>Q=Pt+=o@&X&+)1GK(DQ7@Sd4IU*?Iuwn$qVf3s)T`K1A#JwX4qp6pk z5jd(T+uwi>qnTg@q8lUMhw}B<^Y?A4E_9uyp|Pu~CTri_;!I?f%)k1nt4q-7cR7Bq zSkSAfAlOJkMZ<%hMYY9SMAf<4c&k5^tI`>tSFCh@BM2<$!j-%u^TTe`{?vE&Jpo}cELBB4SVZ*u4ifnoy-mj zprg*rLi*}=Jy^6qXe^J2Mj#_%k_-c~fjog;G&Gj(k9-ZFT8!i!G>e;l(;L=uN?<#_VQne>qt*J7 z4+KZs+B(<}41NQBBeZ z#e@3Okc7M8_+zrEiN*fbyHNT{vv>1Y(Yt;g-J2o>lARp#9kzmzNONNB=Ls=1|*A z$O_!A7{1|xCt^}(<++V;$vG?RRhq9LR*l8DL@e@P>QL|mwTe`08#hT;=3%HjBu43; zo-;ia-n!h5*pT;G`1+l$HG!4~!caC)jYueO>BXnQcRk6MLox+84|$9b4ktPSw{4ed zb21-9?>7rsDwnEGRtdlbOh8oTYHN(Xy;_@3reBm9qGO=j*jO9qo50^8bfu(4hIdf> zZ+uamInAM;Vb~Z@w4cY#<|zhHIkl;%Vc?y!7d-jQ(Ke%xwDXpYnl{#we)*u%XLybWZnX59D8VfI4APTCc>v}S zE|b0oKUk6c5WCP`qzzAHiPR5c&IeK-;dA&uH$bmVCqe~+j}rEm3e{oMXmFp7H{$lI z%l4~Y=f4azf7j2lJP8oLi<5bx_cnYhwL~q`{1IirVRgWwV=I3KIb5joE!C3d7Yuwm z0z7ORp#w-#Z)|2-2Q`#0wLqFfNf6m@hM-JAzjE>9e3suO#O)$jXv}`1>CSP&SV9DXK{$HUCG5CKcK`;pv9rst+4umt~ z2xQyFc2mqvPyhknzF5PPYrztF!qRPRp1F~E-#)?C*Rpd>nNO+1RJLB1Y(1o`LkS z?l@ct*glx~LImMizi=S_dCxM0L;dtlPl?X=6q(~Z!=taMMqQqb-yfHA)jXjT^IR@n zOnzhPh622T8NyE8+rqYHa8_8F_Y?>J^fHN%G5YDO_-MZzs4JI(JJ8(){CrS=tFfzl zH}+fKj)Ch=x6w^$9lKx$0G>b**%_wSi~}Cu=VzyYOUMk#`A|+VmDa$fC&b&A4PfAB zNisWnv(m6|^U~o9m{(QwvmpQ+A^!JYGZrlptS1AN7#xZ-AZYY$5D62Bu9_BhlM+}Q zD+WCxXP-7^wiu!$Njh(by&2&QQrl+!_g|+G^Cows({e=PtnW9Mu2l*t^w65B_*JQ_ z-Qth?o$*->6&A<-E^|}LNd1n4g@-2=uyz#_gQ$)G^o&D;5`-8r@BH41aTHn4Rznev zeS`;;7(utgU4QV{`Dy#8DOIRyM;kT`5*q`*5p*t!dzwjkO~x-)>c*I$1GcGN(}0Z} zTuSNOX{H<_sJNXROv10Sb65m_@pvcKWEWV3Ue=!yF28YPzP#GV^(_adrf&Cazjt&z zKDz5-fCZ)?;r?P5!0o%d;KPK+dQrrkj!5Krj%jkbxARwa#(8`8yi13 zm_k{zYX%);pP*p?9uT41@+hI#vE2b>=5GAsXJX(tWr5b9+%E99x)zQ zg8uaf_9g{YUH!9%i^qU%chFm^X=+GD|5hud`8J-XPi+;`oe2wvr~N}Uh@K9x4fQ{9 z+?nb6I6f6l36*5FY|NBZd>O_3;q2@<011fA6~YshF#=B3QWT7iZpJ0Iu{ixd)~>NR z(l*S-*2ELrnAoZQC{`wr$(Cx&7`h*biHks!nxx_0vz^mvN5Ee$CYeb&Za- zjYU5u8qh6B$ZnLBkeMU<{K9-_H;dKKeH8B7TcgROMpnxAzXS4fYgR7_U?9G)VWb*0 zm7(E5_m3!X30ZTCPSZw>o6YX8TLJ3rEFeGJKVYU>Sp6rVv2ZVmvA+$w)wRF!nvYy$ zqH#BYfZn?aSI^I9M+`-VmojyB&E$hxM6yAz64lVZ(RWP zGo34i@3V@rv9YRZbEl?b{g%o7RZa1#mqPQBg55 z$ZIX2-S9_kA$dh<;X_QR&ZFwRZpsHGJZ?8XE>s{lB7mwmVGG)Y-5s!8C?|P12_HtE zQ*Gu0^$ML;hWw#VH~L#0vvM&+42ifG0}7NgaTJO1@Ih(jm0G1t$;&vRb5m=xD>ApUTGQKDvWXx{^rV!)!w^zo zz+-$YiLzrFbm;YEdNHR5kBufG=tJ}=FGOVO)SMn`NG?}%krJ#}bSePG^?(;tCOR6h zZYBI6n843)UeiObVFvyf5F`npgC6&r(S9tXAfgwi88MU%xR#F*lW@xNV}qs=9XNNd zw+v*LjOKt_C+I@T8|V@!n^oTn25MqFU46X8l&JLnIs*56-o4iT1pJC)XkhJ_9jYZ5 zHnBf!Rnwr;pqxpkcn72`l>-}OISX}w|J|AdfHgMZr7BtADldHI^56^Z=wR2=u8rT? z94pTiu{NVWCReMMJvzG7D*9}GF7}8mqGODhL!WPKPmA?)Yip8p@ywYim7RtYis7Hw z6Z4$YIIKgm@`htG?)d2 z4Kwpnqj3P*-)tGlgeyk8#db4mmKa>!yCN(>b+(+(|IImug&(}9QP&KU{%miaFbt#Y* zkj?o-xF*PE`Da5Y6gL*`4G!Mg=%lfa;&n7$RD@^;d8U=zJT6zmoZesE+}zsOS+KaY z3=xW{Fz{_NrLn2b3H$u1E*r-w%P$(uHWqzuQ7s#b^$f^b-4Y23CKbx`l`y=*xWrG= z8yEwM;)k+%y|HW));Ipmb@>mPtC*l~lXwRlkdlrfDL1{Jn&uwJI5f6%4w@xYmQ6;* z-jWGS=5T1%6p_hhEzWJnu#NdrQN-C%z+JM(a;WM7F9GLZ()B+E=3&uc1wJO?nTl93Ats~gYsk2 zgQ5e(6wsK0_yF5Xss|+p8Un-)F#pJZz*7AP9~J&jZw>ls(A~C`F;U0)p?A^7X5k6i z_H>L>P{Rcf4JwaQsPLPj2`XS>;;LWP7y>KQ2UNm|6jO=++S>XB{0j;RigpnI|3-2t zkQ+XCH!|g!I`ZKD3M_kn>yz`c?Rfp-gDwpvg(?k|3y#u{-xuU(76851s3-VJlZKQ+ zK^h#)jpF#X^~3$mNX7#LGKMQ~LTaBpjs-XXB#?m6YO6ik&D6Q+-$1Ya9}e|Y`b3uW z^c(<}CNW%AT-?nG8cYLY_@^*d;=%bb!U?K+B7KUDe{bJfaNk}r2|m$PgWXxb2-(Q! zW`BErOG=dl0ADt}jg*#z;wN+p-H@Ju56seBZIEq9MU>&=C7~DzA_9a#2U) z!qQUq@IKk_!&n9lG#(n-@A>rh_a?} zH&t{3wa-I90eDG}TtB zr>FRmM7|guj}yVkl~1c$I)>V~>WR<)Y}Kvkm@kv7tQyD`7}G;TPjc0&5VlY%NO(p7 zpaxHm@Xn8L6BFa38IMlRYik@dn#BWrc5zi>33@gz=!&l1S4`k%nuN2Pi^}Rc46K}j z8esd9w(jQXd4+*ZqXCk|?=KUna+MWDiHSwp=N^IUq(aEcRR4Nn@v7A)Z_gS0{8}hW&x1nW8(lW8XQ?!gbkF8 zlmx2NQv$R3Rry@Z8OX%kSR8YoE#WS|yv7GRg9K^LCkbyar$;wY6F0NJpM9+d`?tHjQx|QX`VK*2Sv<}foE#F-lL>h}wYD9;Kr5p`eTS#mi~xXBs)=!N zDcaE^2LL->PW{>IDkf#9CY#OQzZwgeR-jk|j#FxEjq=r0%$@c zIQYGSEy0;-q!Gz7v5`?Xx0gi+57nKUQ&ZX4`E~8w$uxT&&a7-pSM}}VGBkY{$C7VS z!lw^se_Qc@X7u>>`iU1gqa~kj(C%DEZk4JljUGYYd12}@bHb)X4Y6{1-xo=i&$mx1 z>Iu_4KmIkm*#*Nkmn2HZ#OLz8#&3skVh@DUn3|ftUayxL@jE`=XB9BmrAQtp;ob2Y zrlF$nTVY(KU%&48lJix6o9X0^kU3;x^6B<5L0`VX5NBkF&q=ANv~_!UceVf%6%&n! zj36lZv$~n5Es%lFQ(on_%$t$J%fJd@5I6Y*ce z;)0-^jSE)I81vmJxzf8hSRH|(Av_&00f!M4hL~Hf9REuJ zU6r-@W;mm+lA4`jb$%Varcxu}HxABtkE5wrFGAXRd8pE&GZYnIw=KkJxtUr`&!+dW zjE4FkJxwSgEi-c=VGxtONpWWpxcp-KU%5*IM#PoCkPPZM8 zk$rTat4|UVfW|mVW)kemcISsOHw7SZ!J)yHV}}a%^^uL#TH}9laymLnO$6N*=0Pxu zPP;sp(C&5#yhe6xxrAwG-w4EFapsmv1*nwNt~Xnu)ANhU;*#cf`Fj!@D@#w*$PWKQ z+HI);u5P`3NzGFPBp_Is89ygSQV}w{i-E&SUWn=#bXc>PwLTxw0i_risO(&Av!^uZ zsV7HAJmv97dG3Ua<~dxmTOm#+BIf0)%E}e5f%YaQDM*Nat%#?BfXelvQ(soLFGPp% z?=}CLwMJKa=gOC%rIUgivG!PZ*KuYHZ4q{``8Edc7s|t1A-8^^ZTi|>PpUh@QWHS~ z&y$5(Zoap>``92P^t}HG=#{7+dmV)uyn*-KDvJ36I8)FDC^S74K3eOFw&RQkv2d&A?Ll2H#<2c;l1g6 ztZ+YJDwFj=dh)!&l7jW_>1pHqpzzz#%?wHP2pEmw}9*M!OBScRA)m=i{S6nPhKgKaS9LAAjUxuH_q@wOsr#>aolDs108cYBh}6e*p|_!)Rp%VPRp_nnFud$w$&9$pB889zeo{ zsrLA;{yI)VHU$`HLyOjVqA20(8DQO$kadZZ=zW7(wN!8`Ec9Y34;31pV65We2typ zhZ6>uw578an>Q3Vv0=0@IfR%=DR&HMJ~9e=(5~?an8VbzHJ{Or| zs8^T?wL0zm&kjGg?+!&otv6eLPuCtv`*?h~EK%t$1nI#zwY{%n%${OOmvREoCK(E| zGNk2X934!QtjYuCOIz8Y#>X!&%{kW89c`5ol?VvS?$Ur06BE=DD7JaKXM%`En-U$~ z)-J6u<${clRwukX53t3^sM)wmAC@Xh-rs2nHNCvlTzhr~JP^ zU+0J;EeN8TSlMpYx*+LJc-+LS^1QT^RAiVLZfvfGbwR`Bbo+c8 zaeWN<@-nk>!)+D5Oks~c|FJAXg^4moRZ{xZLDbphop0~#>ayMDy%9gn?tEVayaDx( z1UW$8<+&p>`1{*G_piEXQK@04NaXKi#q?=8;y4PQO3-in57s1RIxHRohN*afB18x8 ze1F{sN3z}5-mJI57P!fZs@`>ZzOSj%7|Z6$)-{m9NJ{-Q8vUu##>PRxjjWdl=C4Z- z_mNk&YR5#u1=SX(OExIc&p`q>=S$qvHV#u6K8RGtxe%fTyO?)9!P)6aquT}jkf*!5 z4sAC_tWriDnS*Gju(j99X~0-*P2_hO%{M8b1^x3Sxq{CNX0`A4S{9m9`rHgBaQ4X z%Ip4&m8#mEu`_0sF^j$Wk2%=ywf23rvKUpW7WOHU-FW4A%Gr_x)M1X(HY6tQyXb+sMNEU{YE$1E7q>AUHvLU>)it>{v)03S{k z@8<6A>wZ{f4~|6xSAlkPgwDbs2leL+3<3fWz+5zIM1Eh8j(2gnR;ea9Ff!@7IC0n3 zlvlsKsVJ#z+3Ydg_5i;lCZ1J|7~&~Yq7h~aQ7DI&l9Eyd8+bV}(^zp0#Vmgd!48xC z$#{FM`OmdjSoVSSE6%46%2h*`k5T@;o`ggZ4CdD}mYF;(oiJu(|NaL9qlP5DQhTJS zG0aWK?el0hB&n$C@m}d}tOYJ%iyr%~jv!sB%;j=v)MiH|QZW4f-o_KE<_xelr7;$l6RG+my9f?1xA(VNEH4fw9rr|qx2m40nTaS!btV=L$f-ZP zOgy+hOAJKef(`Y}`SJsT13qR}gG0g$9MGj>+wrnQO3=cLBhepIPuQ4T{EmUVC_|n- zRDdLbRW;f_Q(YJ3i_I_kP{YPY6ZS^BP$1 z)ibNAv@NS;`;i$N2n9^3&UKywboX7IrkkezFI;f`l35gW9Un*|BYH|IfD2(}Uybi) z$E;Z+tx-6@VZpi<2^ze(7_;eJ_oVdh?tWgeTq0x;k75qg{v+Sh%bQlWyqBqDd3mw7 z4y6siVMp3rU5w`G57^KKY;vk+KX*Bo3nz@ph0?xH%+UkD4FCL@o19eUtMvN1FSZvU zOO5~od2Kk}@)HzL5eY^vl@H}7K!O4T@&CU+^x`EeQl^f2D$=US%FtLXwJPo|IeX&cR>Vtk7&?S$95fua-Cc@&eZH| zm*y*Ii1Jc#8>}^J-zcqfzIRch^rup?P^v3Q{RdKXA9ItBsBd#iYinzB0Lv@+eRXx# zx6{47E%NDSW4F&XyF*W5?PcXOwzZaSetk*93vd$z&qA&*sgHLrT-JJZZMCJLJ&Zgo zLkrc7j$S%!R5Eh1_h2s0Q<+R(p=9*zvQQ(nJ+-ktP%4C~evZhKuvTX`H@i-7@Nizd zqes(7+^+%BP1@h+hlv=_k|9kt{cprVLXj;b?0}1bcX_F= z!&`FqnKb-Hlo9?Ne4amUH=14qy~VK7bZ=4YX^H%Z@>nh^+QR(8t=)5aYT5yMpGm1P z>4bCt^0fb@zrJuJDyl_Z?LRZf>Et@|JqD+qPyZ_c%)Sru*!?v{_r9nyJmHXz6C_8r z00xLS03_aI_5Av>U+Lw6DwT@|{~Zy;U654P^1@>Y{?7hxai|t_dKXACO-Oa>vXf&+ z7D*QUX(T+{-MB3P96>Dpd3t&>kKhiNB2cx9JL$uVxSoqiEb1bTL&yu|Vqt5LwhQ0m z7|hJb4YNQV9Ed{TiTCKUvy9Z-$_UIVPVRh|EvSn9dJ-;N))z?Nq2@pz!QRf|o_J1) zm+ORDSoLmJ@HKKH2T_=UX)xS0Xi zR$^sJVgI}PE_>Xg`QgN%Y-Nu?jt6G(Cl@IoCo%o$d?<3hq#egWW&*0DB8ag^a&qvujts78$BMj;@YHMrdZS~F7%FW22 zqM<0GpcZ0i&vMFN!2G~{+3D~$l}JXHfif&4PMu#+kCQMJbW;<9O#V$MYg(!|F}!~K zo!t>TJ^>hf<9(SjjxmyzlQ6EY(wl}pL1D4oNw{bw<>2jppTg!n`EZrF!!W3XR@3T9 z`6;XFtS&wZdbs03XU3PS9f`$@x5{d1VP?gqRMYxvX4Bn4z?Z94TU`_41(6-x!KkQM zZtdY^MZ-rwvp$Zc3ikTO=XrnF=sYlxe)jv8JaG&Lfm@TQ7Lh2yn=(nKO<8MVLdZZ&Otk;1m6Vp$SfZ6bs^;*JxB4^sX8)q74&VW&IVUeXtkBf`)P1_q3iGYU`TqR;9qHWD)6_)iezEZ7b_3s*i z^9)3mxux}yZo8SFADFG2?d!-~-!D4tAXe%}+#vRC8alcz&wppyg@Ik5fdCLgXA!8` z*f425_I0k!F7M2&VhIr5)*N;TNX|d%W6;pBAz*Yb)76?Iu^d7O4Uhe8Y0t~(4v@hi zv#u?UHandnDP!h*d`&0<$zbF`!|xvSh;l~VUD@6e3Jf0T$vxdI%L`Lm{RXkIsjl8K zS4UNs8RzG=FZ_;5i5;0-sG{d&TAkwEy0goUj7+}yNkGFnNkt^=Fgu77;}Nj7dKvK0 zFx!ox&j))0Lo2YQCz=}oUz?OpF2r z{}_ta&08uCMqqB>g_CDwZ62KkjQ|JMe*bS)3kU=#BCofxZfbRzd&*sir`Ug>>*k*tC?AFaM1AZAZDf&3VLE| zR-%x@vWgmYt;*=KcjJQ|_%Y1H&2_aVaJqaza8{P{7y9IZHjm zNj(b#gZ<`PP+ghZl5J;9c>BWaA}cFv-(;-tdi#>FjEs<)X<=w4q?N!GRCpMng1>`t zG2*i1*zC*YR_7W8GyCD%LMeNW`x&yZ{w9e0E*+)1J3KrsyA&d=Cr)k1HKGEJoY#Te=N0=3Ir{lx~ z6g1>)i(2Du&F00eIx;|2rUjuAI7!Hn4Gq;IBeyg(L5SwFs=-DY@D7dwpK{$F#N!8v zAYQr}p!|!J6z`w;8oYY(>82$lHLM#$is^Fu=B86rT2WS16rep&2(%mUTG#OqHaqPI z2?;$qva1Syvk9VMWQLXb`1^A>zZpfv9xPO6!c#`t|tpZ z1YE?Rb7EYgWWqrqW22O1<(7-cxW8{=24o@YBe?|-3Rjz*ALFBwWAP1No~JS`jSnB4 zp8jEFX2nJfGYM!bY0B!HLDx%>o(K7Tigl=Wi4Lxhhh(Q+UY(KBmiiG{`26_9m?bgH zoy0Yr(v(Y#jiOWj=GaxSkUVG0@V&CYLuK7CMhMPN=qzX$=WJfL*hV2H=la0^`6i6c zmdr4*G^D03eLX&+{&aS>PfS)5Hr6kZG%Us8aS&D(gyED%mJ11#4P9Jfk%{`JkFf-R zu;t`tHXAb{G@NY~L#lx?Q9+~`Q86r&p|}Exj3R@q=*#+g_KVKN-)Hk&rrPPZIm_-^o!Vuc z8P{q6a7We28~4q~=p!J2ECH0{P!v7|ltK}4oLua2ORHg>!w^HjLXkb)9F`)IDamDP zpy3A+fqlLui|_lsKJxnrlR`#q*-DB_c)jm~9NReCR;QZ_{Q=HhZ@RCDBqB95GU|>BFczPOb)4W0PE^YA;c|@?bqzoO8sQJ@ zr{CuV336|Czn61OC#>gaL{gmJ*_1R^DpxE$y99aj;o*@6cPGo-c>AmYcSe~ky@2(^H?{x4jKab0`3|<62J-4gM+Q#4UBk%#_>YwdOEI8y>`ecF4&~7lYQw z$mrSLVU9T=HSv1w?=P?~og{3cC(L6!^u4{!!{z#GUVfhf8n~`#nv9eTtpGP*|8e%vSg`Jxm-{0prb#`^;;^07*;M?sQJ66?9N)}?p>ByP{;w1jj+IA5~7k zhKLg|=^4CWY-|#fofnC&&*??6HaJ)%q8Al;GLv`!sJ$n-ZR;v&%Mitxbs|yk%Ix#^ z&}aG%t#2%#u5p&4i zX`ceBTtANkyZoM(m6;brZSiM-3lPF92a^Rr#4Wv*U@zP5cJ^{3rl4XJR_Xyoqs5EH z%@##1&7)gNu6*q5B~E1$*}?wS_V{S=&%f(zdrm$=yM(kaga%KSN0S{t;M?jF+q-?d zMRhHfg>usIqf6_njN6=zljlkIFN5%n`SIL!^!?#SzBN!qhv7%4frHD)Bj2dA=}J3r z%F{S#W|}T{%b{>^cFttkC^?sHiv!ozG%%ptp(K~n_f}LYtR#2+TNmfoC)rS%`=w*q zISoLUtly@gB9c9&x7q3(DlH!fB? zvfBy-A>cJ7hz|D7#F8Rgg_6kWrZZW%Q)}iZo!nh#T2zu!DkEoS+s)pPb8g}k)IB|( zVwZP5>p1rca=LgZXflHa5z*mG^UHFYVPb zgntGQ*=j5=&c$qFJa*>#K1X6L{BQ!=L{1Tih=|D5W!)cd_jdXD_JthG%`cX@D~1nm zF!Y#Abrv2(RkpvLvH2v%Lf$=s*2m`Djb|l~^JIA=y=h9N1Di(;?xBKBJ;e~pGrA`* z8Y!0>P1Zhx?4Di<5lM&e|6p7I;X{ukPq;cmnwP$f%RyrOF-@&ToLoWUWM3_osRRD zKQF7fl@z~`BK6xqg0)eM0!sZy|7?ZK&*i% zb7oe|8v%^L8~@r(i;W?|xql&?%oKnGMihl4cO*~ouZ9d@ZRwvAISolt4#G$T-$Sj| z9<1V4J0Q|^{Py|wiuvB=RaD`4+!GXqu+|e%|BYY#-WP~vpl4PFV81|=G?>oq(OYBe z(f3CZA3mD+DpGlkiS+gg*ZfcUd&MqKPt#bpJhWduoT6{c(!alBL5!8xtc- zOJB^O{4H2!rtv!UEBEc)&NXauYPz$qNT%ByJDdvEij1XxAOdtoCsAz^#OUa1KEnwJ zV*)m{aCyEyH@UNbbkUA6#L1}GIdsxeUj+PjJk%qd9v*K6^-*Dl==pZUPznkQB-~vJ zJow9nZQp!p>2r)X{ z`OJeaFOOphCMO-w!mg&UMFxnW3Bd@983;p$xRif;NKZ9>a)|Olynm$7yh%#i*%y@* zVP{7I7e13SSZ6${6ofN-zL?x7tmk1MBM^nOJIK zBBJS<(j}#F*M1amwWJY!Z5o9Z?NdkGH~%~Xo&Er!N9S{?{E zIq7-Q^;O*4jHuduv~_Kq{xwK05F!Qii3y7HWLsD<*6T#~(wk`W*Wy_sfoK|YbPl!@ zXO2N4T(8wXTxG71yoPB=%aJoNqod&HPPL>5n)1G}QmqdUyYS2RTY`f#f*y=>RZQ_! z0vaJ}sc{5TG<5I4L|smM0Zx!Rba&bj6(}TA^clRr`VU4fxdC|LB8CS&kLtbJGKgL^ z2TKzs=Ym-pv`{+D>vMDF{1g&V)N508rMx%cnI6v}i@(}9Zm6S2{RR%8YsD9L~_l}ZWc&SbeSR^2jiQglV!(>=|Cd zQ|~#Y-^7GP#U*_T(eUu|xGqnz|7-!!??kLN1v~DA!{k7n9UY;yu{U*jW5i%5HY&xJ zJt!bCwA7Tpd_fd0Sy?$ny~$OM&?d*G$FaU$1^srA=FOQ*zIj-mEo2e)3*965Mt(Sj z1?A=R3l^OU+yP$rH8FxeDI%zjo=`zd?)fRBS*xr;Z)O`X&8#FeMN3mGAB4v=T|9EN z-NoWEVTG;P=C2@-69F7iyA$^ta|^jZ5T}V`^UY9@-RBY$zpLh1DVsm z$9&vVAoRi>)BK*KiRRY9IjGg?*;3ER` zO#}cqzth)l2vp2^iR}`s>=5lQ<#g>7SXl!xCzFt_*QFI~%OA^KGhbWMiVpSKT24^` z1u%kmpg!%cx;wqIqpO!jT3&#z$lxm`ORA=Narr<~LyLr@G?*E*F^r$G_%D!IkYeGJ z0kq#gUMA=k5o>3^kwEC)-smSV!ZM7rgYr!Em?sYux^Dehpn)?CPwIH~g~7DDXQ8?> z($iU)Svj)ZB6#sgvGamq?6|l*J?O6wQG#x9e%k-=mp{K72x9FBVnUhcFmc% zczC$QiT&#D6nDBjWiVE8wxz!T;AB2Kx%Q+Q5C5P5tST*OvIQ_9K^O&o2^6q9x-SP6 zQA5B0B^?zNB^4bdRexz^WG=lS(j^uE%6?aZf_)WbJ|;5+vE1_EpeITZBM*uAyGk|8W0*REH(E+ zA@cK0w<(QxVc94Jj+h7fJ^Rj*qov(nhsjVxd;-;52{AF#Zs`$mK`3~H{91^Um3b3Y z6;VSnII5Ir<$vXsIdB98)56#O%fd^*O%MJ#{_X`VEgwx=e8J7^)FUJJ(#z5 z7kxb4%qB&CkM{OFJ(lF9gybL~(4RuTA!|L?*VxL)jfroJ1imY#C=#%qu5V>nC}5ad z54SY0VwC>2MMsDbvfxzu)Z~=;!o2(z1S0NX->bcP?7{At89o5*MMouC*1iEv@FH0} zGKi1mL$&6j=mlX=z^B)tr4i)2UtXZV^K*<_FQ!pFFxGOx@sj>oYI4&4TWYQ^gxbdX z`pxsIwTX@WzhV)a!l)*vlx_h90e$r`2l*9;wq3dL4Gv>t4bFD5lH;xH>_uTDkQ8uo z`dU&QKq6KQrT^hVULl%>48wrC_NgHPPTHo%k@dd5{@hYr)~6`LW|Jks84VjGzp=K? z_cNm?i!n8|v|ul`Z+!?BBQArm2aK@HQed{M z<_QHvYDRDONUGvRDacRTPepv{ejF)W_?eT`pfvSDG<>$vQCuci$LmD&|4lR6Fo*4ePopElV5&}VD zU||t4W>id;Tini137Ed)a%nwqw{l84m65;(@EevNV_{+j@||5E1>{xAl_=C-vPTZu zi(jhv zZGhp{9BTePD$A$Kt-}J@x@Syk#nCP`K@_Fk>V63_6j`q zpmRLAwY4_GvKsQ$^~Zd}RKWpa@4sim|bWdGNUc#lQ?h4vx&gfLy>lcmT*@Da$L4PLAii@$9Ft z4@n1ys42=p#%2Y7^ySc_)1Q;rRH%+AQtZeN7HROQMk(@8fUxJDT$eCT% z8Wg>b(#p&*{&>HGgn;V`GcRO8`iBfipis$XjP$3gD-;zLZl9m~8s!OE(~?CzVDCXP zyN*KNJ=&tf&P2%;?L|XE07w6qhDWivcO$N&E>50(Cb@G08&$9OM|m&^>kHthc1p5FwTITq>OBEaO=fP1NX8l zJCUI9OJTQQte(O5=c`QM-NUn-n#YSBlhAKuqOFZny6aat>WT3Q1tn94kv;P~HS%95L9WMk10OKbW4l9n`hDOVDVKt$dIc z0joWB_#O03(RI`79q1vfqzJmP_zQtMhaf+!2vl6yYMHTMuOcd8uVSJSkRabyk6ib( zv`kyuwO->t0G(4LkHg?{3Q`;cFTU9?OuIO79>qWqv-8uE(riud6h7-O2*KOu`zKuq zs4!Jd!d@u}LH@XX4o=31h=7WVl^#WTqCx_WHySV7yq?_40P5jizh#0SQIJ3ubz+d> zzE()U+$bfWKxMb1)(fKp1$qyF5I_ zK}8~oH+W{%^$0i4(8*HREr zM^-i{d*|FNfA@Zm9^JX*CZaf=Z~FW^r9Y5_(FQA58&D|?8KcJn*)uZXCJ>>Sv%~^= zeTo0CY$QLI@bEi?h0S!aRZ>~4 z&SNd_1co`W5cjC({h3h(=`KHz=LBm6k_A;Is>wq;?Y8xMx-7oAZ{1 z71Nt@xrkUkb8}Jvc zPwgyidvqj_z72yrKu|^$Jdkyn6}`-ASXk(Qy=B=}Sl0ha|8caZERqs@$7gWBUOCO) zP9|-{kWUH~L1F)=R6)3az!GAT>4FLZF3^x*;J}|qJq{dy%nypFfIvVY-Uvh(eEVC6 zt%>$@`Eq)3c^S2oqGG3O>m|qYnq%j!`)DEsL#qn{RP6TqS`JhzTbuAa&W;aX7a@mi z7kLLuI;%-<&!7TX+K&(pn_=CkgwngMq~bla>tnXa48B{@*m!RqEq_}iaPkkhGl%=- zUm548=1#AJ-%OA`Pcki%#pwJ`Gr9=-P=8JU6CAU71ll}8ARA|+<+Lt0{}zpu&t=o?eMhz_}FniK?xv1&sW<*LxiHt4np!^?e8H^&iK>a zt}8ZrIAIrP@OtS_-YsvppNDq`zY2$Op805|0FFGN7D{2~1LVl=wU)TK6(fV58wNL& zCcsmv8bEm;*4%qQJ_;jZ6x8iMmo%n?58I*J?F6l+!pNGOnm8cdK8w){Tqu!QNVvMjMWfaF7t8+}9=}+Hx`@=Jv^9+e9uZy% zmKM_=sF2J460_^T<6EV5Gc~Y#8i_(z_%G&b=a?t4%;EM{M>lUbpn6Jd-5(&-`u>=t z4COAVn9!u)qKfWLG-5qxVQDcs?^lRrBDGbmRqwDGrqIi)2cD3YHi^aM@_D5xVnJx0 zN~<07%yD^xSPadHQDplk9#<$JP{ekp#nOXX80)@gVFi&p-ga?m?da?boEPW*MT}yr z(ZnYSXiX123eQyHQWDEN%V zkwb{0-CfD@^7So670~X9wZxx0hJbrtpI*+Xvaz+y&EoPtR*-SFSD1sSg~!=5 zxOC_IWS+Ebc{COlf;LFP0N<;iAu`gN{?(Wk6FW3L`SY>s z)3)3SO{Ce1ge{e#p}vNSwtWaOL!GQt@>^4L%`en@=01}-Ed5_8#-#=pKzUy2bonIg zv?pdWAND^k zozozp;LMRXt~dE2k{$BwVeAYj9_mJa*~bfxRM^bvscD^E9Rn8?B%OSdtKJOi292ds z_7VL8mscCyCvae;>Ic^~He&EN4H!lYeh__sHiJ{P>GG-6wKUaimuVj298ZKToLJag zm+sN*>+9z;{pRn7q)z2&ct;Fs8WSD8v^FQxO9UHNYPz4iEYG#GBxNKW z99_&UtWbS#NpC5(S z?mj!)*0z?=zKN#0JcMsnDn?Ac=HQ7v&s`<1xf%Qof{LHa=!a+ z);zs3#7v%XpH91!J69V+C*zQatR9|?9W61uRe`EQqPg7S0A2U|`Thd*q!4krDWZ!? zOWNDK#HGptG8-Eu&DbpR4++i5N=*1eTXOsp4>VRY*ukR_;bBUG%IycY5PEQjf`eIE zTk^!*#G!T%I+xE>XC^Rd0^$QKS3KIlGThy(FgBVr#3wL=6{V$w3dgv0@Zt(d$ z7ALKq)9g_;wYO?djs&J>9+mOoZH5}q5S|gtDa&)YzquTE868^Qa1tan<#N$!-UrIJ zvbAIIzI$u(Ut}$pXRF<|8{Db20$grxuR!t!R^5if{S2#XJL(a->LzR<=i+RG$WKAE zL4|O&y8v311qbCf8L5PXNz(iHn<+fgKYuRcMim2n^Ypu-b8}(K@Lt&vR~uAvergEK zgJ$sX>Kd0T781;7Wc0|ZRfJ5YSIpd>Aj;v7Ve7RC8->Y?PeZ?{(*NP zu(h4Nh8k&IKvWYmqT0XrcGGzp4t_H6#rjnH2;6R;kPxib47M4nZ^5jv;{CO*WzRWz z3P2s?<59|dUhN(P13qi5-Mn$NZ8g%QFCVF5CWrUw@mBn73}%jh6abLu!ILS7@XvaU zD!Zw>CWb~bs)u6uL{R#&A3Smto^IVL#l3fr*a7Axlg)@8ZEJm=D0)`f90_^8Uj-W* zYx@ZaqS?;!B$**@c=PgdI;RpfTEhfqJ6KvCb`%FN&){>0%=W$S-qR zY(Fw0`^*RvS~9j@WG^wf$7x$;Cw(Kf9FT&(<#RiKJjn-@4;(ULSBfg~YB zC?hdeM!k7KFflYNg>!7ZD2WvlJ`gRE)=PN;5KHlKd`JL#D5En|N>r%34;Qg-m^K3! zxA1k=XTU;tyIlrPI$_%zIdv;M)rH*ZjqMGJSqVGhFI;jtK$M+IUKc4|IH=Ys^#nQj zDkS7hp>n*g5k4QWm&GMPxl(+D>4F+A2jnl?tMWghd7A~m7#vi|B``@0M8e`Bv!k6} z>iV2s69_H=CN2Rc?(XjH?(Po3-CcvbI{|_P2=4Cg?gaSAT5HQWcinHF z=N^8{r+RdEb=9a*{HS*bcHy9yB<3}AH!e25j*g1R;m>*j8WnZHm-jO@r)s2hW+tD+ z=Bz9x;1n?>;ia%+J=5CC%1uR8bQyonyT|{ekXfmQA$cGKB;)eTM2*F6*=%{|H_u|F z7k9(Ua(8zU<%&HvbOq#W9(SU9&>s?N=|QWk)+-3)-IG)CC|@awxnHcGLmr$AZ3#S| zTJE-ju4^YH&?LT5XIp$30k0W4I0$TMDJHnic`|kW873Z>o|%I6txfV;9QXs`GFqM@ zmfbI#sf7Ra>FJ)~!s~P74J2!$pVHK@&{y$fsPvB}xJPe@AEx0^NFR=l;$``dXmwh; z5Z7pF<74ir?DPZsCn>x?=tqPpXh}%)>(@y75XZk)6{bL+G~@o14AlxmoV#t0SvGc+ z2uj9`a(K_X^t?j(bow^5wBDeuF(zm@$9-4@Piw!iLHHhC&!y-=EMoD3h9%}gNtFLy zF;&>-c^I)LhD!bT;HJD{&h}-__RVuxwbbWr0Myg<9DR8%V{R_#vnsX}Q(0wc?tA@b z&#xGYiVEoR;h`57fe#OzGdJHEvHaf;-F0oIf4}~{E$fNiuDxk(E-J&Js^mRQ)Tz~b z?FGE_BW9}$-E?;Ewzx2NKDp48=IY|+>a2v}XzS$?gF`cTeY8mb+z)=<{ey#Mcfl(rwo&8#{FQ258-nOiK=}MX zL2CCu`okf@8QT~+Iyo5YTO+==Yz-_B8Cd~zfcFL$7f{gF%GN>APT$b@_k|-ffS!)| z{iXsB4^U0kz{1$j>0K^v%?My*`AaP4U~8yo?4${l6B2p9qi<>~s0|c$a}rf_(s%mZ zi>M;~doPTCKF-7Q&yTYG^P}{%jPK7WntK>)($g`~0hsC7w1FZNVjNmpvdVH=T4K_o zT3YnXG%Q+L`VLN7T0%lnbo7jLjI>VXCX_%WcRORCteiMd#n{2o+|~y0-W^cT!PZXU zeYBc%?_c>>r@zG_=5EGD|4lAu>uB!u{;no7BP|;p105X$fSH|{mXVE(os|{vFCi;E zD=h;n8#_A#;P>~L=-JuX7y*AT{Z|LtJnw)1oHzSF=PhMyW9noEU}XS`m|Hm+I{-zj z-c=@KY-np_{O7d((m7B-z}8KZj+XAX(pl)(X<1p>*;&}$(^WBR0~L%NZJiwqjU54h ztfK-@(b>T1w@yTC9jyQO`K=)UJy2TT$-&%Blm5Mrcb)wH)&8wF0ORkKcwgbai~>$9 zY|0ghMM(^}N43MiP&-r4#Kf#f2O=OyEXXa1%7_Z7Xvhr@E~p5pD5=P8Xh=*Pw4+7q zJ+5U4nc`K&N9zXy4wBA^@cMU__P)6PvNRhfV;d(&0MmO!3_#(}`O3YU4^U3u;r#}H z{r9u)ax-UZ0|46}RtQwE{cThLI-s<%k-7dKN_l^Xg`VzDRs3Gw|3mmM{V@HW^!r5r zu@rw?|3|wQv9R3}-%kH!-zXWoIsK1z%^%A8bI5;L4+|ST^WWIdzb|Kj>T>k* z;79MA7?_&{1nTtFc4FD>V~z`IlL3vB0CF`u3W+Td6!iEjwbw{5dl_wwluI^T0G-|~ z(|b0)a@j17TML)lD{cDPgvG5ggy9k61eiCc)PJj1;DU*!ylxuEy$)eAYN`UllXBb9027}Xrf)j%k=QkDcuRiRf_WngxS$&MT<=1 z)p#Dw7A#SV@t@>R?!!!5w#>)Ur1TgVw)6_GMjFT@jvy670sokv(s7lc_0D z9p3k6>bj}mHRSl2cb?Wvp3)OyJiE;eGGj?Wl8e*F`DW(iUJjRwb0xf=Z zipk!L2Oe!yJ>uqNCh?7pe8UOtEYd6W_Nq23+}0J*9fVfH;-R@SlN&tO^IH5xJG>sh z(Mv(TXAE1gI!IzZy)Xt&uG71UG9JGzGg*81gQ=;APLM}pIR?d^YZL{lA(`esJ z8P{nH zgJUOHo>fJeg%=V(>c|mf!8^mJ_DDprveDS0*44xDWR}XVb`m{RP*um(Dz{Mjx|YTK z!XlfV*3l#?e+t$Da06#qonIsJzZPl{$|b){v#vKlCcn1^F3Pj{U%Hb zCmtj(6p$2JX|}ZU0y5UJ67gH(1D{C2i4sk5SPa%;s}s@`ifffR7UnU@7&V{5aq+P3 zF^f)ocWModkN1sS-C|F4q&O{u!3gp!^AUVZC;WFdID`ngq)XLcKldSqr9aDhju
  • 6c**4Hd!i)z+qYxQ_UT@RBoP}RFf$?SWa4yH z^0}Q6KL2`B#X_mxJIOaoif8UZDj1U3eGb6fbsXKaO)pat{G@$QK^krto*gtASwa*y zP>7SGV51Md%RmN`37cG)PF9AGyisQtbkkAPA_f)sS(fqfT{mQ(4@rw>x@%Fr*h-J; zi)vEx$0ta3FAoe?&0(*sEuGOOPrd=qeSEA)&trAtxns8|iQGPlJ1OYd62($q$sY=O zFe95&fUL-Yf>z+s_CL0Ad@O0GdpJ=#)+&)wLhx{&CZZgW z`WjU|U+~n5no35wJ1U$|etWm7VNh#J+T!?98Dbj0KTQ#lb4Zny&mpYGX36DN#lbEm zU+?uE?00&?vz(BZ^<23Q56x`nlJC$iq|TUu-egXn`Qvwa zb`LcM!;hPJp$7U6Dfpb6uio9PTgMVVu9jpJTxl6Z68wnTlt?i2kVHu*?Bt9igj*D;)EE#9R^($+q1J07K$i(qBK{mSlbv3n{wmp>eS;0*R zLDsR@mV2YRO;N}Y_CtUI<5@jk7}l?5pST>EPFpNI;;Mg4vt`b-LlD_VYc_jyW#4kQcn!1J+kXMXG(4>a$Y zvc<#Ec?tlvKpA$%Nt4y-$kGP26V{UX^t8nFLG? zRm@4uZFayHw}FIcOF(&Dg@Y<1C;dpqZnt#B`^v^rAg(1o(`e&53jz#hBT9a9dzN=Y zi#8SSpWw?c$o|Sp`395?7vgT^_K+_Kowb=o{H82?<=;iew50=!9)eDSbw=3yBt7kT zzBAH2{!Nu6`KPg%tmh`(5`mf)fk+<979Q3b?@9CNgFp#HdtHC&)nqRE!9iqwL1^Vd zYe=EOuY_T}S!wHMDyny2%{uU?D3jIyIJbQAw=?i_jzeG4^F`R1vc( z*_3?aXFR3&ezH3Q_d2|s(BbpEt_)KHRGu@9g}K|MCTT*kRp zKsO0p=bv&haN-TU^Ab=b!<3Yuv>)0EF~@Oj0aZJgkPD`#EkSq&S(J2Vdf;f^ga%pc zhK9YIkX*S2?XeOqu3s}M5^#2C6{Ep?l33YLnb6p>hv~eicCPxUC8+N_F7EDEJ@1^x zqSpNI%85ON3H@px?1Q$9@B)aYjU{BozN-`QQoW=QN8?D4mvs$fmsGdu%7?P*yuO&2 zyDpL`S5}EICN41OfI;^VujeMZT=w8N`<_r`%ti5I2P|@bDR@nwPniMOwQBMCVNAdX zgk-RjDT9QBpOP^*d@8+O*J#KsALC$J1e&=+W*m6AnQ6E_gASs#Z>axDT#@MQp zY=o_M=unxsjar68-xVeb(~c|KgSs?{`++8Gd+_CU0K1|-s-6{BQ$g3d%p_X;TN8rM z8`6&pkA~zr8^6Z6(+=dQk-fSh%TYRiBi_Z83GDZ}qrpbC5VsSsx@jUv8WZl)m=R9J zb>@316cK~)<(>Fg{YV|Kqf6* zXj#;#&&00naK1%+H)7or)?jUDcgK=n+J4>OdD8QQj?8M-&?;XK-on;sky4FPkR7w6 z@wMI4u!R_lCK;?^ki{ln!s`7-c(w%YM10Owdd%<{c;n(j!-^e=Fa*0ihxT3+X9lVk zGW!zsCXRjk+jwzVPZ{4#%s`4+Yt-e3?yw<Q%VCkQ=cLRgjC?D0iQxzaj66NP?WXbvLrWe5t3v7=L=N_ypRIqyCk7dY}^% zL1RCC%QemQZs%k`H@v^ck`}XWrYS*dWX{FMam1DAg;pdv#q-WOK%;8MftJZJxs=_C z2O;kCv{MRgn^ET3=;?uOI~YSgh(vE70ITi+CBTe-iXie^t606Xd975X9~z^^^chxG zQ%r++s;g!SmsZgrB03)fJN7UAPD<9oC2^*sxa+)|IHI{7H|3L2D_=V9#kaZJNfGX0 zs$7V(T;(Zt$eehJEBm1ih7<&zUdROPN1KPAx4$+lnjU`2oAr1J(W-`wlEQoWaor(( zc-wONAyHILItROD3~^HVi{8=cNn56$Xk&8I75dy4Gw&R=UDM00#qzuOkABBxep8C57ipbHCU=2LKCVDjB@?xcQ^`d8Et=#i$sHOa{ckt0P z7_Jrk28E)b{T-gh7GM#vJ|~=H-ea!SMX4C zzmvtJH$XKZ-|dcA^LRUbrLNS9!&OL@_7Oj~7m3A%^-6Fl;$>}nw3K5>F(unK+D*_* z1s;h79mL+65)X|gWg^|qc}CloTv%wf>lS?pVt40E!~JcXx=V*DK>jTk@huRP^>S=@ z`mD*bR)9|g49ZTJNLhr|zRXz#BR=ocg+f5I`RG=*y*%qZ^}!i|$ho|xl>)eMCiMAW zk7wK1SL?}G8(kL^r(#jsps%oxX4Lw8Tcsf{uSCa?@18%u`j9jp%$O^@jo`zuVWhv` z5qMIKEHBDh$km{@A%q0J^qOmuttn%Q1ZmIykOeLi6Mi&0I49^PV&^~wrP8@MOUqc~ z_2KsKb_~r58n>W+CKR=tZ2(cFxhg)hMfiQ0SAvfqv@x?G-Eya=p8=^z7-;dR< zHH~7-YyA6&(5h{6T^eRAm3e89te2E!fDg!I;43MN#_EDK8b2Yv4KdJUlC-Byvv{^> zCYVsBCZf*an5Cy9tJS~Ii`IGGkm!WF&0o>TW+RW^Rri^<9``(;Wp%=JJdBgcew>BQ z&7+YonJ#ilKtK>iW44Ni0TKBG@M|$DbhSKiA`_Jp*>Q_A zGpS-NLwUO2lG`@!I;2z@VQT@p>lNXwG&|?bdHs338|Dm=;xO7zH_u=?4@6cu9?;US6QBKyRFGt9=ykW0SR*YVJTu9w>ZU>DU2& zo~vav@MG~Q?(o#;7o7)4=cP{m_}y_If|5F#y$`-pC6DKk9f2`rO_T~=Oi9~;9cC)T zaG(;N2WS7*J=}sKbf|lPZjrcqxa7&bN?Wu-={UQjO~@6y)O=lolkS zDX*nZ^dtoGgfV40Z+NfolCbG~ROUKKNsrY1vRhNZ8$bnAN0FCAyj@G@nNFP^S2llL z6dc_rJ6OFcgy%M(LSZKKH>>gYI+vOm`vguH4^G4e^5eHOS% z-Kd{(a-hDvZ)5gs!~3uZtF3{vq5tW5|0X^|hX=!w;#Zy~stK2(#eG=S}RaF!$I0pHBu#63j?9n74<|;}sSKaDaTN5KVUM^p_ zwR)1=Ha?2cbNk+593c>^yUdTPS%4)Eo2?;wOKXVjl?Qd76mlCAmNKnv$x|<1SRME? zMQj`{x6c%s5Y6%7hZykF_!t!@n@B&fbz+S!`U48} zA~riw1k9%sWmNuC-+3#gdNUZ6!xyI$ckso}ywOu^7;CG%>oz-jQm3Hc+d3-MA8O8x zE5#%AS6<|+M^5tBu$^$cS3?a$5DX3c&!@j1kZT{iyz-adtLccRH=sOhP+;psf#qz| zRz6*Lsi6#X!m!hTg4w3j+)znvU_sL2Jq^Ythl-_=ZnGESq1j^dW>? z`F|QHTj1bg_!u`|ox~2iMY&FK&Cy4{v9NZcx3iR8Q*1^!5EeA*v89VIhaqYFMpQc1 zxWg-?#`LoE19C3kc+r_{Ig_}uIoAmchu*zoCIacJLX8_pnT~|g+X_nt|9DNGmySP2 zBn3_@ngEAHaEm0z*jAM-mVY<;S0OaQrl#hbI@HOBV_r_ol1st zK}B*C&Gq*5fxZZ^YZC2aArxDrO5!jZzG|R{%P~zv)^Z;IL?akk#ps?JR;b-nK1m=J zYCM6r2?-eF4|dmuDPs5jwo-lg@Pq|x z3+|un#jvT%^A1>DD2V$cpuNR0(H3fYok9{76wXM?1Kj#yslg#+)WE18VR#VSLTBJ> zapDF{o{mSvV=z@vh)7*mp~jqagCSiMHa=SMJ)LzbL{bW#Y}mO-;Y3R{kFawcNM zd%7pdwKJ%p;e9^j*2GUY6lC%HWm2q(Frm&6nV+FXyi(H)v5OJ;9!BCY7jOk2Zp|9- z6BNt}C_}mW6k@o6twOMCqgcW3Q_J5p(|e?#<4ZIlv-Y;>hxXO44eGs=Vu>HRG+VcPv-g8lJLwZY{Ab-0?)iuOdG#6bI}gE02= z_rjd;;wvbf*;`q%8kBw7?}VRT&1~&&HToN${#WHN|1EX?2Rr_)cY48phlmD7din-> zMn-5imzR!?_cJb1l<+VtjdYHnu;H(>dD^d9J$Ve(=1o z?Kp4y9^fZ;-h6hh*NSMce%d+go$_J>AL#Ji*moI}FtiC$AhsoI#Wi_dK5rcrqyF)x zL0SWNt8GU595+iEoh&JVM3B?$t6S|wuPLS}sGQ~`{Jf$~b2a)6^e`)?Ndn-L2cI~qEe z+d0|3bNu(qccgF32w?h?TmKgP$?yR*4D@UOb~bhZE6Y27|9wM7-`W@`B_$&wD*l<@ z$VE<(nT?cXtB@pxR$;CKe_jzoX%AUe3hE z0{lIO-)$NOI!2(llfIR?A-|2O)o)@hXr}M*?x#TkbbXJ{`RAtjch8T1@c7>j3V+Ar z|Dfmp;PHRr@Bbx_|I6{J08T_9U5cwbQcoqQbKipgX zA9?)$;myI!M920Y{_RAnN;_`1eC&7@m3o9Jq)sR4s$OYdOSiA%q@1r(bX401iHq$+ z5T=4)!#N)x1g`n47I0-Nmnl@m^CO$FwCR5LdDuKWy7P9=(&DwQE-M+D{>4%B-97cN zBr@TIgT@(+yy3vk+4H>qcu99aKA!x68T(CTzO%FK<-*dT<68CdRxv7Pde*j#`xB?D zUAeQsuo8h*y+#H}n8WNHQ@A1y^ZoN|o80L~RO5mqf~HL)(U3wiiP`$45W8Q;r>EwP zRe4fbGLuExmBWK2-O#vs(40QC#NH6W8D_WD#orUak4eJI6?A!D#G+eF+A2c=u-IH& z9G7S4x`jozs&Vo^5wj(oJ`QtkYQ<1bqX-xXNaJ>K7c&?$}2&$sS6%EI#J^ zbaDFdiKczfsI20?p#1qbR6(%so{yo$6KN9iT4}!WLg| zljgdttPvkoS#D38wTCukYCN@%9*x@lo|&Rk_lJd_s7O>brL+MtHlpV~HW4-k0E-_a zfhJeaSK5CE0F(8-n;5_ zP8oTM<`b8YIuil^i*FOj`g=q&>pdFNcP=p_X^mLCZ@JZ#n{Q^GP58vAED)@B*U>1C zocr-vGi@G0Q7n9iKMAAKI~;Y7Z7lqltPz5A0ELbGGDUG(_?J`x;2ozr?biwl&F2Y9 zgZ?tr`BXnq*k8sWR_sTGtCX!{^wU@OY>75Pn(#x4c1a=m!xUmpLT9h6n8NDHP%;R# z7rr~uCBOitsfss6087#~(uM1HUXcqWYtq|*!(;gnbLENm%s`y95&{fey5hq2)CM34 zFYydk8ZHqCI)mq*@=<8fBy zm+3`U!?|+c#j^q;1uaE3of^v)g0-@CpQA9dx`QFVAB_x0YpqJ_y9u?TWce7gQpyTr zklV^m-|<8y2t~M?Yso9Fl1X{gcBA`8>`1 zEtTz9{8>bNwhI$@r5%@yZtk3s22{$=DNzkUQq0HmbvC}u{1jPVK0+b8^J^0DRJ^5` zMPahDKP^Z*7SmBh86Ys;l-M@_-Q@G0{P2)ZKU6c9eX>Pax^lvqy9e534YfF~gquAl z;JL0ql&ETpt*`1Q`GrKUQhnJz<>cBIg7_@K;l5lI$_G!n3NHS#G9>W<*Y2_+^!U6c z^1!T6zHJ$^R$z-z|5wcCGxxrOm7T|Q$A0m!5Yg} zWx|+ZDFk@50?;yY=cQUC#DGNj3zL~*_NE+ zpUGh5+#{IcA7Q3phDDLTqBnq%ri!j=RIzNp{EAuhj|^pHA11tNPO~61qoqk}dIb|o z#aoKfGdtuGva8Cl#MK_|yJ_5S#bMw&2?2-q`xI9XO36rpH30)8q-vB)y)MmGWmW6J z*|R*2)?Y-vQ;CJIlus%!SBz`0t0D-rvMaDRvdG5%prjxmm)YeP+`^#@y&O7hc0yVJ z&uJy$#^?TBy+7V6b&rgj7V0SXc<*k&O2nfX@hs2GTLr22Cv1+d{dSbo#fg_0T-s7p zuhR}s_2vPq_iNX~R=yE}1!8dgg@qR;9Ngn)-Df6a%8^2zCYJO`X`W4@i~eR=y?Hbc z*fGafMnnfgxKXC5{bj+=wMQp|d2G`rHm>Rtqbb*^<|1!vci%{37}DaaQNy8g2tjOn zD>~*w5U{p|QEfN3$1 ztyI!cqHw2WWT{+N1@np$%?EioWvs^tr+U@I%(LfVf@@L!UzlMfZEEb zafOTNj1fWl>pWITje0#so@6Pon2fpJ#2al?!iz0pt}Zmq;!QBAVn`fnFTHI`)a_T)c1?8oeK!9n{H~7SK3IMG1Wu7cJ`|EnZVt zC)3solCdN}V+WVS``0G|fMZB2M4nlM{GA6VGo%rnDubtNc?Vm1dk_lGpul%s44dHL z(4&J99S_>Rh{=lMkCZoGguICmyH`mFiJJ_|u#$DQJD0%kk;7A0nj(|e2wjJ@+DusC zm7I*Fqqs9O8?wWl88V(sWZ4j_M*tzxj)!kvaCt%{u{l~urZ~%+5-I$NkevEzH6rG8 zeWmzWc-;Py+4pZjjR-mT_q~r5skuBG9a_@M+T-S3ukqD#4g?dk zrjP8mY3j)3?Dp|OS^UlcOR3(BWLBEzn9qJ&!ll7VCYzJIXS#!X~#Y_TJW7;|KTH&2rn69VUGb5 zZt`|_izS0g0gIC-zN#E*K1%cqyzYuT_Oo>+hp5JTEO2z>j85D(%P{U(9kW5?4}!xu zW69$PyP4;hUMjDiB7`7R?A*(56ffqdRBd#mFvuWgrE#=Da|4J)3!KE}`%LJ_#rBo42Hdj=}g%Et$0+#FqvfV-I+xKZ@x~xSrSXl@H;h(5?YaPm$gpxR| z*1qGBpB%W+l}r;DbY!;|ccMQk0w}7oOhI@0270I#y;f+dVd#6#L+Zn0F1|*bI|^y3 z@lW4&`rv^uDtHLwnv4|FkRrTted1$$OYtTTrf%&}J-n(cb3)0@vx~_b>IPYR79AfQ z*m^A`w3P@(5ioC$&f$Je)U;2s6ATSQNI*svwcYB7-jlw1IND#Hy&+?WCSR|E^tDaa zzFN)Xlyoo&vfq&fL+3vYZDbzhP;Eem?OI{Ra>G@aMa1XZH_Je_Q_03q69MqqD#mnK zqni4NL;mc?g)`^VqqMd!CKBVqj964dltK}tN{ovwO(9rb^-yd0i~zf(&@E#Z^=M8# zq&+p!GsE<$+}!U>0mtcS6?s!1f%Rz;+1JvID;U8~sSrYLMFGsJ_U!Bt1Tv+cI~U^V z73Dqk;Gko3t2U%$Ll|BW|H|qOXyO?TjH$LViG0MJ-Y2#)b zqVESL9LHkWsQ`>le3o}-M{jWRSZSBn;G%3+kuh#ayb-i|ua@>{IMQbuV*wy`y#7xs6&9Hn{Mgl})}(H=+|t(MXDZ43=UZ$a#gt zGd4zRL7*ulihYzA5RAON*0eaYqzv@40gcAckOhm1{0d8+EZxxi2E|yKSWhzXh|1$D zz!-=9;OnW&i{=$ljL67_v>rbt(tRGBXO=&?r65*D+g??Q7u)I5HTU7dAJt0y*H>`6 z`X@?6J)3IKu!vaCO`^xV4_sZsm_Y_RKImw*FRA)onVyk|!AzKfhZ+d#8!&Ke$T7ve z_F9NKn=qp^b~Du)krhp@FnjBCbr~!SjTUR7-%0tIm(Mh}&}{=C?eNlMS{(*uo zaoc$S>Sp(n<@?#>6k!TBr~Etev)-`6rVSm&C#I%IipLm6)acFhO|3x-IDsXiR>y%^ zqhvg?;5PFv&-6LSu91L!Z0UZE65}2(!J=3BDp>cxH}eI+?JY!5A2SJ+;F~DjyxJcW zkmxi$HfN0^ZR?FCy*+Fuf)@q&SV+;t5Rp2F8CldxFr&M1Y2xkK7TB0C#&N=Xy&%PE zQ?yU#QU^PQniOKjs=uNX39sv}8Yv{ko#7f2loNKzl_B>XnyV7MG7bd4`WV{Wz~n=b zXmS~rkb58C71ntI-FW(Eo2pnJ5qbv?&yonmnPOvvaAiUn;%$U zm?Wax!PT1t??obPauqJE?(YCcud}o+i?47CY%+u zDPn5$Xt+Kj$^27gMD|8vKg9A{-FzgDUK4wM2~OmREt{-)dlN$nJS|mi=6p+}F;%j| z%kr`cC=U<{o)=j@7r23Xt2a4$x{ZN-bClbLAw&-`?*tDvQ|5Oi3WmXn`q6xq{qE`H z8O~5KWY3KLOW5HTU*El9XeLxEY;ZV!!zSXdbepL?V?Rplc7v0kyEeFHuKu4Ii6ec1 z-~DDjFb)^oy}EFAZGA}vb@MNm$#moNkw;n3)|Fkj&K5~Wm;1LXBfr4QZ&u-Ka* z?FXxA3Z~s`aUjw%--ig4cn#YjH;R?QOR;HwaYf1kCq9nSHY$kn4b1SJScknsyZo6C ztih_{r(;7CNcHCH)(o)vIuyp3EJZU=xiOpCtv%cq->Sp!aXw>pWUSwP^--X1IeKl9 zC8)ckjx4?7l01zh1D8e}^);W(btls+Qk~l0q-dn7cFNFSteRfxiFl-ymam57*GZXI*XYRGIc6gmC~70 zzEU8;?!mdJrp};pquY)SngG6~n{r^2Z=*qL=w+($)2s1jlg3X=jzJv@+OvR6gEG|+ zVr}BzpP+4eN1XH1%T!PF?8q9|f9hZxoYd3k*FMdqV3sR{T8YL$bgwbv;x@bV z!?SFjE(hIiA^{#vnhdhdJ4ehW&&Dg1JBSab`=>RIN1x_0?^b-z0u>Y2Oe|VJxGqwW zPAR+8Xe^_l`*{>1{K*U&6ZJQt3=hq_tR9K>?%%ek9J$&n5Y+p%jsp#m$vAy6fkx_gdJg`Qfhp&&F8 zololGBUg~b+^L+4E{a*{A!bWcLpbHb!6RKjf@WWrzKHAb4dyTV=_)&19b3VbJ)Cqg z!KJ`2H-_4K%-GrqamLoO!1Tdb1g`I}vh@j0rS?4APeKxGmOn1G;v2SbN(nLQBy$FD z>?9n*T_B8;{F6h@r={CT)89@2YJKwxv7`vB5Qlf|q`2tLa^JgLZnQ>Y}T1-J2HLJ{e)0>wOq-!T}(NphI z9|nvtVIDCQ@TZ!$uD)mFC`^jrl%z0RYD*z2>MeGN7!3cMWEh3LD9;n$LZFgDOUge9 zksr}4vKX+mv`+r4-&>8P50*Vuh|nQVuJVPV!R4du+X79oQ#{f6Fa3jMbt45>X3SCb zZx+p~k%{4eE5~0277*a*`a8a3bk7Vo5SS0Z$2W5j!w)mM46{rVe#6hZ0fqeCwotdn?gOqIctlFnss;L-{&zugF`b1F$MGB2k{1p>)}WwIMvn1 z=Z)ZR1`hY*L_vv{)>`%9j@A~e6cR*1$)*%(kK?Xp4h>X?ruYQ}fAI__kPC6L)5X>C zsOlm3@V9gRecb*(K@5K==RGLxABcg5o`vN%0APC$nqva6vi^6<@b4P=i!!jY{U^-8 z#PAQyz{37tFaz`d4KrB(8)jhs9pd-rTm;{T`A68_``G>mnBmXRy8j(!VE!xi^dFev zkL!Po=KmV*`VY+TJ9zc4Sik=@arl1@`};##e-8Pt4Zu#vO#dG+Lxrkz%zM~h$9N2# z6A5@kugx8Fnr3^yN1R~YpbRqL4xbH0RIplTh{V|awIBa(AF5P!R$79w9v%V)Au^-# zsH$4Ex@>)Qb;YOpMxycz-=?g>VX4yL2{T#5*jPR-N!5KXva$SI@93#(zP5G0ayDi0 z#k|_9W@Z2A=xIpKj0Nl3PWd;)*GyX;GG6K1nP|l}+V|Yv86G)R5j_)aXH$*CQVQoe z{U&`?qh$rYU1dW~b=Cw5r{>4G*y%PFj|H{raVP0jVOOKg+kVqV262(r>xI1QFG=lG zKT5qCRnjWUW>=pVlby8VmM9kt(5((kK3h?;ULg{K$dX_HC=nd0Q)KBa5W zev%E)@`ERM0_F95Sh2n5B$OCoNjyZ80owZWJsLM<;Mt{Dw#r-Z!J+n1o+9}B!v`x;JsTWYpG z`n*h!T(lx(E1^-h^KGm3LTM}Khnc0HVCRWdAcKz7kWlC11r*JfGVo2iU9I1=*R5r7 zwe%BUFS~(c>>pnCO$1G?W%wj`NMjoAemecSK0ST(8qQJeR%k*>(G1_{&bh96^$(wXjgrtAad0^DpOZVWytQduiMB=EXpAL$?X;jJrNQd z8Zwl`d4;6rvkhM1&@S>7;@Hk5jFvxfv?hmILlPrZZ*myWYJ+{S6nb*@jLG-OKkiEe zWJ=$8K4vQOKV^s!?0y*bG%S-)aw)q|tPg^OD>dJmJ(*VbQm9{hFIQB?RO*hz9K1!U z;uE|2g~XCx#Q-H!h;3ey3v*#N5J^>H+FW)qAJV0m_`(wHZP-4)l=s(@`o$ZNdt zO28N9M@>>$scf=h#*ms>+upofJrEc94reqyO%=|=qVxr4$SWF^VbaD>oKnH(2hx)hh%Ck=QEFV^AtCbBy=JbPZ`ueypb^6eRj$&0eh@$T3ejVq z@4^=Z_%X?N-bJ$3X)Bv_dsx*1i~GhHJ%kf7&QjB_0L=F*jhb}-y@M}BjCUXYh@qay zwFf`m8GU$pil7#?9Vuis23mC{o#?8vg??ae_Vx3xXtTa`1NWW8;}n|9b%APW>fS{S z15g$pN%l=-Ihja$t(eRaGPpq54|7X$l5&|(kETn4xGiFxxp0?12SD5lA*Pw)4yLd` znM=R*dmcIxT*`2VeqqVzv!^#GMr^;+++Y{w-6fK(+VLvkBNVw{HxNR?2qkaNgz+=7 zEGNaEB#1-7EZYFJ52g*@==QDR0u=p(la=f6KT$jYaXw((9#{COPVMI_@zmB)(7Y*8 zos3 z;p3Ieor3vgG?mAi@}~7Q93T2@A*ZKI{4*^BJ`qAaX;a7y^cGz@^~qC>uh}<>I@w9p zwcxS+`jzs}{)58%luWT4cYa)t)Sz72s!GI@U--zP^7>9n@i=}!(pwu2>sV?|$;Lqz z02(M$IKd)O4pN6K?W=1-J zGOf7_cL&k}yX4XC;a5DQjZrpzlX%?+d9Y#FXWa~aM0Tm~SdoW8qy#qtJ)x0GA3Lt# zwi%I%mkcBpt{r92CMw?Iv4mAXos(Q4=Ro#y_Z<^?h77(PFR|igXO_^WbYK6O@MQ?XRPIFAZ(kPJEADKn6WrBsC*r5g5cA4Xjr{m^82FE(r7BrKoo z;NA-vJKM_PmD78-)t3o;g_H<(XX*CK@7S}}8@Eny{uNF=lkpY0E! zp)_2Dx_B4GWRF$AZcvuEpKx-2PlTxy*<+S&lZq9yxrAx&q<*$HcF7RkMCAjFU0WiJ zmD3@0-|+cRRz~-QxWs_cKb%|AI#v4_-ZguWU0M0?Sj+&1kK|Cl^y0z1Q(i@aD`^WD zk{=l5;U}$DCr!)=NB0!Z?piesQ8Mb*H8y1o~6VxS-u3=jR?rI|h z4cBhxmLY<5&gyq=4s`9~{Mgjfo$krShG+K6GxZ}bK+|oOr%^H>838YOf$W$j z=*B;$S@-*ONMe~b9tY#XK=s`*v)M6Wp#>y7r2`pur#cDUe{2KR1K7)`l8?0UmC?W6 z7#*1L3_Z zBNtYv0CJ;eUS)FH!c36PjO5KhE1l~~@~8kPvs_V>-VE0^M0%0*myJ?_ZVLIHG}FEX z&v6tD8d|qN60}QYpq2Iq4L%fxPTw{TNu;3I!p#%2oy!*tCtwI(@GHC?L2UaS*TQk3 zVJ{Wtsqma1+J-@_H`I81mzANyT(f4W%p@ZRsw$#W^af|3phk&ue#)ygcp_=)=zCr# zU!ZZR4%{0Lr062I-`O5!svUfG_v3pNbJ8&LBW1!LiYLa`ocDE*{f+awxpx_OR$%5S z3P3ml`|P#4>eO!_NWyM@UDVq}t`jneLE=_fr_i%+ZUTQdFJD6wLS-wlr}gOy?Ekf# z$Knv?c&4$A=pG6g;Dc<-$G^@y1RXP~x7z<^zd2(e0x3;6#FdM|Q$^$oki?;72uk5L z6^3DOGb^rwQ@?>Spm$Vt5INlOz&n3=O*gn;Zam@uQ!_id#D-4UcbihTDVglcBGzJ- zQhBYO9bh38&?`sNq^EpMxTIRP!sf_6e+(9mn&2P-&ct3Oma(iSRgc+8iVgotP$@zL z{0ScIaUg?X{Re{dYB4zE(hZ5Vnp@H7T62Kz*|#aN0A}zlNPTiPDlkQIauY!Yq#CMW zJRYjdrY*~$P80HYIsUNH;NH z%@E&&Zj3?nUUhv>i#&sYRQNWjq+nbm+NdQiza>gL)x)7THAZgT6ujEz=7!wtGMeVm z=NJr4;&%+Fb@OP3xR_cRznEeNzh{kxeYfe`eyi=QLXHM!16@E%GSXKw!Cs+HECro7 znlQ(<*bQLqS2Xmi?|Uq4kg4SEQ!E)7;x54essUBd2WA?U#(>Q=xam@Z>~lLeqyC8<_9s%wu>i@Uh!99-FOnm4FNB)Rtw(X?>!PvqMOn2g?YBm$ zrDEa|H?pTLyE&3llu0YGPH{cpDx1(+lfmwjy4+qXC(g0fh5Rf|v9IAL5ltv3Vn2wn zVn-8keZCsZ_KCj{Tq6!*H|wgvx!HJ|rVYPhYkGA%19O%2L{!|3>vk?ULHyj;C_9SM zP!`jX_MPFvOlRrU%6k%a^*R37LVMLn>WoanN6^Ta=L9pKt&PSz1iW}KusA9IWecC-6x~whiqLv#tfzxwFZK@bRbA}J6}p6 z5t(z${j(H>DCR@QlS`E?;R=H@qZs)XcewW6t0jWOzfD$ecg3D(97Y!Jx}802w|`l# zT_+<$+YS{E-r+Tay-Z;~j79d^X3Ux!5vf>Q=zdr7LX1Vb^?2$k`;6NXoB zW@rf69heSv&9`&sNSYTpu-Z~!hBt_Q8S&bAXT((JZB1Ko4cQ5a)l~bqmSInifikA^w`>}Gf^W5Uj9Uz(0E00b*J)jS>TTE zBoZ(b><9Wt`K<3qLmCt~stjrS3$#}rh9M$M?f}q!C3nO4u}RQ6*e1RA4zE-wLc+dj zI50}R;S)zd5o!(PaMDXyr}BvgG=~Q_tz%fNJbro*bC$+<7bMWT?#6cvA@>!dMr#-xzs;q!b$c+p%CYx5X+-tK^JTQkm z4|T4>Vil=wT*z)$Ui0x$8YK2R1-mSAZT(N>P^_P1y0vBeRKslFEt=uAxdj>oZKL0O zqIiiYvCEIpR4q(Y9WX&?Wc@F$8i>_f2d`?HhrV@9NDz=MI%zvu~ zd*dftX{}?Kz;`X`;k}v&@PSB0j-Dxz*MmyB7CotPiZVJy64$K&J{z&5747fBok;@O z7eeqa!9eM6X&%<_*_0Nf9AjYg#pg~Q#EiLCt5C74cA`73_DYs9WwSsONbljDNr7gS zpYGB@I6jv)p}DFWu^*FGoda&h^vV*szg@bprU(~#1V!uPHlY@M)a`tJ8e~puHQVo! zuTxGV*jezPB*pFe*-FWIbXpHHsX7Y8QUT^K(9z{BUA>;iDCDsMe^p?87_Pk^Dhi(p zV?Mc(d9aR!O$gsk)F?kvRAX_ZS^}?vNY9>xnlpoRi`r%H0xV7e$&*OkK!&&>w&M#~ ze!c@7mx8t93+k(Ji+?)tmcV1-kBYPgQ+xjzjQAKdRoICs28l#Fq$Wyvk+0_vKVa}1 zu@Nugu3gjz0{ADWC$0yc4HS@U^qgpFuctpCxbE z%kFQNCnAlf#ktLYc@^FK5sHhgQHaDH1B101Ump(~EI?x*vBSL+P?XECyW+pN?pBi7;w6ohv6lNsyL-|%a*Dl!Fb2g&l0qj&oK;iHn!-0Np;36w_Q zPF{FL6k=vMqbWj=nY|=e79-k@#nCAFfjPB@YnPywLT){rw#6PwVYfm(mdliju)e_} z#l|gqN323n=s584%V5E2GB^rfX*@-nPpX)n_!5};n9JM3x5Su?)AGK#cN{VT`X*Ql zvb!ANi~>jak+8X52mC=S^8G!r&?>P#>Nk&4NsGW=Wu_Yd2iW&rX(6ONs%JUEP%WLX zPMnCI;`GV5#dqGHlP$LwYD*iqN(wlMtBF4cP~Pn0`|k9kb`8pJ=3~=oPg#5q+3|-C z@+`LVhh}>iTb7>tvD68Y%n>L>?Uoz;8I_`nOX98!XUpE>BDV}{ml&^YYQ~PnWW8Si z{EH|ex=pTErh~_A;}hO01h@#K@Oq<(xXBiMpJO5{3W6s!gJ_LgOPN{UQD_K3d*Nhj z?Q~RL=0Y7{q@)Y8h`4zB9Q`uH>*@hk&NNrTvR^}A_2i{&KwUj%z6!P0x6kcsm2r;B zoYNJX<5J`v4zRLGl+2D=mjt54aBn+DVl3)}t)>tTH5hy8iGQ@>)rfiJa0A4Ixq_BM zhnHDJ{Aqm*ar3yCbgTbZ7GgFwKggN{5v^E!A9pT(f9xVZ35gvnFf3ku;&@`OhqaZX z<2LDF4hcDzFFzZWa>JRy)6iSIA}DOpObLHiGR49EQQMCBd`V9ZpT!r$#xM56cq&>$*cZk^De97x*-dkC8y2|(<} zdJs&$TPbLEOFod&qM!}#4F&X7Ipl{o?PMfCYx)TN! z@1@HcEW2qCqhhD>EjwVSfT-d}vuFkPQt|5bkhW1k=DYyO=94z#qCaR|AEO0Ya(`aZ(84Z7UiC?sgR<$g!GZsLMIsA{)$&K_*iJtk= zc>u8?NW>~GI0HQ{=iRGG!*`0k={zPccKT6_ItQ_Ee`~ZYu#L;t1aERkeceD$^%GD@ z%##gvNyxs)8<(taS@w%FhpJcwC;*$M-4O#VY#i3S z5c6MY3$WF`!Zo*K)#%ejnfa4ggQ;Y|Q#q})Sfd$swS*qH3T+2tapm+|xM@I=J?7 zoRSe)GXRPqw`KYE>m1xuclnvid44DZ#w@nJ<+F1G@na|Jf_lIa60*lBazK2wj8z;w z>&~`*mtj%rJ%UXIHq!}`$CW?nXFO#Cm1s(S1kqq*s>*Nv>`W0ol_H+xTD_jC5mLlm7_n?zgG+XKv@3MB1X$o?k)HgN-6V`9AI$Nrig%)s~{zEsB2&Y^nD{ z%;oHdqjp^2j4vM4-A`;P7zpJmArxNq@r>#l$MHjrsRxr-!=>tJ!LL%D>W1;h&Kxjp zc6Tj~>Lf-I+CE>om$pgRU#ldk(c;fqMlK&WbauX_eAkb}Vs9Y*)@zWkU`=5pngD~t z5q8wLb~f-nF+99&V=Av@m*6GaB5;AcB;tT&Q4iX58P`D=G>vrU8|kd4z~sR&A8bS3 zB@RRVs)Q;)rz;$l9F%oY?n-omL7SUbvlc<*N4<1#OsRcME5Siob_^Sr`#zx_aDEuQ z^et3dMrqgV6Xf7hg(p4#HO;b_upEeI&`n`Pf`sC^OS3I5=ls)LoUBt)ve);07(ixB z4sx8>E28UtN!4o?2ao1e++HU!esp*qsxec2R2{KVDoy66M62YwJ`fN7$7C6RZx7)p zT-qnLZa`^h%M(}exV#vvH_uS0A3WMa7(4>BQL#LR|7%u}8NRmW2ApAK(uUO9x{u z`y0h3Y`+-WWp7&$P&)EyL8T@uVUAbB47_9gVGZ1PA=Gtcq!uT4wrW$KFC5*qU!iHe}j;+ zOw!K`suzEJT<{sWIDcClLt9H^*dVrrggI*%UE3X7!|)+6@u`wLo}>}D_snCuI3YzmDNKKU-AsydbOFW*^J|5Ltf?7|0m~21gam=5 znPS=>Yv6m??b;GB7KnG7_yTavv%Lp`yFBx5vZgGOb*xIu^9oVH{QT-0kc^Opx)%rp zB#R{BZ$Ity5mK)N-<{4B+PR43rHHrZgupw&Vt%(fuazT*1e7&D`IqIy__bhu z93dZOc^*5>*+lzcxGeiQ%ykhB%@Pa_z80DYK^V&zQrF3)v7hy6yF0A zvg(?BpiC`N$HpN0;EpWeAqJ&dHSIQQm3bS^2xq16hpLUeA-!Wwzc3BjMUwYxEA5Nv zT3|#(v^s5#=5#0+uM=|0{rt8j&cm95#WB;7?UCSTJc_4tt}fEmo9RN_SzoqvH?$A& zorZ~sdQaIEDTWe27yaQEjW7sB?ZpkgFq9jOPms<1((DtCQ0`9zjT#%hV~>raKhf91 z{sNJQ#DAp}@=k2lJB|gU<0SbIvq>@TTA8+5OCRQqsO#ieWrjg%y_4i zdK0FzMH}oB?rNQm#A7gXB}8>An#3s?E4vJvH7_k`8l-kFZ; zFzd+owsstCOYd7UDWzEwi`@NbXtZwF^l%v zaiAAoD^47N4K4Cku$3vo#mZMj_KB%zgoWZAoJ<5Z`+Zy<-aA9%(dt5~?VIR04XMJx z!&Zv`Nn{xG`X$=iJH&JIzZ!JRAb;l$Lo1H|^7ISqIOP@()UiX?=$RA*4AP#6*+!N8-bd znRF((dR%yGE2eYI76$1|ijQ4IqSHJJAown5jh-cufF6rw@sFmp+awTfjQHU^?G&J^ zY~7r*Q@kMLR3lb_P9oxn&!%ZvPRIx0-q2491TSmZE>;ee;Kk5`V8vF@VOaLs)Qa&( z^6zsNNtTqjQv{3NUZC=>;l}Z`2du0_Fr4hEl{Qihu(Hg1b=Jde8k5v9rt;C30~<(i zJl;Ah>y3qlK~)XJgr$ogF*gIPdpQGE6EzojCR_SJBb%P{G_OGQCKRj4U|(_>JXOhB zJj(7P8GQ?}*V zMOp+`Tne5qs_REt;2kNC%c^>bcR*ZurIfXuimtgbOH+VkK*liQ9Fs*G7MQg?rpKj( zy7r2rIc|o=4F0&~V89ZHzv|+7)rErySJbnLxkz_DdU#YL_FW_M*9d`^{)Tmwba%b8 z9-%sHwM>uLdQYC5154s79uwyxt({Z3Oy*;;6P-c?S%)|btItvSjQP4XcBLm}XZ}F; z#ly_m$t1rGo>SK;Kiyd5(JogJl5JdJz*yArdI{A65LoE2SJkak0eJV%xYm^SS?Q}` zf1rjryBT1O^;Uxp4W0XBXdEXDA1P7su~C$4KH_jBj~pn7@d|pJ+V44Nx;TyUzHD&@ z+u2~PbGtjy_8u0kSS21oLH~jH*c(JexsY=x;!x)hv3e?6aRmt~Byu8M79vv=mrP8V zHB6|leMSdIv9fQtn8P-*N&f}da-_xsNH1$z@i>vJiGZb)lb|mVb`7Z&a5EvZvHu~X z*6aRSZ3$g6gKS;~PZYj1Y4E8|znz}knEOZE>BAcg*!^HxmiYJHKv`m@kVfe)tS$WG zkKw_5g%093ZA;?23wT#KoIG!I>3-g|eB2eStykPNGk0=SZFPS}4%h&vL(TW`&IFfd zz{c8KbFmYsd~PiuA4=qjhjAFz6*Q}3KVZ~I(hS`udgtHQ(|Q~Cxm)hlp2SNX66=DI z5j1xaFv~OF;Hrx+_(^Kd(X0oB^p%39$cR0b)T#eUvkq!Tk0eI4>FLB;=!&yY;IID)i^2&mAt&^2?rJs#;E6b!b4JmIib!8UOBWUq{ z@sfh|=Hdp%N1REY+zk*Jz3Wv!+#q-gvfBMew$@)8?Ehjo{YhT>L&D%6!sh?jNd2c^ z`2R939E|5m+c{T~TTBMHNHD~+E6 z@7;pq;t%4m$U3Nv3&KbvXa=dILWVHJtRVHfq zjy;>k;DJrz^sDVkyqI-4-r{93PiSO@=;T)VEfIdW(M?Hj#Ghm8+ax(7uZ)ys|~m#m>(Y{gb} z86KrwP(?CK7P@rvkJ}E5jltfxQ~V1nG;@~fr{0=Qi)FqJ%ho-lcG@Y`%#^JSV5V9N z94)4bFc|G1w3H9fh4goo6|L3P18%m|1(}G3%b$K7IgvjQ1&zO$QaKzvfyt_F9#Bp; zTBh4C_^#r4g77`dTF=|n*_&MMF*?)OsENf38LsZIp|(fp#@-jRHPe<%tk=X&_;wR6 zo$VOj-N>2huhfY!8xh@Ea@$3Sn4%ET_N|W+nw!P-1+|R3s*LPqdmFriyp8G!ag4(> zgg#REccatbR&t%osMg`tak)u;r&qknOcp&oG0;b{7uJBsk7F*g0`IdDRFtL**Sd}9 z^BWyRf9A}8OkA;FWnM~UXJ)iB&JqZ{JRzTflWL3muKoN7`p9ox6fjyo)Ab?KbMa_a zc49tIZO)}g`()LppSUmyK0Fxg>6A)pXmKOSqFfMx%}tnBd^{KN6A|K@&p=0EV{-vt zx80IwRz1+{m_R?rNozb8nBT<$(QSCUz8bYB1Rp~QlFc@62a=a698E>3B^MsGY~N!i zxs8GqbV^R7>_G|wW?qWdCJ*9IVF@?7?M71X3y>)yFcP{RR+Omp6*t7UZHBg9ero~sPkvFKgN;Cb z;X7*=BnQyUe;@Qh>4cFcw^D$8`x_Yf(nyDdI=b|6QGetF2qLvJ-8aa3rk_$C+)Jp-B+osYXKk<7bl1?~(FL@jUd=pp}7Sh0e%e4!p6) zVJRH%+%=kbhz?KI?LTByE)kJ79P0|dH5)-3&iQ}3XR=+X)G7(^*$hbhPo?uRo##Vd4Y+NIUNCn5t>!m==f!&nPxtT8pUB4Dw zzvjk*9d>N5B1L~mF8_W2ZI_2NE0YRypzfshaaaZW-82I zt`x2wyeI+OJWsg>)NuuP7)4-germ`#H0ZkwU(>z#_RpA>$0TqV%+6T#CmqP!veTc8 zFU{5XnV)Re^9xT9HrP_TH=$+-FdpKUtoR%#O-!>&VhMuo*K#3Ku;lf7*O2(&Azc_o zL__OuUH$C9iJ)qW9z|&_C5(Hf*gGb4n{>KPvp(JHX{$z0V{o~1;8w{)#y>P_^JhUs zY0h2|cyFlL^IlXhfL)%YRtajc@;hR77+mNNP!vEThW10)^ajO_3xHk0%37p85;QTd zxVMQBQn!>Zis-DlYgC4) za|`j<4??G5xB}&KHXvRdv}Af~EE=veRL&3a&v3CD1@5=QFxqayty$otWAS@M+vGf# zxer(h@cnw?er^zgd5M`sc5hfCjtHai<(o>5@FieVKBicyZTAP~C|+nT-xb8J(1<&& zD_f`YSwWvt+If@@-DK5&DN+-es%*Q94urMh2WjWa-7H`Zseyw1kp2$sAr|-Rmc%)& zkd<|@u?N$N;+>32^bDVft=wL2(^R#u%#vr|rsS}l=xOZkAM3bG_kZEb1JR6Xj*qy@ z4^vl@)ry3)qfL2Tr4Fc{#&4W}am<;29&`32C^QUjoyJmd${K-2Pd|~{^=KDfFf3U~ zi*C(12e|+vaQVHsJUtWPa|O#anQyu%7 zGDYw@ys>!qdBIVlz;n*^gFOP7tMrxX90>+s&dQPOStCyd3ca&nx1q`uUwXP>A)jOd zn+AYCQY0lV?9+QcI#xSCQ5rJd-HU>^O2l4m(BZ9-Kzpi_ZXFzW(3b)?#xnAK zN5BO#;LG_X-J=N8Av`45B>?MTxv-CoeRW#3)P+fvL3eRNAu||cPRfQA2~58>?pQUR z-!4h8k%B|#K&lY9sZ}6L&}Z%o+Q)Gl0+kF znmaLkMbMGnuulaZL-XjDTlNHPa_qcU)k2PP$#4|V;eJiCjzjn#;Y8E%`AZs_9a?=p z2}qcw@Qk~1b&j1Pn=8P~JrEKsLG~g|{9Pqr$?9|$RFh8mKhkiE2g;BvwPP#eePF_> z0Q9nYJC!sGiKWhE=0)3m_RtCb(v|NZveOS>1?pJr0TI0-wwaKr2&rvs#vRAWypx%N z)AXxwb?7DTZS8fDl4k_WE?&OLnk-ZpT;dLv@k0xYBewcE^-B(ADO_aZqSiVhb0v7j z{kn`34Bl~I_IhBXi|$aRmUB|lwO|{;S!8m)ebE;9tkNg*U41n-vz5Dkr_U972Easy z=jgqKiA)y8PN(nB(OO|>3&urzn%L{2!LD5-=;!w_QbGvf&5WD6|)djsT zM5f*ko}LJ27j+ogDwa(;6z+$%pSN&-<;(W;z@*9^!-E{+^->(C!TGH;u&E5gYf*Q$ zJX-*V$43=MOYI2o2;Q2jl)FU?kurB)t;TVLyUBm1If%X!wrRv~iB=zExYQG6%Tp+xujzH@Vw-{|IyLmC4nY(?0rlB;ovzV zr||TYSGc=9Qp<_S*Z9+1&m|pi^4oWM1jkUMs5ni}B|#TrAY zN|+w;o`^da`t}K-pa;!=%+>ds8d?yFzcF8gd_Ng6NU<%u{n&RHpF@5D^4VnW_fI+V zxfY{PiIGjEjUwNKti@t-`_t0%Txv~)l^kRVy5flK7M6_m z-Z&LPA9!kJZ8}A>vk?0accTx-twlI%{^>&%;r5(Iv8B>5av~hH`Gi=6< zXu(&I1;TgCGuM`V+R|RFOp=mRWqT$H>^7(`-;M;lWa{Kkyz={f5ZICEj}(^HYgm{T5|N4uX!1|IbKA(Q=@~- zrvB!{rp-m$V&V%tuWfU`wi+*FahO~r$L>39Q&0T1DWLP&Bjqh$M^~Z*2YA8gCz4!| zN(Jf3fcdEXFV$1O#m0<93-(LE@u{fH6wSC-XZVgbl%hs5Tp)Kd&`gtYV+>J9%h=_WqyRym=e~P%wdov!4(1hLGIEMS)5(VKrpPtc| z+Pk3$u68PjJp9SaYR!AZoZ!>TEg*sBdY*<*>jc#1B))eF#>gptUodf%?^I2XWO&JV@QmnLaVs zMi5|r$#F=U%p=CN<%bC|TryU?3J%DXp?c02$3+@A=ov}1?Y_;nlZk$=)MiUgI*RnZ zaajl{7xk>f_u_Q3NYQSOllSk$ac!X924Cj0lT>f?y*0gE#}0eF)zJnRM}a?OU1gh_ ziu(1q@`RDA}awl08)RE?0Mas+XsG@)Ce959oV*F)s#IPQtF;TMzJ{fih129Hc{>xFUkNS z4o8AGLtbQNdq?5bn=QG?E#sJ#2}2~ju^D~53n*yvTLj)(hyr&qxe}oeZuqxZiQ5%* zQaifv-Ze|d^+C^^8oiu*xsUy!I~s__fUP|Pk$$tn_6vw&le+6qV|mi$&_w1F4sB8XKT7O}ubHjK!&qc%FXduVY|L}*P!1L3pq%;;l0 zC+lXLU0HXqLB=B-Bu*sir0i8*bby+G+$Sq=kgkfUc3P>0EkzM|q3){lb|J#UGdv}7 zT(f#WXi~LaptK#VhVjHytEfiPNvbz2>BDW)cV@cc_HwBZQtDDuniQ?~WQm8qtPsfQ zC?Py$-f^#CCvOM!v2C0`vX81C47rDE+z{G{c`P1CACU_ro-a6?nV zN*nLPCGH?)Alk1W;2cC5Mq0sJ^V-MUPMzzyN1~MkxYVB=f8#&ieQC77GqRi<>KM`- z@=zVFuB%NW>h!2I8iFP1wls4pwJ}B#LILkcJX2@}b8ahAwcAXs2g!cSc*pTT95|_% z{Z4#rz10@YRTDx3y{`^k*cmUqAAXHaWT^*6S^Km)n}34PRx;;!9ex;;$}OjF862Ht z?5;}|an~jNLuTcBIyf4GuH=nAaAGdRh_-5lKc)4~U{cElBt6?!kfJlI%Dq$-YVQc< zNd&~MA2LfbJ5wO5ALRYh8(LdYRRxmj*L!6TnK`*0Hc}R@4<+n|H}dHiTHe-T$*S^= zbB*m)kbW2#i4kRN~EEL?l*?@z6G_1%$6Td60}WJ4nN^fyxmYoG)&*lyo!-I;yR^)~+C4O5SRK@0(SGu4NiZlhQLe<$D&I7Qn2Zfs|WFbERp6e*?$!It(pT32H z&2e1gQGI({G4CF*P>m?{7mbHQ#nir#706>e^79sx&^8O9Lzt+FHQn&H7?)C1H5C*6 za;sw!L%}@;mC<;@`V`Jyp>KMGUvvNw8Le^8$&DaovNMCKLeGhb@oR}tl=Y(8PnIdv zwgPe~`Mi=Uf~K$~@}!zeNiE1cke>xQJBR6)d5RMU=NoyxyxKj!(=P8Td5z)s7O=^O zKI}Y|;MVW<$Tl&jz8jxswON#pkFo8^J5d1eT@+2;54Lz(3O}+*(jhIFAuSFdY@=md z+HAPrzncgWhIE~SSnyuwdorS#6IU5u=wKw*ROx!gOzcO5MW4fpR1<5yRZuXCV6J(b zTYzyHO#`J5K^SHOHiA8P7TM?~fUI*( zf7S+a4s8|q_vl=yT$L+khavjyOTLYSDf}Y4F8;n4HuvzsR=8j;dl4-^RFCS-x6Z`_ zxF`>hugR^Z%&-nvGHb@n;u}Qb{nSBTQFDM&7eosxkQpRp^Uu{q!ez_zxY#td6ed$* zeO_!BV)RXN7~Ww@?sWf3EYsw2mI4d@)JDw$5D)brrW15Y7F&Q(D5i6X`zsNV%%o#)r{oyq z3=*cBVZ&?wj<}p=`JiBwrR(HzyP#It8~)^csO);<#hv4POKusfnw(%A)Z5e+AnlrI zDsD;i21#|WZmKhb49W`xr8HTV19b{L!w&waUkyVOdL=nn_STBm`+=UbI_XA4bEHKZI>TQ6FbBm7=NSYzxbm+gUPdQ#u* zTW@d+olD)S|DjC-aOwiX268>lpwL=hBH2t;@?V417x z!U{ui9sZ;tdX1+scsxOMB51976EM4tKQMp8K08p_nC0&Hok&49R@8I%+(vwz&3x7Kcc87?eSp|IX1`n9eLDO0bG zLi#B%!{#w9;pIXUGlZCJemUPKu8HU(r3{O((H7*0x3giTg8F?9e?&F(T-yZJ+y6Pt z>$WpvS@)&#>u@%e`H$iNlPOHbMc_401(uaTyq{a+#mvzqmfA8^$D3&JzKla6*1a%e2N}smfPP5l{_#-kp=p>!|LOl3XT% z1vu6uYsn&G-(4%B#4oM4&I%niyJod}H)0UhzH4a*c%d3e`AM~EUuIAbNG{Lds_9bt zfOm48QX_%xHBn|XT_%D3L+Cya%i#Oe2EQ8TFR)#;4&eMxF)z_ zj#vrB5@_H)aA&e!ekmmNTG}SlC=MtX%k;3I5O3j$>3AS~DOD4iB*-0WcLlkg*>Axm=FK)D#0>(Qdu_>swiWpvt$} zTyb7W<9ZK}o;F7~&SBXJ4>1~M)lqO-c;@JFZ*zsZbr-;q-}>ntBIb$6?88yUD}xOl zFgRDPeM#3+aJ`ah^y5jlAs?Z27B2=itTH$wiR6mMqtnv#E%krfo$~AyJhX zPJ>xlIF^{;WjS_aYe=^}joNo7TT#2JN)?uvj@ngn7W4g#sEhDi49uP@MiW)W1!h<> z+>hB8=l+q*ff;}Un1_z#tvR{Icd>&yRIx*{y`L{h2~VCf2~ZoIm)u^YeIXHeu%$-C zsJ6;2E%;kiVU1>~>3~+D3PFS9J1VC*%b#%9^i`anJPikx$Q6Qa`}NnuzC<-oCK@S@ zqylieGdLe|He6_RNMl$vSp9QqphPxNfHT@<$kGG+Q%dkvXq1@Mi}}dBkwGBX2kTK< zcArYeAjDpWw#ud&1U-DK6(ZlaS!)QAfw1-`sXwtKY6$EQ^fgU0CljP{_~RD~wWlmY zc?gGB53_|94sFB~yAYb>hbSw_hHnHI7`o0vQ#wp8qvoM`A?C1~?vIOcjylBf%quFv zi?lnKsRD^g8b#q8mZKLEC&m}7#l%D`ljTTzeB1lEFwjTY-26kzha~J__mFz5o22G^f=h5rlqn-cik7tFfWV&G zQ^H+aGP-<+Nl_?g5JF#3QPd6j#qg~}LkI0W&Id~mxc@<8(=|g0yg5|A_|pSkn}9J9 zbWhbNdynOhn30s_-VXASax@*zhPa1OXJ*0pln6+U;ptR>HpFI*91sd85(^!wSQ=uJDC|6DToOYFtV_G*s}kY z4blEDEsy`Ezs77T5W}-&njAGyEY?7SQh1VPL!%b;z35HP0((X z6P-iV)P>(ey@6D}j^z^z_>`|y4&*>W+06tg8S8cVYBQOu)EkK1w5%y9bO>_NXqULL zFZjU6A8Tby7YH|tihk6JrRbj@fD7ZE%GORvf#%fA(VjoXZPwsgK`cfTp)zBOz2f(Q zITf8TyOAB1p!u`P={2^;)t$r)Ia7v`hsV9`Tt64Sv&YRd#m(eK_f$~OI4M3&hFQj& zQ_Jf4egF2Q2XDo(+b^B-`{(ONkM{NT^L?)pzE9vlh_)pLaL`6phX3-8|0sRX4&Ye- z((Czy{UGw63>=u50S4BVmU;xttN%Dx9brYBH13NSRWz7}-cueaeH)aL+~RU+TgHg-{nFsT`EiD#&+ zpP~Zp@z4{??;K8Rm<^L27W(c+(*v?b;ar|$jnPbF_`^JrM^j}quByf=*)+{M*^<=R zsr?@YNx>pQt|zJ=js$XOmJk1F-mv~9vGd1}|CX=$kB$Vaf3i~kk_%#_|5Gi9fu5fJ zzsUtjiAc)-X-dE*Ebw8c@rNk^>;M01L4P3OUoiNMfxn3dvHmY^AipCA{2NgOe?UzE zAgE_=^v5rxrT^EE&Ft(Qew!Z3NQ~GNF z3H}&g1%RrVp@XTtCOgybp#J>%8|xpXzoYgqE0X`p#eYTo(S((NmA%=&d+^WDW&SGV zHpah>kGdR;Y#;6u>>q#O!#LsN_1|Syh7X~skM@5bOza;w>EB5DDF42WgZa-of1LmO zx{tanAEHoyZ|6_={jJeI0{`9M??Bi;j*mwFDF5!nM`h-ZhX4Kg_xFEy^pEq8Ykrr1 zXX*EKe{}x$>%Z#$4)ovG{Sol*pnkVaPyhE^{zdHi@3|!SQ^V>X;FJF|gUmwAB5cfm zOCiJGGWb8_@9*jRt>5(*vFnf2{z>fmuW0@P>EC=u*nXq(AEIDvY#-D7ql4&5nz!qR^?Pv#(2AI0qm>O9DjLn>k0Ac_+fW48^$ElUsN7X+}Qvmj6E&w+pyMNqR z{u?Kaf%!jKclbC~|erQt7Im47Xw4D|o$ z8R`F~pLds+PWJG>`*|%xGYDBeA_EL1_3~l=bNy;U-K}wd{rULw+c9v%M`xL!eRf=; zXMsDp!TaFB$wWoTB-%1yyvV-4lQ!R12#`F>`K=3Nx4dEXS6+Vt#Q>55C2aYR;s5oB z_kYGb>z`E0zgD)7dp0cx(+9ng`9s0&Lk#bqkQWvDW2VaJIhZQCTI&BF);9JJod4T! zhmD!x-FPEc_XDjYr>*x0=&rpyt z(TI?stUf*#9)5yor7J%^BcJV_Hks?&=O?8xf_!J#H92)bbiH=wTZn7psWW>L9(bBZ zfciv;gvhF|U+$d9}q=0NM1y?iaaQSEs{nAgDI&))L$rmM8YE2GJSX>QevdSS;AXl-3zs4zePS* zQbt{RE~~?-&eLQNx*GT4_x5Vi272DjqBhFieYZZ190s$n!>HXiS5BArB}<;_GxYWy zoh)#W9VQ|#1adXu@+S(^?1@j5BYh|P;=a)qXka+4;BFJ>U$L>M;01Y(5WvAMLhk$D z2Trmo5k-vL2Z)u)r7RYPn=O%W)^biAlPIFiHrBzHBvfCC$r)P4P5mGhgaY!=PVNnC zk-!8HJ?$|dDIxdo03tE%1zNOBC*)iubga%b85sy)cT>>AAufrn0JNC&`1*7+4H>_N z?zES1&hV?xUVp9NxzRjl&2t|%pE4j-f4=-0R8yRG$o1kmat@wBt_!0m#(M9{W}9HK z+#)kIxtmW4&r6MN6)V9Tqz_Ki$$vD-zxj!W2?0J6$Joc?!mQF&gr4@(zAZU;*(Kqk zK&K&Ouz+hW5o@Ss-5I{;Hw_I@^A3AUwg+Bt>~t1u*k1wHjZ4QQF9An^hq*neJt%%6 z7{0vR98B%}@O$0iG3fMg!4nL%vPyeKd3JCcFFVvW-|D;2a17xy*uNO-LSTVRs=H1w z=$n0dxOY4)#qxXOJU-zE3)?4eJwUpVltGjDgSNC*%y{nqv5V;{!`-!JB1Xfql zT(^ky8;Z(P={sDmE+X2d=Jin*eDJfFSw;$e)bMkf3;s?e=A;_^r}mTHaD_q`Ztg5y zW~*`H8-WM}5j*x~2xU|u#d%YTRofqtp$LFj@I$yIdoaD}U<3gWbX&1)1(yJ{qMnv=yf9LCz&OSh9z^ZCY#OypeL zY3l?TVkf&WUp+4AiQ@^x<=v`A?6VX#dc2(;m#doNaoiG3^Z7+9JL>hLa=Y!bnZC~N zyCN1SFV%V)v7O_QK=MVr`(UV+GJ0O~+3`VM8#(Yj^Z_|vst?lb0-x}O@TAI|QjoSUnPx;|2E#+Mu)t5|{%6uzl1=YSB zG5+#-#z8!TBpK~}iGww{twp?PR#^^xrMSs&w+o`pcp zPkktA?n;VT5bbDN2-45bJ7Iw}yh_M!)H-GOgUnPM#R!PhUZA4b&r;xK8dJ@Kta1GX<|my+v4S7LYBy5_7^OKwUvXcQIYyyR-5HahVSR&k_t4kzA(eg>5AMJs7X155 zW=D49#_RG!3gA!5+cz$#$uY<0@`)r=Y1L%js($KBH7I=~NMz)RI`$4J>Dim7=1BY) zuRI`KCCyq|$ZTXT+^Y6{o7VqD@f}V_Z?MwO{c4pjvo}A9ws5b$t()lVv=RRpM%+q! zX?Wh*>|t>da_~kIviQRV-{SJ+6~n?h^^KErRJg$pDN%18${wKzOE;Okr{nG(Uh7ye zb+@p&$;xHs!SYK3Wz>TTTjL!vtC+k^1li}6-d9GCv>#f(WiMpz(2tjO-E23}ySoZ$-I;X85+=3`R~TB#BMw2Rdt(qBk6{B>YkoeM`5)D5iI46O1yOzY^U-eBLj=&uoc8O2@u@tGIp+992p zGqI$HXQx@o)$+)mpDo_$*mUJv^3Y0AaO=@L+#J}apmgVa=FJL&ZEx$%Bh~Gr2=zz9 z_xKRO=}O{n?g`Ocvp@tl%%F8|V)Ar(Kixfc;!ao7#J#$cu!wg>v1dj--OqwZek~`NZ4MlJc34ZRlNFBK4Io+UCNY6{3 zX8c*R)yJ?zQX9+K<@tUI{~ixl3zeZb`aX80c&1E$vc85pUZk!Qb3XFZj+v>DaW-bA ztTo1Y;e-eIl6)LcieXnNCF|byHqGlkG)}ZnPM~;6gV=0Z8Vw`sujQOKayu+rOT|>4H<<~W>VkPp8kljO`|IZQc@gyn<*oR7`F>-etwqj#!Rw8A{rVxxw!%9YYULKMxvhgFO$8{cr`?s zeNb3hK<7Z%Y?aPouhr*$G6%cm3x;mV;-$CMr&aKFU&hhp$xjnZ4{}f+WDgICscK9a%9%MD_I?PEs<0(0j4%)CVsbidNoahu*0iu4gLa8X)gy`xw$#;5Y5? z=mdsYJ7V#Em9lYCW`@eL&sdnDtKE&vL12&pjoj_0@%C4Q zvu|)dz7iip+u3k1w|h71erRXtn3ZQw2sd~U$1*_ww1J#M>_MNG%FTMImlK>XwSv{<|6#$>;vG-_b9IK;OBqiL|dp$6@0ItR$?lC|3KTK#|-u^=&19o%aSz}D|(_P2li-$E;YO|Jhl)CKuj zSm5GUVF_dt_$wUp8(axMf1-Q|)OD$F<;8^DQpqs8iEb?DY zmv`i5pihXVlxs+1`{wD*iYHSN4?tF{qOsa)MZvdAA!C=C7f*CV+_g^PUhSzggp`>* zlul?mae`T|kXpsG!c{6lWeM4#yb!jk_CiJm=dl`T`JP9-a7J=MUc4(lg!;&F!PHTT zmp0K}*Z3+rUO&o{Hu_?)ob)V8z-@HWkb6g2I+tNJ+1j9VerBC`UNR?9?gi5$KBMQ? zy3;)k!?h==*Hu(=tC^f8#``i7f;?ZNaa7N;jLn<1i;(8dt3HKaP7N%~v|->b)nqS| zI;oq*m?y6#@TpMgZ@E*AQr{agDLK+JdPntE$~!q(Bzt5Bb6glt*8GYS*BNBb7Hsagl@IIbN285HWNVXeHIdaC|q|G#l8< z$}C+;b9G}^nM`Xu#898Z0uclf4{~wvd|fYhQ-0gtrzU}pcgQDZFb^WsWe0`sGOuzF%(tY)c1UJOhM#FE0$_9a_ScN}ja*l_m zY$Q{W26l+PU2P!0=WO@1CV+3%X%t-5MG*t*u8W>(8dZ2%b0spLAule!ygUxiG)6xJ z`#R;Zbf1&HG-rZGB$Iq);`HHH$&9h<=K<{pVnhv$^8C!sn1*#TnrN|NC!<@f z%h}vRmc`em(U0IKoKvz!dMf-S4X)Qwb<26>ic_v_s`WUx$JOYm&9;W#nXU(nB=>)@_GCF%G%lkb8R@43TZl)f)k`a^@iP^dKRUzr6lRn&Z z<*X-#fmSaKj;@tA<=Mri%!DIJ1Nby!t~Z@2Ve56U=Xie#xnY|@_S~UaENeN|!(enq zZLEFs{$z2WhvEC42hc5u#n&q-`R&d$N{J|DQ@bNfik|k0!H$);WLu;w=A1$(Y6sYl zq~5)FK(MAr!E<1QXHY&`No>Hy&q?KKF8H!dGad9KM;X(S9G-~Q}Jd+Uh>X==>ZvcYPs{p@N6mn>FEwH-?*Y}cRUMG zrdo^7pG@*M+!uiq_{U6AEhZ2|ZVg?Y-06AS`XUy0Hl2y%P+~`OK&u9r3fx1*S!vl? z#OY*Rao{$-qF`RjhQWAd6L(>X!T5@;HN#OD)&q1}l886T^cG9SLB^1s z`yFk*!f<)9Gil2^x1Bt)%C#N4FO8F(7Y{hcXzUSelELD!SUJVA#=EjqGy1xoA#8ze zbhPJvM15dbS2fy+Jt%Z6n}b1UZvM?-GRN-d(Wk7)uRV7r-etP^sgElTG#)&c4%Vp% z%fH?kax;_3;MhR+*Ux-e&Zv#{k)zk{KM0I`@cI76#Ld^aujbFomnJnT(*`77nW3^( zawl^;gZ3mFJug_R=h@x^4hd;wgpt0h_Zr+3POadv6V|J#1aYl59{1do1Lcv6G?!wj zmyXC4ABeb~GM&smVQr@zV!b-ZwR4V!8p`J`Xy4AE1DuyrF#0uifSg=Kq>1c-jSM_Pj!m(#doGtDP z>eCMbEZ>rT-8mp4onQ8;D4Ce3@TT>tKiSm}K>qtwhxpT`{Qu5Wr|3`xqMm>Z+CRv1 zJI4Mvd$ynvmlaW>D7}45>LU39R(P!L@X@;3*2thU2m3cPTp2}rKx-!tnV9Hz7$}na zepYf^v@|yt1{bOf;-A~rIhA!f8)S;Pk~F==%tcIT<;R%3)OW>bU?C?nb?oOhaS@$O z6|pa~^;ge_En-k}E_bEM3yCI=tH=TeXsE}=`k<*G|xuM$Z=@O>x96$ zBloth*GF1V2Zig^(%-&vjcs(L@qF+LQl5o&kK7?ax9T@WzHc|eS- zdQ?hAxDfH1{fl+#&XfGW6sG|4?4pq?VDQQQD+dsqSZ& z{kN5AFeF58JF>v_x5f*O})B zN`o(2L}h+*dAcK-yVd0^ULP(CXKtv;i}8@Zr6`Kdmj!c&`tsE1r*SGj3VA_lLH*)l z>Qs%BwA?ARe3*AbZnTk})9`UU8P($xZh={+${Fw6DE5+albF8Y>irD2;F-H3tSP*C@G4FZ5X@;-!ot9;vN1vWCrfomk``iS`S_ zVUF5zI!0FF2FLRqjMT1W>>RcHykmJkn>YV1MUaRT%GK*ovf60=-EI4mwuj0$O?R_< z+3CI%hPU>)>RM`@Nk;2he`VZpTc{knPSkr<(C<&C^uvY!Zc3#QluT7Do~8UVE{4sC<$06mllA1f0*|Kn7`gHYg&%P^eX3J5+;e>K z(dq+Q&rbU;Lt;|lG@q;qC)yb)SXwImQ74m%bEby%hzoC8?~R5peVUgiX1u+i4Wa8& zE>=4insWEr>xwz^&os|%dVtv6P#fpR`6c@O{3L!^VH<2UO=6nd5{%armK|DeeOPlR z)}op&c&FU7mYPtdqaW~g+(a={Mvh{x;MuFg=8(j;g)ID+-mts^jd}aB`e9GyH!a-d zmWaTk_1v1cudle*IMRI2iN3@rgD&J=xO9Y)`(FMvDz2wY?)X5Z8eHlsYxNM7Gt(t9 z(UeD}51$-PtK`wYms-y6pj?3n9Ih}rA{%|ucYQh|tXiM{@p6AA43_2fq~^)()l@N( zyl!^YdNWN&N{+MxoN4D7qYh;U=RLhUA*H@<0nvLto@{g~klLT7s**x!M7tq3Oo*h3 z$|E!}R;^S*`)tL41u_&L+K?bI=sFgWv%yN#+0wf8CsX?k??L=?rlzBKHvmLESJj+n z`&og`PXye~<7~CU4^LG-b`f;pno<{*TQO_vOyoM7+NAj6?lt}Ol(=gHg>#}_OWQ3X zv)b}j@<#BcYrc6F#MSX$;KSG=X+J;QzTQc&2#P>d-@Bl|l8 z?8Dm`cyVLZ)Z4@0w)C%0IMNK|FWs2X_76yv@vtvw9}jC+j!%mZdK>LQtU+XK zr~1ex#!i#xe2sqXorrN`Th63)Z_f>p7asPk_LO<4=jHBlXe#M!kLaH>Cf|CNn$wU% zE*7wHz*KCL$?H5U#~ zZIDNc#BG4dXio{on3{mKbmibJ)c!B71*KU` zSZ3u7m6FZ8Me+n)U7gch`ONEOirz*ZMf9gWI|uee&q=gMzUmylOG23{B+%$(fUIsE zM<>(LyJ{3kNTIE}O&zuKeX<#yK<;S3Nui4M0fpv{9L$pFY?5#HoL=+O#Nbu9=!xCPn zUvdv2gIf$OGp8Jyr}AMqLv{2nSSq00u4^*zu`{Wi@uLVS%aL~cMeG%m7Fu7DYR2kt z%4HW2=OJUF7OETalCRhgMN1hb8ik0iJr~g*nzm*q>9tX-MkST5m&i{Lxwte6W*z$! zCM0{mBIe)#w`kZIwTA&)yHe~2!lq9|52xlFRoft8%xL1`R=9qlO1c#z)s<;7uQ+TDXhUoIKZ z|H-<3pq}5XOHk;aTYfb;Re=t7@;yn&IqES+uQM7&d5P>*x<4X?)}Y=f$HbS-rK;QOL8XHU;0iD#042 z*O}|Ol5I;iC$crZela&-I{e$XJiDotCTrYYr<_7|9-{$ZGa_50EyR+cT9#ByGp z3l1+`{pBFCWJ>Go;>c3Bn01TN)3C1DSlRe7sSd7lxN)NKA+vnjfOwV87E9CgqOY3Qty?sQFZ(1% z?T}pZl$=!x2)Yi9#vH^S)Q*>7WFnF<(-w`ov`hI4?eUJS4WzT667Mjam^c(=vQuw& zD)yd-+OE1vhb!~-o784)o^^;%FBrS+44$py=muiI{bKc!F_Z}2!j+KwpLTRbOLwTG zB{P3#0tnA!4ZLmmb*i?1H;DTqW%zAa$wE^z@k zBeaO)=!&X5-o_|}^3)z}kF9+?JCCF)JIU#C_7ySYd3e5`n?X`9b?9t9Ys9<6)rl`9 zmyk(EE?9vI$u(3vz3*h@WXd-Xz22o4^c{%Q2i|Zm-90*^S5WfGPWHfv;`EK!9qK@D zGFQ##w!@AFn{s+|XJW5JsM(H%s~d1`hNgUYCGna}r%_fl?|>(bRf z(Q^GU zlN$`3mmo(U5-;D+&7@?A=YxK5K5Z7vhS%Y#8rM#1U_T~U?#Ku4ac8LuA1dRr$X9PX zcik~iRQ7T~Nh24Z3XY)%L1dZLNz(=2EYb;lWuk4;?8k@V(=Q_~QUi|(@CPV^KPCBr zhTiMGfEqX4(qK-I(U8X~w2t_#RyCd?W3+#XW36*vIm* z%JX8h=DE*Hn7KDb9CM@CbQQ%%SyZ9#+=rwt;zNDmT4!r`Eqq^ zUbfeuyL;PJsc7Mru{-YMDO5VEUte)lJl1{X_3lVO_PKC}R1V*m_V?M@xAmXK=;-9< z80$=P*ZJN5aRyP z8XEHN-70aPg78lo~+KQCq6z9wms$3{KAE9i86L2>CJNmP{j=Vxg*5qI?XPv(kmr+;V~y5>le zey`8Mlag!EN_ocpR9%j z9@7Rd&6CZps+mnX_yl`X#d=b{GkCr&XuxwUuB5MF>LV(dsxxR7H(q=|&~+fy8D@b~ z{~BTHIomU5OSJUJvg1$2`PhR%NldY0ny-!a{6fHPj_8!+WYVUT% zv!z1K$Y>z;-lA6OQgp*(`3nZjVGDT}`-4}cTyHA1X)SXGq|$nfx`zZGUq_Ff z4;c8sKl#kx=)LHTgzoI&A`P7B)*4+YHYa9ioD=slI%Fk}2ghk2FS^!gxj6$I2Qr-x z9(U>y&2mnQ(NWjL&Ek|jPJvmozwRpDsgsvClK;d|62;jaR-gFj+1+?jO-Hz-+E;Nb zZF4zO$-F$1WiI{g@#b&MHmmzw&(IN9> z@LM@|bk1Ksdqem}gjVqD)H{r0xo!EA$5(4R9(yr|d?*J=_YJHK+TTv*+BwSFS)I*w zW}an4cKJTfC}Vk-p0{U9k)$QPio5@cXw;UjS{3Gmq}U-ER9R?JW_Q!1j>uVfdKYG|GDW1a6eWc zJNCQJS6ba-yD-s2r|7O?v%6KBFGny5C!Qr*_>jgBljcYx?>*;V7;uzK!jcU5U2Zp`?;HQK(0QRj!C%J+o^e%%B?; z5fBpYYhtNs%rHwccg7PZ)w#l%U0#oVcc6>@#@eX25dN?+x%tSeUgB->RYY<=|4W(& zSxI`9qmm*z{d_U7thg%`W*>Ua7V#cR_p<)Dp9{Pd!1S5^)06S~3*^*Xg&Z`(n zsc4n#Zi+kkZ*^K!(~YAU&D+d7{ESw{ZfrOVRlD6^z-8P5JCuvr>tNjH9mZ^B5{4#u}f zb#rk`<`fR9>xRV{h2)C0+s}rR@-FzbyX`n+%AmX#@q*s|tDO|L_!{ruuj2^N%yq?j zW4v=V95RxwH+Xqt1BI4%deht|-Nrf0lgh4U8!xY=9M!2+b_$v;?h(Yzlt{D)4dFK& z$~Hdi7S0nB)rASw{mICF+mnDJ{;@rYST#!j3)C|1$%DJRU+w4HuW0rbo0`RO2ilmP zXmF@hAmV^rftri_tE3uAcl{9+3YCHYql&=*A%*;(AO7Q<)o_FZuDL zql{u(;MS;yRr<1GLNeYd7)#zds4XdHUjEGMM5^LgQ^1+$9;gbK zN*ZGEcsU5jGQrwr_bT|r1&N5h5(URrURj+H@kg_fs3L@+DpI8kp>wAo7%U(kh*$18 zVJd=+$Ulc2h#S-D*YV2KT8h?mns7%2ATtLOYmAGn zgCz*cFUYl5r-P}Pu)2)gw;F*fF)ni~)=n4ENpE+do7z>vvtCX zaXC3zmmUuBBNk%~65@v-&A>YUZ?vVP^sn`0ZM|plCU#0n!ZNmKCu?9@xFqw02>d2k!sI zI`enUwIG)-(3iUb3es}yEY(s zAe+Bw0uaEnC7=nS0KnxJ8bS~X0X$SfngF0`0^09wAOr=Fzw;r1Wh7x)p}kkIeyL9Y z2LG+B089W1bQl5OURDtTn$TVnk6&phpsxUE;Fq$1!-pU&i$cIrgnB^%4??H|AWzAj z`4QBI0+NIf&`?mqcA-cZfMWg12hb2ekNr{>pa}{Q$Oq5>sR{Yu5Fr9tph!4iz=V9j z88`&b0o`*`ers0<2_-Bm1Q*b@8nCPPTUo$j2xu@U3_{p0Fkcbc6&wZ?L=fr# z21V|nSii^zgQ5uM2sj)H`Hc?_g%OS|fDeu&Yy$vS_5itG+5l*KC>a3_n5urG3Bm~V z2aL)+{O*^s2nZBGV9#)1m;)KA2>FnN^9#U-`ppi286QX)M^F}+A_!$hKn0P6c7T8i zp?;GEv6oJSpgsgJ7zk;AS^f5G0tmvf08AQtnUe_W697{F5YhwzLnWjELz=M95kN|k z-)O*~{!Ir$sJ&RWz0ddG4>$rP@SAN2Kwx`_>o0r)5Fx@j8xG9kzuB_@kg1KZ4FMQ( z5AOZ7zP%p)jV81g-$%fQ`pr&(;kK8T>sLPDH{drKQsDRJKmr{?SQaU;7Y+AoeMo_y w=Q^x|i3NZH0+5F&Si`~%1N=|}s}D*?ERf*kuitJcRFH;}lT%hjj)wC80LshP@&Et; diff --git a/crates/pithos_lib/tests/data/dummy_dir/literature/pithos-placeholder.pdf b/crates/pithos_lib/tests/data/dummy_dir/literature/pithos-placeholder.pdf new file mode 100644 index 0000000000000000000000000000000000000000..d967804c714275c94c75e54d7eb0bc3ce1910102 GIT binary patch literal 327 zcmZXQ%?^Sv5QOi2in(yqQlW^6;ouJ!V>If`aHydYjG?B~1bup2G@@bC%g%nYldZR1 zqysh#34;%Hx{&D<^zgVz(77<;stVApC}fS11$IahBCFhiuKzc@Il3zI8ppulncbr9 zl>z%Px=G*YVzjjAJBO}iUW%D&aP%!d9@&@uENQen6CWY*zlDhNfa8o#o1mSd>NFnr nG{$(R87<1I)a^g)%cn#Lx>L%X@VUP>LYp?9#S!tm*PI{5#hqO+NcnZ2u#tCfSj3xneSmqq%Il{B(9bunmj- zmezPNpOYR0`ZGTa^oQaqGvFy!z57|-D-pWyM$O+DP873td~zE6Npu7D%` zoXfrHEVf%ywfbtVO}vuQRP<88g5-jsp&|d|lw|eLz(C9IWr$MZ&$n5sQ0Xr=D29`k zTWUZ7L$uY2fb7+W+_c*4;Lhss#3t<6(B9x&?C9vo=FH6A=+fXbbf3iAK56$ao7B+e z%*5){AbzR6nFaWDaa<4Rv_mFXJ?ks74KIr@0aZk_-eWuBO@WEmK2lQ{LUj? zg6+&IN7-4NqK6cim=t0Hsi{9|TI9)Vio^ESr`^7#J(b@I zj2E}Ws%GUCk7i9!Py`};3(8HCmcHckr{!M`14w02=LEkTb51?XHBRa6v)d~Nvfa0? z^veQ zPn3%kIx_X@PizxqOYmx9=g(P(d-YZ-aXI*}2NlOQ(^995sN+sIr5km^nx@fv`1jti z0|)o_mR!Td9)rV=6%OjJ>^lkMrNP7om)7=M?U0o z1A@@{=1zz0WZT8xLjgulX<1oW${Q`|kUdyr0>MS+J=l;(pR2~Q#+{P+uoWH?`L)Nj z6T+Gmh0&zg-T;BuB-=%f>C{!UlCwP{bd}A_YUlIzp=8?0r$8fxFuwNFVna^w8+F@t z`J2&7Qzge{fwi+~hP&Fqm0(g!`*IIG&MT0nXp})B=`2NL!tJrbb8h!N%1Wc@<+4g7$MYl_ z?_nhT66#YruGZmiD=~1{DIjS<$x=s_i=>d`QC1cFRZ~PzBrW1#DHFhgQxQh0C#dA7 zSmX!MN*JtdyEtH@Qm2>~iehy9?u!!#we}^-eFr*IUchDp7sNOb5|yeN3wDk0^lmXt zO-xHz(~7s(&z{pqKv7dowb`tMJz5$^3pBp?@j=-2;S>35?U$1qT{j%1I9L+8n?-Do zuE?LD<&Z(r;9`D?L952qAe#>?9d0t@p04s1`*dQmC)<6;?bc{V#rLlUZ(ob?)l@NK zEu=z6mLi2z#_RGQ#8)eUVQhtPGRWq{+SGq=?0z*6omoALGX|09g=WOtN&6M~+NHAe ztzx9YS(vawgO`Y-k^|v9$XIDps;NV$mo>I^K-WqR&ZfZPE0lJl08LT(<>jUV2$3GE}#A`@&=~Y2<@iO zMkQ*3@n8j=q&|>in@%_B#n!JNS8=>Pxgm8Z9$9GG%I4h)w7hF)Fwy^0O z9~hT}9$`b%**F=t?qe=pOqZ4yH#V9;6Z@&~N*ELzWmpO+5`$mlbHU&4{bNosE-=;@ zitxq_hx?Mh2C(qA>88J_sOrx3%3*JdYL|Uq2DBt*#M`l)xwLxT3f&aC$I|@MB2WpQ zPLiedrAH0Kx&#gZR)lq?0vG2(S|FyCnc)2vg>o6>HQ9R+8Xzmr>QQ0Y<*DQ3OHhp- zmrvKq!mAJ!HFDIZr^X>o9!rN=cZ)Hn-&Q6i?l9MB-AjOPZ@_UnWy-l|oI;N&qgN&8 z(BdYLE6yU)*$9UFn?xlnRpw=CsQNu_cU8sQ1c$ljKkfv@DLJVs&FPFA95XgIM%t>| zsdROs1^j4?6kMA{40J**3S)t<^tqvgrUPBV1j>%K*A&C9ySXpJ6eWjLR-0k-IHCX? z^AejVteka<$!Xj-yV+iqm|84WPHJwJG#od!$d-~a$@+q`Hozi^FACkf=Ae?kNLoA? z#`I?&9j^a%((pL0P@bR|b9W%3v$VyClnMTll=fJ2Qv!TAkqA$#+-Cx_NI7D**~B=8 z0*16wTn0#pksNcH+c2S|jM%NuKe7-N+J*P5`yDn7iR3Zi@*G^(@;;%vbivbQN(}}w zMm>cK8w)tjme#$onU~!wdfr9yrqn`WZGjS&MtDl+0)ee)o*%Xukj}dq1463)VVji| z#_1YEMoHUS3eu~M+ufoPDZe7Fw(n|z*4di!@>pr1dxj*;4yag2q^4pT`JDi_NfZnu z`a#tY#1p@BtBjZ4MUGTFT+F4bbtuRO@zdr4GVs^LeuiLT8GHfCw7)5g zzQ$}oMO+|Ak`@;D@Uo#zEujg;I0FR+$zgfeC263-FU9T~Y-6lD^~vUW6Iwt^KO!UP1RWqfV_an@?9fF{ zNPdDMt9g8j@8f%zPw?tSNl9599}YgCT=YMqe0d-fo!dG;t&*S8tWd60HHns*j=f&e zWxcavFDlDLNVXvL|5d$4ivsqAr|~&_XJve3Lytr^$!S+toigTtW*`4#oc9TzQ|BhLI^o#PEB;}) zYU~VGe=LR_uC*@Gv4mN|LZrV)`jWQHPPnj5FX>0gNp{Z^_zPAgP7$7Uexfw%gXIAl zp&hK8PNMGp0}TurB32MyKp&J~GK48EG0YUs=9PrR*mEx{MO|D#DSPRxqmnOe`t+v$uNpmzrQD@;*Ups%F$S^kJE2VW1ByCYvR~Xx|Oa6v9Sp5Ut54 zoukY=D$2=N6rPlN`WeI6lI0lx?S5880Zs?yi{NyMg3v53m*sDU77v(bpVc7lnb~9( zApT6$%Fu?({&v{dM|p3AhXMXG^09ZJ_I%8_cpz%%p!_gLTM7v4Q>eFSvSK@E#!i$| zJQ;fPjp931Z(p_;o*AQ;&AOQst1shB!}Pfz@V&%jG*nCi1yHJNt?f2K!;HPt(b;JI z5VK}0qh1dC{L!oc#6*Z5TP!|}O`P+F+M!^lvq#K6yg8#qjHFDqFFlbg9O9;WPJHOV z;WWIV(?gOmJAs@bHT4IlJ#WX;;&eU#v#0DsO}vQAHY(=sF@-Z7>7uh0`G6e}j;&3~ zrBn4AP!zA*{w`D^G(2*mxM8-Z`zEc#6;(}YJ7So&F3?r>sb`calDXQ6X z2_|=oD1+^Sl^fd>oer&DoHrGJZzSAod-^TfPAiV`yLh>5A@k5k>zMaXEp1cvgsd>!{1FoC+# zj~-3Mk~GQfJUK4LO3=+f*r@F*J=G3uO>5#meY){?)RM)^zwZa;-ZJ7!T!5BZl^7AO zYIIQW@l#~CcHE!iQY&g)8T4G^Rgy)Xp%vRo2sah4p%4@UDp&hI>|3?VkN3;?NxQ&h_tq5K>xe%8dJ z=upZS9ZhRvKOoI@yVc`7_K5nDKLDu+lIB@QZpd41EbG>Pi?P}Fm+P(MyC$WrIB-C% z!3jD1sGhl|mRPBDy}-t9Trd>~aLb({SJZ#ns>xD=MA>5l03C;8kq0vMz8TS?qU@4c zG+ZhW9((>#nJdr?!0yD638$|3?iIbzy`aJxmb`VeuR^&nIa!V@95p*Lns3!P^ zwiv-1qi^7mv%iK*V6#hu7#-9g9!jfJqs~>KXAA+2R;N%x=rDU7<8@frS&_qVj=v)5 zMMl=CRi`tz$UB%Lyt6STq4J&|_JPz6)j5IM;xKDLGH$H$p_3qT=?WDerpDg8P%~F0 z$5^D&B}?@)rwt43C>NETSYlnl*JqiYzbxk{dkO=8U7T|76rjzfS zS#3FJ><%VF0z7!@%|H0^HxYzu{4Qy!iuH27k-t^p11hU ze0B!hr49YX@9@4AU>yPL=}E;ZHMU%$z17cf|LeR!GmVyJAG~uSWNvdE^HmJ%RAp{y zwc+R^tSuc@XgxAmYRnGI9okqrYMMzibz9I4LX$D!YAuZO5<4ZJA+kdUVKG1mr)wc9EC>wWD}Bg);C#*fY}_SJsuUjR9T)Yb1UyDHs2^C!$BXLSlf&r z9~zu|9zg%;&0P3o5`j&A1kyJ;uhL#YxkC~j3$<_Kq+SV|7^s*_3uY6<_6^}RVaB=a z14mKZSw!l3hvzVuw#-PL+P2SYg*7nohXIcLCN&HOhUM0Fz!?6MU3*-|dKhkSn|yl+ zH7Om_-MS1V7T%9V$1UDp7JAxIim*5()>?SUH=0x)aut#29~w69G#i8FZnJG`40nEa z>H&Pzo1B64$mxyPnSuyY7b`#u6K4`A@=mFH#8hZ}WHGV?; z{K6MM&}YA=mt<-SHIYwbw;alDtxpk9eyLmjKMSD#ms3vd{RJ-tK|h2QE{~Dzl3*QR ztOeG&gv87(i=+10@{=Ze?rp3V6?u=AJfQC)%I=$;f?;0nY_T|4tuL$<1CTo+F7rN& zZKj?ZfD*nOx(t@4I1{h;Jx*e|T!qhW+hb*WaIl`9`mU3^u3`9}pFbc(q--mYg7AZQ<0za^XFOxnK_Z8SknD4YP?;m9*ofEEKy_TtL#jB-2 zGuNxOIw&_DTLEhAkF4#nvdi?C*P6}7R0U|9LwPx3oFO!(;@L@j<_zHxGzerdSPWhh zfNrHE5mi8yM@P&?=f#Y0sBpIK^Pm1L#xPzBx?;hQ0>H`ZWh}rh{a=$N# zHXDVH@YwNpjTn|l7Dl!-P|zsNeiRdDa@#+DDGrKxrz>uwk=6v@3guxd4K zgvU4Q<-_lzJKS4jJcOMUCCvsyl1|{Y)NqRv1>`Q{B->WBW}735i6US+o|N%Y&unn3777R9YR&B|D9g(m;lkV}m&mLbV zyVMb+3p3x5w>F33S}J1qYkulK=Xspsi!WP4Ur@^EF@VRHCuR(1uM+XG zlOr43-(8l^ETreeX$jg0xzmfQA_=hf8rs3tIGO7SW1^wc1M!lhguU7AMz3JY4pWJ9 zH{Z0&Xytztz0w;{w6wgDVsSwYT1@&+MrSH$9wcMI_EF8??HEboAe2Y9N9GV^?nW!W zr`#J2;?0pL)rFB?nmn(-w z&S;=`RDdAwRoL7pl7oKCcr_s~Fo4|mE>~ZCczr&||9}a>qGHkW8)U~=J;m4xAdEz# zPt#Qu%P#Dv_j=EB1!A|#0u!MC*0YW@sIHdd%ml06d2#GEvJJY5QCADNs?6pT$kCdb zEU`HY>eMEkjss;PkSmRgp2S5>UsTtjHRnj3J!!b>8AdeQD%RskHd-BFfQ7bq7JWe+!!}zVGZaS1d`A zAYsJwHH8(Y^<-;drQ2=$Ve7YmrLZ0yxoZ>3Whv(isVg~~2?!rev=*kzjd(ay;m96- zN;JzA&e5}7xHJRW|FCB>vZl)QwQu!}wAlH7GA2elbYqunu$=og$V$dtU%7+#ocvYB ze(TExHegS^aJz*Qz^(Q;)(Y|y*T9=hgY);r0LbZUFx8dS`8=eoGmL70Y+(% z(rXke(KTt>KfGmySf(oCQyw9sORw#cXtH(cLpZft!vYU(XatMMs6Z<9ZW52|3{4s9 z(BHs6-yQ6yzCI;lB<`wu^1j5I9w7|05J4_z5Nq5#=h@lg1W*R+UIvL4`HR&PJjT&I zlY~2}xG{Q5JM4b|175mc3yS^=M~ftoCh}s*kW{RZ)=ZgnNJ6!IyvTCGSRIf`wW?&2 z(M-`m2hUykecWlUmB~2%o}50(EA@NJ{1Olw2YwADM5LVHzyEFc!5o(T(@xN_U$La` zMjg2^DDpPj?}_=uW2S>}y?1O!K>~6U7dq8rEw`SpE@YOwQ`iaI+jnhsm`lSc<%-M*@kp6cRT%m>4268@2g& zb|^MBDsWap5<}u<9d424vSHltoHCnu7XfjbbkjSDX=&4QJfjs2RV%5D%oducsX&mg zgy~OrSZ{#X>KT#A5ruYC0ybM1vo4BHx@4m5tVCwU>UhQu?oXYTQz5tLb}jVjC}zUitiuK_07~W#XSA z?`U$onYakqm(P0~kuOWXuxRGPhX_1}$*BH?DjTp|E7KXaXz@7ZWzG-A0)Bel;LQFf zkm|%;3|y>D6kG_*@r;bp!XWcd8TD?F24N%C-|}mI^n3fiM!Pey$*CQp9S>qJ(Vd z*YWpwiujh7inv;FN)lVE*NQFw<5(#`m!r`f34$Ymp*`V#|>SeMCrT` zkO0QjQ}TSmBFIy&i6%#k%XI$q=3}FeNzyk+2K1Rfx1!wcdbK}JGx+I?V{2!%(ds@9 z&mj2y`d02<6LW{G#V^;G!gy@1K6Kd^al3s>M|T=p8|VcX^Jy6E^W0BC%c&r;Z&OA`(vY2PP%8v#9Atg-x5i)b^B{Q|sYM1J{8<_)g2P-&K4i%5ao z8hZI8HWc%8*09NBL$Oego2|Yt!unnmL$UQ}@%i92c3!V5qitZ&t}k3BNNU1mC6B?? z-%hvraDMQ$?EaOHxUb36IjQWfqiC;TlaTs@V)jeZYDXf>Vz|B@0649#5Ic`?N}GIP zL;)H;NuLmUM?UXbgNrrK7myec~|2jNb+xk7Xe2Ij98ZM}qaKY_|)4 zxGrv{0F4_TROo`dlo0X$r2V2fsDJuZ+}=d(XaC~6!}u}zKs5cysxSn4yTu^ITz^SX zbAR@>js{eLls~;gi5=VT82FMb1ob&4g|EK`f89PU<-bPeO8eBz*!{GJ9*w|}5cBw< zy3TQDG+^l^m6x=k@?p4uX=dD}64ZYS0s7@c3S`7_Ka<4ki4+9fUw*21`1SYWmkR=4 z2SABl1UIxY)XrAs4!>aE3uF5S#y?F4^ZzMw(j*;|*bUZiPm4JOC{glt-S5C}o@9pY zO$509xvi*$+mrA-FRy`cHRihiHqqa8f%`_I^&R8~c@5+TVTfLD9AAC(w3Oe4_zL~| zQ_6fJd;>EUG9Z_-H{GV&(co-r8lEUW@>3=WDa!rcWxaJ|8FkdzNX|o4&fNNlPC zAwN6m5laPFBv7E_@H!C4=Kl3M^;K#I{?^fN&tX!s*a~hl?^xroY6O`V_J6Y|rhKJFz~k#cd@0L z@-&E8kKG3MOwd)oa41m^PNjQyKh{f~Svy*+>l@sg!T$TX@A{x`L=}Xvm(riUZ{w;MWEb~q>mx_j<6UZfRZ#C&j7?!ry#kYa znV4Q4Ia<=tB`+30)_&%vMPd*3AP$uLDYf9*k8JAYnOwFzTy+CjYH;K<(vN!_Z#$?e zx-!e#D+xCg$qcDSLq$ykk~CvI15H80R!c2QW>-0&pcG*hCyUFNx~lrSs-ahJ`*(*N zmh{a-LUONA<@OSVTYi}@Y?`qY5eJN2GEXDr0HbQs6HHaMYlmg3Ycwxe9PRm8m%P5L zEt_m|hF+Ra+cDga6DeQ^Gh;E|EkuX(5Bs>V8?g+x1Y(4wzYo+0^Ht5$2=59vEDL6joqDY3Z%|7*~B>V47+u_*l`s>S^r*~*&2r7?U z(38pfJ1FuDZ;GcUFtV8_+BkgJYO?%NTbqNXdQt`S4JI$CMu)bWcpwM4bhLWMX7n{aEfN`R$v?y|Gub2?o%# z{wFJJX#ML6Tc59fP^QV5NC-66POg8og;e#`H0dY~MoWS(GBR>VBP(_BOr*Yn5XxDX zeUWj{r1|st->0c$yspEM#!ex6(`p*B=aJ>+A|&kU=qk=;D38M_*|Lqoy{8r|>gCJ- zZix245_|M7xmAMfp?NQi>V*Vxax?+z!M`#SCqYSk46ZP+LKQ9_b$pf;g#b{`%4hu3eS|$~`vl&BA}wFLxm#*_l=CnG&uAPV zpj!oe+IRnyQUwZNyD$vb{(Wq{pPK|y+@Hm3to&fb%FWMAU6SzKLewI9l7#mZvy(pP zi~aI4Iq;s6;9HoTS=IK70x!tCbe47q<1arickY6dpf6m8`DMC;3}InQJq5jG@!{1) z;-cQ8TZmwWTfl?w+-(m~b7y(t-FP{|RPZjW!@1{P1yG}0=<+`TkczI@^WlF%fqP}a z^-r03Ip z&5u&As$nH|tzZi<3+EoMkI&C&y?EIkW-ISihJGT2ejY7?a!cc0&DDgE^-kjgd3Q%5 zP3YfF0c>KW3Wc*8u_9b5jK6iJquY$1M^)ti;daoi0!0**@5`&++r#9roru z0=svX0@}XJGcfelaRL435o)^tyzsKic$IEz$Y=ktoA=NBK@agDueWC8ME5liviEBD zm2dSINzXiQH08r=!<8Rpx!#u~Zl|H4jEUo4>SOub{(fMniFrbwQ{Rb!)Ks9-2Gv5z zforyn#v8juEYdrC{kLv*xI7mY^quxZ8Rh6l{XX5|V7b{%32*+n0etlmJa8}+5*`Nq z?fSp&9zb)g1kVoSPD)`YP}}mhG^;gCa^H@VW2T(DE1&o*pSXg`?>M#7CJhFm#QV&bmIMlazN0`j z8^Sj^r;OSG7!`i3MJJALsZ4#NpI=c&chzu|r{Ya8L^C)GJNFjLwaeK*KlfKTa@|q6iI;=8 zO?hX`TaIRZ9+lg6(fiP5E9yA51?R{+Ciy$XR92;4H9r zew(hHSOSX7aXIUk+&|2-5N<#O9rLLwM-lg)#oQh2=j+6ZY_)s-CrqQQVb2$&Qv+yv z4mhLnh!TR6!=k;R+!-WRS+LV=FfwH0IQ^0d+&RP4_vhAsIwb%ON;UWvbUAbxBW0y! z5=C8F(TQA`P&v1vhv{r^b%tuY*ulJ`kpHr{(%x21dco67Z+V-Zi8P~vf)vvZH*MH! z@}s1{Ol`B#K_9~wmCf~7!9f8WI%(IeIee+2_r{$C8LZ~CQICtsG$&q!OqY$oMe2&K zOYMs7zxN8p1I{lM8|lMV)z(ZOl(6(0n=rK>c0G%tRu8lvu*_+udGu6j?@o9JZzQL& zN?n2t>mAvr`YKmRC2}MbJ7DC4po2Q8RJVf>n84NG`Z%Y*sQR4YFmhx;uG*R^3iI?o#s5n_s8=hZQ3a!kI4Bf1Av>y1ORC94|IhXJui~ zaGO_kxqAXkO_;1|RX5NhExaCz=!_)ngv6R&Lwjzvkr865JEz{5G5Vv$a`x_+ zt*tJAZ>_Wnh`#HRbkXD>koma7VTdTahAD00|3)eH<$&W>C4XX}?}V}|4t&h-{Y!J% zFUUHAM@COg5KE8z_5ytb4@LwZ0_}M}aXtSKN5w7Xjz)iY3^t`d?c}1~ShVit@p#-R zQeuU{+!m-HeB;UY+Zbkmj(ItKM3`@+jRf;^3KCpW5xMa$tlAK-r36-iXbuf=+f~7S zQRpNds?YdodZ8SMh{u}@^)L^GC%gI2z-@ZHL!nRk>C2XW3=LD3z6{QT$IQIpTO2p| zBPa+ue&#yjfKxw(MK-*3Jw}>6I(%>bQleIgdFg}Ba)br!C&A_WJ&NDGvUeD3A(rC76@nAz6#t$ml2%(~Tdk01sf4W9magBs@dpM?g9 zb`cHG032gJ$lR})@5)rdHK;T7r;`}34Jn7^SFtTGd_^xlM~bpO=F472#JLbAIKugV7i4z(8Sn5r)@Fw7H(Aryy0AEkW{2k(phXNt()dKyQ4uXFGT^ z?%Ppy`-rbYzKLgD;VTV~nr}vt`Ham%#X8hU9#Dd(bo}`N*i*uNc@$}`_ zZ!@4aoELvIPS`h8h|sFcU#L+SCq?K5`bGG7z$FIX(;vuU@5ulJPB9d_<`%B*ymd=a zQM#WGSqsKkf5)MP7jgVb3g`p)Ug@`)!5F#e!>#VULE(2sUD2bqQaqk?Pk1H`vnqGi zUg!7b|57cY`$lct_sD2CI-;Ng6%1^`;0H1?L5I(wa=4^+(41w}R z04LTJ=3*d=>F?`y)BhVaDSWg}0i-1l>mMuOJdGKT`5U@?xd#ictcARXomQOe`@>Xs z&mENDyEh)LZ-SR^?QY{O88Xj=XrX`Y)&gEJ6eKZI&^B4@?AD`9y|3kyq`n5P7GkyZ zgl`f98(|Wk-4^fSdBp6786IwGbszTqJs$ zgS-Ui=RzB`xrsW?tpcb~;eg+8`xT(9gEC*Cd^}cmvNS5XIC@Bwk$4E-xc_x-8R$TB zysYFLc&1t9U&7DdAM$iRT9pz0T;rsh>`dT86%`c`#7n*|@y(ZP3;NAY%hvvb279g6 zfAkHN6H~Kh^#iR2t%3zfZ!FK7VUA124h8DU?swj3V(UeZDJoH^2gUd8Ua}`26_YO4 zXm}TlQss@7PW#HI=&7_)SFP1>KtfoWTqc@peSR<3t{cK}DC|mBX%#i+y~>j)m`4ke zh?L1wXgE3Y=?v6~pN&!4EqOjn+j7>N60v;UI6TV4?ofx>bd1SEc~J zK!+Th;3WtD4GiXTLc;@C1Z`5)*Ate&JcD-hY>YC3aRf#nOLt59dB*>H)oP8?l*MPN zGZVfh!@;El4M`DEOg5g?qZdwF!o$Y%z-dligNCnPP4HX=AIJ#afKP zN4v+#7@^yC!qleVJmBemWaf4>v($+UfGK{eI z2aT{tPIoqaqA>-o)8ebcjezbjo+nfeUuTZS3~P=7AV!(m3t~_f#6r&nw8quQ;)|et zQ}djx+ysYlLm!18e$tn3U`1w0>f^VCaBW^yY~t^TuD-gLGdBi0Uk6=u|pKrspIoVP|q!7gy{2j89-w zi$7iS_mYvuJqPoBdlV1aLm0k^lapABf4e_!*j&Apd zJ+>IT3A_!xN-!Gum+D*)lCH5Iv5--Q1|Um6s~i90xwc9$hq6NhcY}bd0)NX%gYS4V z6ad22ho=GT^7KK0yvEX8pb7bmYmg|fHtbdbhU|*we1Ot}S=id|$FE$!rzgg+r4`37 z%aEwPhsn0%36D&+HB$ZrZYf3|R#04zQSBD$ZPGV}*tf_I$Aq}iW*m{Vi$IFGN68pN z%Bb~+eZu~YpuJ(W%p6|j61IGq^${G~*>uwzoTD_pJ*U|iI z!3T39i_@nb67r(rQs{Ln&?z}sRPHjfv~?RGzdbf3#U%@ zexrKvpDW+8VKSTj-q`9ct*>)^|Li%w;M1Kn#VfL?Ki=mBE$a%l(B5yG$+}7Ks`1&* z_*pfH$I(T+*ELAF8(BB_bc*tNlA6lxq1p97SNCvc1Hi@|j@+Oc%&+iH+Wt>9+~Oh! z#Gn8iqhwH)*9Sf(OC~1%#~g=sItn^eqNjSsC|~sM15QeX5p7ZuF-CNYOc%Co@pJ`| zdR;l6)@*LhGk0%L3En;!ie{`}?P`Yt+T7p6o#3^~JV5aMv*yC)Z*EoE)X|#zd8+)k zHCRW4%LBOtuT5)bSh*z>+zBss+23oepDcH7Q}rNKB-mDETHK}$tlO2WJ;eB1=%?AJ zeDY&DXhW+pN+@bj{i)IcEsIw{r>Pz!Vyizx)&chjMpnTfLl&p;_$F(T2F23ZQr;pb%&(lcJ_uIFcyvU^%S zbd$Iwe=9%F7H6z?3^_+5N<_=LOitwU!~`W`)WI-tJitlTifA>$qp#6tvC^^zX> zL*FjGvXq;J?d`O-Boasf{&StGtT|j7(r$L+YtWwi9~7N4>~EKnfz{h$eP5bL2(W{oK~hdd&0To z7-i71@^nMt8#y(_l3h+#vr&=ksgd$o_0Cy%94voEP*a1SX;*Yt1>xI!$iwt*p?$!;yb&ux9DDb~=*vK1gfNRJ+MUEpL=ivd`STe&NoH z$qhaF84o*U{DV*;6BRIrb3t#38g&P>8$HBPjbnH(IHvbDS1=mWG-D{>p*wu67FUfA zAIu=0?x1Le`TZ=X>uC4j!@GlBNi5BKZ^g2QUo@uX0lzcVt6{irL=yIL8}i`lLvBAH z$sboJ>}WX^*u2p*7LOPS_jD6$e;1!Y|G=8!y>mY?zUeD%_>&v5*+RDHXdDe2u>|FB zq2@|h#8<_+=KL1Mpxw%SnL28cu|5wIdt@rO_|ys-$i#2gT||-@cYSP+JwLjw&s>|L z2~EQ-^o;LFD{$51DcCLAF(2S9ezx(f9bRw0K2I>^G(0U*HpaDQZ4W<3?1Ir>;@5tj zBE6SWOOwAc%#eiNx-MV7_$GKU6wv#XQzF)WRs<;+Su503qJ~2$sj)OnM)J~6s}ui? z>pqdt!eVI7S*Loe!CthzxvDilwePu~xyDJbJK@s9N9`;GPY9%{T!Q1AbaVd8R2Kpx zIk}mR^_%GUcOO;7LGF*=29P&jzn!p3ocC4dAwfqu6+4@6-bL#?jxQ5|F%RD*K`djI zRPhR9o!_*qAb|3lkX(Crb;#w z3>v|4t`^!J2uJ%32=SVF@2Jtk^Tll=oOOiThLARzd5Ym9Yd02gCQxN^;}^jVSj;Ur zd1orc_E;@~JCGb=jW3_fcoU=HtM0})+m!Avhg*@RuH&iPbOwk>KT_TJ5yihTQ#5%O z_SepqrH7hBZQd_XbON5;(L>xh5c|&LPZe68ti!6SqARzAE^u6Bj;C2D*Xx}=1`MOn z9j6iY`vS+6Oay|OebhU-_6mnmv=^Sj+C=}L{BxO;*ur_ukDr1Ch@f2$KLNP2JDeSD zx5ZO3uvg1-P*4_6tA?qjI=5W~P1q^r~Cfrjq^ zoTiWa8icZ~V~N2@W6J*;YM7qtK(bP)c8P*Z98?!Zz-LnUh86>BN{cJkRPJxq*p3Lz zm!GDzDBjb3vtS}dxPcje&d3@WJ&N;;9{Z`rQG zaeCYeBz?JB8WdS+7VrUt!7x2U`xSfHl0PAI>>+c(f;xG37g0p80+8 zn4`Xq+I`{}w-CT{C;_KZ{_L&o?Dt_WY%5cmST_}GtSc^%Tvz?$mBgzm&2L}(>S4=- zob}om;loG+-;131<6?_y9Ychg=BEwBYP2B}tNMK0-@ze80!pkA$PBGZ&5S;qk$i1H zWtIsaX*DPGpJtkU)TeD_m)};cDrr{Z)t-qoqx?=PW@dFXyL+xww(M$(Qu2dH|2I<5 zKRt$~wWv+jvBjRkHF%QdhY0Ui^vz=Sa~ee&ic{mT`O3Bd^s!)QLa1Qa`&%(}fe5jq zOuTXc(!X`Q1bFoP@&M#7+veQdI)RrOSuNtr*=wjc5MNxp$0)aj!NfYm?WjV2BYbQz z_ZdGsf9$1Ii^*9-1A2az6UTR?94DR*t0>N2j!9XXBJyp`;h1k?!wuv4>#o(SNu-=1p^|Mis&&uxF za~J%d>(5(}0$M2?wHEm}IJ(44wn(2WJ!`Z&G-!Q`$z2dCgEGH5br)-D-*ExO3F zZZ!|V1dAe1cB?|UIfzeKf)3BnlZ~Fvn$vlPx{3@84DBd4W76a*`dnT6(b^U!%VsY} zQVu1D^^zQ`u=5cY7!<77;GtFS!fal@Ms*h-N4Al;&!gM+@+)f|W46qwIQ8qa*q4a7 zZ|Q{9w?x7rUkvo}uCii1^@9r{}09}8VgOEmOF3W9oH-_p(+ifX{>oHF55T!<%cuI*d`Y0>t?Yf)h6d(x)>0!n4=- z0hLHjV;+DEv!5U)EU8LK=3lVtlDrJ1G**WV{-JHD{?tSf)bbMBjurcO)C5&JdK0&1 zS{uVbIyj)={~kP6T^B6I$B&8A)gBTvquXgRE*2qdGU-*qqkY5=LqL_L3ca;iqJEav zw3f0B&rdRanoa*jjkxusdzvP0f0E>krSN!FNyt6n?ayAn1Yc`pbmgDAuN+{VxW3H0 znXa@tuNC5xUb{{y5}LHLvj~l}D#xfMl?jEN>EoNRw%HtFjtf6sOVPt3C&Oi_y(E<~ zo0_$|q?9g0izNLBbVPY=Ar1`W+VD=kCg>?XnbRorfD&Sd7Pa7~=uG8>aI;B55kzp+ z`;QSeG}(sn++yba=3||TR;nj+^5R>sAti&)i?$Z&RU>he28fTis_HFr*N z7TA$|T(K|{AG6h}GGzdBX`d=CZRVkUoL+M3zT{Dk5kwffYk}=rzdkxr8Tu#Jr3Q|_|st`3KD9RdKdx+U;B5lG- z1a1LYR>zO}veJ0S*L^rWtKWxfFSlxt44pfbzU94Q<`IqibaHmEhRxL{#(lv^^N$C= zf3wh*a`Y$IH1Z?4U6eRLeJQbQYID;!1gdBQ8m)6F-EStK>KA@_{i0dj zCB|bDipai}C_*;IGT>z&tc{)vb$hUr`?{J_)0W$JbGqJ}VBT^XyKrqKC;FbE2*jPA zyxVlGo0iE6a3I|?c&h(K}TUQ4GTT~UIhyX61U!4507#u*`#d1 z8$7w4hW!eSt2!6^{C!0KBONv4&L!;0Clg*qJmlLLD+}%Te*h~$)V~+8%#0a{ke{ybNKFARDi}fD)K7(!$mb8@f$7ZCTq)Q3rDzEx~cuE#UZu@BHQc zB5dPfFZe*M=e6J98}Oq0R9B;!a2I)d41Ozh@qCemB;I$8Qe1WAk+SkdP#C6?~A@_i?dPy#JL(i81@H<+HdqSl;R@XLah`4^q9?2?M8t z5D%9np|;!6osZ#(do#i2S6X|$YdFFkSNJI@x%1*&8W}O5^q`XhU}VrqvOe2f3NnZG z{}5@8(UkyQ*6yg|O#2b&~Gbwv&!+8`CrM&HMd1KdV+%?b`b} zYn{?Co$fi{q?9l=h-zQh%NW5jE2-2{Z-G%xDk+3b4mLQT09zSnQLGRoE&(P~DX39f zum>(3?rG8wTaD#GB4nZSW}eAYao`p9?oJ?1j6nmVWoKvQ)q(Vt=<5ax$a zMxC3ClXzB-Iz-%*55j2#h2j9Jm_fAOeAhDR>6n)ykP3`W>e=3 zCBA{-M7mvR;r^)@v|kN7hhLDZ?^v29rpfq?sU@0d31n@?0o+2@{zVP~F=j{tR@ zCP2f>8>d&gDuJAR14g#B!d^r8^PQGk(tWMLHDa(;P{S2xoO>J^!p1$l{GX~TSC|6R z&q4j8cXRsCT!8`TU-JQGO_y@)!} z1^B&?k&z(JJ+{00=ZYY8y@^H`3{&5(d=dnLg<%r*)wfd*b z*S2NncUEPepIEnl%E2ftz2PLcI`#iO@`tctLUrDPh-Y>kqi|875eynoBC^@8hO7hN z==d+jzF*7=I6v9CJ3X|hUw_PZ4?%4S#7bOsXF?NofZ?e)l1>jkz@$vgCzYsp`erf# z;e^3+74B7|2OxzK)g4T}RjjkEQKGeH%2=d&HgKfS z2Q$#w^Jk_r(h;vzmcU^UljvChRaE%e4hl@7wN}cK$iVO$uP&Y3e4(%RRd`$^fHVKh zwu6X85Kp)(pZ7HZ3*S{TvwROjhJ37)G|)xG2z!~F3hD>$1I?mW=Rw7Rq85tS4uILZ zb(&kjiq6;9H4Sm$`^PP=T0`g=H!541D7@ENs@57;N63pZ8@%^dh(aDLf z{WAHS(-Pb}=VtlKmAgzlwoHc83ya>&Va0G#M*U8I%l}(Q4+kz=_~)-*clz)_1YEm+ zvkSA_oi9E84O_<8PUdwyK6TmlDe&E%<}vm-z-S_$aku=bhOoUM+P{C=P6Nys z`5>Zm-7y}{8F?+&ZOo$28&iMOWUbvYu%XS(KIc8`7rP_}=O zGI%+04l?ul588XMDa)>z6FP6+@2fF=>JB@vJcoGueKL1_t@D@RJK{sT)($g_Idi$> zBW?6~KOdMr-9&9fK^)2H*@CDk&xgU*CDZ=;lXthhLD`rRZT@{ zDlqauyL2weqCjg`+EBx|^o0X~_sIXXHT?Y#TfD$Kq*#rjhv+`6acDyx+l}Wt$aC4J?(2JK43`ZX?JMr{!VwCxZK7U5Zl4FlW?_k_WPE-5 zB;)=oItiOJeAEG{<=Q~2q>#J2d)4nhqOcJ~5yUV~>Gi||@L-@;Q1k6bB;Z1-i&7Ep z=v>h%%SKzM_2*yYc$04;kb)y_YZ-V^K7w`)bm3y=%?mYa?DU=S6m>%frmZcSQIlvNpyzt&`9IB{D{QeMLRnFh0tVrb%g{SLG|s=6!cumIPxWq zhDdWK^jXcg6FEAw`rZ&FT^9@pkny!VDpep7@2Ei(00ok{#VogEsQ^1P<|GQXWspZG z35JC%5-3^-bXHVA0KqjYdido1Bqs+QimGx-9BU1s^^tT4?=>uKhY~OBC@xH{exrq~ zlnKM%>mw$f^Guo6jBlwGV?)%l&RH~JfXFv&DZvgOa18V!E6(Hsoz3!#iv}@$Gvq6r z%a~A-BY=^iAFWu*E?H8JF>apGIGQE3KNCP)Fk#OtGPtS9aojzj%UYaNlOUY#_~;tq zBKn>Dl;e(Rzi!N&|9q{TIiK73oLI&AgynucHt=+l`!xBY@81*prz^dmqdmac?j2kr zkGBWfooe=O|3b&9_e1vUG5#gKt}DMO|924UbZ)QG8^IWVcFo7TyZ1%j>cKwI&FEHl za#Ob-;0`mlr@@({knHU=Q{T6KakVqg~+0DP^`2mrn+~GLU`Q^5=)p^O0sOWZ8 zBqNJ{Khc_DJ1XzVd2^X8j`K$ClKaVpLCN0uOQiKhzYSCcnBWg=67c8xB)N%}*(>x|FFcSybc3-9Vk`4eq|K>6V1jp67$+lb%&(!Bloy6(DsE3aSEzS4PH z2)gm0L4SXxX>0Tai(qo2H{R)dbAD!=@aQ$#j&#G?{&!vH(+_y&Dxt~UWb^#7mN)-Y z=gk%lT7d^cq_vLZpezAA zU3)mu-@0Fv`KRHjwkdOi#IjbS2kmKV+&H@GIP&c_Rfb_Ng+Pt`AR1QUCU@R%vY%3C zGTdz|-#d;kA~Q6V*Kco7=f+_uCf+A~y7^vdJ@4L_|LaYJ>tn8X)%ZZ-<=AkYInR{$ zX8n%qY3S&fcN?F-&ARLAfk59oUNfC=WymY8VXQajdDOQ3u-ks837W7sNTDNm^?;DD z`?XwWsTDcg){$EM(^og{&*z7s*_k)q72rBrp-@AW8?!x*uwklnER z!@RGR;Q%#NiTl|0e7)Uk>)hka|E~Y(?;vx!Z zG`!xj&YLfOb+oy_7IQ~c=X{YGW)YKxss%xfdwclZ2$!V9T;v5z0X26~Knc_{ELb&l z(zZzm)OA|3SyW@iFiPlw!EAEj7`h0GTF8-iHBD_mq2q4^RwXJTT1oHNp5^1C(g6Us z)Go2HG{H?o6dZwb+cgUG{;!suPD<-iKk8CNdegk{4JA65`Xvw*x?~dXhb^a+Rb5uA z@^JGM8Ivcsrk`D?uwlzpGa}R)waHXFiE_|mdn>%duTjk;pou1XO59T1PsvhP)l0vk zlroqBr5p*noI&hBW~3QTYlkI}0$PW24BmXCdz_|HXsuCTUYC&3*0eYpE%u_p?SP#T zelFDbb8SM&<;nTE;7^)yZ?DWBW^4pB#hNwAV$9jG*#p?x=He3K-L4%flNuZ9ros)< zM;QT5N&qaQ=ID-91{t4QUgcEXGnz2*?_R5q&W9qP68t zggY3U<}bnl$(}$-nGj~GqY9j1ry~n?ymoeqF9Wy2&B@AQ^ zL7!TPvE?8*CA?<&2`^#0K4^KK)a-bkWN(HShUoIigA@_|K3?Xy){lF?%fAPIe2(7F zo(I$I3`~`0xlGyr_#M+Ra3AbV-&@V9I`P894X_Uaj#gcf4s!sUGDA& zQ#SMvj$Y)qN6ExL*AtuSib!S+62~1-7yYcDsUoMC2nKi8ck>71i_l>xC=gV<<*y6mK zV&$w9*e+}Ryh_53SoypY*bSk5+KIvApN>qw7*3T?pjdf?*Kzd>R`7pF*Li>W+78+k zJ)H0*+G_u>?l2c;R2J|e8_xG9(0}vEos)lSS~RmiIA5f81QHtYFjwfp+-yW&&Zt>HxN z++Ow1j-cD#VTb3Tx{LmsrTm?taC4`n6Z4z1?5SXptpv^1iB-v?Y$%3YP}-z!YI$Nr^~bndngq+v*LZ9FD5Q^?s&m}J@2>h zGCjz4-2{)f?Nl>1HR7nD6d| z{z{bT<0dmIFW#CyC;H@=zI-O~F<5_yy@3;-3BA0?v=>Er)T1t_emj+~>CFQC7F@di&fVECo&z zf)=5=!B>7`1CgzxuXVs?o$tkIq6DE~x~`k&JP+YA!3U_$~3OC)ul1iX{)}&xi>h#$lg^Q zZNt#vnWOYksQ@ZoA#7AJlmn;C1)MC+p<~BA>QpCJ zt!$TO%2%zU;%2@Rd;iVP>Kaqneurc4r8Sbm4~H7*Y>npm&NWL@+n_>iV3iz&(?0E; zRFXjrdE8#1O7ky=?vGsdS#0^9HUIHO8!A$^l6gz_s2}{+SG1I5?czh;9Opbqbhe2U zx#iP1#?9G-k=n^*+zD|xCx-b;f3>b*n5&?pruSlGBlXmD;zTqQ~4zTeYM$cjZZOl(v5*l zX>HhK&BT**5W&>3fhi#7f+jlHRH1PC*kp(!5*hj-nmM+9tz>RH`GP!vY z;Vg-3y1pYjQ;%?n>TxMEGnU4M1d}vzR+1(LMu!s&kN77Fj?x6U6M{^sYrO>1C@t1v zt*T^`JTJ0B_xeCE?s0OsKyi{|%QChkljA(CD=F4s*B5N5ntr@sHKd&^l_;=8Oh|Bs z=h;eouk&~VNHXl+FUu15+flivBlAxWyWB73M{kJ!pa(Ih2k#U9U0im)2Nc4up+`K6 zmeq#iBj~0@NLipSi-O^?_n1ymKgFBoC*=($Joh$;97y|4ArnX`p6g zFd@__TJc+z?-J%%YozQt7@aK+baOH0#n&>BQG~-VmaTcG(xAW9J{h%Cg_b25A+g9! z)YKzu&4IL$LJGEjG5@z_nqkanv09CB%Q_!=EO_HYLa7s>6@`zC`JpC_ zN(&$zzyb8_bq?T4m=)xitxNzc0@F|lBxZL1gOdOXPi)I|*CN#n3fT5s=x0J*3?f^PEKR3F!c3 z2wJ>^BZlF?w00q>7!`XAYUGnNRRAd8vlKOph|6>U=ptfN5^8?pOhG8`+5Pl8t@^J( zfsVQ!Z-$PDz4gU~oBNGNT05=X6W)J;K+y9{8K`@2E&n=1uwb4uA{8DF$Uu3wHtJ8r zhBU+uaJA{0v@x@7!ARG2q~Bx*aF#TSrhbbtJ{_6|I8xH4(HAIL?!6S(p`gV*W z`-5oiWty~=+Z;6-`(o!fVG=7b;CsT9ICY>Pv1)C<XJ_K*>E7jfYG}_gc=tv5Oi0-I z&T{F0$++RL$=L2bh`{gIOMTvzwwv>Bv2O5x6t85MhopQNr2)D+N)_OZS^!i6sG%N! z{b&V7gU;~!q~3lgh6Pk9q~h@>CMQXRS&<}))Z%zi*m=HoBzX2JCZ`m5vnqSX+OMR4 zQzHVuLne}1QoI0lpLK!(Ap?#R+uOur@Yit-#S#bWR8l%LM%AU~?LQNV*D3OEt`JhstqQUIyB5!6)UBuLpbqiRZ;)C%JIAtp~@ zO6~x<;I2=KSYTl*iA716hUyI=OC2>N+y#AZDJjN90;$080W)TZ&oxBZsq0%qJ~m@Q16o zidrHivSkq{*=+mfHvRRGDVA-Trwu2Yn;yApt zwIyj&k~GX4HunG~Lq<*LP>|du=u%^$-^dA`J2EN}Nm>Zc?zmYu%RKi*t*xYjKQTT$ z{`IR#m%{Q4rxb-C7lbTusUL-DUZ~_SIpwTKmFVC`rqlz9)PtO7l38+nOg%DUtdoEm zYZRDhOh5>VMs!nNSyRc7admknZo)raunxRNGRN1mEE&6Klq-|Oof!e4);Z{*S%;>9 z#fl3j2?9h_Qi_XH%BxA(q~;S!mNuc_kdbVPSuiOXO2k?I zsR>zXBnjkO)0Q?26vp3GTu_#|5=>1+jK5Et!lp^2oS|XN;0i*CF@XGfq+;63)!#{S z_dEDHQ7U%-3VRKMiKxQyIm`p&Zvu1U3wjv-{%872e8r$C)$y+>dv*&B^2ZKyvof1C zVh$vpb>jqSYPq{9BTPhp028Wg2217ukoSIu3;_v=l#0o24l<42KQpj-bnNt7Fa%)+qZ-H2QbPPQk*g;Q(7&X%#y&+86*iLavZ-E$iElNY`8W&gwoMbtj4Dlip*^Y#eD#ceu;01Y{O9IUx|Hmr+n=_J)`^Ai6 zOv`f*nZ6F7?z?yeCCn2KTKr1XJWu5hsntn;^Kc*At7$6PD{9wg#7W~6LV~DqC%LB? z^#-U)D_l}ERW6Qh*X>Lnr@TqW;=+mAf-hSCMql2R_$1NM2kMz3>im0N`bB?$n5Yt` z&2BAh?jUymwm~XW9BlKD8IB?LoW(+=dKJ2KyKke4r-{g@j=eEUuxlEql2;Tp$9dt* z6%bUQ8Z)1v!<29OouojK3IN%gpd`q{CkK;B)Zcn~+#DgMXV9q2VWygut}Ukkk;z;m zTyOz1!sL}7uBavOYgD<4UL=i+R7!L6k@hp4;yoN z4pc#+X9;n*U&NoLyELqX*IZa0zLySU{)+*L17%8u#U`YQ0?oW>VltFaI?1~m40L2# zk*!>EvThs;kymW5ar_cYW&43HO@M8jFc+w}aEAErz1V_5)B|RUL^es}gyABVLIClF zj{U{d&ILT2Tmk|W0!hjnd8Ha{ou=OB%Y+56&16j`cCm_Y)uJ*|64i-w#;n(B0qLT@ zMK!|coHWw20r!XqbFyergQW73uU2v~qlzGa13x%sH=+ff$4T z(}lm24HN1-B?dD84aea`Mp!P|oVLud=dnFZodr8fHHrsvC6qGjL>5bGuS-I&cr3m7 zq@hhqub@I|<2Budrugjl0msajXuX5QHG&u^(4+S{|AyC{s?T+2K1C{KNH1<|8k}g} zE?LX>%6d**O=0}OfME?0Y)lvtA$66B$XI{}?~;a!N)KK*YWuoo)8>G8xN@F&XPj%C zp!Mk#h{`oVGuk=rv&&EIaTz8kA+}tebitDkuaq>6!8g>YeWXH3w?rzKDMpEAq^78o z%pwui+Q)8kea@1q1_czr;h|=!cAL*9O=~iRF(Mn5fTXoQgdLtx#YWHNi2_p?s1tJ) z+5;bt6@W!7W-ZJ!VWk;hT^fDR?Ja1OGhWQv3_9keDov#6X-fe%rR-&VgV#&qmintv z?Wh8WA}mIyr)PT7=$CRGb4DYSWLP>ZIcY%8xJ{)hNakFPKXNUP3WWlM4OK*WPEji6 zKB+WggH|4r$DV#q_c|VhBv;ywXM9L3L3G2fsT?SdJWYT^3h&XdFaw}AScnA=r#&>b zDH%Jp2e0bt(p_+sj@NlH80DY)d z;Y$SoNse84_Wy~T|JKuIdqc0}1P>B_%GqTbhJYevW0psshGNvK0Yqdu8C>NOLLGzt z6ydu4_zH6_oyPB&Pbm23d7>}b28|LGy1C9-pj&P$8pmH3f#45_kOt9*1w~f&gUs%( zo&_f;T}+df07+HBWFC=7m`Vs20BH*ND#ItGAWTkiOIFvI)cCCQ9x7@nn`FELzxxnP zdIBf`=vuO*B26Ad+(S0Y9iTQVMU6o~kZUs3&{&>6;nmU7MrS;+GK(ICvgFml%7*6} zL<|mGGBda}ikaidnS=+zT_rA&sMbPTF25KY>+n&G7gM)Wl&aQ=!KA<`t_3!kEES{* zQ$>mfP3lgXj7wFg0>V8h;N;HkL*lpEaD!jWTBZnqm&DvhCE5hueH#q zgn+?0Ol->4cQ80JE?~ix<kU3oI?_pw-wtSi+|nXh*cW%0G&K3p=J_GC90x=lVN8=SZU_& z;wwwq5^7A$@hHuEVV=`lqxy+UScBuhLR1-$j(~h{?6+ZkUApF;L}3vNBl1F}Id$fQ zq+DPOKOC{0LkrK z)!mZ-WiM(IxWZfVoZ3S$<4Q8o{!Vy;Af&zrf4GLA#hl=ZNqLbTP+Q4V6F?Ch{+j9A z4^{5$El1qtEGU2grcHSNT$N0^{GY7)Z#g|54SgqLE@rP~va{k)*z69OA{I3z*mgn; zx4}q7hz4_nAoPEa`jgWII}ukwtu7b~olGu^ z%PKa83U0Q>D4#H%6{%S~(oJbx?E+6hrbI#Sc@~1x#~LqQf;KXVrcN(junHg)<;t-9 z(?SmTV~GoQDN##m_XorxF1Tc<$!8Mu>IyDxRfT4RanpDM+ZGdcKA?y~AYO7S%Vq38uI(E^F|(tF)~` z)xH-7urz`TH-H&JnlUjUXBZevSnDAWk{ROxJn(zJkFN}28}|pt_JaKbJaR!)RAFMs zsDhc7Lq%1}%WJ=tayt&fd4PHg3j z9vF)B%kM_=nBj-H6g{n?;)tU9<^&jh=D8?XxC9YSJWzKB^wn%)g$%aHJ(DoxbbP>E z=8W7Pofc0D?tpkW8#6gX;2mf=mkLRn7AD$KtTlxbFS(priIu`CP8dmEjJL%)u7od& zl|0%qKTWPa8Y|pQm?C)x$aJgo4^lK^7floJJG;6CdrTSP^j( zHR5ca@j;HbBU@{36#VSJOjQ=4%yY9%!zt-S6)Lt)wz-hLw4fent&(I`)}2>KMLcBw%nDiuHeizQ#eQ$bL!uUnjWZadfutOv znDUNs5hPIDArD2{4<(tT@0=j(AXbCsc9_3jDEH6H_q?FDdAV4Bp5w_Ax%-yC^M$}G zqwmRqrTb~v`1J~P`Fwm(jboOI^#7Fhg^ACG%DOn+{`Dzg;f~upmohm_Byspdf&ZTG zwOvSBP757hpgrv|OqK2>#fQJ96ECXSUa$|8hr zyT3xN-Y-Tt%|PUX2x;+SanS>WDYauxQp}{VsL|V;)j`w4G_PJJJuMXy93mo-r23V~ zit6J0&}H1hdc9p<5Bqv&E>K9T1mGPbE9oSZ{~{5!hRrg~uv5YsgS}S^%~GfYNK74- zu&K&PHSQN-dikIzJZBZ4mM7|!6`N7bl42SPLM5RQ*Mc46xgFaT6V+1_r4Q{<-iK0E zW#KB4YKBc*CG1evfUwj`Vl#?rEYZh4On((BO@fvA04Ms9sZTdj_8QS9XIQ|7lL}HP zMf~|q&g^&C%R!7htD#AW*bbtms$pjjci;`KdTL0}BsUYd5JbJOQ4)YO;T4t9IE%fE z97=3}A6J5{*!f zpi~vv5Y$ZN;p3H=7aS`U$4Kjg&%CO_Mkc3Br7~x;$28(y&rl27WtE838@h@dDN6PXDJmZaoFR<-L1E#Ww$kK0a{ zjxE;a{N0I9|6?z*jDTRJF7zCaX#Z&Q-g84npuc>J5JtBB0&B(NfHK23Z1e|-ah9?5 ziqEd5&f}Q=Eh$J?iX8PN%xYg=#1_w~9f#Wi`lix4Ygcrx#FXDKV63cLYRQL#K z!x64ErFSlGjk^E3+Ub~&+okWm|4u=n(0dOQXDA$`(poMygz1=r2?``!nLhpuK8S}9 z`>sLqBbd|Jn%*Ox%q4VNL|Pr4HVX7FYpA88A8?Rhe-9e9DhN!^a1Ui9W!go8aMda8 zxDpkOM7VKC2iB#b*s-TW0UAfwLm*N`7DLNI(FoKd-cpDonhwVPtW-goz$7^1p9cF_ z;L12DR6X3J1hZB^HZ(ETTQ<~jAEfb4bZeTFB{niC+PGuPJUg=yxNx*ENw^dTw&ak9 zj;ba|L+AuucqCyX)YOctSRL-Q6nNn(LT!nPMxJnJI2oCQ zeO943aTIs(lc5KsJ zLbAC4?MDJk)YzQFOgu(nxD>Lql#hu8oRAFsZBYRYekOYJZU&Xr>?~5C2^XVgW0Wfy zfjqWc!jfTAjvIP0nGhXfXbqrLLQ`Zuy@ZA|jt!Canu_KilbW@0tVu_xwZg)BP_!!a zI<0-90MonLnqy{7$!l2HD&q?4KBmZl-X?EO7Jo`b64sX5f)ifaFqCqf)~W9J5JL%_ zd3L7RB$7YDO4w1L`z{RGX=_C0Wg=XED2#{{@pYx@*}slI@q;U>l`HyB*SNafGZz2* zlK(F2*X{7eRxWJbqu~;b>r^n%eCq55vz@_I7)ReD+5bu>#En?+mrwwqW6V4cT_PZ&!S& z#{N`C1;6d_S|>Q~Pt=`O%QA|$?#nx_?#i#f-cPlkqh)^G9Fpa6J^DnDgkc-~E1V9T z7C?$|5cdavmH3c>NmA~*yR)QmiX_fqlO7WaM|Oqu9AGB*N;x%eGMs2=ZVucS4R#5= zV|#+t?oUurGX%Y+N5)EKbL|KFS1i2Xc6g(A!a5x#!UxC zN)Xz>cI8k@K_m^J4%LXXLjptC3~q^=l|+M$2cc9lmnft&M|1I^m>9iWt+^Iev*5exF9kyu3>naf@{F+(pHePY*OJH4G;@+a8H z`t!*@=%9v-z#5<@381eK*z78-mjaXqkXQht(qgWQ7*rkzmASba4Cqi}tYpel0R&8# zdn+%M*3UB#L9BjnzB-+k67f~>{y10dkEt=d^HoRP6Lp>2EPhu*?#`22-8qaMR<0ji zs=Qu*$J=jpTkG3ieQIBxhdrO*nYyY4fZPq+ss<_Ey`5MsnN#0+Ezp`qu1p26tbVYxq1m_KjCVSA%cJ-m4c6}d<+Nw)& z`bKT~*PS$Bs>Z|Q=TEJdRal|4gNn6|D;p~7^Mf=hs~I~hI6SR2XkTjS(cfYOC7{Aq zv5}d>OW2FYzC9a@h9?kd2)Mn~!w7Pe27uJ2M@*NS`iz;Qf{#ALq~Rl)-Pc3x*f9~6 zv491sSyT*B+IWV&yb_~Ny?Nc%1z(Ac?C0EeXUQ@n6}qV7aZ;s@m?0#4ydw9ImxJE2 zD2WH}YiOYp94)aNEhL7pm6Ne>6fPua7y8VETt(0$LIt}{#49JE039{{`bQ)lmFg0z zr4Vc6OrNo|2ttZSNj)G`l$#d@WBJ5|q@;&NER+(3_&P!jwdhxFxppcx{1{g?2M{#| z9UH~4w1+eJ2FV(+k1ND+gi1Jeh-$B*!{8VCK%)^5GbT2^Q40Gmfiuz?{1E2@^Dbe)!@N`ru!0ypbHZu~Tj z*fK^59m7%plA0JpMl%o{JhDeaj_fw(m+@L(0#z(3(iKud!jkeK`jDJrH$|v>fq5K+RcmoppcE zbBN01yrkj1BMnvaXX{4_q|y26&?1=&$jnMD2_k89go*;FB{6V-$k7MfVq0zka+%MUAG-8P<{QM zHVzscI(DF>(sWUDqRgDA|2JY4myK_DWxFRkffqQk-;J7DY+RG)-|uCI=8|h<_?`

    J5x??*h?{1gT}zB zb$+igw)}>+`pttiN}Q@1+zxlWXI4R0kpZj>ixo40j>X(e0hgQzk|Y)_;M8eDgmI6> ztib{>o@7=l){(0*vAiKF(*xNNDL1sMP8SCqZj+mnUM$dBl>`;m1%IM+?x@pS=}^C^G0bejXQJT8?H>P6P&-fx zTGl^-3})0hZ8nBzlp{ee)5tFz^b-CHGsRPY7_o(P!5x?{a#$Mt^kk?iRHd{Q@b+Vv z(p3Q1@l1wF6+re_`KV!@sl?QPh5h3*X!OZE9A1vbpU9VwQig?0RLTeNU;aGZT)q!0 zDiyuZVZCU55?fkt`{u^lsez`1O=2ka22#U1$amMA6aW>8unmJ4osfl}$ z+`k>7&vz&nR)w`;8wKeZt}oy%CzKY8vbun`p2n21*_s;42fywyZdFqO=t~-4o+zb! z33(0NP;fBnXo{W4bE_yceqJTarAW+?gj`zsPP*hsbm=JhJIU=&?;N~!buPei6@7c2 z+8*p~tYBq8qLeEFwY}AA?iq;?mZsj7@huOH1)n$7j5hhZ+jAB-zm{)Y*Wdt841ek7 z&QMqW{;#eXdiK9Ou(3|A*0Oes*22itjJjw4*Tyvtlcsj^fpzt3o4hKcQ@j7UEz!#U z$znfcj3_|q6vvvdNr6iCN<<}8R45CnbSlWtcs$Y+B2qaLNBX1Eg`6YijtMmK@H0^8 zND zTN=AZ;@ch>4JPtW?b`Fi`p^+i^90|xrp{*0j5T$C8oW95jmHm8)hQT`vs+OljNzDN zw?8$mu^UvGse=O}U%aVp^egY}-hU)t5$~H!WG9;ag4Nmq`RDbexS>_R4XpxhFcxux z@v^wVSi%j)PlOwY*DLeiy#2s48^ZBz-@JYQGn3)xP4$}xx;8c1O?AT6fot<3ZrJuc zgd6_;_LZNV&Skbdyi&YEiSQiM$&Zs^DEkJA5+RIRcn#}dY50SgIIVXq%oc#Iq=o+( zLWt6#L!os!r{&=g2^ur6bS?jUW~suhc3%ck2RvmDkPBnwa3fGndAbTQB0#)2r}TnGBomJ9uAcx7r~2#opKgqi@+s4 zw&$V67b!V}7D=^UBPA3XB`|f3Rx%UU)nB))N=4CfN@2*2?#ivYdsVr)W%uYgJOMT1 zQdp6j=x_7&jJiT@8EfM#ZiCNj4lTkxysCF7(8~vU zdjmY7hH`%kL=;&z%NkMwqUandpLMyY&T+#vBMM=5XR3%OhFs;ddEo(zmBEz|Me6?$ zP(0MI``N9HJ6C2jG6|tp$&*9VElnF*JdvS2J$oU&q!L;!-zmU}#6U7L(OsoL8PEhJ zRcDs(Y956)xT|3_+dSPLy=jcL8Zp;1HI$!B#2oSuf^>eQ-vFyY}>rNbR5rw4)w zjn%*!wMwX%(7==V;z)=ksy{K7ApS+CeMrW+g+7}X-=rZdN9WJMIa23}I^A)n<{`!~^8}cw z1An&!OE`==hsC5Kx}{1X4_|iSzBw#03H8CF0+zUCF_u6mwc{j2shlbGBFJLpSO&rb zh%w7Ymw?p84jW;w!ASy`IF|QhpmI2!7v^1xO&4PN%K=l5mncov@^3A@?8#Q>(x-Jm8AqS>%pg9K; zIpE8ITmf(9bxNB}c}o&(NrGibkV%3_5H@cWnuPX zUIBkmW@h7YDo_L|Eti0l$m+Kvk=3i;6;6)`ctvZwK*$q7qi&q(FwL#qPM8B-9r zbhSd@l3SDG*EHC{lf#eiSqxk>T3V&y)B<#2)!qA_5uwX{>oRMbdV4`rXUY$1GEz^L;O2JvZ`Fh!Yn@krjmNsCd8 zc9CcCQXsQ1W7t7nqTZj<#Rv921CdNqg-!-_p-fSh8>pOkU}ZU;d}Lkz;Z?z^EnmHU z;I?r-$R6=Dt;;ozXKiK(`2t$o$X-FR?tQ$hALTie}ORnn??2Mtw{(1>uVND7E=gG^2)S zW(uJhHRPSm*N$d>UJK2`ZBOQtnH|sWK-cF&?dvlwlg*xB`}!=p;-&*XIMm&E*Y^(| z_`!qS{9SK8vinQxLYXa(j>Bs>yX8?-+G+M<@N+VVxv|Wg7lb?R7Q7&h$8D3;CgcVA z9?Eb=t9VR_oFI)S^1?gF3G%%w_&`e8YvAV!Xp|Y9dfC4+dL=Yu6|#?L&S=$|oUGI& zgf@|=Tz5c$tL#SB$STP%Nae*KC;WWl5R*j6Dy$BhE@6_1c@M}PB@amARJB18n?ml7 zLCqLYoH^P~3i&X4rfogW7HA!RDr#e3Cym4s4gRX=K3 zS*a}hZ@Fa)h~Hq|?k9Jy^UY-!v*_B8mpmThA#TDb)3N4>*rRK3+QJ=*Q| zWXnxfj|FvqR_Rz3 zI2dJ&7F6;==Q2Z;W2tBgm9@sq617>UHLEF0yfy4fhfNlbg;JY2jh<0y95$V|d3{6G zaC1~aP-RW4;I0nLULfu!eNY+4MK4F(tto)n6G4i?Dhl8kUSkMa|KJz)S1!UcxcS`}KcPYkh(%MW$H9kPA6~&`~H%S>Y-hc6z z2%k*M{%^tr@kvv$s~*addTI`zWQzEtrT`dTS>EtIWQy_)rUm?T1!y8nVDhep_w&LO z%Cv}&7DSRRMJ0ht-5qF8itTR+Q|qqBM&cvQQI&)gb%ck>L`BOyRC8P(u;I+9=-Fd^xq>^E1g+{B=ThyeARi)SM zZ&yLm#0Gb_b=*e==e?=*pBnUfv%QhkQwog%(x@NOs1MQzMRyuXG3OM-3@%*H z%azvi?3DjcOW*>pWUc4(>=fny(`AqXc}1kqS2q60me%dd>jP|f?2)aTJ~JBrrX@9y z%Jo-ht;vCOZXm|u+P!Za>Vxoc&%-0P{==cZt_R+@f7cV&#TxR5SHNquK7SZWh)+Xp zTn^Df4C^cTKD4JDEQ^_FPFn!{Deq*M{ZQ1uD0cFiAOunPgyX$v1h2@$F%tH}IgPs@ zj56o>sJ^1-qnwh{l8Y};G-^FUQ@?nR)hkKF(?CW+Y^yanw33&k@ZCnOStFHd%v!CP zk>Y>6Lk=`fBWuvm)RP2Bpk#mPgs-OvC=gG*mX@VHk1d;!QCMepdc}{ac&bf^yCnufB(F{5fC6|C( zC0a+h6V+lqpwiD2@R_H<7$|eQ-;x%)VoTqmG&YmYZZatG^)y=)jTbk&@MRZ&jQk%^ z2SgX@fGpGjWr?x`(g8I^9gq=pK&;_CXlceam_GbVIv|!Q>Htr}dwF3BW%}@%b-?9( z-hvVkZPRFW+hY@TYun0blr4a|j(h6H8Z)b#e2%864rgW9i`0M#NA;40;?pzZmX^?( zLu=z;`N-X46>6PMu4FAbjX@>VLGhLtsHtd+n@N%|xIh7_0jMBYt5!bqTO5z{??C)8 zA?{tI1%#8YMnYcOTa`>_}m_z-GTKU^(SYG{RuQ*suKgjUhD>-Tq< z%PNL<_s2RDwxD_aJ%c`fT@NNov<8tXBSo7L5<`1TjnyNXsDn-Kdpy#bv*lB!v(kkm zdH3yFksh9=_XQ_+&DnhUl+<*#B<8Q(y-hB4Z#OXE)~%1M4|iq*j2xHH5Ui({RSsO= z;RU)xuB&YAz8-gU_1{c%?O2}nsJ~`S^(Sk3EA?E(vYNJOyte6qJ9dv$$<>U?Y}c7J zl#)?pCib@?)fZd7tF7~ZK&|P!DzAg`-Cx%migs4pyrsxE<;HYa22ZnR@S$>Yvs*d?r{IEWh-u8=Ee^k{@@#HN1i$V<TGh zbKiG%iq}$foH9bYu(~J*+&Ih2jV!ICZG+S>N`N~fbhkp@OeM_=H-#*?ze-SCDJR<= zp$VnKs1<_Z5S0N`u=_LeNdv3ZMd|7n`7sVy3XF~KMtLPlZesATy~;e>X@PZu5D-V3gp7>(%Lw=lyMkz4x>>86yp!wygm}?$#^_Lw~V#x)kap$ z_^~q|Z^!_D71{Dy{n&KT zkNpS1SR6urY*6bjbr!ChsD8`f%`XRbq<$WrE?g(R_D^FxJK&XSEf-^+!J=Ag9j?3R-<+MtQE_5 zx7TkTsHJhLYI!j#{xL~XIsqk2&7%ZW2_^jbxIyq?Lqdz%e^4=eMWArJ=*OmcI=^7_ z<*>qndRteFjgva<8#0~X&z4<0I~N;&gVMQU4sWGX{RYRO*!UQD8XO2YXMO?oBm@ge z8E{J&yODF+tStT^G^C2*@qcpTKe@O}8nsnNdiU66`|(I_Nzq7S*Ocy zLa}o2rK@30y^|EeM>Be@-J-WL5+M@&^f+Tfk>Cdgg-Gy|@9o<6d@&MS zH__~yj|4+Xa99M;0wv5#3f%ik1@5D$IFlcR0-|6+>3Y-$LWk0GE%m^%?>D*Mb=$+wmiz5g#Mdbn8R z0zFR>uUBk){=h9yPetPS=lAc0*XPyNNL^2ScuT#((Ky*rGh81s$?^LS|MBR=if7J0 z@yL1M`i+UlZX8aV`wxCQ|C#Ua&-j|xTz@xKifTiB9|KBZ6?{Gi@Y#XS2E10_vjCqN zpdv;_5EfQPi|RVXg)Vq#LE->I#K*#7rQ)zi@~~L{E-aEfEEbLm7a*EYJ4|SjiAMj@ ztQe?)*SC+uXR@d;IwSmEy8kkIgi!KL4re^U0u~vax#Ql!FjD~1l8Q%mr&xyVOyspl zwb0fj@^=F5vB-Qqu+=4_Gs{TAY%Zva7ac zi@@og=`(os)qzvUBhxzc8Uv%&`@ALrIPHD#$n^qn5-i|z#6t)Jjjrcx}bp&n}eJZTYiIx_@v4Htx$KxXFn$Xv55G9kfF_I zP?^Li2$ln15cmSXZvz1<@CxG7ZvuWJ2pB*>4+1(6U;tzi9|e3Qh**IjOSMH=8a0^U z3xkUhd;JxA{QdGY>H%nr#hAkE1>O!vFlgH|h^H7-jFdr2kU`b@m=|#t3u0tZl9Et? zp>0nJn ze+GU;agK6_yVAit#Oi0hTm-B;+<}?jm5Px92t6Do4n@-lEjm6$LDYKTdtUs1UCb17 zFwPN=Ld4pDT`0=bU>XEdLL(hQkg2DNA~h|_%`_^|O2ciHegV=n1Q#g;?}pGG45|C7 zrmOZ<5mk1y5O%Z>c42ed(dM?lfLCJ}e2Rz~P6|E9xB_acgSuY|0d$CR%QGLiJOCwa zECI)>i?}ZWn4gHd^R?A)7dL8g8PGb+<$sj(Y~GXkAdh+AJ{6vJ31}9Yar`CR{Fk#p zQkqYa1p=Pf^p)uhYc{EstVPSBG!VPR)v|Fe4P+iLi)kPbFN=8~5HG)h6(b7gh^Iun z{3~JIxNr=VQ6MZ-AT0}kfC99jE#yMW*8&B#swAWBw^V}K%FfEol|-ZxKw%B75lpRi zVJM&@Y%gJbet80|PZmjJI5mq?*EF8Z^Cgb~!eH?msH8L3*hVyLg2#svX9%7B1azDxk|vN;fcRcF`LQXo>L42Z}; zW4TU_G9c(s{b2PqWI!xCh`+G>8;?J*DN8e2tU(!xT8jQ1dAe!zk_!R*%)*6MUD&g zMUJ8RBCl%pcweP%y`;X#t9i{H%KK^_zEuL@{!2mlB4-=^PH^zHk>S1lfzSZD_5~ld zRJ4>=wU%+#cuStqHVMg^WMT!?%gVt9dw}h^~dtD_N3X zam55s1HMtIo++bQnj&N>#pkQ)JqkYS&qiG`QZ6AR+Olj%D7Uu3sfw-W+yZ))ng{I; zQfXir1IN;j#(VjcsiID2&`C5##$eT&^=em5Z`9j5ys2eUsDsxCxn>pohP|bniYWLI zuWIY`wbl6A+I%&H+FSsC;4PTCtBf~TI?9gx(D6$L?r>1?R~q*i{#IltL2O5AFOySY z?Q)bI$zu-2fg1=otug*ezHtv__}e*#6D9P5{*saf6|a^6rqqJy_#9e7z1j(URitlM zTSa%;g?wHLrNmX$9PS=8Rd&QXWD4ZgQAk^dR@T+~a+Pjqpx^|hEYEfX8dlfadwW8y z37fuVWt~gOvQk=O)EaDB9b1>SSGX978e(RhQqq)-r8pyJva7Tzxzeap+Y;@OjtvY> z*eiLs1O2mc{3PD8Bwhx-Uh;(9f#PMR*efDVxLL9HYVk4?6MH4UVCWJ^acw!BL^Xa= zsk%|d2%A7M^G$RD52vIOV23N~GuuLD<&Cs@<}Cj41*66G`BG(#&y(=`Y0P?^0-Tme z;XSEbG81xvFU$y5b1USU<#-OFiSbfK=Ydkj1bB6L{n5s81YoK3M#5WYM*w5!eAm<%XMl0bF0q~>t0BG?!+=_2jb zfrLe?;S69GL{+3(HS^!On5k`3tEZA*-|UsCS%m@`Tq;f_)o9fkPb%b!JJm9+0rVJk zGPO}{cb&k2GdqlQ^BAPTV{qrhFuv&bJQ_?F!1E^*CX=F~0FLlROo1hs1pd1ONIX|o zMtNRl_td?#fEufJE{ThkC7#Q}fvo4{JiCXgd+7@FD2WO3FUQ63$9!G+j^*2VtJ0C^ z+p)}8QRY(0kj#^reQ|qDU$qIiMzd{e>nsndU5P-n$HDp1{$SFhiKW-Jhtk*G+aI5t z7;f}aGNsa>XZ0#dCX@ObS5)fkK7LibC+X!(y4KN@(EzP(9BYA8xeHPSA(joL3`$59 zlmn54FL*;x!#jC=mIwTE=n%-u<$FvoNf#4kK^6q(;h!rW0C_oIzDH_$DKEW*Cgc($ zYpK64Ml39jb(hka=$~HZNXOi?gv4p3#2l{irw0-if(b;c?QIivR)6PqL^R7GT<1^v z{7J7y?M?(D-TxGyS~uM2gI1S9q14bG1Fcl}`H?C)tCD#eS5&9hwTEl2yEoG~T4Oc@ zt8MOtmoq}D)Xe@nK8Zg%ubX+T4s$3NGcX^~>~ng7_Xu?dB*9Mj2UNWB_A8=pK9bk$ zqr6A*@Xc$@kHjZEZCg7!H#IrkEn7SLw(^#Pj63D`Cfp3?O?pDrP8DeHxqVetY{hN; zowtrAQe(Gt)~v|bZ8bwREn`W&BRd3lc;)QJ;7sN5R~%;+dk^RBFzj9 zaHyIuTUd~S4zlrEWEEE zw*wpioTQLmu7exU0_mB;mY4MM+ozCTjv~cO@{F7@0^^a;K9v)0z9YnbMB6HXxC?89 zKn;B)4+jy6BY8MxPDA@YiH5eQ6f@T;A^687G<~8%j?Iy3sWYOVI~gZ&o(>5ApA9_Mx%2&spfr79ngJU zWG_L3R50gu!at=5#F89H_>|`Ovw3IuYx&X>DBagfELeQNlB9a_Djts1V&cvFH=xF5 zLLE~Lo?ODOk;vsTTUlM1x6-N61{=zoq>5ruv&QzWM6Ep$wo6E;)&W5&e5vL@&0yNf zd8-|v`mo_!(XNDDu4WnYf3Wu@;B8%HqW50y`@V12vKBA0BwOC?#FlqE-fBC}o>Z1) zTZt^Wl5E$Z3l!Q;DNBdPW0n%S+P0$ISQmM)_muoOAzk&cFZX>PqKWZCaH^skW49OdhKO?dz`6$@r0)@{n4m z;>&ePnMy8}>NE;V^&YRIuF@gqTPyH>$okiRPwk=Ji8J@?X6C+w%iOniYK?xID@X<( zVl(nB(zk3T?;Cdgw#!4QBHU}t-nydf%|qFHk6gBo-Bwk!@qP4oisXxgKjkS+V3w6!C;F?qtfZg zDVahp-*`s-llR+U*RfO+L z0ltd9AI0qE2d^N5!aeM+x!YHqx@+8BC)#_i-{Wu&o<5?l_BrI}Q7&5RR$6Pjt8}Dm zv<7Z-O?QXdT~WKrSz~up+mzp~ne6aZkLL$RZl7rph-5O2TCJD!MIxcIzSpPKyBdaS z8(MU#x}lIsOIIU#l!H8IkVh5BgIUQ~sxOd#_Xr`E6AefnrZQJ6*?OPdE=z=uQ1 z1KY(>oVD$(_oV>c7k(h+gYfadc4_R`+IIUr{*FB@Y1hDw2krG$b~&s>Bo2eVAgunrJ)@Tc-JE5hPdJFkhfdyI&Y)d>DAQ8)iJdsi_Bhm2alTVL)n$lm9Epb^xaZ-GToTA9zO0ouAP`RPE)OW zg?pt{L1l2|mh|!Iy}{t#>Er2JRz$wJ8MluJT?}2+ut&+UCtP0rg+x8He@qW#(#?Mw zeVsngf%T6i5qn6vyP-Z(=V#w_?7NA5=dO#k)wjQIzb-Ovt$VkP&&|hsE?i$9elL3Y zZKO6*>p`WBraJidaHO_2LQSC8%U0Aty|sXQ`P1S0x;hV8TVG#GehFRNIEr3>i?H8| zO1$^gp%hpf+4xQ*Qu{Jg$iIQYG#c$q@IrpBF4S;&080Ne9ImHm7HLBSg&(1A|5zWc z4?+RF#ZFzMo)i3-5{l2^ljjaoPg9>2{G5oeXK@l{A3C8ztr)94QTuW&&sJ-ztt?yD zUZCzcSGg|EvtLn1M-P*sOUz*vI#sQ!_9L}C1%s_ETTdbU^Ww_&lsM0QL%rxBk5E*6 zzhN{PaVm5B{M?BS_*>Uact>UGvyj58I>rN|Z$1*KKX%(_G+(8bC}k3>%sSN`8gDII zN;sNAUb)&Jk;!>ev_md3nKY5v|8ZjG1F05wxzeqtoobPq_70rvz55P{LM@WYjVNxt zi~1&YQ1Ang#RV*jcY$o8f*%kzcDGM~0_`0s*GmY!U|pfm@Yl^3sN2tI@Lr$n&H1|e zGM0f{Ay`iVj+wun;;_se9a~OqDB7^JIE_KwcuF(F7~scj&|8ZSCWA23>fpz5Tj6!%q!LSwrurhrX`)?1RB0 ze|7l5hmSV(>J%CYkAJj%ZeLZ+NTO-aJbItoIPEd(JPl=P(^uAyz5itE*Pi@by4oui zij{hh!2yuL8kWHqu^se*PqcyzdYL`PA0yhygXcZ9o?35X;s)(uOOuJQ3 zi5$bkZslA!BOMHbIkXe+j>n)^wwy_^9nK3(g$nyw4xxp?HGWWG#$&HkkZ%cF$0&mv^q^&>oi!ZBt>?bZbN+wMyDnZ}mH^a*eNHUt|4vqg|}hNu@@8 zk5V9WH2ADOmj$jh4uS0WAiD`HyDwpTa#PRnLxLYd{ma;%*my&@2`|xu_i6R4mpkNg ztpl9ooVKuoC@l0ZG55uu_0%>5Glzja36e2mCjPRmSsElEg7yE`1fkm89o3hE`1gild?sqo&7 zuj7b-BLu8_nN^uAiK@$DL`+L zuBT9k^CA6uDztf)O|`c8xnl8(wW*=J`1CFfcMvGM{-<6-`&o+bPjiAI?og8}r%!nHmccZh2UW zWnA8X+$3m{@thT?pM?7;8N(0UwPST4&QnB?EUEV)1P|x z_PolEpMVT`@gI)84_=?%mPmnCcEv#0iAv_P#4{e@6XYR6LpUH@NC}N+T_^-&H=tFJ zym+m2iCZfPA;tra0X%}k+Ok$caZa;MJIo{Oat*%Gzs1`>(>o?c9U2tt&Gul&I-w!aHvP2byPBVC1R9NL?>dbwQxS@K>Q7@47cgVrW9?B$g@=UN4;ML=WK)C^n zWsNU4YSj*#LO3KfIjZcoRzB3K&@HhZ7$cl>}ExuMDRtoU+%Q@Pn6Zt zD(Yn>bx6gJ+r#C$jVB9&d(o`rkz#j4Po=*j?2(IYp0M*_qZV{gE93E=(>N;8nu%Qp zK%Q^q_5dXhoi}TQ+I7dKA;6vF`en5J<$0k)yPk3sDZg$P%@?9S&Uq8Cs1RHfXlwvS zK)AnpolPV7I{L5}B0rzNZNMb(l;psCtAx}CdGF&1R8EuCp&)tGD-tRCq+co_ zH#R7)Ouc}XG8A7Ty8I=o8SROH9@}4jnrcUZ6h^BwE$OF@ z@t5$^7QEL4^~|}HN>Kg;^$d|B1k|^Pj%wtaZ}=$X7~Oxvb>rz?m#goFiSbOY>rIA0 zG!TjURK`FTynB_@#S}gJYFn;|9GVRT$K6vWrt>fjj(<^;PZXNg3POlTR)P*95lERj$=qI4~Xfo!85q}a)H7jHreNvTD_UT8BYN`Ag zvDH#$e4k7~#w~ii#cH6gUy$ovCWGh}Sg}Yiw;EtpFWExvqrc@+#NEIhAQXvL)95NR$ttC*Lkq&)ZhUJiDfW9S^6IwvAy#TZyJX@P!VNQ(jN$YHmS*CQMq@Q(qe&xdSAH4l-9d2<8N|<;Fm}_W zxdFx#n&pWZ`PEiFj9gb;H~AJ7~`{ zA|ySJ$X{^t6Oy>u7GX2A%+$Xj^-Ho$jy(c3-n$LL9yiwIoZav&nl(3K9-Q&4nrEzd zqqi%>QiaYfZdKVkjFSv6=)}*3LbsVXU2!~nFMs_Rr#BlG=yKr*-!|*JjYWx7Ce{Tb zZn})xbPjl(Su3FYutBxrB9!vlA(V z%g1_?jMcJLFr0Ufy7vn$pO$CTGp@-#9kSrSS6Bp7vPHVaWEsYcR2hhJzB+OR38kjj z!wP1V>`1h-RI-edTHbK=X5$K0xqTR`Q2H32EsYhfU0qpasHWN&?+Afvd9`Zi@0yIy zM@=L1*pa;Q#Zxk&YB5_lH_r(!=`0s&7A`@ozHBDV--N1v$Qx2? z)q_1OGMf>t=n7y*sTd_h0&b~6I~<;#qvYQUqPDlwyRTZ}P?lw=}hWm6q+PbY>ZHtWdG25{>S5D)~S-|2)4u704>O|WpCM(un*)qOf zSXK*GGFiG9=F5?A1D9PCB*VW4`ap%>sh}{#kTiT3HAWT1KLWcwSagFynin5kGHR&d zAQvwqKS&z4WVY_2RE`+xMOpZaXrMY^KAhDhQ}cEUFIKwIq3#?mC*7$@60FE2wOe<` z*$z&151YS>K{gi7SeE9f5cri=_@5zons$Yw8ww@FJ17~O2e9(B_io$ut8i|Q z)bO;eWCam9!p)kgjSj#Dv%ZQ;u0>-uwPj{pUhQ2<=aJmU@VT%d1)NK>@pmy1UNJoL#*OR^?@_pk`wYVJm7cy`;21 zvwmg8E6XeBLC_yADnoqh2!S%jd9H{0(o;bWzwUbof9>S6om2OqUN0dUTOmN~N~}eH zPl!XBU>3rXIFON&&^KK>c|Ju#zTQdtbG_4R&&J(^5~H|H%OMiCJ;|5@U)Yva8BUfAdb+V`-4 z5+~W*9favO&C}(_<$%jy=;m*)w!c0DcnJ>m?KZY&?SCn@K;lCx@gX;hTwf}=qHLXA zm3qGO5?sEx+EqNQ^;GnPzT*E@yZ1w0^qTB@g5F^x#pL32zUjsN^wBecmHK1mzW@_* zNm~V|jZz0K0l6Z0S#Iv9WYtvjKPbHOBX_@;1vM(EKU}psh}Bke*?I{F1?=QR$@n@? z@ILH+-hAw2ev$@)UqWB;{z(ja5&je=;MMkW$Lo(jBQc9vkM)d!it&9{|EQlO;n;S2 z?>Xq@au|KUTj58?3BMfu<7dW)tCvP6&oXOq^67Gg9QZ5IHFi zg6#SFn}-kR)!0sbl}3-(3ml*dp#S`5B2?|A@_S8B%j3>rx1z|r6_G+d6fAg=B zfp%5n%Q3YdcNVi}?75$QFx$?%@ViEKuP2$r>|1^d#hK8yd_lYDG$H7Kg@0@QNBKW+ zH`f2<@4c5{cPDvC!PfmKpSXC0j;BjF19`>WJ(Pj$i`hC|-2GX;!NeCS#vgdsJDUd`OI`udT4w?)7-?%Xsr0)L(i@AB?v+C;1?5Mg7A49#?#~?OeIabw}RP3 znx8}-Xip|p1Y&LowtgHA*_JZc4ADga;svQ|2C6Iw5U_yKL>{8nkLovv#WadgA0*4b z42fUS-!$Pd`X+UkhK&8m6=AuqG0@9PBYIOdgMPNBM&A{wTV5bv^qh;EEjT=G}1^rxWsXs zns`NhKfRw~(w8x;RG9Ar+8T`*B!H42hx{aM)ZVi92^$qph;AT9dZh6#)rjiQ#$r2b zkWC(dF6Wi$opO#4C5!BoK>G(Ns8{r~Ut#PI%AnJ)s+vR~F*YldyLlCB zS+x*V>_Z&*r)gf+0E&hpeOc_W_l31aqOge zASaJ);F2iv-Ow0uXf-i(;fT7ID7Kd2M!N>OK^AJtEV8r-wkG;b)l?cMZMp2XUs2L& z7$fW)YfMe0bxVZO>iTpTip?->jA{I_Ru)xOGc{Q0jEz7~nMQAuI%_lFG5PQ?E>Zbb z*c+pEZlEXPn{7SYhkKJK&j5DHAk%sA@a=IG!?Y&Te(sz!x)!)uA#9bEal166EhhGf zMi%a(Tw~!$0GyaG4J6%gT{Pi$_00hc43B*QNx&?D*~$0)Q>Iv#;F^mlwQBCR98f(=>u;syh!8T_6WV@16h_Ccy zhVY=I_C+8c8>Ak&WbchC$Mh#|K`3>bk%{~@Xtfrd96}!;G&11?J>WNf@w5;ZWwM$_ zgI-2OwJWAGDyKn-^y8MKggZ^CcdnTuY`h>JXUBuBe7AT&%n1r-u~MIpGQNehaaz^Z z4r{E(@$dkd1lgmjS5?Fju=ds$ldOBd#7RWqC?I7F1-LM$A;?W`u%?)K0Ur~$ z=9tuWiR@+L{l~vGs+V}My#YVRr*J5z8neO4BWesehwJ$-$s2#9ERa4np*~WL+__EQ zKNv7cL++~C*6(hKo0&f=n`m7S$helSy=RSPMB0rfZg8$nMQof|(lme$!4ASTYVI1E zwA9_E4Y#vLA4pt8WHGg1Y#9EK;Tfx|1*pk*V7iW)D+qj7DnG7oMT%L_sTCj8hb5RE&nqlXkIYU$7mN5|kkx;*Njc%s1s> ziKBB_&PX%T$rQPE#A%gH1G~0Li&EZp`Y!Ksf*5NWeTxY7ZICrq+wE<%~5mJ#4|ifA?zUmky?yJgUYRa?h8PftMELL>*v>IV}Yu zvpfZ^gEodN3PHH)?>f>-kk}l(+m_NR^h^7IS$mBvinNAT#PRfUdJ_ktQ{Su z1W8myV5Ht$R;D8-EapwYW_+&{k16pN?Xmx)d$m>pMU zKDwY^V#27|{xk~;=K;(3<~QIYq+VOGtS%rY`JjCpu}-+mBM@CNLLZ1R4n7dk=VdM5 z|A-sVF1Ds}gq@rQ&yh18L*RQsXol`}rw$AAcI2DzFDkYZhzpwbD4q^)3IiHA`~?}SbkLUO&SOBJe%nH#g1vMaDeS(Ul) zcg2ElljiV8c$k*q4fEXhyM(0b%7A8Yj@?e|61~VYs>Q`1`V4|(YKztbpn$BIx91^G z4tcoIJKxnx?L#nwa(L4gBx$>KRZYV-474{6Bj!}tkD(;d#`B;jNdBzo#ce_OnwY0k zV%t<+AncxJsssEKADeNcZ2>EoHqHm@sLN;_*!nJ<0WYh3=b+Z%-Iz2HoH?}W5(F)} zA)Yf_>6{}Glm^Gb9dsqZjh3&}d;?JfIy^x7LT zv7`rUGW5);rJc-c#WHLu@}~2GTp3bxDB~}|-Z426hsbNmL)Hq6acSe^s%Q1ZBA%8u zRD$0}mzgF#xRTT5N6y?$E$AXWf0-ZJ|9s4EEdk+0lBT2k1zzvWgTfD4Eu0?A!du6a zDSRhGN9yEU;hxnllX4|BQ>jn3gR&HBVKNoFjX0aBKFR7hqB!E(2ngTuF%o80VPL`sf zosc)J^aQ#7-H|Q6un~5bI6aheza1o7D=I+ypq`AfkUny<+}ja0M23NZyum+R`tSyX z(~~L_)P~T~;9iBgQHsIe{_JpHi8iM+(Sj@17(Pi5W0w)$$>jJn{g4MgEuAvAjq!VS zVkxpJhUpgqRk#UzSbBZXO+%WR~7C8e-EGslc`<5l|gt_n6 zH$y71ATadw_PCoN3_}e<+IV<5(&JR+dh7lg&GvGwcwQEnps=`a*g-DwbOp0!MvdXa z>W`@V;FV4K>34)TWQ(}7Aa7^J#2C1`F=|7O$$6q3{F&p27`HFQ+E$b-nc*jak=nC_ z&M-$SDNJ{3tMdbzlx*OxAqt?(&A72g$4nN89QrnLDO$RkQe^%1znTesC>sIDT7{|o zKX1p>6`7Gma8)wpAx_tfZD9dvil<6!uxbbX3I0!*WsLDUGRJqo%_;2T2xf6av{hO1 zNwni%fPn0-AiOF1(K}G%7@A+~{;&2Z{=TXh2{*yS6RHOD##}j?P#Zf%e2GxfMA8z#Tvg%Vcf(G!@*c_QQ^s0cmigSsH& zry{rnW``UFtjKl3P_vM!)wLvwf-1+A$BT!onb4<*9S{6p4w zVXn}-dWH=FBvEGZY4WE26r|XHiculhXs@abki#9N^HeBD;vF4VJ^bwu1;IQ641$bJ z!SA)v1MwY^?1wydvBoitA+dw6oK2wub+BQ{Lcm$g%vNR~i78ViP4>qz8@~TxhS50l zjtVawN0lf6i*5_n@=SLK3Pt>O2y2MO*3gs zod(K{v=8(*^l(Jzrl=E0NeRf`1ano#^fbldv_oKRPzA#c@xRdq3qGAvN(e7gBTaU2 zPfKvXR-I1^TMC}q|KToTTbCsZ&IrrXXANaYS#o4doO1%OUWn;`HcMQ+(ZoUt1-2wm zO0}KI;I{)%rV^`gm{SIHC+PCLfr+d8?e)ZBjpnb~_)%1_)YQf~m9QW$8v+cxoXhMk zgj{pd-{s~HP>dFWc)NL<__<(R{QUWV9IU_=zX+(P@YB;pBG0PWm>B$mC-|60h|}$$ z1C;K>#QRkUUJfQco>e%Tpu(-eH!mphw=?=vb}lFnA4DD`sBG-Qxt|vbLEVYwb@AKb zSD0;x6neA`{&Q_oBDCxv>xaXJUraXOn(he zaZmA5!lM`=oGos6G*3rDE}p2;h%4yd6hNOXqk~X#s5j`6(}4QD99{E^-;fgpI)mL) zDt#(X_RjXC0#7311s~mQE0|GC0dwlV5dio5#W#@r~ROPubTeqXI=_?dD+e@?Jm3w`DawJqrhc%cQ@x zqi|J{@ZtWBcccb7Ui9Z#FIvK3zhZH4zw}rr%*$l_?@9k1*0IOM;t*N{NcQtoyyTSs zj_0o{M$l*SJ1y`)_br)C!HdUO!`^uLIJ7^bqdOP9>->DZLi7}%q2IJ=dV~W0c?AWr6-z0W>XlKf)8TvjR>Y7HUahKhmxpws0s={Vf9dDQQ|YW zhG4{)gOWTx`Q(_9ahyzojd27nG4mm)V|o^uTAnkrM@CmNOI@fxCMT5?w|5ji zx^k6_x}+Bsw`a>puYXtuKRNxb?$T3u`r|9<6ZT~Fn`qmvT->9#FlYZm+`$YCL;30J z#_ME;PZ=IfaL8ZZ%%0?*iXShAixJVeqgsy9Bae_~F(zmq+9^RIQ zNTr70)30u~5^Ud`%dao~+8s{TtM-%4#;)0?JKhsEr%{S^$h7N;Yph?kU$RgRefoSr z^>2WNfPMshdqm7ENGSFIK~`WHG}Lq%=mZ2*wtoN-unq+3BOrhR2=g;O6g0?PA4pID zzf1wLqJjs}^zFLo|3Sv=5~nd|v4)X_$8?5<>JA8i!Q=o9671{yju{0C z)!#>mjX4DgCG9Ur4jhAog7)v1)K@?NPzUt$V%{}gV*%yLfv4X9o?zfwmL82DoSGyk z{{1Zagpg2JefrEmj&Fb&@F1MNT_T{l7^?yowE(N44rVLpR3}!{2F3#vl##z6DKHHZ zpcK%L&9ROEhye>S)VD{&YJmwjtUSrku zmH|(p08{<};J`m1P=#gkV;zHHMwAH%09TM8;l5pJpgRcXLth?basGN=9}H%gf4_h} z5EnBF5=ya801RMk--M2N0R|-=AczHQ0|(@Q29dL5hZJD3usZWr;{bt>P%V9ZkeJV) zLE-<1M+)|Lk97!`*8hrUpsxT0pa&1y=-Z_Nx`(prV=jlejBNL{W&T~F4FsG(ABcb{ z0uuCmjrFJS$3Go8?$ZYWVtxY<`wKDw)gjA$`ttsE5(=|SYSnqj`x@%#)$vTjmW-BHtuXDt*}yPmz@X~; z^ofA#uz(hjpzgk1N+9KDP5_9SLEd-3D|k?TpFR(ekpHsluS$W;WdC+vXe2+Q4+I*j z=AW->2rTHOPoE693IVwF@3;7;*^3~5HT-7n%{ApjJB^zyiUR`x^Y1s-M}!6>1oTVk z*TG_5{QUz42$BKIAfX86zRAGh1U~eI7`d)ePJ!b@zZe?Tg|9l(;$lvLLyh<8V*yuT z0FU57%>T5r3>-?Y{>jg6rm2#7Jp8W3CRx@ke=?9pfX{aXKk7lAwe-cpXc{ptF+MRq ztfB-AEIGJI1NItndX!}VcN3wwY5wbr59-_moy?4=WSM2)U^O`jWW^ePzW2M zLQ+&`Vwote+pq!ECwLxnqG*MZ{$~vH0SqU7nZX+hca8!T;z@|lrnBam05z)_92sNm{MnXXmnn7x*uvkZklqI}pp?chxEac!t z95bzoKIovg6J;78z&Ken+eph9Fma5ABs!!TmX0X7Xf>xwXLSRAovml6i_lk!XH zqihAE-e-`HG8pRDW?#CW;qDXyxY_4W-w*Bx#6`kD=Fb)K8O1b?DWlME;ex@T@+>wK zHPR$NT;2pblNtlNTei}!3O-$?9i;OSA08uFWs$P^658JVQ(J;(RUXt}a;;t({> zU}?!Gh*#3k110w7k?0F`5DY2D4awr~VJQh;YUECdM1`ne17n*VxmZ9ec9|6D%L%Zk zA^}(!qV0H_FzC=GH*oRaCS)mzmyk@hY_g>kSt@z-9HY7tVj7aL^J65#q2VA{i;}Qx zbVkd+m7N>>B*jny$pb;hFu2(qzS`%5z~Uy#MigZ;etn&2qZaM&DSyR*EGlMqRK!l5 zqsIuu#<%#@7Kh8qp@VI%B2%Z31bIM;CfxU!3^i&g^K+E+%g7dODbHC9K0C%S9SbqZ zsd^DUS?RafYSmOBJ-S;Fswyb@;5;Mq-EBg1-I@&>VBLVm3iO6@Q8-zpOq_@SUx$c* zYrtqxU^;rDP+t%}HOO9&4-ZTe2DyMXHQJGLrsf+&%}f$!Qsz9v5<+znrEeV+{r97a@t~vrU13L6ShWBH2DAhy1jD`CtmMm9NVPb)B@Qa|;4=5k z+!QB4M9x&mZq+0R4awv?v2{Rr_@q$Vg{uMVSWHfXoB8ur zDEg~qz@0LBr3C1>xd8Zkm^JrOxgviM-Z z0s}i)CNQS#kSjd-j@-hIk{7hX%GdxVx5gTpt-|{k=Dot^s;f)~JwC>s!;8I_s2E9@ zp&=0{S_CX(Z1gP$6p>*>ILDAb6S9)C)RIe4IY!{9pa+jIX&NGlKcYr?TMP_20ExPC zkWcf6^^y3-UU@zbe0z*G#VGA4?I6uyq_a%qTI8A%+mA?m0E_#R(NH3}B<-;Cn_t*&4clNi9^i%O4ET#xnBkbkX2+Xa2VY|h+f{fAVlpk(SS zaj|rIgyFjlj63coyRZe;PFWKvq-nBTMJoyXZSh=Rk_7xsbCb;Gma=BPno5sQFAH_9 z3=PDlBL7m3xL;Wj7HQQok%3C|8oj|VBb5Q)_r1GCB$py?G|``7FewZ_$)dHFo^g|A zSw;54^hIirgZ83`gw__>9lx&$^^EnpWaa4|I{re+wUZbMM@vnmnh_dya1;)lNNNpL zZMWQA(6{X2&X2i;Wc6hAjFwxWtte5GWgt*hR7}uK(Vb#AzVo2en?_8}zi`n4q|C+% zv#$X|Qs(#f_mxEXmtOp|GXyLVv#?#@lp;`->rshf(aMPfv5S;{YU`5BDVs6Tkb1KX zoHYG*hUkHsxK%fD=zkTZ%D*OA)lw>;ArDi$9gvY98iLh zyoIzM4kXB=q*pD>w{&jD)z;8L{H3*o$q_^7E?%Z+ly8a4)Y?=~QD0wCP@v+GFSYwe zF=nJY9!U^Soa5%lZO#_SIpLGn4^|u2geT+#J*r+A-Jv%B0m#jltn}s zzfM+u3k)GupwM~$J)2e)h(?PkE7yzmiRH%^F{jJ)kmP|`0|3R0d7+MASuCM~^P&=c zMWZ1q6zByrf%D#F%ArNl=*fl7N&VKweYg!YV*(l#$SOEAd@G7dOQssti{V8|K$tRs z=ZZt%+0f6%V#%b-3KA#@3`|7^Qm`A4NgJ{>7_9PnIEe(wiTOiVaLc4(JUS&E7#E@q z^N=OoY@}5Abci;17gP8(FAt*)hrdUdg4>*AtoQkWl2H-r*lJVi81QkUmFKu|bfJ~! zQ%tygEyH(bxO_AB3zpxk___y!dui`K+k)-YC4{E_k@<=EizlHlA850eQg0<^;h*eYaC{HknjI|eH%vVyr5Kn-J zhG%=0i-?Rzaqr?Z4Bt@^{<{BR4;>%WUD$oeA;DEadUH77S8Qg<5j%BaF&`TH=K&>s zpBGIs2(z_}Zn2ad_IoFfrBiA;6GN?~`H7JnAUAyNdR@Fz;`F8#uucgZ9PqwW$2KDJ4oJZZ_B1%3NsTb)s#j ziSLq@J`^6H=g9F$v;Az~Sb7X=*Jl4pJo6;SKT-FQYW&&8Matndau7OPFV6hx#fSXo zab$9Y0qCII`*BWB!o8O5Nq{hk27CL$AEG^bgWuY_fx!v}j@qlg@-YW`cqu2GsbB6$#~o65bxGva0? zUG#!db`tzlI`fGc{Lj%Z$!XX&^0l2DXTBMmN@SxiLW!uRgbTt$-9ffm6M649QPYeu zBefhPwWkdWMxJTICv`KNtHp?v8=bSY=dcOY&*kCoo;ih2W`6e`f#`f;4scK9Smvx8 zHBVI&NQxvK65nd4oNP~9?A-eUG}wH}qpG5)Yq=RVdyRiOV{zX&2E3H9YQ2wOj5{bV zpKlDxpuk1)Bh;&%vw3HpzIWqv@TO!sUJN0qv3Z@*Wj8O{DF3+j7w*I>v9h60mEm8% zTp>%aK`=1;wdC?N3`gOD3}MVPvZbMJ&en8i%6-OU)u<2MpDD4GIiK!UY15;!Ve3WP z!p?PsXv|FZsFeHy$3p7-BjsXka>QG6yEOJdt0FP-5*-)Yqqox_TW`xudqSi?da0tV zK4HD-xP82*xjG4lMzMGVoV{gN&jH=kogw%X{>RNqNoOmLWb2o-K z9v)#%Zf}|^!F`~2p5KJ;5&FgNEZSH)HXXH5wT1$@S<4f$b!d{7uXiJbOzYgid81g; zV)?_S8lWo4M{&uRR1kWiJI@=ebMNlryEV$-{82Kzxs1=)E*lR%oB5lEmDyW<))~Iq z%Ug)MTi2=k1AMg2e-+f_4cf{6x`3JH#^qhQk-2|w9t(9yi#c!K{7RUXyZ7*P4!)&$`)fl%e+sv0_H}WHUl3A6X*}OWF@p4}1 z@!}_TzeH`OU7TaB!G`*y{Ah)b?7I4=_2$9wwO#EBP`mz4Qfj4O1Eck{&_u7>)uN_l zn(4N|g3qi0jnjSPYri0ETW}mchH+>k(Iulx&1O3cgjn*M+d`r<`z6gCkZrw2MP^t> zr-PUC$kM(ywL1P}L@gzE)2m6H8RvYE*DDr?9=T1R< z)3(9KJ*-%#cic6_m8~TvC8ov4hISD5QWS}ZjqTADDSxM3pf;_pv?mH@UaymjNOW zU}JcukR{#qZaB!GT$d{Cr+@O|G*ZHj0Cr+-wG`W>c+kk(mI_ApiYJlW~pW#ooW>B;o>Z?iO=l-c)(; z)^iVEw?C7n>mfb1#5O`RY2&HYe6Mx5RWmZy`C2< zq2%vo`Q%LPj*pt;NAH&uD)I&@@W&)1zV4jH=#XIFt>?%``;EbXeiC0C$4v*(%B7D$ z$&IZNoj7ee4o;h+f-~uTKcDu4$&|N_*0<)LI$^G+{IZ%Vbzh4uD0ekA@SA*YQ^hkm zi8RsJnpsjJU9rnE2Hd!YK|VS$9cEEudvx55`ct6|V`M~b>}IJ+rFsz$-juBqRSeH@ zvEnPT58FDOuC6Z@uH&ts*Wag{_3;5+>zBvlPw5z`r>scIb4cPH6tlElNQ;BTUzpjJ zYv*Ig*E-8jp7b~zH=5bmZ`2>kmCudEGw@_TQrex@ztzgaKUt{NSJ%PR=_8pd(1LBo z(JNn2?$kaVpt~X;$+c%Kbv{`OPP}P}RZ9^Kp9f;zAPb^Q6~adtg*>^ZHtIKKMwnJ0Mr9-U_yC?q%jm(DW+s z=-#!0XS^QiQF=bFj!m$1wia!H7LsZ$n__M0iBWDg$8_B;^L6BuYtt?|@`;h^hwq*2 zWMjD9{4RK3o$D(1MRv>H=!O6JQofRt;h~;>C5^{85&H$v_{_vg0v@Yz>q^^mb{Dvz z5zG+PfXd!H>6EzLKT0+thu;Yr<-eY3_dJR|f zmK(FWyY>ngqt`CnIsn=OJ{50H__uTtuC(-m;RO zbrh2J$z@T;HP3ixDm9hP4b}c)HOjS~h2O%zt<5Y|iEHbXzKYAjdkdY$y^ATfK#O^! zMu)V+UCzgC#a&TJ?d@f-oh9$B_IEy73`ERpAaviuZ|j^0+v@@fWp%0{izA4`RUw-~h~BY2>2fmL9WhpBX1A7z%k!~ixQ>gLopoM) zYTp#jQQVDGt$3BimRxaDS!lXe+k(8l&ZQ@7i@iA3ibsF0KZ#%Hd>p>qmbZo#n5E0g zoDZvCR`8MY?)P|`dk|}w6+gy5P+ld)%$A@nwB1<696fGeT6p*ty-toF=yVt}pmgvU zo~?UVAi}59VVmR6()eMq*SRpNHNN7q8;c&;jdt|hYP9_6>V6rrG1JS=n8jhaF7fM{ zvwGi`lze_BY@$(JSbbz5u>Bx{E0?QEls6u5U%A-}BAm7VP7w_slHtYSW5scAUXygyTDV zwQ_Lq%OJ^QE5MQrIXY~5-F+xhFq`6cirtlg^K|uc*W)2~?YGFxA8bQ)x;|YpH^E^GnfIyA2!-%XQ#hTst0;w~p?3@+_z4g|+YZf_tI4SyqBX6wM%R?9>&jH~ zncoIBjhT3fC`sFQ)^frVqJSP61>4oXPl86& z;IDJE9@ab?BbUnP7wFAx-y)?$I$P%ZzuEZ(BmmB0;uAs3#viqwjs%a@%%Rqq=})lSQ#9UxR~{%EpJgZtI=O z;45(om1K2;#z#31FRE9YubXzAZI7%n2fzjfiYdcdI!c9shw!`f-2pUhv(^EMoBNY* zjVrmh(@gqQ0K6W4W-JOO!grPpF7}=hr?0|ol*-@dTdH)gKHL2sHFywg1 zPk$?Z;FIncK5N3d!RcQbwFVe^PHlx7Q@e37f+Pypjn~k+4?vx z%b$HavRuUW%AMyt&S80$RCIXX%y>Q_Uozue&bH$yWq-XYIC2^6zR~egQ(1wu8MkWB zCAcZ0r18-GxO~l@?e@B!XW3}C@+G~oJ~JLK3Rjo#+?yuf8{WPg|&Hl?9cjldH_ z<<-8(Q60c?{X9lsKmW-}V3a{Nz=)oK61Y<3hHsGRNTIj<&~KXSW_2g;aVRreQ`Jr< zZKub5Zg5%kF@C#J@BVq;Q>66#;PPX*0%( z(Hp;K^9Ol{ZI)?htTL_bVXlJsAz&6Oa0AukY&m3_{^Wj6Rql> zul$`zk=9aM@j57RWmYbQ)I<%VP}}Qr--Xp`ahhCxO^EpmDZLYtgPoil-XBgycV^nU zc6D*>mz`!?S{2=dXCLc!;#rA!$c#RjzV9ghcwdrC?ht<1=7QHecBA_`ZyKGUt=z`R zfxqZEI5zm;O#i@N3emAApiH-VS9SWFiTWuzxuf%2^zPKjwtV;Skr(h8@=BFks$M;Q zb}(D%I z>_yW?$S@mMEcUFHF6_GZSSLYil2?uMS3oJWUvDKH4ySum;WR*bD;V-xCX+z@9z)E8 z05_A3jNEl)?x!JRP&>+{(J`&d94>x;A_jurm;I4YxBBHvklVB`^%Y7_wR_;s;iBQk zuO5x^CAV!hc9(G-yVpOrF;!Nz6hAxo5Daa!_ouQ3yJl$9Ov+ryNe}H0y63O%nU`uz zew)@3{M1@8@fpMGZTZ%oeeU3Su~CJWm9HD<@^g>1=;#UAm(aik(Qs{uHG3Us~R^u^3yekSjZY6MK zA+)-zCI=0#!*cgQHh_<02D#AdP>O2}bK1uK2))gFO{d;RNU_Fx@DwY6PdDf&h~7F4 zZ-}g`#7A~h$(f7G^EYH~dWNNw4?|?py|=d1j<~QQkH@s1*WT+(E;#o#?zkj9QFiQn z7-w2o?duHOeF4cZO8G%W`+A$hXs|OFi6QYMQv7S43gVSegq*U*) zdoVLQtIkI^)U$1db=~E6x~gc%Mrz*G1BvN*+vlrGZ0g87Xkm;jubXA*^oK%!dXJGR zuuA)$iQHP}^F7#hJ#y+|GMpew?@TtaLSB?1=i3FVNoR(EORaxJKDwZov{Nw)^J-3- z^AwUYX|cs%xm=P(&2#Nm*w=eD8(LEkJP*<9_(%jl48VzLvyr|ISEm(`jleJ{OIvx0op zI>Kh}2yJ7F&17Ui(Cd&1ueb1xuA6x)+P3F&$SAlAJGsq2yN!@W&PhqmJq~LSSa6~5 zF;tRSD~#VZ^T_l1(4SqZSGYe(YopsJ6i=HA$7YOsrPE?LrntcODW#L-3!F{8q!a#|iLJ4d zv!jWD&A-sj&=UR|6FUnbknmrqMaaa&_Sg7-T}-STgv>xTLMAr$zZNTy{r{T(&HM*5 zvH$NpY`}kPRwhnDHfA6p3kL@w3+q3a{qJP_FDDB-Aqx{TA;;ehvHcVM%fZUb^iPD1 znUID3@8o3t8)5$I;ROD__Q}G@^0(qI3nRzBPgWo^Au}V}-`fB9nOOc)_0N-ok@-J4 z%>T&+5;C*05;Ajg5;C)}|A&R0`9G}eEINe$Q!m#4>czvuAZB6hY~siuW^LeXB4T1> zXKcbCV`6LOY);6+24rUXUj`!~BNHnd2at~s{@Z``ch9=WhSg45dVS8l>$wI1;il>a z>i_@%A#kt&rVU6;7)A+G7(~-n+zL>#FPG7hA?a>WPf1ztG3SgjO;c5E*_`;j-Q}`T z-PM|$N!MZy>^)qO8ipL)K+q1 z(Q5m)+9MAds7FPI&;B|vfH&xW$p%KLm!@Dfz2@3Q2=8A#tO?(U!N$gq^m5D&a$*Oe z5S80qG^~D~b|CG4zAgIIe3vn+OT?wLfA?mkGw3WF%I@itOhK{hx z+~?9M4hUGBJ};RwICs&y@9)b*waS8{lvjyGIa@|RAh!VYq`*)HSDg^J|FQUmTAz|4 zHVrNa)<> zv?(!tTU~N|6^Pq(p)%A}ZtO>D`X?&YznzMOh(SYQP6 z`ojG(2J&omweNv-9@l(1}WwpP5e+lBoX&-cz_>P0hFgF>EbZP3%<$MmB2 zfw3ZzdSMZPgb+x&m&$cU5y_VJIofu*!;z4?4r(oixo&=$_NUx@3t|49&88N6$z+%D z0dn%Cv~%jTR{Nl+3!3_Hc^y@+0^en7_HezB@~8FlGGfgvXHFYc{B+XxS z-v|DpaK{Uk?S2S0Oj#SnAkp?7YjJ;Hss zeJAv*RkVX(1n&CaZk-({$b(RL!wb9uwt#ynyig{(= zb!!Vz$gf$WA^S3v*-(>#!ZP+grW>WY$di7iNf2KB2gtU-N?fZ_fV#i+W+EWwH9q`o8qs{yzXcK*GPy zeawS=)Oyr0sAEyB1Ne*>xVHQL-_o~FeBu+I_{1kZ@rh4-;uD|v#3w%SiBEju6QB6R zCqD6sPkiDNpZLTlXBYQlzxN1y;*;N5M32VMUy@>vVGdKG6C}134dr0te=7=8T1W&7 z42D7&1r<;Yc9;wcpc$6Ha@YP-1~*2h=#FMy9%f53lNh5mk4iTw9V#>zU0 z|FtZb11)grImB<4{K@|+m?(}FH;Hy}k?0Ulh?5~0A~C9FNWr;UVK|J$UP^JclVJ+X z!1m`efdw%gi)L{wl~uDTY&vUVt!xQf!LDKJ*mZ0(dxRZiM8)rDS)Lj3me1o zStTgh_j1qQpZ*uXFfVxiUYG#?TBxQx?lTK%Cdec%%<}4OrAG2qM@;mp}}cHQ(1o>r?EkHGxqniq368Q zJZ+xhd<8T^SIKF~EN8TsNl3sN#?r|7 zB}OM}FqS&YmdtM}Z79L^+SRJjrqOd%nHkWoQe#q$DQBvwxt*nsWO71CEiG&pz+cO| zb&AQQjdPsUHIC8}LqbBmY=hBqTTbO@r=Q%GaRDDQTx)F4IM}wXLkF`O(lv8Ta~h{P zoMIzZY7CQA$i8JluW05%Jxy}qzNvSj4gpE~B?CF3x70Ei2vF!xl;7y%h z_O~>8EXriv3E)XS#hx%>xi<|sHXI?&FCl@C<=PG#%)*RwRgHu4Mwn%Afz6s;?-Uw% z$w6;Pn4Oob@|N^eZ!jhBr7CUkd@q?F>0C9-n3;jmm%o$o8_SJO(cCa=_I%F9xoxJB z5{g@$!)YtQgssst*V6VZEBZG!;3O9CFw{7lR#USx)Kp9kXfX0MSWxSbt9Yt8Lq|KI zVYa81(^^`>$7L*SYbc=s^0rJhj(w2pKH5ImXxN>LM^Zf>hckRME{eId%`s=5Grqwv z2lwYZqr;HkwAEwy>P?Qh^?a2~I%nEZ>@Gp>QLcq^JI&kc%xCVG>~C}k2C<&678;CY z_+cs@js-egE;-9rw0OAD!3^Ma!(KcdJk_rq%!tXO%XyK=tBx)=B-AI6{EHzOJY!NM zJN^5#r9(r{n5dt#4voC|fYOYmb4&URuwSDJ&j3BG4LnYP$HLPw*6`=cS?(<@uCzLIM_h zGs0-xgJtOk@3PAIF><~qTYg$`g}2z)=5MO3ZR3rbJPm>ouOOVX(p=GaZGP}zyfn-3 zbTgGTn($s!*4EhJUNx(&-DYcRE^U}!$lEYg%xN>#I))qMu}*ZXFkH;L3WiEnSy!By zfoD%~yNRu>X}7VpwNo7XbO6S+bq<%ngwYMf_3cSm=GbS%vt70d+{%qSW8@j$(nQSo z%l?LaHh@)fsU#cZ{Ok?}vd!OXV=%izAe+u>6VN7+%_iHp;37rN#}MOjTWXxcm*>*@ z`E3pL{NjReTuS_APG%YjPQf&?oe4^fQ)QZ4>{OeIxxI+ni^#6zc0b%0ESzC*_?gt! zV8U|>H2Q~-F1$H!-mfK1l(%VP^UPY0qJ=5E0V{e-x#h9XddHSHM^0I z1MCi7*DtwZc0F!eZ-eMk;S9hQ0z56CN10rQZ%3>&8`lOGP_DlUGtO1@&iZuTsbfLC zyq$GUC^r@2vQwK1Gw;Y+-xh4jk*`g>6jaG;I0A5NQ0pLz0khayJq5{6gCn19!lKy? zMvS%uvukmy(^;xAkYz3&iIRD)tSW=25cssjWVKf146tB>_|4O53%?{4$$s_qGzryFk-1MOJryuISX9Bsg5PU!sND4_-b_sgOf)USlZ6jjZpjaeo!aDzMm+qf zz8Wp2d~aocenzN0Z5$%2AD_7f<1QxGb-3>{El=npnHeU2bKn~j4Eu0nLw(!n2ItiD z%nbk2ty3Ug3fP>-NW zVx~9<@*!T#@GxB*o24m}ZD9E?3OtZPJXBYGRLMI+WQVhtlGg`Zq;vGE|4rxL-?9-obVOlpXAv|_>zoY5I!e-M);KQ3E>}vzY~tR z;sOvKQ|=?ehpyOQ#8Fpl6ygI{tQGM-;XT5;gm=j0ZOXkx_#5F(!W)FY5{?jFC;Wx* zXTocQR|&7sAPy5=CcH#=k-Gg8dA>k+p70#uS;8U0GlZuJPZ16ho}@OPAUsaS1BCsA z#|Zlf9fU^-dkK#a_7HXxT!eO4Ob)_H*yW1JMLbM+h_I8egYcj$CL8ep;eM)jAK_lY zJ%qancMUR*f6Sfhy5^g8lMon)aY^LhB5^f=EBHT>4iRx`6+(@{Aa6REV!Un=0 zsp<7peI4OiLL1>4!dj}ZhHy1uHQ_44m4qu?hQWx-39ASz36~L85H2NLLb#Z)oUn|r zl(2-*N@yW0CM+UcL}(^7xuS<4E+i}@TtGOV@CU*I!hFJcgn5LyggJ!Sgjs|}LIYtY zVFqD3VH#m7VG5z%6*Uy$AWSAqBG?Iagj&KxLJgssFo7_hP(>I=s3eRfR1nGuV+du0 zQbGw~G@+OWQfcaX~CS8 zj21#BA%l=k7(_@Tq!LmHW>2m#clKf#ZTN`itQ5k!JOU<8oC+=OmI z7vX2ZDZ)v@PlO){Kgiv(@8ub>6J-33@GapR!qqN_d5EnD8>;C93x#HT@^T3pDuW3C~e~&$_~k z5r?S4Gt|-3H0r0Qv4eys2~Q9nCmbN`Cp<Ai$U2z#ib-2@k*ow{`r zb`c(?rXM2gBkh$mEA_3TPU}gJZ~l3LKQXL#%OyPDfnr3LT4B?FyZWxQfcIB+o0z<#O9D zVv2cpx%#qAL_lR z_n_X5dKc=QsCS@lN8N_HRW(0;GwQ9Vx1er9y&3f;)QzY&qTYadJ?eD<^W!(5{tN?a>0m9G1Ntguj!YSl@h-WKZA^dW@%oWVHRtsUVOV78~BEm(4W-e8Nyd9$^S!Fd>(a;|k*YG@Fn`2qx$WIzkX3kf3$pqI56~K}}E* z0to&DKbMv-wbC{f`A^hN)GtxLK>Zx`GhEIOP~S&=5A|KtcTnF(eG8Z6Z>VpgK7sl; z>H*aKsE?uEj!SVHY6n|Ik-pfa=UaX`1!oyyDPakrl`xu6OmP@Run~#~BMBpD%3*|1 zLI}q`Q4|H2E&lE&L;+v!1UQ5$ia=wzgiuSXK9NRNL#QT9AdDwe5ylZJ31bNrgmS_d zLK&fyP(nx~B+%fEgm^+6A(jwBFc6{%QG`gEW&|PJwguUR`ZMY&)RU+`;R64N`UC3s zs3%arL;V(4^&8Z$Q9nWb2kPHZkD-2y`Vs1fxVnc?Uq*ci^+nV_p}v6nJnD0(&!Qee zeFn7y^-)~$y{M0%?m^v+x`nU4&_yv`LAaE#z@^7WAvT|I9))EdVJ=|~VK!kFp^?x) zm`RvHm`<2Rm`a#Js3$lGlL?auc0wJ&O0ZCF1SCW3+> z5k!JOV01--?M|c{wHx(Q4D1`Ie?>il`a0@gQ2&hj8tSX4uVA3}p{^E_7P0C@4T}Wy+Px?# zwTycYSri^!rdyP?$hJtlsJzMEyraoJp{c29Wz)8%Cz}*2n>I8FyD%l#ngXYrOs~OQD?8U)YvEPsIgD5jJJ)eXOP2Ua_OxUR}<{Sjz0BVqQGnK_Cvlo8wl+9gmaL4YAF!LUZiV*yFKcbIkD= zVWoiuMX!wB5G@Abhd>`u@lhM1wngoVQUu8fQPUi}Dp+XNuhI)ydYk?={ZYLH`t5ol zXhYDppj|=YgrJ#0Uj?~?q+LO5SKyO@uLg<}0%ry`1&Tp|JTK~OftKvDAZ@(XHpZ$I zhg-Ep+6h{5gO=H}mYg!1HYug7NHak*QzLHEFq_7lR`!+3trBc1EchzG9U!;^7>LZs z7%&||^vAW?!@}ar#K#%`2U7(wb{*8ES9bWhCssQBtEW2IT4!=C|FqRiaVpn3!9HcG zqn)uo*0(cZbe%Jl|7=~(uU@|%Vv8%Cv9%7DxP5zUaebw86;Io2a@x()fG+jvGZwcl zUfhztI2}JwXDmibODlfMVEBn_ZQ&&?iveBI&m!DoF-NQHw77LUHzMB*xK9^H&YU6tvVq@LfbJXn{0_q9_;nlX zfJfjlcnV&G*Wp{Hf(BR(Pr}FWIeY`B@VfP5VJwEFox|@moTGGKsaObF@gOK60>FLJ z{aN<|?$7Xe2<&6I5pxlex!2+jc6Xj`>%OtOqx%)58gz19o$wM`kF!qqNuh}6-FZAO ztl??7#&N%`-MhNC^&3tzEP_^82Fu}MxCB5lHo+}$D{O`>7{}XTD{S+W@%&c&x>+veC3nKT@Br*Y-UWBVJ#Zh~ zkNF2N`a9uaw2_7K520l{+=146(aOEJZ5Mtyp&eYX8}{IuQQn*DfP=6X9!2iM)!YvU z;Bj~Y*YO~(?9;N1TfF(Rd(!7Ka0s4-=iqsG0se$L<0W_*4#O+(>aQw!w#UMAufd<; zFSyr^z+d4FcoW`&ci?^a0FJ^(xa+<=qx^5^@izLr=kfXwz5WiLp=T$0k{|iJE0=yM zw{QgO9))AfA5Jg(Np&&xy&Wkd%CC9-iXo+ z|Jo-y@Aiq|-{Bu}WGH%6+8aIYaSVNUlzAKdWBU=-rRejz+}^hiFMAvFpW*57B}SU_ z3wb5JfKPhTpLohU;h*pooRB|{!`FD8d<)-U{(H0>$K09L(=9*X*N^ZMoWvFU8M^x9 zx=t(Uf^J-6e86A=6QR2|)oYbiVhX0jGtHj`Fcs4*WMV!C^MzQJC9@c+VHR8fCl#LxFJt?mc&fDj+EZKcAcVno z_mA%4?mN27@HhmhJuyRN{7V z9$p6@O5@;CdhtkG8#`K=S`ps4Hp_nyO#!@&jC!_4|#=Np~6uIKax*3KiHS=mgV zpqDi?Q1J6pDou$NVMvNOFE=-5q%dT#*_0S4$hN_GLr03aIdOs*>a~m%c%F&x{5(M{ z?Mf1sCk(62RMiF>wvQ%>FfbeO7V`P$YxmGPG0{m}JjSEZ4ju}y@4bmzC4H3~X ze*XGE)u8gmu3Mv%BUGx0S@B)LD`qYh%@ zOdak|ZLvI+tkIexwGhq%!_8`yDNzNIiRn$|WPIzi#o5%LVZowClM%5FBHUi5nEHF{PV|TJN2|=)MbvB%~VsBh&fs zYOe>74&s#?4DrxJQF;!rwt&DIgFHP0|EY*gQOBp(nO@YH z&gcroZF7zlFc$nWHYFsQhv*0Aw6}3 zA5Wio)AH5VT(EI=wy@dNwY@SWLzNaDy!i(kKq$AF5q4*t1_FFQ5~LPN6Ew1)}+Y8PT@cAu_=3p zjaxdM&(<1TP++y{_|7)${SUNATcdwC7cOOzm;8A9dP<7v^j!fad^}ODFQk;$PN`Xf` z@$w7l5H{MvwXqNv>z5kDs{A4~S~kv4r^e)CY%+wnkMG5FNJx~j!+mr&dMM?|2C{L= z4mNdtp=$N{tI|8vdu?4!PM@}BzMNa6(57^8Z&J|Ix7kkgAb z@i{5x+&C>pv5{M)JLA$KHQjfsB2(kyQlr(~acZ4fsl*TIri>JI)F6yih5K`9iy{e% z;4O;P?wFXMNd8L)kP@_CxD^KT?axOZi6h^wm3e%(hLff2NlYxTj@-|zc*j(EHirr) zW(z2&4V5=XDF4Z&%{s}mIry2u4^avqo;{tI@m^&AE4_H5+3!Hg8>+{%TE2zKF~;yp zeQv4}h9p26pe<}z?U-@P1%<=@aMP5GNy#UIL;1#JkLaR8RAHkU&RZ~K%ZUf4G&p~{ zwXW^F5`#u6jU5!FN*a_jYUzD*n;uwH7#hklGV@~05o&dKd}vo!TxN7ksH*;fZ#Qr2 zYM&92V2;V9{VJ`*`=u2Qd+(N3+AYbRU6bUY+C!CxfZT&LY4~r8L?o#Zcr%L_G%-nz zOVWNe8*HG#TW%=N2WjFpf(CDN{cdgYJuRIFj#-a%=IH3=#>HR@__a-WFMoN{_T9g9 zGYCVQHz}>u#^sn(a$~jKF`78q!`ir9b4pH}h8@$!=BAi);Z%jeh_=ruVn@D42)@XjZ@Al^Vjn_Lec9YgSp7UzUgz(XI+ zF9v)%%;5vGsR#1v8Kv2r!UhPc=h-7HRO!c9c(^oD9iNvrsvyd*d-)lWy~Honn3Iw^ zI9?MR)qOj=GCUwftye2m*y_Bl%{@oOGisWry6Y{$tktU|w5s$;Dc#ntN7D@8xj)t6 z?4zN=8|yGUZq*PFG%-xxJYoD_EP8Jd%z9X!;@Qgk-#U6W3_tej@OGf;+Le&$nTD2a zLWAPMxHN+XZw#Bfi*o8KbyOOyjPfG97Yv6tX>2yNHY+Q_YE@YxBcnTnIeU__H5wHr z9)+a5nkbDrazD$&jc#!t-=i}LkzY0V zbeNMv*OES0$3OH1Bdoc(`dpm!BmdD(`t4d1DFv?8*?icdYD9h!0Lyn~KB6Vz5 za#B`|M(Dmq3Xac8Ow5W87Q1g2)Nxj{$Ex!(cUp?Gj2adxu|#crT7Gi7AtkEM=8rvf z47Y-);2R|7)W<#c%X9OBOa+5}?h@Ic!la-;tmU~QOC5?}7y-L!751hCsVqT3q5OA= z;w(9cJrI{aF^$I|ILIuFOG~vRYIHoMQ7eNw*osH-j?eFw7XHs`-mM~E6rK8lf^U#6ww4{)q!_!mE!Aul2F_xr6OO$F_1Rl15$wfKB^t=_r$~TPbnyRAf zP$gY!wQA#rq;#jG*Vb02mfcb+%v9+#3WWwwAOWh~pDCgg$q<51Jv~pop~BPn)EkGN zDv0WRcAJJ5UacvT-h}vtrM|b=533r~{ar-IfjPZB=uy@{lOGi~)eAm=JJg}^& z{L*{)lRfSt5wGPKNQ28fPj*Sl{dmOcAy#K_^Jy$_YX~WD$BOiwk)m7=MU3zoMs%>Wy|#p!2=7VaolXI+dCrke`H`sr zgVFx08tHQqrHB^sjNpVv#EJY_)(qdqIrWRCK+-8$F$L0uDD7&vWWS8*MsmT2qzJeq}1?=E}llr6+&-OSw{9H|~muXOCX&7#6LT z@YO0Xx4Nk!Yuu2Stg2b_XH{jDwr;DpOsgIl>ZcGyKdoAwRW@yCx-C7-I$_TIIped~ zRr5BVH#j^#F*@54KPXzAkeU!NXrwu#C_6oCguSI^`ugdXz{t4JzzEa-5ceg(ZCq!X zed6fG4K(iiXxukQ5IjLrBAOsTQlxH)qAu#PWl6SV$&wsfwk01)9LKgMiIY@3IchSQ z${~`HnaFVtYjWB%S;v!1T(Ohcc+2Z-w3YFAeHjsfDs9TB7HMi?(C|f)ZO;Ev zX#C3Ti_l0{`p{ejHJ5?b@QKeN{>T@V&%zu6rbLxzDWT}3@W)`8Nqo^3G|Aq(S~n!E z%oMcQLnay)OwORXL~!$eNeNDH+n9NXS5RF6XIsnZYRcc|0=r z40b-R$s1+akIFvVRFdq?*Iv9=w*# zuq;n3fgqdKR=Kt6m0m9rlfM%&p`LVZbwES@Gj@g4_JRDSUy!uo6> z{|u_iOJS)rq1i|7Lxf5R;w>*Km~Ciss#k^gL~@Q*^e?@TL+-H9d{P;7$B6eWYEx@YT^kwW;ghm|L=P#ORhM|!e>=$`mgf`#fNBzJ1Me_Ga z>`)R-CdEG3Tc_HcoEe&Y7N%A`FQw4-Y#KDoP@3FN%-;Wl^WI~q2dc;tzA#;r~<;Z<#)n~Jvv z^=NPVHZJ8-cv<{$)@$$C!}(X|LNbZJYdw0(7IUhuF`js3r!(!{iEU0LW@;KO5_rmw{;}fxxL#R=%47!lw15V zF(tyk98B1(M&+u9UU=}Kmmcj`80^+$%ElTgi_?rBYY^7F`8~Mv`!7 z&w|>V1qdz!gSV zk%00!>}g=eU8n{j5f3W8WC8_&DM7(*+}1t18MOUEVt-Z0^=t$(aB9fFEya;FdFFy9 zta(IY0g>x0nT3?nB6WK3wt<1un^KAOCkl=p%VjZ;GSG$?J!y9_de7QT+oI?G<^1s4 z&;I6Y>8{OLhQ`l0{03;iiOu(}S%2Tgh+O^(#dytTFCz`ODuX7USguz|xkr9}=Khx+ zFKewfU9_qdLQlXGCo&fx<8`mVczu2vF@z>$Gw8v*)azY{gP18|&5I}uU9hKui zG5i%&kPnh35A#ZW_4#Q^HNlS( zg{j~Y%+n(vs`HPQ+u)-ea}UJdl**w+$feh>BS!GOPmo{5SmYbP)kPR^2EgIDnuM=ip^Sm5M9Jp?BD5gfbcBiefzxfu zOWpD~6^*(xAk^>6s}Xn8?x>!|ZR~E_K2=qud}6~Jp(fV>*I0~@5rT#Kv_1HO~UngbaxvawC7ob0c4c+z5b3RIH`~S-mXMXX?a_9Xo zJT-jpmPB^^^y+mFY>8*b?}c$2`0F#kx@;i(N?u?^mw}I};ov3_H3*OLDUy`=;5)Nh z)(~=DuIE*Y$p~0)lZ{0-{!H7EuiSBf_sX)?Ex+J z+W05N6P4#0r79BBnV|#gvx6X0u3R+7x{&Ia-vDDygE-XlZ2%}=n@+02>I~dU%eJW@ zJX^zRkQ`36wHb0?@E4)k2u{I|zsfrR6Z%yn&iI!g59+C+Kpo8Gh;PW8tG zvikMz1nc&v3Oj&AEWj4F16yz+jtd9@Lh_VeNvLMf?386U#T#s`aEG~3gLXah%JS`M z)pZ!|{7IYqxI`nJ95)*17%=lu+w>pHrOWg zWHlYo=XS$=0Rgsl{q)8^5Of=hQ;;+~{gxr>iL{nM` z9aiZm92Bz}{L`MG4)&NRul)&^tF$yBW`Z7}`WKCZ@wFrC=hgkA^XvZ4=hdL>p9oC| z#rs^MRj>M&EBfDmRX<89J)PT&6^4cu+o?1r}1g%xg;@CAc5Cy3V z@wq<@Z25V-Gk~fF(*<{!u2=tJMmvb>_z?6P9UiPK<}1m#P3I1XRy_Il^Qpo|6xE4GqMbNErVv@E)*&3 zPsn6q-Ju!On791ijpBE^Kkmlz?qT0o4AtD^BFIFI z`%KKB19>@HwDhS;Ii5|-!L53EvRq>mVLy%`SIrfmDlUD%uV|np{mfzj6j z)uDen{@giE&9(-2@Z3J7q)AV0A*`ZxCmf2Qb5qB12S$<#cXOcfud52F;XS~2IS`p$ z_^&`2{MmYcoc}rODBmCFKpgcBKrh^is4WuTK{+G|Op&J1ax%itJ_7eM=B4FopIf-z zD0d}QZIt^^Uo@{{t`^fpLP!_QV~SijDK@qiH^laS;nr;Li7)RAlzW@>QWDpx)NF=J z?c8Thm(!Vn7FI5miHUESJw~O$Wme_yo<4f+`FlGRM!Q~V^q5-X!0bNtICopo=U^R_ zg=esVq{IYy0y&1f7E0I)jZ*`TgU%=df`^0`69;Y+>tGQY?K zkWv80#`=Jtzj*Ej%T`kyVrlU(^Mt53uxwSFI79qH&$;53yUT8wHR<&wtuhVU!X|e# z*M5~+=}do2Z}m|kT&GsKQ>&9zSr(hTK$a2$Cnfrp{`I`%^9LSSt#T_Y#^yUFPi8mw zgw*)Pd{_IvkM5d%l@iJe3Z3mPHre`mXTMpyLp+S3rclliYx67h9!qQ7>@W*Du>0(^ zIqWif3`)Jr$ZPBDcTeO<(v**M-?2VPQZkKN&=C16@(A(~PkakOvh_Hr8JoQ^3)2WN$wb))6p z8Ps|~(#O|0UFoZ-x|(<@h)+WKmIc+10A5^CLsiuUi!n^Kw*+Qz`UXW1wPcQ+52`n< zJ@lH%klFXt-rbLn$9@P!sM9_lYtuQ68nK9y5VYDE%h^k}aqit(9hB}atuN>EwD=i= zS4v_!mCBu1m0aKtjoy<#a2Dtgzhvr;bRQW_#5X^%evmQ*Y}t6_$XLHvDi$+dYb2po z$i(c>i9P7Wc-9s$P?_lJh+gl{dBUAur2%?UU5kPzxf*>AXt6~EaeM$o=NHI>2n*_f zC#pC^TXL5%xW_4u-JYkku0A^Fw-Aa@4TV7n?9WS$#S9M}8SvY5UQs4X)g>(ff`$B5 z2&mEz_!W&EBaM*|xKJI;@w)Tz3d`azP{yF$88FeSKectw$2SGiJ0Bk(IN46~P;QmI z+q|neSqL+l;Hr!{nQnHvYp}hmI0BG*7liYUHuMiQ44=)c;*uj1O)UpTQ%ZL>0Hsz0 zQo96lnlO?<@oF!kR_ltub8W&IP*ec9x$uZKf>|QZ6EJ@;$k8%Fs0geyOl&8xr-^S7 z7(rO$GeRpNe9t?fZzSiyq3iHrJ#5A-z;ro-95Ye6H0(zI7Ue1 zt(%Yb9G^PT)_LdOA2|GlJ&CvRE#rwo+=QX;#v-|~ZnxH;6>D53y+f~57!B(7li#`f z`1j8C^&ES8oAbb3-j2~YKq1rhcd<`{IMI$Asp>?pLM$*yBtkxF2DME!7tHL;T-&(g zQa-^sODe9`9*}|&;7WS-O8N@lS3=#~-z@CgY{Z_q>ah+u0waxmnvhT+ikRF6i{C8! zl9Y#%FUhPauQzF<4{Nm~xOK=|KEB4^7l3*8TQ-kYES8AXzP9j)0130P8i->r2{4(cxl(TO13NWce^YcENMr&zf!sbF3NhX|6i1HQdKkP-0%A`_&)+;B3g0>{jktY!i@-*^==_j8&^fc7c#na>C#Z9~5)gkSn zt}s1=ZO==K&f+2PIt}W)vzMCA-Fy0J?sGt4(;w?Qyz4a7nED)dXK8X8ejR?aHP*91HLF%iZw=;t~G_evOtlTX=uwsyxJvl zdXd_9)eLDTt%Q{`Fr-IBz@>3D+ZJ56*pL<*RnO*!w#2~S0tdGx1HcCRM+*Y>5x)br zvjf>OT`0Imz3`a0m`o4zFKVq@IeI9)r*vd0GE?BabWE?VQ z00IzRfd-J~8S|<`&q@D7sm?WboA@0NrrP9WTGIdnp$fj-pn?UCyQB&R*z&P8;ev@) zN(kcZ1Q3dd8~wn-RL$Yr;-ze6YmG)}$-|X4o(fFc+BqQh?d>VE@+BWw(2U!s3I1q)@XUg8U_q0lPkr zJU0E%L%W{b4RP@B=xAqo1B_(5&hN_bc-SfH+y!2T!%!cNlSdCf$vp;b{NBFZ8xBJ~ zC{J<+N@JxmXE5hVKFB1Lr_DumUjf1(S%X2yk}U^Bv~*R**bAdx?sGSB^hj^z_y!j2F`0j@${ZDzzz+>^%xSX;)z@w-H|S z_eGsK7&dyjbg3rLdf0&E$yiOSwf2pF;(bTcm=mrj;}yhBED=T)>p(OhcV$DttV^MA zWrLxt3+SNg1(oR=*!*g?YVszRE(fCvP!SXN&#B0!Yfhl=vHqJ@(K6<(P%Mf&OK6bC zK>W`j1`Q*<(+-EHkAk6R*yMrco=e9E`n1s8y9%CqFgI~zO)GU9f4K@_VPk=o=kz)eQzh>%m+WHlx*+FoS>#U!nD$D6H? z3B=xBrcM_Sps}p~F{OfAWL4CUtDN6h{~YJ1QPLuI=)?&6Q$fOF7TEt|$VaC;JI#ZT z=(ybIw0^+-Q$b8rRQ0|QUpxg{z{(F#Q*nbyyD|g2o$vsfg>k5M$2QDsV`|h_W z{E*v7a;}oD^dL~n1#z{==l5<^k-baEi5ke)g-KDpxAf-Wd?^*ZKh23*DC?|s0LZ%X z&E@?Aaw$R13%Qg=>IEI}mF^s2`}|P7Vj8H|=+ax^Ka9{e;kz2uYf|$>@5x4etn4*; zsB6J%8gyMKZ+UoYRIUJphRy2H0w5HreC?ri%bNg#BbLe&_!seCBAv(}GL9nE{`Vne zLX5X~is|C>Mch$Di~sgh8ETZFvY(Ek_EFS0ijMyKiwtUDP=rx2n37?(x8Q$m=R(d% z&$B%k(u4NA*iuw(K~?yc7xK;_el=};+gR7tu`!Sl^NW*kof7;y_ACC4Uu3i7*EK$v zE}}Pn_S~b}dtT_lh#pkA(&M+y4Pf5$$7-v5z!LO&!CF|q2qXe}Lv{B~Ez8KlRe|Pg zZEZkr00{z`Ve9Lru#=nh`xSWgH~fow)qcG;v*Y2l;Xy{GO~-yyI=&{{di0yeZvWf8 zakVSq2*;blo=|q@LnEQG3$>`3%6EtReJwuCaDi>{Y1_E2DYHW>n%J5fOz80KiI}m& zHF(!(m{G{RdYccE;J)5%?LEiVr@Z-1O|JHA%3v6Zx9#wIcJ>dRS{J3Hk;-4WVN*Eg zST$q}WoOq%6Bw!WIPI!b#=ypT<{ntW3;3^*4v-g&Pp0g{GuXCiM4>?XApI;4*pQKy z{#56%o$z$SgHEHxff;mlp7NBazv#4HEvB8h{`+S%45u~zl^lXy;^8A#YfO+joo-wc z_bt`w(=@Nc7Bkd+bTK90eb+Z`ZQDQEq=s~_NG6s=xQX7@wape^clT<4jinD1`nVu% z3OF1=BejU94CMQ_Ur=%mA5rCUUR?7 zYpV*$HM=t{2iByN?q>eHg^z$->Lv0pl0t5&?VqASHkAs2!hZXdJ!ER!Pgc%L^Uba>~36tu2s?) za?AC1(T|gZ2!ptg-rBB9J@!4siZIxASVW)3vMG_X5o`NM{u_m2w89X_Tf=IfBvzNJ_TDP4|7mEQL+?e0#>2WvX^}>8*B=J(FMl7_|BZx_YMym!tK6r zr(fH?_mOhl|A7Wl$URjJgtSoPb{+;0y9%D=L8E2fuVppArh#9FOy2&wh%~%_s=IP) zVtyZgcypM*+z1ou0{hJ94KQJzP=GLY07T%fzex~477O-aALDzCkImnQZO%)T#rn2t zYeUP;GW)Ps`h%O;hfNUeCudF_|JKo#j+4)vI`OSLS}L*7`pE*J?(wKhP1>mCsEc&+Q*cD&cCzE4}@xHG2e) z!7lUtd$(0RhE+n!Rfd?Al!J;>xE!b8S~3mKdO=6?R6bl}m5j5W;Z42@CgX68`byPT zs2hHK!IJ7y&cRDZgk2^ia!R5z*)>ck3S4A?Yj`?aTC8%r(@2v9h7WjSW(wxt-uB4s z%Zr`kP^z0%;$kT!V}uij-o)MlbMHsqnA>d~tM9g6m3JdDB8J9Z&w?OAy`j$LAp&Hb zSq$g*NGsdXc6g?Ph2JOrI@}{&1ZA%rEg^bTMd;tESp+dq@iwLne@|*eawOIgIsAdxP3j zXZb2vX;L2Sp6J#pmD=tJqI?D^pMnRwTl3aqeJxuL^!eY24ZJZD1FsF~*VzNb4R7_8 z&njWZoQZoF*27Z19_9(P^n74 zo&B}5Gs#RclaoyTB-iW$aa zD>#uc)`1K@@f*Y;W5lZ+B?YGO6{<~c28Kg6tS3QuG(k)%O3-+4WOlgGxrt{>~RNE_O=~blRX*4Env&1Oty)h5pvhXhxlKcDqJ;% z^I%0-VLt`q z579XsON>vP?`x))DIA9oGXLfzA}ps-T4w+8S zglg8uwc8ynXNI)NwSO?sUtc-K?0u{2?pte2)ocWw6)JqeJq;~>)#tiWI*Mq5mL8&i zf^?|&7bQMeV~F>WvAaD<6H9lOxGgw+5mgsD)=>v%tkq_jzf-RD0n1#N_i?q;BZ-|c zeTnk;ZW>06_Ve!-;Ur z`7Vb2YD`+9JXXW{%6$-~k9U{0TW+rsQ?uEIO& zsbWfL4(3JBSyCFYe7Rnd!%EsB$W@Pz&gq5{uZ zm4<&U4_xX}N2)S;#7#K6Kwe<&-odvRwO0{Aw=hcN(FLPjayDK0ee1&CpQ_;0T_zo! zAw*_{LG+Pz&jd2QJ1O=*7M5@arsKjp3S#hRHN_sX_|&>rnb4u&t?0!ZoZza$Q4&wZ8VK(D4r51N`BV8&=}0Z(0XS6|zXF?UF40?3|S8 zE6Hi>qJ}8pgnSWmhM$}g&M4Zpu+4lNm_z&mRLY(rt$G!inF5m;skdsKs736%laeb; z8pUPEuTYqplSwJbveb`x_~1^CMO{_pKZT2xzzreLbZ47ZWzu7q*YG?{bPZe|GbwHO zSp{H;z*sA+Z=i^LL+{R<%T(7BbHS5b_5;5gG~MaQGcDG1^%L%1jgZ z>#<^%(n+hj?RC@s^ETUlniB;Yt{^_Ek_IBc+2)l1XQ9kD{`h77D&|Le;sCIVep%`a z?C$hrj%K`qWgEeb7E@`pp>ow(IdCUCtZ^s{v^9%?{$VXO^yuX__xBZ+6_>G_=YEV9 zqbD7NNYY_3K|iroha!)ewrw8Rfg9M#PHp^4P&sHB#~6ELVUG&l*cT=GYmp0+?!C)v zttkdal7OtXB3_N^x(2&7JvEodODaV?q$2p_@p$Zy?}n;2ux^#Suz)>+dX>3g*YY7E z^6tQSQh(=Bcw?_ob5Yl8#P%L5WoT4~iR@g1QjpYH$y@|@^c`L56v>w^B$ z^2#FCOso{2q6*QZAjWB6TAH}X1*$E%M2HvgJJ!q4hkK)G-(OxOwbh75Wd>WTraZqt z>8X(g045@4gq-Yo3Fh{Et+^M*&_@Fv6xxXG(GG0PtF7JKQ^LwJcge?(9Ts*Z%M!+- zb%|y(nbXkMfW@V+03T&Tk^P;E7N$H~zvyhefuA5{cz(Kh{0r@p$ZgTvCo_{+v?#Q+*RrHM$t`H1JWH?=r7hRo`{m9394qD@;sJ|!%Wdivpg zdcSH|7o(+(WVvSx!Yl?M$_P;CU``N+-wG8t%nzRfz-IU}I^Vz@`>ywJ=-bQaTXcwpve2TGCOD7?f~Y_AnaTWt3)N1Jjf(Up-pqyDuKuYMFC?j>d_ zv$c5pjVYm~`S&!gocBT9W`S#tDOS*^)!|;-_Lu(ZvQL8k=|&gPYMDduv_e|dE6$#5 zJJzIO#344^h~TMB6J_w1?eO6^&9;1`d}yndc-uAdJhzrywLko=O5{q>Ah|9jJtcQ@ zAi~a&WMpzDOJG1+?Vh_Ft4I`I=CoO!7UygIu*#;MnN&I!r(917bGd|AemxfQv;CNA zN3;^Lo}6R=q)vVvZw}trIwd%XhC((y$t#I3tHlU*-P;o7`};~kwdBr`t1xn zJ@3^PE2m-nVV2K}bx*sJQJIX#lG3Bt2KATv>fgpw0OMfq(wp>@qsK9dFKel_!a~l$ zL>AE$6=vpJs*K$T?p1q-*>@`Ej7Fc~!@8`i-sQx9%-0(@-c0~^am_`JJ;&`FJ$MT= zL$_4EsY_4z>O0g2cz4m5C{dyYCSuqlvxj1L<;O0<2 z-TPCNEo{2S=RwFC(Ub&N3>v8sneT~_ z4Bps;A~_ai>9K{BGE1So(sz$V@I#zdOvA8}VltZH&S|b!`6i?XE{Yf!Xb%0jC~+gV zSFP%JcD#FV#dmD)lni^Y*m>oTh!aBOqsg64{)PGmW5WXzmuHOqVJLT#koA4(a->R% zsm$#0wkUlvK?{wCul3YKv!fb8o(9yHtL<}v3nTp=rqrDX4m{sbs3Ydr5BWz^t0#1( zzqDDT9yA25_ou3{lm98S+*d7!7KM<`MSpm{3bNd@OOO&l0n$Cc%&2WX)!@|Jw~Yu# z>KZCWCRT}4EW$H~LrWLEYe-@GJ9^%1mW5~ze6bb-lFU`%Hk>jTc_$FOawwS-qtI^iO=-gfu`zU-O3n5% z@a@d^djfWJcmr;!svY7;3{F#0+e+tzytfz|81S=tjci9}zRm5wpWkwXmW;z$UcW-> z4EQqeg7%C)S*(!^rCmmGB_W^{==p$>>u9kd#HyiA{GelpJt* z#)sn~KPz%WXcg%s(UyS0_);c<)G61)qctH!iZi zrhgB-c`4JXf8QRE3M)oSDU?T>b~5M`4*Snm?^|<4f;6a4)47wGz{CjC`dF0^qwdkq zF-he+45+aQ=nxvvWY%-i8+zE!LYpmNnaCdQr5|GA^fZcbwW_kXEE^+a{wuQk%Sjs^ zCK<6Kx$K5G*(tD{Z9(%JI5uR$3|51-VwSdVY(JCH)mc8$Kd?f5)WoZOdM^S7=L92Y z=isrD5VesPehi~z3teL4tKL>$$tZFq?MyB!6>DbWTAe`R5x}eoXWC7)lo-=80t-_t zW#M8DM~zi!TFq4mq=(QVUAbp+Px40f9GPdFI)utYQnd0YHuBn&O!_Bz{XE930(_X2 zz-p<@_4-O%47V~xZ~rSMFae9c4Q5BJBG#(swVpv-deCOdqKNbj>vVT0*Eo4ZCpkJ|3~Wfrcb4UQ&B9<*B{GS2L4H}ph`71 zL!P`_7R+NVs42&PSL&yc?$jlyczIUog!!A?m)qI_q>{5F0H1@yRr^v&00EZ=p*?=G znCVs_MxT3vBfM+T;)dNHurOY3X@WCWY4*P14#gIhFis#`$}dWS)fwp?O`HkH=&I|RuBMlE6#Jn{V5LQ73Z z?U0bf42P_gVv~`L81W|fJfGUK$md4d+-6m#URYv5ZpunnQpAk;D~uUZV)IVNKJ^LS zrpr%1RXk@xqj=2p@@s2*@@wh!IwGQ7P8yu{b{8C8vwoQGiDDBn=f?^Ef-zxx<2o67 ze$FIGEtmoGZu|${kK-!Ii@1uE&o!8BU|!+$Wst&{tquLtU5o?EI*7_0O^SJ`7jjan zq``ZD)9{40>gOtsQpKuY#q7yW4_(|CFsHmcw+U-2S}3|K6^_0Ro9lU|ob-zJ7A# z$nHr~okdmZN7ah@j@SOfY5`3HOy~lApK@Oakm0Kg8)x(ucaaiKP3DU&@hkHfm#IRD zph=E<-M zVMm)o=~lI}`{}e}z{8n@03Ktm*}u+!-+ES=5Q;U9PYmV30A->Wo+9PCJXJI*5)cI$ z85R~fJ*kG4Id6y7r{fM!FyJo35vn+Wk$-25S+SvB9(Y3i)1eZ{M1d`BB~xGy8dnc0 z9VmZs7~xG9Ybh@$qYu9IE7uvlG&~zo96=cRnuu+WpQ5(>q7FZ8aHQS-Z5+|!-mQW= zA3j&IKfvQO(2{AK1IjPmd~Y}Z$+mI0urQ{I?7~?LAY(yO{qPrH+YO2d(y^{2yaj~2 z2vn~O*io9C^77=h)|nmpXb=P*suT|^Qhl74saVuMVfG>ySZ-3bzyDFy#I@#r9D!@~ z@zSfgE?F(MIv3VDtS~e%X`IyU;c_^U3#N6#ICAO z^f~e)*?@{ta*!eBpFePeI-nd2eZ)g&m_b{LP+<`JHjd_%h?zKoBEpbWK(78#ytkmt zC$G8`<(TB>On>gDr}e}BVa6RpG{<$~dPsGs>y$YCoe)2&Uh#0am!bHE7}`Jcn8TcpuQGpT0DSx@0qAe^OODhX}Dupd4EPL_lJ7B zyfMPE{~S4}o5U%}itXC^gPJCLa8WIT5I>X`e~7I&y!c)ZUDYQl$fvr$r$?VAz>>n( z8}{&QQJ;M&L(YDBTnzBZbTZPvJlZk;3($dL%f8;dzya?XlV~+bhMK=xoTDmQ-_-yr zyR{xgsv+G>LJ?HG%#DGZv?GUzz<$l$-D_lQfhSIqQ(BZfeiH*&5Ae*ab#nSZ*NwVr z%G=Q7U)w_^vALg$oJm%yQHke}yu?wzxzTiS*O^e$$gta4K;hl;dr1L!X2vLQ8qy^x zleBVGr!Hkl6fv79hWmzXSC&?la~ik_FC9H{Qg!bd*hd7;oHB|+y3$ixUral-@vL`( z17SXbrBVhEor0Swz9`Z5wt6dNEhCLxaNSjGWm_ozISGFv zHPWp93B8poQ<*5wry5#9TYE3aD3mHtXj94 z2lp&+q>bHFC>0xNx$e2{W?XbVg!>17km+=FMuv6t5HTX$EW<`?ekSH)0v>1T!S1kW zeF|a#W2>$dqO#t6T`S>jZ6D$NL*Ma7lC(WGja+-%mgwB24ZirNn|@=^AT z#Pp9F=a}2`$m}X}ja;-8EsOcBpSgzj0`aoXpVDrv%Lz{n!?P$15Yh}Tb%!kP^w3m! z!M<|}dP_(3NI51szA3`9obp8RXi$e}L8scm6o3Dfqu_Ka{#_1AuBP93^`+*|>_9a& z^&J!M;kc83Z&R%fpoco(rR>4MJfC#{=A+QeiU&pW2;bfnt zE{e2j>hLiC=~8v8mEXKfJK1b?g}l}Ic$#eBiqe@|6Iv;nJ*d;P+z6^~KbLH4D&i`v z@%)_z-#&B8&@Q5T=LJx^eu3DR)qc1)AsX@vbk8y&w25lxZabsoO3B_@_t#>_BJPfdvo)75eL8hxj49;-J0F* z^icgrXTCTrRp&Xy*Ufod5H&X-Yk^u?#&N?!rrs>-B>;tD$Css<@F} zG5=G!04RHqEcHfb*C417j)NwbAs%w?5vovYWV{i-dc+9*Ps_Y%Oo9px3i^WxnJ$%8 zAC8ik{D45lVIqzYIkdk8rgY1f2{3V(iI~5o#OF$)N+te922~8D7c)U%$GpaxtZlQ> zn`-;?5XgO4ehvsuhA9>bowU$dTmZbGwbI6=Yl@}f{r)Liv=bqck|J>IOcbnrK3*Dl z_GEr61y9=kt%%$p*{ylma5=D@iDR{D%UyT3!L_ysj?zoQr7xdmpiUy<{f|fFlOj00 zGysA|pJfaUxL-3+2;N1A`ulf$MgcbQ(e1Ka-8M9$ge~%@pko6AJ6A`^1Wg#71Z{sB z#CYs8?@wKa?{*h<068@4=x^+_Bb02aM$X`SS154V+ETuLzu)UIKo z^(ve-&8tpd;8fN{BvC2ke;1S=@!*kM&@2Nja92w^RNNHkI}xJfkeiK6!K zNMRhvAZ`)WN^W@?9D+lM2B5d;lfDI|H?NTF8AzTCWUd{HVeBP4bB)R?Vthb_HXCb) zv*mjLb7*rv zj6;}xX{y@xx-AIe@3Hz09OUnExxD_PlVAdwW<#Z?*n=@tSwI_WI27)1!*7B?xUOQ! zdbroYI4d^9v8?JX1oGZdq?=Zd>B4+wnqaK5)cz6wo#*Fsn!E$5sG6_TQ#!X!?CQ59 z;e(Q4;@CVKrAFF~9;Ih|m+%COc*r+T_3z)Nr1L9YmU}MmXt*zG#JMoVwov*+M6OKU zESSQ`L-8zs3bo$|IUm`xs=JtQx|z1(KGq(FIfP(rLLf8cKRt$^32mcXxo$C&>U+;5 zH4_UIii#t1aVQB&zy@kg|2Ve_BC}pIJv!ig-hkM>bl!WG(Kn6x{BsH%jQlgwQ8TKVc(xya}8$S5A$~3?*nst z%(*-T9mlUN?!K?-*>~8J*WMl2jl?nWdK!8|+ONE#8-$d=Vzg}SZCVc_vjwwWnZAia zkuZEF9_de83K)$yT70lt4!L+HkY-7)UWrHzL1v^ zA`B3S{Th7KdMw%cd6Q`{nv!&BmDo-1vX=u;cHwMc4U?Kiv|Nl(rIJKh% zyp2ypN1l=tK3(!WuwLD;qScmnw>hIMJ2cEO#mI{qE=pyN(ZePbGUS{FVDWmi<>s?` zF`7Oej`#M1J!!R8yX~Z5JPf+kGBcYjSPdOQ#iDaDds zZePY@Fjp7+UDZId<_B!6CA0^PEOaMFInsrM1|`EjgNJj}R2A_xIMMSvPE@Ym=`*IX z;_&4H-t7=sAI@09B%hS<3ckN3fiY3}KgFiRhx&(qt%w@i92cf{ zVPctbb^qhi#a+HC1(7+?ns_3LmF~Qenih|gmQS=ApzueDeunPOhLE$L<+v@8!?dr1 zv|RUD%MGVvzk5EmsAM9wzEWSomtas^Vd7GFYY~lQMG<*^1u$}fFfuYp0n+V z?`|LY9)`A)3@YLi6fc)z(zco68qH}8Dz>kKmh1IjpdtSi z5HfP_HF70Z%)obl1MRLXfNJiS@JtJPZ<$BBtdL0_M-k>jy_i3izPflOki9I7WDS+) z2_tGm2_Jl&u(=Kvc0&^zd~0ub{%P zxI$|@O&*3!sYN|utY|oK$!Wn-FhY15RN`m@D<&xBAmV2LqtF1zs^YQ!kKU5jY?#7t zyM=-7Ojve*Ad=Goty(EQej{UJ<(it<$Rf*cbLKz8;hLqm8dCt*C0jZ_O_M z+1hGiS(WDJ0iysnuwvDtfT#vDU$wEy$_w@9x%%t9BznbC{gRdlCvre@DxyW9iiS4^ z)LBeB1gZ!`PU^HZQ@cR(@+G4H%K@{)~{j+ zRLCRhyyQK_h$TX3GJ3|4$qAW#vaFGz2!3?cJAP4V9>Q@p5EJfkoLCI9145ez!KHoe zBbPBeV&$V>F>S^ebBJbOJWxY|grJ{lhzdacj4G~F@ zB-Q+{G0y5v_%w%RG;Wtdx4JIXtwOEa=b%|xV(uCQJoy%1_?LH2;0&hCZp>y>wtaB$%~ zNLO8Ix1cu@p>7lABk_(FHc*?lnLFU`ro!r`mF zqOe!ajzRZMq!!!{?$+Q+kH8fGvS8vPa7Lv;Op+?@)CqtY`Ra?N$Mi=`4t{ft;NOs} zGt*5O9H%)am{}X$&H{?X?%K@(@?>YHz~5zUoaaQ#&h@hf>}7UOM^>b4{+f9!RIaq) zlZAAVorGIy_Y+uaYdc|8C`bFOFatZ!I+tk%N8OcIVo4qd$_L+wN5 z^n#hC6iL2KCi>48SMK$OFh1(=St7E(nz&h0vE4@-`mQai6LQbH{3eM1ZEZ z2iu$&vg7t-((8tBJtu6kb&LyT#)=UVsdTO`q*gj_Wy1M}_JPKQvR@G^aDbVb8-1?< zYZ`dgjTFlqHH&QWuoRRwcbUg@jdp#MCsW;=Sg%1#MH>Vtu^fXK&mA>x`i*wEAe3hj zk^%l%5#*m=JA42mP~6&)6I=R!hJBcRHOPNAr>{CS^luEK{5P;D<%;eJ?yfXP_R<=% zr0s+#$@7C`eT2-nEbJIg4_2v{i`Dzwdd(i+?8+)G;EG0UnBA*P(c17*?nKj@iw!}l z=zI{K2>*z`3UUlU^)H%(BW5KMZ-hA57mR^N%I8~0loonL|48U~*WcHUGplf5(8b1--@k#7HKY`zfwYJYwIOkejjoTMF+zY+lMa!S$ zR9E5xgR3Y|L-Pjl1LKR4fLQS~XyPVPF^k48 zmXdhj>)0E7kiJ=k#!^v|43^19p17mGV6)0ng{M&mCIm{grK~Tb4=~9nXqnZn%ybM7y7j`=pYZw8ifC2LjERv!X8S z)phveAH2hMsZg}hV}P~lIu_8LTbh}J?0EZyvS84aF*C=5>x#G{E)!o`4TuOsZyfHr;GN3w+`qvf zIW9opA=h2^RJXObv|s#-kcG`h;BU2ANw1N$S%bf&q&YA~7}v?pTwF{1OU7758fsqq zRPpj>v=s^!n3~8`2wX(~)H6C5mZmDnD)HsrYp8PTmL#s=s92A7#}oF-dR-1>>qKpf zH}*~vFPaq;M#Y?hhb|m0)XkC086GjRqw!gim8=!UL~+BTe|W~Rn!9q889d==#3BW) zsLj>KWCB>SC@v!8?`>Z3kaJjP2O4Y9T@3sFS$`|TCOs(%LHu9?FL$^VDFC9 zi*9HLHt#Z9Vw-d*@)NPtB-3ZPR7OU3JX(F;=3=^auBx>HtI0gVfOhN`s`@ps6T)JQ z{}!rU^#T358hLEA{u_uR*mXOK%D-JejexYw@FS6YtMV6uU)!~syA9tHQX17qvq}22 zZ`q_dU8{On?_Six^k~bnuwVSoJ>Xq27cw7H(Kl|BEu@nzEIs3Hz!vW~lwG{Zd%!<3 zhnoh{P2z?kDaR`T|G5RwPG{)=q5SRC9}A0ofHjIHHlcq$ivWAKP`^)}jXR9AT{5bs z1jfOY+su?iL(*JR=hsi~tdt&cw$n-0gU_uSw;4+gl5d;(1zP7*sBR|?&uwa6{=j^k zrxVUKxTQDey!KU{nlmPMCOM9i^;f_RAOQb4ctU@AqQR8PM8DyR%W`++vMUBT zL^C@@HI+hn%xNOCdShxU6aatP@?6lcACkHOpDCBEY|Vm!gND6UygUba78g-nCp{S$ z-UF{Gb!JsYA3i;oINi!|c6nyIFxuCO+F$0oV0fhE{K)#q(_<%Ke4yo+KZ|&t6sHvr zKb?nHkX>LvDynHLqBd5+S_a)*IBovhM%Zhf7N$_tnbk9&goO(O7lDl)&yrgOJwknW z-KWPex5`g}HJr;+gqKU)N6qI>VT*lDS#Ov5EQ$a|>Y?S9=@?D~0hZvbgBv&Qq;c2cNhhf9)iQo%q?JwY3FlbK`Ec8`Ul{8Vu!@b358#P~udU-xZ*P>> zZlv^-jSGjfRdyvkS9Wu+Ka2}ED*3&&dLPb{xsL4Mx-X^A#C};RuxGIX6{dG!&{y-e ztX23|*N8}N(JsnQWa=0`p}ZhgX|JrG)+f59AOJx{t>Ph0QJlL;!a>E1U$X>^sXmS! zw{RAZm6wiBtTyqO<5)U{J!fE+b^OR~v)P`%!H@>+8U1L(9i{y(;Yh{y57W)#R-2E0 zt7v91>i)2Bc}waFFdK{&Vm9Z%Q!7WaJk9cn*d$d=OJiHJ;aLmb{}Y2w(ky2}?L7Y{JcZwz~&{_C_A@!-NW1tc@=@)+k`iM*E}V6=%Ws-38t^0>#= z(Fx>b^9jP!s^ny9b0Qk{c(WKFdt4R4Rka+&uL`ucGDxqseuBiV))e*|&C&`d%gF1U zHXpJJ5J7lA&MDd@P4y5dQ=TsJ<#G4~SMs9`C<|L6z=(~4fgUi)pZPmnEBpmLbnlgj zMf%y3{rA>);fFioUAo9O(7X(d#|V?B5XXgA!nXT+t^F9XA^J=1vpPmBbNzOS z`)TBa-i9U0W)D|9M(-B^XX}ahEe*|^T~{y;($!Q(QsfY{_XYi9JT0IgjHY$U)_61X z%CcE*hEnz4z&oFpl~%RYf~pqlnazw%C4ESeITG3OSk?22P}DSKE#4ZN!X!_Z#fw;_ zNU`#m%er+9fPOMrRYySsXlRvwW}R+6o@DU|6G2TvF=;w)a={Xct7>4P=x~T>-VdtT zmTtapQILXYE-Tpd30s~PkcyD_-UxMT@a8Tw{5t!WJDSWgW<+0^Dn+Kgy=ll*TgOth zoODIJFOw@b4{$7fLrlvqa$QCC3r#aUlZSfBwdKw#Q zDwavKkxO$%g4udiGh4)~Y{G(unsH-h%E4*xktvg^m}Cp?Pxo5yXOcF+C>dy2n88=D zAYq|dqF4)`J>2lX2%1I@aJ-WP>*FH+tCAfOk4<0wlXvYwr(o7lj zIUk-+_4>9hE)x<`Vn*M&4BT}enB!Q!F05z?yXQTX60w={? z#xreR?T0#O$4-(Eegk6X5YC?}eQ60@!swWlTi-`&A3f`}0Y7Ld)vL^@7oc)NRKFB8 z>HO7uqP$+m59BvgpWX{g-TCsL#9JBO>uaQ2LsUF(k{u`M_~@bY_L7Tgo%L-Fj>`Q5 zv|9P?N}1{9#YJK?J>6y#mmgE#Cvf|-vIu_R#=v_zI;`CopU?T$~d%i#7KdK5L zS@9`qSlLSY)zj2u%6MH1HY*_52)Wn>DcdJ^wu=kvOmr%%yfvlC%03F);;Y)cOi+OD zuOBoghjc}vce+%FL2s!77F1wWF+@B0hqzr?@uQvbD}+|Ml$1*qU-{(N+&umHUh_Lw zN*IAjUti`{V(WX@@~p}ETX?`#+~cbjE8s7rq@=sKdhnfppSzKb1mdzV?4tnemNj*i zc~vdwtr?6}WZ!vipC_jm{iFp%Eom7tQ$`nJJvLAb6LJwG-qS)WEVvS&kd_sN zz$`g97sYC9ZQuu#W5ycre&iI#G^U(sZG}3#9UQ3|0*-=)&d$D;6Q9a;Nj-$Hj*TNN zwT=L!20*-J=O&q*qMr^;!g&0tnCU52w`gv9zS=DM7BEOyP_Wi?QGPi_#Yzzk>2kTh z2*Zv)3?TN9naZBa&Vyi_G>Ks#OfOyfeG zGNoXl-7QnNYIPP?tp-H5()5|upJu&mc;u=~gfYB4q@AwatSfCjpsg;H3&4_DDu*;q zr_oN;Mbe{`{v$sQe4#`$t($om=N%;oeCgixd`!RJj@_xSe$oA;`zPmT>Nly(zvedj zol4=?;O^pZs&beT7TM{V*c^C;zSmWN$Qi3q94ByAUXO_fQ`CZ=CV`rIh~>UmB(cp> z%(qcQ$*meUo_!jByErI$EY4ORVA${r)x|sEHSQ2`lVyJvg~RU@32b zRn4~Our(#>N^bVWF0pdpmo@+#{ITsu+$?6>%&@_y;DWaaLV}S&9+XbJiZqmVNXUjts6S8Bd9^u368Wfn8`Zisr@ zhn&Ra*Z>nWQ_1+%eLGXPQt`j)#p5vVFO_J{E@lgkTh8y{W?Bh+R z+q3N7_|SF6qtPN^MEZRU%pIdVPlUc#^ylbe9A&~J{QKE5!r|qR;}!kq`LUGRk95)L zZF~BnRyXqOc7&xO`fOMXx{OL?Wtl7*UD!Scdra&g?90if2Z8+1A63Xc7?E28Gz*>z znnlmv>P_D4P_|t^WzW;Tkwg!dO@|!L*s2&n4pXJc*ql70jWt|TDHn;GD;R;OgnL#| zr(={VoE}KUN|{m(!vwlQYw6G#p1m}uONkW)tG=c#d&rh3O;IwtJZ{5u!*WSASEzzN zL9UHPVtRyns>*f;-QuR;m;kA-hMQH;w&^kgvo(;Icw-&^+7_8~DE#t{Oot z6?qja+}=3f<=L~DtapC@Xmw{l+A645(b~vi=+(tk}i3vs0ahYiDKu}(2z%Uw6OK)c6r!4yD{ZxRB3vKUv36w7r!xu2-T>ft+u4iinhRC zJujTZbh6b~M9V}>sl*#CUo>Y=S6`E}?XGQWFQ-E5>gv~19;ahX@a6!VfW#pLUbq^Nrrz&XEef*f_Y~A+{SWq?D_ikE{W)j)B~z% z{hrd0Q?#qpz7+~s)$2HZ%5@9%-pI1&U9+VM6?G^UPBJUBV3%AhyM>Zd+BI{edFwlZ zw7>GC&ei5z$t-{QWMvw8?7Mx*FkI9IBH5IA1=+fbYv|NP6bf%Nxts7_G@gm2GhRDT1b-fLYcT~p#neQ{$kwGENUcuYw7#=L?xtK*)78r4DFYS{NosBErA?SR`V+~)kb?675Q-kQ3$ zmkdTrqeh*ZYVIgijWe-$U ziZhSY0o$Y#n_-X3?k~d;TS)-Mns#@Q`j74&<8aM&pMfqZliI7alST&{T!LiAl*S2f z8*M$=rOVUTOzGIBGNn(3iC z=RG0@ycy+wVZFR6kXl)o2ADh7PFIh0pdP~#=l|nM%&Ht+2WiG`zss;!Ya#MezB-mN z+zC40GO9bia)KezNE6k7ik<1ilXL^B7}x*sx)R0$OfF0b|WJq~a=$sF3icWUXzd>A8Om{BD`@<)d z^NLTW<@1V{Rplq)=l4GgI^`;bJy3WR9@F#3ZKY~JoGO~?&XyYsAwTDl?A_|ws$S%? zFgPE$Tsw|M4xzJ~AIL}RLmUgKMZeq(wyOL~56`OedQ(ZK{#^l|X_6n|$MzqjcO7Zj zN#5$hYH{!lg?a;v!3Uden_4?(va@vAP+8^7o?Ty$4;oMxcdl4xYHnt3C+=FgxlSD) zomDsH?U!eBWM|dd<`?%}yv*JO0>m-ZLrYhe4hU_d8Xf3=rE{w~c5pIqpPrKff%viu zdDixmdp^QiVLe21_S5zc_NRA!cWZYwzuy)gqoOAv6NyvUEt@+cmAvAe-|rfEF}LQ-2V*)2-3a1GkrIgH*?U;y`J>)=AOM;dzp{3 zJLL25G=*G+!2Vyj^V+=Z|215f_$5VkG30J$kGPry0PBID9J+-0Z;2%R47_KGu=DKF z8yGB8fZ`2=J$zkyEP-Jeu)7O2{vWKtpONvEGgqUnyIr#b6H?n*XSdMDIV z%ZEviq_1Q}29oHl=lcoaANmV39WMiynz4{$I!P1h!g<= z2*p>AnI>ItfZoaDR0=1rlZo7{cKRZ-`u{&wvy!<|>zO)#(;X_{Iud9u!;%uF^zWjK z9H4~wF;l?Z=*UMEMf!IWkP4D4GzKrurHB|VYHOY;>SRwVX6gI-D0~ZccN3Ijxr9(` zVSZmIdd3La_+}96;2fW*k#x4;dz0{`Ba9J?IDE)3ycyAtfrQiC-uuXHabHAYy1wg|fzh?)bek z!tV4vv>^4xcq$>^_uJuxY>=cJ48t(@GD98X$56>S`rN_p_$I9)Gg;*r9C(C(C!|SW z6Kfj|$6BUq3NMQ`#y zq_%T#CQR@y8r!yQ+Z%I}Y_zd$z2O_%b~d(cTbpEK+r}H`e!p|ixpn@(HC;7N*Gy0M z)6><}aKDe?wH#*g=6|U{OtVYWJ2jA>rX>r@cEk7#65VuYS9+aZfPXowA)@Od`p=xj z91O0p4<8_4ZYd|{$wjK88`m%=Iuo6L{%lKLRcf)I&$Q2*c1k!k4vb&;<%hV9yDyA? zaB_)3y-oH~p1(@q<3JS{mh7B%gBiSj^l|AdHlTAVHYiWBPss;yyt0eI5syoChxTQV z*Y^p5U-=}I!zVEG6I39#Z3|WB7=nLJW?Ow_n0pFzzKO@=_Z=2&P<+SqkdwZ6?Ql&4 zJ;@gGK3mKpO8nhX`8fMlM$s&8x)oK#A%T+XE{~Bk;0NYNWk+g-M4CC180E=uUi==0{pIZ?o8LHtirxW{NJR88x%T3;MT(Yw5q%wB$Jup7rjhkUQT4nBYw`^+vi43I zPsP$u(55xY>|M@y1?HcOm|vnff*6AMC+kE8dxJI@XYM_4J~AGv;QCPa`9dZPHh(xF zU6p*71#6A7TtS3HRgSGD)1zksdWg2X)U@XG%p2J|X^(TpL*KBah zl)BrY0>7w3w_zd8*ZV1b9ok zmubcFcXc5uiP?#u;hkeOO>O0KYl`T6!wMXXR>c|W9AsNS^q6c46eb6pbI)=*i+)uF zqfj`9dx_4)qMQ;FhVblntOu56Ftp3)g!O1XMPPilX`Wn<9*l4|y{*g#tkzsTid_a_DKmmSichcbU zS?^EtO1Ubn-7Efws#3Wy*q=g44Y>86Mbg15@6fz+qeM*qI7yIOj8$Fw3cj?Utc~2S zoRUs>f@+QGU=ljif$tgkt&Ryn+Gs)Z)|So<={Q;+=NExjkV9byy4peYypqkqu;z;u zwxa^?KttXI&wgZ2u@ z*Pwn`FjBc*&lG+ZHyhvN!Ju_kbz>R@x_jPrKAkX&#BAF1)W|58tA zwDXf4_!zKbsRep3EBcc1pxQ@`C5?Gm6Q3KnmB>L3vxGEQCX6GB*+*x}TiY|Ue<`Yn za`y>JHhjT-4_`9`dJP1U%z>H^|MQ`HTV0<;3bGK z0)&Lmk%FLt*nQe)@iO9I*icagNyFd+<7u&ibqWLVJw`c6;YiGIA@BHkTG@zE-`1?~ zUSK4e$gtdE-%pIq#$gd6!an9Ra@|MP@cU?c(VoUu%SnGi?e`B6R+*SSko>J$!3^Xe z;pjt)fu&$Gw=&))4b5W*<|v|>QqHq!LnY26Z@SOVnuI~v|3{v4LW)=NUNSF83} z=IKWm@vt2*H4j6i{{R8-z=-=3gqZM*@(x0C#{^FzgN(k)h4+5eK&Z4b5sC zUb!px_7JqysRD)0?_N)Y=Rfff*=+{}EyXX;H!qQc5hTEY(<1G=%n!qp14)<1R(wSH z(PNYi%UJ}5tAtH+h?G}se`x&KF!Ex(DO_P~g%)>h+Zi8`SOCHenNDyiC_PEjCWwi$ zY+$p&@oQ@pw-8oWWL;nmSYs&B*$o`sM%(RuQBTk&=uRfH<%w2caoz>dq$0vb@<2Bc zwD=*nMOb)-0cvsmUBSKbpBV#0m9t|gy}P=%qgC|^BmoT?fS7(TyJkMz+u0J?RyajS zaz`id4^etQKB#Rq9J&(;$`ZlpTk!>#M5r&6*7%VkaMqI)2*~~DK|R6)#%gNp^=xu1He&6~_oy$w*0&By@g3y?WA#mUjM;txACB z&;nw3F!-%{9B&@uWQ}HjAQZZ^cIbnb5e)DxQi1{zjE+YJ0;e?3pnrgHD7HHtY>)tq z%Fu92K5 z%n1lucDKca1;zR@K-7F5IrM+7c-N*(;-7kA=?95}(qrKz)1DOd(w z!<%mcgj4qusL_Cwp(C0?b0 z($DVX7OQK7dsBVn*@4s@6c^h->9EYrC2)Ek%Cq7G+LesA1#oUaAB4#m*?Wpp4Hh3z zGFgnP9Gw#nB_ls|cD)y@uN@J%#j0pD_|@JS3t5sXu1~~ybdKq7VFu?qc(@G;3vetW zUiuiV8?QL!6ShC1JfXx7H3}lx+ucz8T5b569S|R^bcHDP#C+|PkuKY;o%uYl`yMw^ zVde683Np*p5AGpi1X~hbJtL?z@ev3Y-~}bAQnE#7yB(BlEbk~}1sKwPT#$IiBAMcC zQ4SqAlxu>^g8(L{JS*Cc*c)davBsf~9WqAV)yjD65;TF%Lp_^J+Dn%N(PFrI`FIyf zlM9O*{BW+Zdjl1@SLcrkn~Tb9)K0`z^gMg$S>`}-gTbWIk`uqnqSll7J_(_Uw4!s#tIPCckF#K09gs?$y z@f(hmx|+Pg89lLi$@DL1^Mb5$j>>3)66tc{e=Ih<*nbB9QM8EZD-iz?5!aX7lG_s7 zQreOwE^#O%30>#>$6IkyVSbMLZ}FdF2Kc$YMGy}Ah81^HE)Tov`sAi^I$K{+RVMGL z<}v8e;#kQ$hhNDjLv)_}SpN;-80k~>c}Dtcgx~)TvFnF@+Z%dp)X7%@-c23QhI8uywk_V@g?T#*u?-mjDhdE; z_P;~-`(XX6AyNy5CCdv0hpW916;uz@ptVZy8Qw?A#V3)%#d#V z@oxPwZnvQUq!8VrFx`F7-QSVVKV1%s6a(6@Zh0WK(f=o|YuN7S0FXex59018#I7&` zARnq*6r#HhY+F=KYayaDP*at6q2r&D8~v_3#I|z3AQa#~#zedN5dRZVJmj`6_>Cpl z4KDc2fAr+~-zocj5N>3)-=+)zvG>12_P=xN`a$02 zm3S-?Fd((+{;@5GbVJ?ULI8lkw$XNbAZ`s%0gkZUhEUxyU^kcnps@fDb^p7-?x*nX zC+4oc`L+?)^bJ1Z1C6PrGG}&%DErWK*EC7L5B{z&7(lcer|#GcVcYkADxnR0o4-F= zMce_h8x?H(Kfa6sAm)Dx*5HoVyYArI!~ve@|Kscitoy$jIQd`4_LGY0DWS8)lcx|i zSeLEAB+%W90lY8(QH1}B2+#)shzH;PkMV!G=I(!|waGLtFT>~L?B`;~jEPBN|wyi30vVr&%~NAvsBItSef8F9M3#5e`)J3ZB*qQJ(hkCK_N zqB2c;O;?1dR<5ni5$S1kQKOOIU#WKp5h(kYE5XSI6%}=t-`@{Q*qt~H~ z9Jpl37^%)sU#3^KTP5dKgjB}FJZL-L;~ zOA(b77rLsU(}o1c=*dmOht9+C!qW?gDQ6``juVk3MDWXSvN`EUP{OC7uUp-s(5WFY z9hdcQneBI7cSbL=WqO4!)>GcQSMP#4DD!zTiQWO<0rJCCc!eEa@6RfIjvUl9+8mk6 z(GAi9X@UX*!$?WdX=`yqWg!?-uNtndY3QYis&u$C{U+sNuMaFZ}*O{E0JD#(dRM|jCH3XE> z2=O=Elzm{R8ib_y%}5S}3`a#dnH2qUl6VxwMlt02G^E2^JwxgiuULO_Piz?_0GfWDxcr?O)NR4JsFKY0?td%)XyNof$;2$Ra ziMzPy7g(~*bYfW0;QC^9vv4zUR#8?aBlRX|A(A1or;3oiG^&wosZ1mKqLE<9!{}u+ zFt$nk?JSX!A#!H*2oz#o4B&+r#bOr0jB>@rG)R+1vA};iG9v@Oy0NA|$W>hu%Gn}W z5Kja|mKXodBpF-`ds9E1RwvxDS;PHX%Yf#B4jsd zVEBOAgJf}Xl^8Low)hbI$gJS}v1H)<9$|15zar#Ieq{am)3A`A4S~SZLdAB+Ghu(~ zrQ3*{ilkmq7-)lqv;;|Ery~ir@oDrdnIW4Lv1{;X(v_m=@7ME3cmxG=o)8vxekC6V z*}ep>@`eip;@zphF{-4p7vm%@Wq>pF9=Lnq`hA}04dq_^3qi4@#bqb>9xr9%&#z#L zT@z%Ffa=|WEUz+43 z04Ir4j-YZPlLB|dQ zQ*5Hf!H!IfiMC{KM?rB819R$DQ5m=-39TwBhf)~bF<_B=i)ec4g zU8qWJ+>d|=xi9u}wj^;t?GK+!{4$mx0d(m+%pY-CvQPtlu|qO0u!1US5RU(cl1{RV6HGQFbiz&dHEKsx z@_k~O2Ut-={Yf_JL?VUB8iA)iO*SOoH>y!93^d0*afj4^{2+vsbA!;f#II}^0E{0V z1`42uv&zG0C#!y81n;2rP}vjt+<`CRJj2_nGITFCgB{<LB-lhW2!D;;jX<)DQ4Ha)b~Rf&k^_Qh=-hEI&`tol zRJ{d^Zw=3xDipcA2ztJPEf)n2UW~d+nl2WG%+6j_UY_q&9a>xO(S!Lfz7IoXQ??aaMBvg^E!QKD(DlV0~)LUY)rV@WAfMVm3-goC@_NUxq6+wep~X z=EN^d4n)^RtHW8uZjryNujc@Z{KIgGR6@=}%rn6Y=4p==VI81g7NML-UU#mdep_Vw zhcSpf;+2t*ddc)JUt~!ionn$I*?6+CS7nBIu2lfB)o9}{O*Po^3t1E6E0^7jq@b2E zMRd#diQFLZe>amw{fQ;)V$kDTfm1;eVsc@2WIO<=aDF&y@rCO=)Yj-9Z+~6nbY{}W z2JS_xN%JMb>7zs+I^}by6rN;f1{{x?XcXNSw++zv?z?wQ*<&! zU?gB_WJo?>p~k3WA~M2Y&lACiuJDgNI(k1Pf1UpAIFrBYVD-L6yC5~HzCt6!H?O|R zKqEA+z9JwZj_Vl>T3I!6it|TY)%Ga`>izrn;I@E_Sp)yFJq+xdG)d-aM9Uj>pctW> zVZLe-QXD*b;93r5U_lAP=L(UaQv8)LRDn)5oG+Uz8*;}sM*w^9(38j?(V!UnMy^yV zAjbIH!Zcq?#&m~Fyl=-S97j?{8s8w3GXcGe?=fiy{ks%7Rsp96C_`wNHm9B&eR~TL zLhC7Hh{D9IJVBHJP+dA+rR?Cg^~xB)Mfa*`AzKSd08A@(bN~u59BO%gz~e_Em{Fe# zeWy+h7D^lx7U5B%K{qEAOWq@(M8(7*m{?JqTOn)69i_+bu(_czH|s0Qxwmi8GI6X9 z3(ksiNN+68gU=ID&ecoMkS`dGs2tvZ+KvPH(1ME48LnKNJN~YiVrg5Y6%`_*grkvg~W3@6tgv5X+_D9{75w zIt7Z~;`NTdn#jl=L+SI(&~y3JG`KDx8XSwXwP`^>{CM1ELMj;BLK(@d+3*#2KjFY+ z{YrOiT}(B=MId?O;hJF8!{Dc#UvZwFYCQ}UaY!c!F3q*##}qz~@)DW*AUC0Qtjjq> zqJ7F~;%kc;BfJ#stF^sMZy}or*c}qefUpP&unM4G`q>{p%6QYA*DikA>3DL*TeR8= zqidawPa=f$gP;?jVaBDUvr_ookgkB#;&a}%v@@$L#9S{_74h;1*J{s_oD&V?yW)Go zHCA9r4q-~^?;SuxBZO4LP3VnfUXtmCKd&*2gZ^g$OD|}eHVK>^9ts2Lz>;DSoi{RO zcfl@9AR<2gTLX5W;al95s4!$*kVHkM93$1`FdvRSyik(2zMo5>Zr9@0{Bt^O6pZdc zd-LOJDmUwgODN}D<`I)mBR0Jd=jiET+G-pgd6|vPNyV)6NGy5Sn>%Zfn+89)dQyZ` zK8UT3r1LHy6YG_)o~P_y$gOhWy~y>uTHPz|vP?5kW{-t+CLU*bLNlMXZ=i|yoIyS5 zF$cEEW1O{|FW(b*5n}q6q3K5E2~-C(@-`MKyU)6ymhgHzhhCFYqKS`+^CX&=&A&TU zSu8D~$CVT+L4D^prSNEq9pwJ+%Gi>AW;^aJko% z9GfV#j6f&|JL8hqYUn36cCAr5;cn;o<0>`G*qi>5sjol)R9YrYFtL=VyJZr#9U)8G^R_=-BjpMi^AdzO?I}; z^`Wm4!}9*X38w8*?f909mu}|6E1|)`5X|{*dwCFQL2D{ZB`Z>k$Q~y#MAtfNm6;pk zRg>>*)LmrWg8M>dP|FTdkdsZkM^!moD8&j;aa5<}U7donAxdA9%1ctDCFc z4_7~C`E~i6Wn`T{LbRkQ59&mA6_k951*s-S-D}mG(%QO6ST(QWu~3G zzDX*bv%NCRUQGf&K*f;UthlX?FON?7h0Co)#1~^JqTQ)w;95XWMzwYGypn^_%f13n zI04t0_vOxVc>fCFYB~o)opg+=nqflcff%aDtr@_-Em_J3MGhwO%AA<4NKj)Dn*f$zUkL=rN{ zT0wi_LMVa$g-t}Lus??9f2*8q-B!=%`dOizsCs^bHnd1@cYyd88&~?6$MS%msvAo* zlXn|UZd6kVDo&ah*9xm{)Q2yNjcx56ow#=H{3wZKZ#zE_IbC_GR#cz4F4a9Hq7Y@Y z3_YISx#mq2wx3^@AyA)^oZF-~9rk1CS}%pUYc0CA=l?M?!<-Y76P$UJ20t8Hq{`Gx zS@sHhE>ACdvN!yqE|R9a9hCQeACmNwD(ab)ns)c_xjOC`yC_9Xi7Vg!2&(2NbJ>>& z+bKTsi6>se>Rn-b^Bbw))yf_F2Ms9B1F5=hrRn2M#B-FN^Zu0bvc==HK+wx|y#D>H&EMl{ zyUo#~G*ybF&_KO!1th!wv94MR9xEVBk&OUz)s(6Kv%sBe zWAQ6!F>dX(nRzAY5m;4&&JX5DHU>?E*@{!%M8f@aI1d(QSA@?T^WAmCF0+p15f7yE8EvJcmAo+>mQ z+Q;~L5~odx6Tu&BZ5^iwF`GNlFX;+}{SD-Q+<66c2SqHTl{s|9Bn>J7ZF|<2|C=^s zRvr#>Eh|a9qi(KowsO5VQ07Tq(;oXRTL^Tqb+mi4aWY3cnn`z=AG zE7nU{7xvSix9q>Rw3|y^lSi|ayr&IG0TLtk*+4|io zW!&9tZtUEAi99~FxII5+UpDTE1ry3+{SXs2>n2c}W_uSV1)*Iz1&!LfPg+OGpIxkEX-d<%`g&bO(VVh?yyOnsa4fZL1TI6wh2!r-AUu+spmbG z;$w+H{ghwdd_rE77|L-h8F_{Pm*dG-QaSn3frv6bx^_x3lq11o!Q!=VkXJ)9z8?j# z`;oTTZ6Z9JLj>YJ6(*YYw}Ns^+>mPZQ{Qy>fL{FgI+HfliP_WTQ%CZh$_>?YCUkn% zXw7YOF(YxGtNqM#bSX%NcrpU1oex#(qM+t5^MT3PW}~|?>Gl1eF86LTrg;vkEeHn&LYiG-oca>1KpRbn8$?_3u`w=UrDB29qc+a+VF)z#UMt&iRa$LoSrJC+E zwnqG>{ocP-VO&HSVmR)KoT6JAqr}hCOXl@wdn)eZbb-;wC8UkWhUP3@_SRs+a(QOn z@~;hVQhlO_Wm5(DlA~ej@VdU9#xx%L8NVCtT+i_u4&_@4Q2>&LPQhKI%QJ1Z0{O#S zR))L(T7o9;7)ZlG(PB7Kj`QX_)9KT5X*d90IQ&_qo@}%Vdb4>c=HLF)LS^jKIq_WY z%Rf}v8=-K2KI-x1Z<$+f03%h2URCBj3RQzz6)GV;38*x!Yk zu1-11!3P(SbyFgpOZYpECN7regdJzh`Y)qxC#MfNo4UzGb)^Ku=rX;!HLiZ5b({Ss z@e4<-XV6ujb5huJ#64eKwm0RcH`$r!mN37M|Cya`BXwj|H8}r5n_Vt88+eW;7GRyt z@5Khh9sFp&F*^66cyJ>>d0G4DsrpB9vtHh8lY{?&$Ir0CYWnf?k@wW*M$)^KZ2m91uv#z%J{Td@KRU}Imf7mf#^dj{+x6*fwtRRWrH~rMASINn zMBreYG!xeNXE(C!)?<9T2~AK0JpW!F`QdF`0QewGbfmU21}rk8q8VBF3vj35)(IF( z%*W-iD|QK%Be?y}?Ge&>Hc?u><3(wqqxW??=9}AXZ5lgh$oly4Ns(9*=Gm7+@yAxA z38ju;N%F#cYn(iaHn*hJowWiAVoLnT|PuDh>}p}pD8p%iqHXr~@dW00~W zw2n*MF)TyYW_D*+-zUG^*Fa&21PDI&e&s%Ozs`L9+1EC|-6>?5$#CdLWW%6pGHW}l z_4&2OX2vY$dhnae)}qeM_lxc|Ko#ceQj=X87q_bKx)bRH&ZhD$_y6ql&jtvEs5tUDkLK z_SRi?(dsedA`=zoXnS@)#fUB}NW7q#m)cqSZVZZ12Ej|ohsi`Qhq)L!pzJaUo3?(< zcSA9G9*3mj1lLFEOgr-mtc$-pC*1p8pRoNBs8bLdw<@$C@Ec_$! z(SU+d?A04|gD2$*vR)HVs)zfi)Anw!x4tZJWlRUhf*Hl2&-4lhcdkR}QC}|BhV$RM zU8((@4d175_g0*lfbqA&MnfoQ&UV;R*~W+1ZE+$kxqPFn`S~=i-Cb97`TRgH#=(w6 z)uT8=Y-gwE@`7Vg_*ME@dhcNivW3oA6|o8AQT;x?vwgyG*I^|;q%oCOephCYg6GC7 zbfHc+!9G2iS7M~_E~X4Dr#7l=ebUU@yb}b>`eC$0c^91@R^VEah-$2Hjd_VgcMOU# zmpm4Uf4d$`tQ)Uc;uNz+N3nFG(Pqe#k%Yz~T>+lunn;Fg+p&kg*3 z=$X+ly<7QBREdqWBaF&(G5OYHG`~%AY8*JV1ns92nu6~@>C*li`ESVN!e{97>|Iux zMr=6B5j9QR`+2L(#S5>N{&iXK=gJW4^tg>rS5(pR)2INgxclhtP*dlWW)IUV!}@*N zs2?UziDZb|fu)13wGK9Z`{J`hZQa=A#irK*0v~0jK%_9rKeczqvGX`47S`2f;5LFG zDzD{NbN0K|yK0BS-VWydWSz7$OW#?Gnw-e|15a?cXzcARtrJ^J@%1YZw$`wZnD1jx z*53yD+&4^2G12s|xu_SoCXxbI7zD*eWzZ?gPJ|`cN}4_{MjG+;$5oU6c9(isq51hbCX|j-+l3?y4~o5k(HaVwE4Yyzld63@=Zg+uUNW(X4~PPD|v>O2$l+kI8rYTGKzHlKi`*eU_6PRN*3f3jwQ| z{p#kACKu2qw?^GyE~Fn|+3|QfXVdn$>Uzbzp_`L#U*SHaJ@*aK^y&Zw*Yg*rm%TzY zk2Y|9=_cQ$@Nv9l_U{X9`ibVVBdP%QLwlTn2d3IC;AS%g-c-=V+jcEKdZ-uo)+vlk zc%IOJk|H=<9bAKw-OAy;&wd#ADVL2zgG*W2m{5!iG=U4AEn$WVqj(-|z#dAP5Q`55 ziWgt$U2&6S-mg(Ut+=TO=Mf$Mo2K|^HkW89T)_L~ek_qG^*msD7hBMlFd~uuau_CP zJ5@_XT-6c9wl|)#mgv~_MZ!8KgtPv_))~*vUC*<;WTkt@oGYKUyqu(}Dp*zvEboh?Z_z?O&til=c?#> zDP8fsam&|mw)$@iZO*_rYV2Ujz$xp}4(U8-DeB8r`6Q*`l&dANGkEG3rpFsyZu7k3 z!m~pvAMZSqo(~9e|2t-}L18LttK-Tj7csS|He4afweD>(nAf62%!)uo+S&ZpwDe>S z$#ML*XpthYBkTz5+7t1NCL{#ht4`>TKVFTr&s{$2of^Ujb^0PBm^+xcy1AGe+y5`< zXkvqmz`@Hy#zywPpe`9N2giTP|DVOp&GWx@9(MNsjsFMz-#7l#^~wHk|GfXlzmO2C zw6&d^xeKebow1v_q`9f1nK`SXxr3#f6&WudAOHWx>|`8VoLqb&BFG5;hYcX-$`{^O zV`L4)d6LsbbGUuUIMLHVMV^pwgh9r1oD?heYY<~vYM6?OTvZkV9`kT&$jeGL59xuCCul-JHX={!lVQ-v9#Wsxy)&P&N-on z%eiTOS~L?$E+pJgtK*#N&83Of8;k}9u_|2V8tW}9(w4|^y_maaDaH-rXIlzGDuhqW z(Ri+A76SnmItIK4QnbuG?QS#oAzJqDx-hy+A4yGHQ$M9#>usTF^fA&ExB$OIe0`k2 zsff{%tEyTh-cRfUeaX={YUXI0ZFgTMUe&=|G^@jfERC9bv&Ha?d@x@b*O`SkO8!c0 z{asz@=GNwxDy4s7DYIvSztdORjve@6^xko*JA&FBI8--sLz|jxRN#Qfl9>pp{A=my<*UP(k8C}?`246z|GbwQ8sDg!0#`@q!u-sHaeWtwn~x>#wbrc+ zRKD#QkYncS2;62i+?Es6RDQ6q^_OJPB_3>r}8f1gq<>ktqbGo-$ULupvQ~VKRpN()%qFS?6YMDkH~jW+oV1Emd3SZEFE@)eKWW7?S5$ zT;)zn=mm{QY;86%oQshe?%evfxwv#zi@mC{m~Z>+(ZWIz6xtp&f`tvABx0I4M>?*K=52UtW!PD@n(qG~$o{u`JAIFV zJxEaKmXUEX;W2B_i*~)GBI%p zi;T65yUYxWyM4`+v8Veb63Vp|b6`hBAp(_0g%8ijyF76BX|0NIO|q`m`|Q%i@TJg1REOG?U_do6~xzj~~rJ#*nD z%r#m17Oo4$_SrccqWhtx7c@({iMM|8kfm&BLwzL(Vi?tM!5B9<0dxJT2*4(bjO=>Y zB^I_w@0|MULX?rvRIDF7I6Zb5!o(qUPCX{BOd6y&F?-2WU0xA(kQEiG?x$0$X&6#f zKQ=YUr{r2WjNzBbS~wogi{_WgS|xdiJFTT(9ocW)VWDwF)8_4VqAl}r;6R|TZ4yVM z5n`pld{$m*drPe`wnF|*K%{Y}oIi%B;-jzeWI*Cy|MP7gfKFwA&goh;dD1>k5NE?8L%SVNQ)1dCI1(+f>@e#%svw(`K!6Y~PYRrdJR z06;MMfn?ZORNSjQc5omfi7ZqdH3{)A#urXN%KT2bPMt9mAET65z_gmpwI~g`v|itM zFoutr0h7#O3XkL?jzOzu>cYnDMF?9%kJo|3*nA6b@VK+?_!<9+17})NSrHayGoyE5 z{;ntXBAaGZ$liBB4>t(fs|80LTVxcy78{;0Ui*F&6bf0aD)6ul*m+z`S6<1uPbId{ zdd)#$X;Mf}HcI%}e8$_4<%-s|1gS)3Cd~a7QZ&TfQ#-_6X5-RbKX6&8d*FCX5T?q{ zn6I0W*`RI`DyVNaRGmxGYCQYFa9JGM2?9Km!gyF7M`5THfBSz+En2;2^u ztqDe#(Wvl}>YAw*dTuB~nh$?WKC&ndcAR;8G|q-qg(E7+>vqU@+8uiOsMCZtIc7kl zs;c)d?jghBEd&bd-(6`GUUDvH5>qe39dc}9=l27_OHyZb?}kjb)t#Bg%a-X(EcLUT z!*V9iYG+xLH0?|-`i6zXak}}uQ#m^&z3fv3p)9K`A4& z;k4lm3}7w*%Bkt7a$&WYao)t$ltHOubTV^vBPMs$kT9Q{AP!HbHE}IQPMUtlf{E>( zUc`C$C33zR{5P04oES=DO7JvN^X+E6JhfJB4t>*FiL>0|-$EBk#;cB2`80)8&d3Sd z6@pa<584@G{obkKeY%EUdKadIzVxmyOPsd7bL|5%sSkBE>l3S}Jr~#Q!C|;*?=C05 zglV+2P+1?--p>I$2k4(I-HvTVJ3Lvq&i1+u^))p$xt_SKEvA22XiRXTJnxJ1E~Ndh z^s%IZ%g5dfK6zgW1N$-ud)Ev6KW0dgTO3&)1>aP>!*ouC-%@xrjE$!i-%rs15sq~7 z)qD6toLS@Il&3axbkW=cINXCaO*%4?^cd=j3la6Jm)a_JQq`Bu(xao&zjLD|=(4FM z&u7n3qDSviQr-2-yRywYnl|6~{t=(g&KeiY{+y@K_c9%eog&OWm*>e^7>#v3k(QDU zk)`o0R*3w(_j_%q%QJdEr?EE#NS$$uBM%8^wS3j@mQI_uZQB5j&B)g&C(8`i*RBVn z29IN+L zo&BtK#Lb^3T+tJg2vgxt;*tF1LCJd*o^naw9fZC9jU(TkSh=GlXFQjp{CU_f`Mj6k zmExU|3e92T#Q_RzvA3%=;!y)u4fSO7+B}3oe0~F9C`Sm#7AM+Nw4e%gyz?%m{27by z$G$_uebF<+J<(#F%R|7?aJuHUm{y_J`Fm7h@j(w|Fpl@AbUivH(;2+Snu&rrdD!hg zj=H6^gcP#SNq75p|002_;&Mq5_O1njVjsK>GFdr#shXI%^Kx z$d(SP3t@W)_7G)Zu62c1g_oCf5hFjh<0V{~ZgjV7S%eTldU|@8s$`1p*Uz{+pj}Ey zx6g0?ch2tm7>{&A?%qM5tU&Ac2Lf6GISCt0W{WB&1}n`?z9xheR>I z&*)79_2YcLuIXAYla{%=(TD}4S>dhM0q#W_hm?PcpNB>?l2I6fKQw`7QU4#K9-c1h zihs6|9FOW{V5RO4Iq1%41n9`v?T>2Pt8B7F#0+N>L!p{B)J1(8Ij_m3D?8s~7fT?s zI-R!jQ9@0o3f@b>R{-F038c>%F6HYRbfmHMZz47-Du#=n-0K=ZyAmaFHQd}IR=Mit)bsch zbNn`og@f+$(K7exZ(**#;>}1HD}Ou*>^YV1Jn70s6&(ly<7pxSN=;Q_Pj#?pd`X-4 zn$Y5!tHzFH8Y^%SQbkjz7UrIn7?cT%*k`YeHadBeKVEns*DcDPj@}&=Cq5X7Hii>r0NFT+|(DCma|F9eySF zR=#bM24zkGwyHba_pXru*)uxy6kJ-8uCIx-4lQiYpui*@RT>cH>swO^3Fy z)eP<=M;PPRtlO-YHGXBwavBY~n_~VVDGW&|hKCQ><8)@dOHsP1Y4h24jf+Wp66>kU zP+r#8P6Y#WcHR@(6>rB&?`YRi9`f9f8p??y_?B=yB`k6!ko4+hDOz^qSmSKULrBd6 z#O1<*8vodx(OH_Z1ucv24Uo92QQQKa#&kXSN;xy-nAHDMg@2krE5*b7K6(HV*U4YU zg;y|6ao?v{GKnc#IO)c&ku!(5Z{z#8E6mTIQV@O%>9yfB;L0R=cRKkILfj*!rz^p)tKb*!7uQi&LmZ% z$xX#fv8JM7p*P*mZI6!|j+-T_Xut3{N|hR?2q(n$RFXo3C;v3KZFIT5np7jFU$S=Z zet(Rkdldm=`$D%ymYwA&>(wwAe*htAqe59ZXR{r<*>EY&#?ikmeLyB#YTq@N9v-M| z8X_(94JP_%)IJh17ReI~e6aXT;A{LK5eh3&we{_uueo@VRCv&wH{w4^yu3&8or*=3 zM<`ewMNO7G<{m0RyWQ_ys2|U#f{4JleL*Dy0~e#R#N~{c_=)>+XelLL zM}8h5UfW;KuAr~6p6ut0A2X@$2OOzopEpu^sa6Vca5Df@K&!uV~1#O8Au}= ztJi(k4P@574)SpTwd%zSi}giyLqX8wq4@bK`Eq{rn~F90dBL_u+~14To#iQQ+arzp z;Cq8t=2r!3Y5b~x^q#0C1ij4PhzQS-Waqnm?mwAMA9kE={iMn9to zaAgf!b@W?dtPnJNh`l)eJQr_)`6L9lEePXuS>|KH`LzzMG#jdJkAp=B?LqxAi>{^_ zz%gcv^R~1*+i-%???uDM8!$^DP989EqIF=v_{Va;M^3hulB-iz`fy5lXgsYLzr!Hqz84eXO6{^#7}LCVtS}Lnx1?}!+QahvBh?Q zabz1Jkw|&tvfgG6AFG+rkE}L=VF8B$p1lGOq4{OFPslGM?OyD^I0a)a1@pTk#EMf$ z4Hg~kJcx?163HaS+@1Mpf2@0tvTR5kFlr2TW1(7b(2JfrM>@h3`bAB_ycT?3h5D^S zBh0RSYR7Dh>Q3j=7Y)`JVKgIJvf3b9|C*Rp^sl(>s`g37H(2Bfw1JlmnyM}a`K(vD z+75rnZy~e@&zIcUktPApLq!5*5AWLa+Qu;ow1RI}7r_1g6<#bt;9vNtcF2|SqxA_0 zivqe=B*Fv2ZJw-7uNDJJ-qwqI{~t=-IXD-fc^5pfZQHi36YIo2v27=BY$qqSZ6|MR z+qUif?)~l6R&CYpA3fDQ-7{T1)zvjs&yzOvKEW<&VlRBtUy!ijNl_!)*9)1IcA9F3 z_P87}=stm9cK?0a`Ny9YRwnUy$r_3RN&WbDb`G1WVc_%P6Rl-0~; zPX;QTCu+Dsubh^{C zBnq!bB7*ki3t~|!KfLS{R;^1!dyfA89P`V}8IaI-XeuQ%E8fuVoIugSPsv1t`?M65 zbmPqM?8|mlMB=y^f8Cqiz3E3YkNl7nh^r5Y*|=o&xr+U}nMEp#A}lr^_ILD9a@4pQtbzX_Ip0l*KPfNMSy4U=qhKRuhkRNOAA< zhWYk!pM5`YUwA*^(eTsgqsu}|mLIXTGgPq1h*{~D5N%BmC8kpum&mLgHpe7wgn}%D z$}oOoHs|;93pv83B1QCzO1w5xbk3yV%{)m@56Xk8iJ<1XuQ5kH8+4V#P!6Oh33Umw zpGN=645lIzK|>_>e(8W+YfDi`${TzNR+AxzUNNsVC()9Qph&?hZn+Lxdxmtbpe$7a ztdz^lmT8iv)Cdu5*~@V1mpWnt9n=gHPk1hV2d*vUx(Mx11&$^MSmob(+Ei>8pY%tehe_)=; zHd_m9Rgvo57fsPam#FGG5GybKv(Wi!wqwve)Hj6iEP)=Y<>1#3kjMWY`fBgMKVMy?6S=un$MEYFc zkRU(->ir$l5w+qTTAOwk0h=Qp#OXRX>w|bhP`!dMOm}1qx@sWL0~;MFc;4uiA3i5c z1j;c_=Ky8ZjlmF}_lhi{T3u-H723;sv9}TN_T&NS*|2$!w%$xhd!uy31F5IETslt> zMui`NM&a)@Mg6m<<57Z@u#L76*|tvTc8C5S_)^bQ?-Y?FyRN0xPesy-pRvDR_J6ru2&Okzh4{U*v=tFED;JVU(~z+3#}f#>|=$__a_hXm6zm9Vnve}c&b z?|5VTgyS`_3z45AOU~~?lvwhZyRV&JX%}6)CuXB>3_}ZpT+~+cmd}b|6ue7f-)QF~ z{7gt)=MA})=kJXNx)lXAJr>D)immK3PW)7;aVvzYBv*rnRGjZ&pID!A{7>x6qc$4C zsE+C{5qQx^dO6{9ZHR;;Sb!1GDZJADAwutS@E~+kEbm%8?e=uV&7CIQEN&j>%jYA1 z7GsG6=08=kNgU^wjotM|_9&eF((5-k$3UTl-L;R^kP^>&Shthl{crmO#WcIHryJ2h zxqdtK;oR!aEh~R%&^xDf9W;8kgxpc@=g$Sw2d6CIb??Jn`!cNWt!AVr*;j@=If(lC zH6)jWoe)T-eZB$YD!;c}2NJPe!O`ymyB?DQrQNCDF-j|MijN6@v;DN&?}i-`cIaHS zo~;x!Kl%Flc0g}i^eAg4Lf@-*yG&vDU!45F_<0_E`BZn>+UB!ll@;~vU=%Q1XO2>L z|FpMx-^Y`46Rx%|`KD>tvX_p9uX?;>&c6_;XM+j1B2iDIIu};^)tvWYbh_-ULjK2`Dh!;- z?onJ5Xq$l-HLlCH|7jnNdZsVA-kT%7Z%45m!nwV{2JeygM`ObKJIJ>MpC%C^!ZcyuSlrgB~UD*L-=#LL`MLSA!h!=1$m<4V0De5>!Ih? zVe^@-__0jt=iV!P@xix55O03-Ho-a@0U`g!?sqkuPKweS0Q@6Qt8 z`i%IR#d6EnfS1TLy9}>j+m|8IONU?+yglomX0OuK=Z^KJ^W&N9;@3{E{?8Z3XE^$m)y+N6pKRN2oW4u&-T^{O8zzoYjCHhCc0UL7$b=4ttQrCu@&a~Nv zqBElUFIVHn>Xi+_hoH_3XI)HH($>(mkkY%bhWV!4USP zBSu4rUZ9wlMFN|VkLZ`H5;JuSm!*%SA2=HdwRPDxR9`t?pXaK?e<9gPY7i`W#37XC3ly;T*M#H z9jg8p(>4Vuj&SJT=qPBkUQlTM{@xdtn^; z6A$d+=ttFQFi5M}&WOI&|alFeCEZlIh+UzkoOibrMx_HWGo&his+; zHdlk_o^dAaOm#%|N=H+DOeSWCf7}iyJrdY5Ml*J1O`gfU6HDrLNd1S2{oTNN!mjd} zpV6JpU=w7M5@csoLA*h5QfOGMSXs~A(Sgs6Ux!Aubs7Pl0R+FYcN%vV{wz;*1pd%A z*j0>n(2!ohN-SeURMAN;6BMJmMH7G>GF>tSbj9$~OAz|~nnX&7DIOi}E%3+Z(TdFa z#QsuTr8@N=7nJy6GqEfmorf8nXL1neSLoyW5gtA!GyZ`Xl|Hm3bXxqSX*R z?1d1V1yLes5BF)WFLmyF#rQHQan3;Knf=QS^{-F|Q8TuQUUV5r0a2O4VtURokR$cg zAZ+b!k%o_Chy@Vnv)X2WkSfeQcqSG50LY2f6?Puf27)9(vgECSe zt1kFqC;ov+t#IzOrZxqIm%MZZ7g6>+SmfOT-@$p1es7T2*Vd2|ivfZI(fs>&6T0tU zZTUL~K`DB!e7-Vh;x?OXTjFWydDa0qI|NTMTFog0&9cn{lBpK z{}}%#j?VMH`B*vsAGkal3+sQE|J!2zFFyw}*MAr0|7rb?`2V5>{BNlJe@<|4{uh$Z z&dvS5BmV4M|3@STC)@uo#sDk(|H#D6%udX~%JqL##m)Txi5;)IoVxIXkNrts#!UB#V9!gA z_jNa0-Y>9UWIfQcDr(AWZTpMvn4qffAn5k$YI3+jFAE>BP?TA4ePN}X;P)Dv&)0sx zT5xrmCo^_Bb{nW_G@+GH1`%s=otOV=oQ9Ok|9WWpb>Df+n9lVRN2IX{q4WhAKx?;c zDSN_bH{OL~=*J{{3yghUZn;rs=mh&1cvspCg#2oUxiRWNeXt~Wz+$?qk?|pXATs(o zP~bCGxUkw<>V;^7%00PC-{~+3w6-dMoZzeLvG)7eU(T%M_Ml}hT-`}r&^c+X?X)!D z?~{Fv-pqk8r`N{k0ETeqq7A;oVYV=}iPsy$^(}(^g5gB=B_Vp6{suj$ZiF}P2_&Zo zzwe_uxkhg z{TR@Q!l8ftY3BzG=^b~T`Eh;!(uiax={Un`#fi9kH#SWZ>!0QCeKt4&n}%K!QqA#L z^e_?HlGF$DGeUd)DK+LhZ@%;F81gd5=VU|c>Tt3^^-d+4Muu7659GM1$ghR zoW1<~bI1h}#WvNOI;_oh&B9u@GwsbAx%ve(_ukKb_ae94><9#<50X=|knGt7;Q2LO znsEQ5Ka+A_ui(w0%xHrbDrf%ey`PA%x$8gH7hXUx$$i@I7W$WL z)sg@Slt#KCR72Nazrs#D%W?>vlq{;Yat_gd3e6R+IwBmYUoRlVF>nQMz zeWtj3oI~S~&_}Qh*5(d^iwbk_0=&6#Z1AiOymo`gE)XzUzzD?!B9O#;hg!z>Yzt>T z<#YnC!ncRFpk!<=2$!FePdB8NN4)ejZ<3@}5d_P?FLI5B&`QQbyN5%Z;zx>4TMf z+x`6RLV!56xDVWGOd4H9{N=D~3mv(?s7e35>|$VI&eN?;;+Kc;!Xe?|c|qrs%u5Ex1j zuFNZWG6t>eW zt}=J~tuN9fQ7y?=Rc9#asqysu)vfEYxlErcl#xnB$AWNb_`(si|3|fyfo;!<>w?=X zQyxD7o@2aQd;h1k4cL=fxBIZTXcFbANQbK(hOG@`WLHC~MRRNZy`(!}4OxG@NoECE zZg#6AZ{`vc$g<|Akurj@t#;=4)PVCOpRTWZX6Y94QS@j+!|+-ZjmT!l)#9K=;r6Ek zST&@EQ6pGcSXohw`f;&bHsU?pTG{wDTfJ{qCCEAH zU8%hRTsY6HptZGgZ!`(MQ!y}ld#w{apKxv9Hfvj5y4qQGm99lGYc;Xl;J2-&Vh?#! zw^{tQ#4iMo(Rrfk{5dXg_-IZL#VOB6%hRL=*zn~ z&uf_A9kUaVj$Z09Dz7P+4vCK!i+pn87G>Pt#s%HTf~K#vBkqYc+43aJ9KVErtUZ`$ z8%&WJui4;Kcc}D$8K!+ycA48B=NeVREd%ZKV z$Y7hL#V@NsA33Txw<<3JiS+e6&ukGEaj|#LpJ*xn1)HyhFSW``h&80Kw6w^{b!k)I zY~(ZT%CWY&XUNu3ySA{nO<$8Zf<3(%73VCgn5@0G8Cxcxsg>0ZtaRaU;)^rqU?KOIMMQwRgb9^=R$*0(&)R2{Jq`)UJ-bmqG8WN;?ZMol6gi>Ara9>C zcv28EU`L+R78y~)Ea%MOxh^n87@fb{MCoW(P+R;|ZGF4AG-Hvd?oBL%8+KIYLw*F@ z9p3MU=OQ$&8p^A_ zqM2|t(pXxsF`&NU4z+e3iX*QR{i|jvMqU8_V7T=UyL+jT)&eG0&Q)RxT6KZ)WZPi$ z>VnH8b@6)(gGfM%N;xr3(Px%EWTHHeg)wt!8d;{}>~Ow3v5rzhL{L=nY9QnALCg3O zCF4x{6oDSY)c!Fj9V3NbBlAPDNcuyvE1NCUq?uq4<%V8xx5LZc6&6E)H-bp@`3wh*bh$%E6 z-APmAL4Z=ys`_4a_CmA~-ztfEIYMO})RHigW}&tfStuFjzhLWNIO=ADh>Y9uV+ZHd z*V&S!%wh-5(EMRG*tG2T-{DIn^W00cLq|S0-xVY%2vjp7zG}k8U$6FUVuh-h&e&@N#vw{e7Xw zHDm~T8Cq+8M+xfLCfDaG=doJ>pT&y=q`3JyLoZIPkE^iR-J1vkV-n%AFmT?9GRWqr zn8*sJ()1FOyLZKys&Zn5{2 zfy;zZ;U`}7B!ifuOnZL7V?tTHeL-M1KpsE{JV1Q~5}I=s1jH+jk{zOg&adC|BriO1 zgF?rEUw;sKGwu`b*#mB%cTjsseB@svZz=Yka*{y@tpOS^-v5=K)F!{)f@brYs_VB$ zU|og-Q+}H0of(h_<^tvg#g4p7^i^PwGICiK+8VmI@Kfixl+MB9fp3yLr#XU1im zbtnY*0g(W{?3?b;-Z1uP+oWC69>KR7Kx6hoDVPoDIAlUfesPcZ+h0a>dkX&^2>=@~ zttj8(Z7H{mJcqQv6u<q5qbvA4aH4`T2S62 z@|N6Iljz8c8hro|1Mr4kL8~FtmGua{HHYbn4!W@dLI6?#4={AlCn)PQ=v*iYlrBnL zlN3!T9WZJDTu}%ke3IYPRG+E0ct8#S3xE}90i+hC$xmetRFcP*rY51L0bl^?!Ffa^ zk)ipGh>vmd6Pa^<0i3BJ3jxqLC(1Rsw_tnTKoQ^$j6WH_92Zaw00o1D+~K{?BnhB! zn6LzyPU*w%Cm4o3aev1>X)W71rR06d?+NbE;2V15K6hK}%@YiLr#|Oglwu#@A(HII zaz(R3vf{GBb|pI(D045|4(R}SnkjQH*bemKJGU+K7wyJ)!dw)p;LEyDwioS2c|t1d zmFfn0@>;x5s+H)5e6m`s{A13?B-IV?%5Yv>rt=Evr>*3h<&0p)sNBpqQvQ1O$$t8a z%0SVC>t*ohpy=nf=8iF@o&(yR6VOzkg!nJRo(d2bNCAj~(F5$DsG~8G%8|%ji<~?+ zNCS9*;-U=sq5tiD%1Hk_V^<&_z+iy$k%atbETAPqM+mf~cGCwMQfNZo0l0(Op)@+u zGJ;0PdL_SLpJW%Al#ufLV&8FUUN_3@^LrxRAvzA~YXxCX$$4k|Na2Lkp~a>HjoLAQ>|#I8z^WOo61* zhqAx$7vU3e=+TbyPv|?%IaL`^;kDEk^pn^k5;OmflrPeg-r|W;uEbVI2igfug_+00e;TKeOr>O@S<$4BT>V9L5w- z|DWEU11JMjfWJjy7>ro@e={RLf)j|y&dx>@QINL{3Ooy717-@a7Ds(p;U(O~Xn1`{Ovt4O%Wakf0zwq{6 zA0F`G@on&8x$CdB;~_-q@ZDT(ZjZf4wn9v0lSygRUUpR_MU6Ni@@9yzck*()pAT`s ziA8bJ!^6NWU~)1Z$|BL5=sJ*CYqc4Ygr#sY$W4@C(rsXBCspdf1;dpx9L`knkTn?& z;N&k@ABPF%&O{$9`atWJ%@p1n?&29S>|jSbIjjdQa4*qtZzMS%hwySv!c9$C!Zq7l zz)g*15LiuF!c2|xu(-^JqKQ!*PC`#@GLtQ4EuixfUJ#`%V8hy4z;hffhLzZtA<0i= zp#BSb9^-M%m&Q96MRT?iZN{?_U9egOWSuKv7+el!qT|1I_tnok_K{dLmP%?XvBl%AWP&CEVPcBh}ji1;e#H;Z6Z zIiG{_`pF4v)MQp1tek5qr6m~wmIzB^xxGA*|A!1D``n2>rgiyd|9VJc`p1A&Xu9$bqR_ zPV-tSpA=M8RY_EBI#T(`8oowe>e8pu>x2ItGLF)eBQRw+&kCug*dVRn znWkrIy@>r z5XuH;?!%iExXq1bHo!m00HCOYpAI~7Agl^VaRBi;6m5g{3#(1*!)-%f1|B&;R0Y%R z(YE2%1X}O$wh`J1+kb=DiGM7m_bSsul$!`_!ngJl+?uz+U59$_#-;Z{*o6r7Us-gc zUEwkNcm6RR5YEP%bHZD{#3UymVM)CUXZt{>F=RiW=YEo}wjF*!S+kCC>U?*B zGWHNhFreD$P~vV_X=fP|r(Zb&sm&j8q-eZ}P)$*DXb}2D>Ge4?4?Wnn7Rk|q(ByYM zYZdO#~cmHVy0hHOozUo(}p zj;2CkOw3OhB}27RbN3YI<`!T0xwjbqxdIJ;1q6Stv~Ad6zrsI^!hhtk^%l&}H-?NC zddFCX-jhJKC*^)e44o8tNHu{a_RbxyNV)}h9c?@(YIS!56F&2Mta6jy$63-^=c)dtywC!QUNZ#5sbwG zQn@`yyp8mJue1rGkNa_``5n1?Un}+w4fY;mYGg5f;{k8)BtP~V4t>&51E@NHVLlxz zmr~LTht8a^$s;4Kn5@yk5BtsrJ~i4nr3zNIaB?QnQ7F0oyin2S*ER1ZNaF6>SB2b+ ziO09luzVmz`c5!Q8)1nES{8WesNlZa{)S{Y{r<~-`E)R-q!@H;aQTa=PGj{r%faF% zbr}Q%92gWMqHu)YCE`DbX&us&3tQfWATM=;@?G5g&#BgJ{<~@}cM|vdzD(cRN1P9X zGq$u;NYHlA$wN+Q+}aa#*!In&e*-DW=vY|jhiQ%yAv{@BYH{auwDbou6cl1q#eG=6`k74MZ{o0Md;R0NFj^I9;+d8# zxie%uJ2#F;39iCl*5!GF6>2q#xK+_%8%3^gbawm;X@^WTE*@zMB`WLr2&}r*E$f~; z1EcJs2+PIX)^w&&0%)25Gn~O=U)r{0~hg=|j_y z>z#+jkdY$|Gj9Mb#NtYjpp3aR)$9m8-NkY9uWMIpm!V6F;>49{LhG7kq?92zT&gFqMdfKR?XF$H_uM5gQ~TGZtjnR!*))#IpZ zhW?VEsWdKKhWpbda1`qf#~93-t`^^2S31#OePnpO8clnGAjs!&Y$PsD!z?)@(&=Q% zjxaYTfx_i=LGR$$r6k$`n{02=QT3gyKO$j*H3-22K32rPeb!*D&a169NHS%pqDeai z=WMLQgrKO>!7P)lVV(iARprx2TD=>ip5|2WZwI_I1NK$@^fNW_>!d{pN&Xv^BfxR8>RC75`M0HFB)y^2n`C z`=pS#Lqz+~4cs+Cd-c_BK@A_1YzqCo+xN{o+NcHEcj#_q0G}PybsLb3jEY+{JSk%e zQ6IbCd?+S-W-{8H*QwN;cR_NHXMc)SKQ$#KnNxtKh_4F+RYrEGm}#VpW|Ygw%Dboh zR|FHpX_6sOXjR`SLbpF%%>~SsYB;J>+6Bs%5<;l15vjG?W$|+bKIu$Q&Z)Gq`y9{&e2fl4kGZYq8NjH25<<0KIX`G}%2=;DTK}kB z!e(d1>4D~BhisNB;k0;vanP^UV$dG)A2GldD2q4xj1Ef9%P$kuL7Tx_xUeN5BZO6A zR3I103Uqg9Bc#k3>T%g-bAg=qMAqnoOoVNZSy_MRK5HL?y_Tr?5wul%Yg2J!ctI^~ zJbAlqQo1J^rJs%l=zN*tU>dbY3t;Caa8Wv2tNfFo^jMU1tdeXt{Q2%CR}GSO@aw&a zzxphK&)kmZq# zO?XYoAfX$QCt5UrZ^l8aq!lTz+Ylgg%-ke;T=5ny2`rYoPQqM}gZeGf(?p|L;OO34 z?($L-+CUOG(+gy18wd>$si!;o>lxi{!B&6JdXOevg4b~ zMtD70$xJ^r&0|DN7*rpR*!3XZm=VTphO1*lEvaGkp2S+<&}f0L5#!eOPr8Q| z&PO8Xz66DD<{U+sUFM_l7H3zczm#DM%h30yIE1tK@zn5!nTfhW4vIZq1}$TCmxY{d zXrOo0#*9y+3L67!hD@zrn})dx9IUk&t$#L(J%NZV%MT5!beA>g>pembmXk2$HpfHo zdDAApV$R4vbDX&U?4V$`+QlGYeRCID) zJ82%eNE(Y4kf)2hn~)SHfqiRRaaf8XhPS`J?mEGlkRkLa!t7J*@rTfyefqNwiwe%n z^*u#Ry1CSvwQ_RDDMQt#pv8OW@qER$mv|1O21_&NCEqzQchU?p)A|(0?f)s+B=gY@cG2(X)4+eRb~AWYP*+|GNbZHLP%gz1vTCOd80wT(SHoX=u+x{V5o&y zf^iGePs>}Ox!S*Zkh4X+YxN#4G}DpMF?P>oZ#0tnntjCpgdVauFAI2jnsf?|(Wlv} zA9|+SE|nCUCe$WL<#K;5!YRUJ`)yj}M(7y0AE;FxN2{b?VmeN?H`a1t8r@!O~U{CbB9?harU6ywJvT;>T%Z~3v1<*u?=s`9~I|LGqfXQ)}kfe z7Tc~aW+$i`co*?GZ~<8Y45 z)bI1{ztK4R(Pj6?kcoXr1@N{OS+M=A0*A-RKRLK<_Hlz zpr0CpUuFNSbml3n{6%H1>c|I4nO3m(ZxuCVC?zdN|8y4CshCEvA6pCUZkUH%J4upp z_ieZ*ih34oeyq>GtRR_|HN({!-UGX$h;NQafO=+R6c@rdNFtB0c<%Vj>A}ev6%0mt z`;gsU`)`n>-ezi^tb(c-F&B+;<=EImEGsYDdKQ6Hwg;Ggym(gEt1CChrB(6bY^U10 z1^c00Gp$J}Yc|-;Y?cW`XG+u|9AuZW-nR)sXFzAjK!J;aLSm4&rdC!DiE;7_ZRSLy zq^#L@+odGB1U}C4 zzL_JfO69$l6l}Vm5$+xEgKsS2qrpOFLFjW4pVws7%sQ%c{Nl6&ktD2MA*ZB8fTJBk zwZOFf_06dmX6ZqEnRqSeGQzmo=nEnzn`xvW@;%E3vV%+xzlM1x^bi{Gv#GkIo}*@U ze-|{(M(Zu!BR{0uz#X=u!v^6l3DN^2tL(IK=viK6GW^yUN#@(8*MOBa$&mTad=IQp z(Xt|(ja}ZGWxGWA9R072h2UdK$%uqHK5R;uk0p(-xY=^V0R@{U3Uz;?zeccSr(PTPe8213&?gHLzuFcklGUQ{^d$ zVw?884q%TZGt9eyhLJU;#A5tiHb1)>E$;faKl#o2-#k_HG{@Bb$cu zE35D39Rq__5+xP~Q?)43`|aY>M2Rqd&(A6L^?)~9_8p>6?|2Yy^TJ-ZyBVydbf~`D zl9?8BHK(QFwEa^?5gajbPjkFr^5)Ni+K_uTSUvmgLJAq8-gM>fN$eDkf>BCpC{>?B zzUkdb#Ro_mnHs#L0xD)2{woHDbC+pQd+i?CTK6M=gR&AN94kMZo|r*Pyv6;qzy}lW zSE^IS#sH7BK-`-K?X{xleoLZ`iK2tO(auR{k6!{|Q1M=;jwgsErCeU*Ol4DjMq8qfR2ximG*bZ-2AkNw&7?uW zLEA$C$dqENVDK&GEFhF=Vw_%Fu zPycHTH9ftjogagxB{w1_CnXn?YdG_ArnI0nE!BKX=8`I*C&EzD3SRq-Cs%Pg$6~Io zGU@$rV$-4hIE2)+&(i*N>w*NU)<}Uq8sOUCZU^TUWIfA``kpn04>=jv_$GgdYfjS) zQ7c^A_;R;vSGlRPF~*5zv#Y=HOU-|NzgovEZrlyAKC)ZOg?hyik;AxBC7woP-bw4i z_+=u#{%xfc=k}NKQtGYZ2XyZ201XKVn0g^FnC3j=DDt9e$4&v3<-W5*NW8t`=GEDW zGlBTx<-z1Y7sWh9io5uR+b(oX6CD0v8;!g8CgNT%PO%bVIcYDbtgPKPP0n^#y^kZF z_A6o8ml}1hZ~Zntuf30lOQ&V+1_liijRx;96SgW$k3YgvSa+j|%o!Dv`hSinGCe#Q zDvg`X)Ab~bSvIVn|DmcmhL9~DAk$#NP-c1oRGr2_eCGAvU_9}t$nD1T)qgIlQdO-G zFk{AT+%rYLwqn+wQd%m~JojDQs?MWuKM6BM`@fr!%Nb3&f`}JwduqDbU0LX^6>4i< z9APnPe+j)I`^9cm&i0KrqS!K8h49}G;&oFW+fL>hednLD(8O~M*Y^j{W?^~Qo;#kn zHY-}0ui-c_bzkDnb&rAoMZlu*4dl!qy9(OMNkZW0L*YIOECP=#0u2mRmw1FKRg3?1 z8b57WCXr}rHdnWo%q$0ROsB8lHN(oC$;-wF{ejNt@-x?Cq~9gFK!E(xmL;``YkHzW z6I-dWj{ zVEiT!WWCZB-uL0>rYc^8%lZN(S%)~*=xV3MgC@77pUL#WQQ0RnWsmhp;rgq(3l@zT zZW-I~nN(A!SKSh(H(jS!SRcmZ0gg}v>(I~HR_7&PaVhY6v6*`(*aUIKxQx-*USi{1 zyG7^(-Wr~>>|(X$8lg&8D`q8CN0ZhZ#;z0Dy^EVBk7d@y_6aH~&H)}V)|2|Kqba~d zUI}yxq2okx!s?KN;{H{GU`@bckAXblVmNKos(xZTPG9Ic*u5RD)+d3vk)3*uZ@I{w z&tj#_47JLg&?Ay-dmTB5OUgMNZ>vu@hJj`ldU6|Qy3AZ{6}No!jHzDi&%IOPe5R{l zulM4E7rNA}(7T-jt34Iil=R7~Dlek5tnk^l^?FeS*S0RsdegLp(iHYDAXmiDoTHBQ zwC+FCH9vVBFSjQ=A~`HEtP6KJ!ir>)4Z8RD6Am^xGXcG|`#Kyc6Lhff*d0h!J!Cd1 zm$tg3Q?@i);qGl5I7hxb1M`H7e(}Ur<&8oYW2;eVo^l=P(DWZ!;R5tWs2%8JGi9+k z92V>?NTeW$WsRhj1XA+{`3=l?sd7Fj*=JVEMI$}Y=Z4+}zROoh_5AtMWZbx^1o_tx zRt91X{93nHIanKJ%7!IGYo%S!$#zkeUNr)07f6FjbnMDm=&zcl8ER<$w-uE(?CJZs zz+!FXTb`of-Vjsv8GfH}LxO5euutS75x3vuy`=l2Y?S>d)-vQ)TUmJyqBQyr&c*Jp zE@kVb3vU%-^EW@lf_q8My+DKA7cs^ZscMNzgev6&m<=@Db*V;LvRuIdey{Zv`^%ci zYQ`=LCwAFsqkO_qx+>XTGj9yME=zx6OO1V@krE{_e&sTi1|jwHDqdM@#>AI046n-V z?Zn6vqpiF2tjh)>03s}euAh5rdk1B8qb9ps?^1g0)cF-s)r5I_l3XJi##h~G;W=^UgoVb5$wWtRPus?;7$VD6#e!Ua%GDC~%jmyUz zIIL%YWl#yBgwfs2T9yex>28;&w7jF2-!WCMp!fOA5A8`cf;sr*oK-c*P_8y72E6_} zPz*=MH?(?psp1X0!?e^KRA{btpCH9_A{*Gn^Dwq!yja*~6*Pa@CI6|EtKV;U=E?YzepFc2+b75qf(e;-3QFI-yiEmhxQp(6V(1q(Js_8 zShwXg1d)L}Hi`mSf$sUuZrI$O{Oz!4@0Qoj(}eMHhb@T2$_-w$1IK2^;EU^@+lwWk z{EV~v`>^QWi(nL-AugiNm!+UZTVoKF=k6%oNsJJ=oN^h-YT~7+Hu#8ex?KjtYv;;# ztH2lK$3!G69>}& zm2!B}B)x`ebUQ4smCWqAGQjkNKUEoJf;aNWvF~1CfIsyu){}k>lFQeWAgP<&ah%!vH(ZwY-uF@NscP1jl=(HuqS%X5?!-c zJDn!wy0iSTSB5Dq$6L2p)lo)(=TAV(bjBT2n>|=6PLOGB_>o!UQFduS+5=@GRn=$y zq)aphoF`0!tp)SMWS8;n>py0sO+!y_PRPkW#s;Ik$IDx=!CZK-=J@F!hkC`WGQGau z{5_@@CD59kK?iD)dQ1cxF9+_}Qvx#KKd{vh4q4t94vV^wBQJV{Qu+NhcABHod{q|c z4Z-k;gg0+U%svI50Z+U_|vOvCpUj>q7w~0V!}zdJ^fZDzz62}z{O$> zp?P_g-vz!Q{0Zq+Ga2}fkB{|3-zl|whZ?|a-Vof9yW@mpY(2F~A+<7oU^*^Gn!~gD zUvaQoAh057<+I!+)NwS_yjEAHS>H6KdnFb5=zZP&m$RgJKWv>%3HJd+S%+2Q_gkLXBlwuIx@^%G&|qyQ?=tBO1Vm4n6z@nh*U3S z842qDsZ7W{+S+){#?VYE^A~}ChvQO&2=TBWr#-n88X6YXqnG*ol=8=XpeDX%mgql? z{4nN$iX+<<@=!3I-FG*av+AvH;Vk@~$qF*<+o36EBzvxH!`2RQ%d(qjHU~4;Y$+32 z!xwF**Rc=JF>+Eov087`(-Xye<1aJ&{Xx$bdO11(pNk$!=g@PT_~@nUHMDCHTD0uZ z!)lpusj^wD?Sd)kH$~hnmqt2dXRuMT@$F<_cJQxYB9H6_XA;-7$+ph+O3r>zyRrSW zvT|&-qN>8ulper!`Xr3yxzkgjd{cREA$7BsV8njKML?*Nq>gjRTRZ^#_&XNAb=`8Z z61@m@UZ|dGUb)|fjI?1hZpcc|lD7Sqd4POHA#hl2<}2$9!6J*OIY0D7_&{G6Bzc4> zixd3*ocis0tg$=R2clTN$Ymcgx61vY{$CP*Vwo?j$V}Ax%IxZNW3O@s*xTQRi;PSv zp)Ly2^sl9C9D|$ZGmMq(P7eL|YLdmQ3jG6D%1^}LUq(!7r+3^3ML`BISNzQ0TRnd? z6c=u7yT9|nq|B&Td)CXU{a&sL$6}+|l#VX9x{MuS=;jif0Q}QS!5fg?5oT^=g9=U% z$zrw=zT^K|z^gDTH)7a1FK~{HOA9vUG4CnToTQ4&X55gk_@w}P+dH-Jv@v} z?+Ia$ruul-s#cDy=g-Z#bB4sj{D0WTDo(b+1->~?9jS2nwz90G-@^&FJ zMVi=w?Q(1ghY6ISaG(+RVR$N|`rlaU$1>U*m)D<1*|*mvwY^~6*qzv*0mhfp`BYd$ir&6!(iI#6saxLGF;37h-8YJv} z>;|A4v~B6Na`;^OSzi@w4oVvwc|z=HhCv3Ymy+K?QtFkfJz%Up8z3aH+}_&vFI;Gg z;Z60^ZI0zev9bN%pK&zbPQAy{FDh%v{#-Xz|BlZhrLnIDAquGW$+SQHLZijwnLZrX z!}3ys9b^p+g>xlK&RyD-Sc`k1+u99D=D(lVy(<@B8aL=KQG*2W=gYN3D!D9WMv`zx z0rH&+HqdN8oeB0-+QZ&&2Apv(o zpg0PU=6tR~?IF8hyZ^8D-ZCtXZC%t2?!nzH!J!-X;O_3)xVu9L!7af;aF?J7?oNUP zcL`2#ce|N$uD$o#Gquk-cirdS`)Bpj(A8fL`QGYL{f<%fqb9nlY|>Z{!phS6jlASP z-S8ZtN%(1gGB}3sHbiOjh%?+~pPb&FB;SBN&O)v|y^5PKKtlL=!1U7|fBvk#JQcEo zs-e#>3xk6iAwHS#ja5#H|5=D;ZvBmS>xH4JW7ywrSe z=goLMf^b7OrTr161(7JNlQom;7F^19BQQ~29qLzBLL~nc;WNv3C>&hJpFH$e>WQu+ z2p4UlN3#cUu?|fTU#X#=x?HF49G9wB3o}TaU3ZL7Bc@_5^fb8(9`Fh%t4pnWU%yT+ zj>xdT8HFLg8%0~#4K9c2M))|L@i6^EkPXLbuFCgwtB;x#%>uW_(pbr=Q$V)hqVImE z(xTXgAI$a8jMp|*)=f$7e5K?HJHquaF3r83(3kX(M_A$>8ugg=lP@7s6f(sGL)KVC zFTzfFYDEUygv_S32yOSkyG1&gZ)nt&exxJCLmq-=-pnl0TIPtVBn%T`lyYMPdWTvE zGjT+TS9(u;`-qn9FI94>kcq=E*G7TD8Br>Qa}qZsc63>LAaKI_RqT#WUy^xH&iiLF z#Dz>(!ziQrhH1bWA`p&n?t|50O=ffKT_A8ibHF_;N><7-e*rx#hD{l^WIuR`HUl^4 zZDk+4k&`3Nw@=slcF>#78~p&5^gPePZF_B^tK8Lyr4+S+ycP_P=Z_hP4Y#$-gNF8c zpYUEmTH}jNX<=r}RCZxedrS8vKj);VPxqhfv-mn<;i&B)$ni=!X(8|!7H3mG+4B|4 z-RoX~Qg||6q`1=8LcGHg4hjsVG!chlMY%9EWA^+S>XDZI*0FLAXLGJ+Iu&1OYk3CR zFN%9ggxT1qcPLZXwB6h9NxnsU8y>NN?5lF+FwE9Iq!>+G3sYa`z6|pJ)bfTiyJ!Xmf;q5MJrjhzR9jc#2%Z|DvPxU)$+s05`!_m zB8NWH?L#It_LWfJcz0|*enWTh$el5SeaFu-|DzGv9!nMgL@5mFm=bBmR{1|F#D5&!G-fqLGwUX}d zN99v2Ng6-ld%AwZ&pyP;@0dZTlR<;jX#55Fk%wkOF^+nQeSSX)%caVqS%EY-ghVrm zDMe3MkUK21!^qrmH82t`R?2b~A$xQ1de18)kW?2(b?{&`Y~3gruXgg3gk{aSm!{@J zL2|}uZUHmic<}KezpKhUS_>Jk^kS%V?>!=~O8AY6O8JEVgqO@;wxN1$s*AQ>vcZLn zHj8-mnA>f&Zprgt*2}3;ir9|$rlX?s+w*NtTVk`p&mCEQQP+d#`RE3L=9aEamRkOP zx3^}^dZl-sEPHke<+Cl-Gjq%D46CiziE`f!G}>&TG~oN`_>UQ%-Em5zK?OU!dBt>^ z;X%IcsC_J%Q0Q-IefTOmvR(er!OSii9)~CpE zw5(hFaD_9%EmLP8T{_g%+V0AJ%?w_3XtU9TDy`24m8$*%@N{l{KbT zKB?Xouge5r;){^>qM<7f)TH;l({|xRj*tmZljEj;n9CraTj0RbWCT?z+ro7cI!0Ua zCusV7ZD}1SU2Vcu#6Js?Eg#jnD9G|j7g~y3)f~AjEGdt{!*-me-zTe7tK|H-|E^_W zqqZ3C%NxSW*%%)UK778{t#36bpeoL_hSaX_=;SYh-we$qJZF4jI%5`(g$uZ<(Ku+$ z3-0)G=GzH(<4fC2>L-A5t5W@Apf?45>Xc9F*H0ut6}Y*PfSo|+E$yzmkICEZ(|vy5 z-BsKcStwI3YhR(K#V7>+=d5aboC0&`h7TChE3QE^h2AY~nh`UBg7z1stZ(oQ(gt3Z zEX>_DY4?&bexVn^nBeOoGwS(@+^7)r;q_)YX61w>PSxPxV)c`45!vCI2K1n9Owjz< z_H#V+`u8&z;zpYOk!86_u1_JBC;i3tQy2?AN%PZJk$mOsG+t!2i&*~ZJk{EukCorM zET%HMLaB+9j~qU!`l1k9OqY{bpJk#I`ZZRGRfhLGOi?yxS}G!VL?7sAVO!1V<1h2p z6}9s<*r}Q?iu`yDyS)r)dwqm}GKkA;Y+`I|1EYUWa@6*=-F|FO z0&7+Nt*+WxQgW|gG!a!itBKZ{z2_}c%XIz&xabSQQd?DoSsC>4^JMHQKZj|n@_DCX zSx~Ufkg=kO{hs@ZAz3?lZLa-D4&T<13J&fEwU=g+Rn?`rkH-vy*y2)-rerkw;yT?DZeb5O}Py6guQ7L%iAz%)br_wlCCqDSi9M7%J`(3o{j6( zs);Dos@p2L7wg*sCE{rI+2hM{eypUx`Uhv4l-KobDCC_-#{tLFnr_hTgKX#Ckq8D``uT8#V}=|(_UqGBP<9;P(8X?|We0o$qT^Bz4KDP!ry6o zV^}I=cKMw$>gLGiNb1kf5U;fvuZqh7{7?atN3l!_3+(4pGq^VE=YoNqrO(kBtwu8yp~{rC}@jA91GVLZr@^{Vw) z+}*(5KBT1N(y-anb5qbKMQBy&iH8ZfDMbRm64ay$hA<(FAYFQ8(y~+iCJLMj zhqeGIs!qAvEB!}-!fKPpJAy(COIu@|kg4f-ew;7I(FN+etL_+AwyGp~m><@WTiFR? zAsGz%w`1G)WfFVJRyD|>`=d+#_}=Y>)u+!%17ttpvuWC}T9w72fCvzsxWC9T~q40X*F4a|~DlysyNUY78lj99!oM%ji99y-|WV)nX%hy8{wHEKSnWlQly~y^<>dnYGfk6FMhGZESqysv=&d3(V0;6>O$Z0 zSWLom_NRIYsZE%Sy|`?x3a%%gya_(4B`sgE*giBV(`Xd}N20I`wuE^y`nNGwPMpmT zB8CpX z-WPU0+e3J*qvNw|1tvTZCRkP4(TYxO(Rv{wj(eQE%p_>RDVK;96(Vou3bIoV?f&j2 zj)6l*3QpCxC>jsPv9Q!;lMt<_iG3@8YYv&HPWsHCj$zHvA}*vzc`GGf?)Q9D^v6se zo-#uwdCH{PO6aVYn*Rq<*48d!5)&n%1~wok3{$0SbDGZ@Y+mdvu75$tu5vimj;8kiWqy4w8&4wRr<_42jl9 z!-EIcz8o=j9V7e7AAMQ18s$NK^Wy7jMa9L)u#4{`0--x|V6?adPKaP*kuNsxO@Do+gU&b|>H6J<8v2CO2ge1_P zWHEZl!&=-xRuNL51@5J~no(XaxnFebJT0xxZU#xA7)Qaf>f`EbBO*fx549146&vKhGuEIV)@Cy z>kW9MAbMC5I@ATK!S*W?e&>&iTwzGg>F1>Atdn5d<@BA9H;ip)YhFa)H)RkqhpB>V#g#LOy=5-W7H6(Qje}5AQ)1kFZ zIa&)VSM@(WwGa?PrTqvo!wN|V8G*y~yz4|m#*cYsQmPH)cWf^1YcOGOlvccs>?(D_0NM^W#WqAYUC zQDnnLIa;5K4e}(uP;R8e9(1DVrY`ls=uMbO zi1kn6Anz~}GH@T6+pFCLl-h?;*+Vk;dy9c{yWmS_+3p_5$(-B`#;d^`A!twOJ+{p* zQKDYwLK;~Zx8!aoPxJz!?Eh9^W0N1*Z6k2g@Aa-vk(73m3CCqSry=v~-U zp;Ut1Zm~f$d(H&!qWMrCzH0a9c;Lu>QGd?<>IY)tOFG3cI#$Hib*H75&ax!#$>TKN zd1dJ5#XuSPUxt)=aQh(jDzEtu|>$TQ}PiQ@Mhw z&Y%zTseHOb;(EUL49?~O;sI}*Gp$63ZU0YKswsr*%)B~1`e7c@0}9erp3QEPZ?cRx z%(g%B^S^}zsIY_;bA9FGlY}U6cs9w$gs%&gjG#_dq)2ALF^D6P(#u(Y*hQ=kdV})w zz2FF0hxv!^?NT3wJ(19-Tre|w_=>WH7-VFpLquf4%!#dnPHZ~EnEd$k0-8?e1{Zpn zdYuw?nL96E>6o5*_*H8yz0~P+YvdktRS|xX^F%OJ^TfZb)4GxMlyx%VAM}kZF`Fwj z@epB@p%hmQ=Zns>wIy9;GULS4WC8DEs)KJbd?{h4&ZS~^#dw%=Bw^Dg9R?fK0;N1# zu)VL1t5QrEeI%UA-sY{xfMbL z*4jksAppyI^D*`^j0yF6vu}FisZrc6<|Qim77H3gT1wH%_LmeWD5qgDKbbBkSoZKF z20ea*Bl6oBlOS1UbYwgb8-jyY<;W^{zwm{kl%>#ED3qn|Q>`chA}tlOsRQNRfY_!0 zBvEo#W>+edRj#Diw;QEK_8dWNTq9qWiM;TV>b6LQ=%SDI2UVvcX& z>dhDgg|PCg$fa5!H_J(?mpkc2iWwG~ix8EocaElIv3aPqH_o@YSySQqSuCum`!0fu z2`TGGkYa&SJ|C)mh}d0rm>(-^Or!0)BYD|r{v+lsR|LW|VH!RI2nV7Ue$3d*ILu-O z+q2ry7XvcWM4*(^_}s589!B(C-*mli=%i@s+w6GRac?5&992;I@*BnD1nIvCidhTM zNRabZVtgs~f1eH!#JjN39f;^5EVN1%lHW<>gb)l%-Sj^0KLs%jkPsEtV7JA{wFRhu zo)T2G5R!2&k4|0Z@XX3-qF?p$o)h(6@8UrpSJXVVsxcK*u}0StB>QLt^R1QjUg?R1ZmlFt@8eq8Yli4m0j9!@EvEGKKZB7c-?wcxHObxp~SZ}|jABJvXB5Psy5Y+5;e zsF0BO9<5=_CR=I#-MdX6MetI+Cw7kU+HCABP8E40eCBgURMgxorf!@1w4jVx2nmEi z%he^lAyG-^dJ^>lH@8fZriqo*HH?*dOboJXS)8mmJ|@A~Bk05^d&pK1VsSC!4?Ibectl~EV(r5meGz7;9q!DN^{V7E zG0-)NHn!PMx&%=I%~%Fv+uBHZ_W6PcKW86C=MmZ&3j#JGSghty-Bk@GTC;h6=z7WO zi=9h@RC?U7YY+x-Sb9pszSEgaqt`+a1lGFMWA`2AtYhAPyI-nlsf)I(53zo^iAk6m zJqJf{F-By)gxbEB^Fxv^@MVEcj|X!>4)I}BO<6uce5xx}xv6khmdMWYcbJ`(=*F3) ziUV(^aMHXV-8+wbI?|)tkq;emF-=JY8iQ4v-nlpy^stNCla~*x(aM%asfsnodR0LW zJREy!qk?xm0xCH51At!z9M`esRe8 z*zo>xHy_85$yS%MFK=vY zXR$x}u6iup;bQh{PIcaE_X)Fvv{m1O%BMmdUjR1sW}FMxd+?GUkN|wf()^wu~0;4y#^#K72~0$yd{73a7}+&Q^Va z(+Mq&gjIlq=z&C!A=62NNGJNLH?%c~bbl7Zz*kH~95N*_)7u+quTh1so_LgsMA3BE z))u&b_82f?#uC5SxScuDn5A||!>$7T5^Px5xlw*0tzk2yg~yl@5#myYglVq+EYezC z#^xO?^^gKyDXsQYXra8Z7zT@TBlwEYOP?^4=b-7-?oX znq(={a9Z6Q`3mQA>;Xu%4dH9b9vlW`L>|!s_?lvfsyu`G*NqbG&+^1^*6&SgU62Yj zp(^AF5hPz1Yvu-}%7QSmHDxD>q~x=E^RaTcWSGZ{ZIH@D3*k`T^^r}`g>-z9{(x8+ zK_5xesF{q&kR*c~K}5re#a#004Q;8$3R;)$C`-zx_tjn$_zCn_Z>gAHsF~^p+g70# zb`T{66}&gC{>-CVO(xAKGuCCU6WoJot6NwOuZEaZ@_`OXzqCFrTM3Eg)7vV%T6r-x z226&kd~s-aj=oOJ7(1$PTaBu0q@MR71U#>D`l*7Ya9Gvfm|)eK$-TO;@9Xi?Z~-wz zQcxljPzj7KjLj8T&%P#@jtf{tDc7Y-rVH7HH8shP(-Y-#oRULc?Aj4F#Y zhr}t^Q9k>sR<)3uR>G69*~Yhl8Z-8W@o13iWW`Eda`?NzW#Xf$aVHyLuS)Gf`XQ%W@+#?}~N6EZEU%t{pEUQhei~aj(hw$?iQO6%7j}N1R&c zM?@Qn#tr2i>8AFwG}t5~bw#mmwk}>GaE9^HtJkm=BE~T6roQr}p$yXbh~^d~WvMF! z4bX92Zt}P&Q%W-NR5Xm9uIc_y42W++h+<@Eu(3{{yx}?JOK^8l%H*F9&0-R@Ycu1G zsW-7+P%vX9Y^ofEIS#ygo5Himu(+Sr6;svbt-9Xm8O*vj%RA3 zS8{v^&5O=pjuZ2);N4joDsRr7&O*A^xO|;moLIRQpRgdeSayVztD7cvt)9Mjc3@Ow`zwl|nr^(F*Z3^RJT_nBReK zcUf#@&HVE@(5%-Y(5N&|CAN_DnAH_JY-=#&!A>fJn&(Qe7&mxd`>8}>sPg=~Zaz>z zqoAP2U|pX*fb_Bue=f|MjRk-07|nS{ARTfSBL2Kz6oxG;es$JZB)na)j~oy0dt7*j zXg5g2Njp9Wy&sk`Kt${TjuHlB^Z_3WnYi^C7&B2a)D+e@fdNW5SQ&2!{R$PU*bo?G;Sc0kNWM;p zB;D64qn%XH{=vc0bXhsZ{yf%MMpm=OPMsYc(oAd|U$f8={BMj?;~X)J(BYy^QBE;o z7t9`oI)z|vHp-Mou@oaYb?`~QzC?%0M8Ch5Ko%7Xhg`o66%Q4gA=4xzn*(#|VszH` z`*p!#!cs!5=45APV_Ss0mo+v-G^%`?O?1~qO(A+7CV4Epur_jq1Z{xXvoYNj8!B9Z zeczyXY~*ye&ez%QnGvM$_}HL%{Ip#Q-Fg+E&wB0Ly3&6UOg@9sRsQxk!W~+1GE5@b?pMB)@FAsKkrl z2UfP^P*N&E`=_*2gXmeG485_jRoixbHy3IsH%cCVFgn^GpK|Q#gVxTNYv>E}&TqL+-kG+7EJ^0KP+xP&w>>)_w zxwb}-J4)5=vpqp$ko1y~@w*=m)_q`TCX|j{8H)o%a{^--C`P2JLTF7E<8qiXst!J6 zZA+**+%li`}$q)^|US1m#@qE?v)B|&SmKV!tN zs(UtV*ue*-ofPB_Q*{WXDaeV>W;0e71VPjO9<&Lg-DzYxcB*IM4qav#G=*N3`^;9r zc=MSir7`KVva=u!H0?m>Y5r280<{ShF&p@KxT?ixgD6#^&umeR>7i`bQncO9G_{k1 zrtqtBq2y4sQJ&c%7(YO3b3fbT{P;$-lWTYZN>ZwUh%sRPU^xk+R2`SWu`d#h%AS$Z)U^%5h|8GueE5mZ2yW-+Ax& z%{Wi~`!$wAM3cGbH}@s?B@HX1_)gPoCislyA$YV=rZzS^mI!BitUCvde)gkS@uGx>V>GbTQEk~+(W_m=g*OmIKeQI*pz4+ z;rL~7gTpeQ>|DEDW~`KmDs8ts#SDFYu&kU8YDn+?4?PA27+f^*xK+3i7-rfFCcNkq zHx?(umr|-dF#^<(&P1hn*-lPWzAm=-y*HsBtY5)HRl%^A zAimj%pok0|Ma?863JM~mR~8jAMMWbri6p$CuuLAc!r`K3ViL~Mr}%~^f*5V<%Ys;>aSwJFP8a@t4*&BQZR){6JsQ~&jHCb?ij@5WQtX6GGPUv1 z4vTI&!>)#3UJXWzp>wNj4 z;+c(zA#n>xzRL@2$mc{tqIbpeM*2a{{Gj`x1Y;#>7coU{O2z#gi-m*T zY8gW?n~X>Vt|*l@xCr(1rlA7!D~0zdyb=`g=y(q12rV1z8@)Cpi4v6r_Te=7zCK`PqhX0{cP@)w-CH)bW5C4;R9D6YhjHY2)MHibUGe zU}2>GfdHG&RGOQ8{XIJSgK-2@hUr(DL4j(HI_JT0LdIw>1Y%;D z9gBbolXCUXkc9CSQR+7@@OMXe(nh2r&YUB)E23?E?2E?QpHmy&730eK4zX%5*Z0HA zsv~U}ao%glt#CT8&XA#C*(Cj7lv>NB;kwtYc)eZc!_MS-!Zax1ra7|Tv0oob_vU>T zgctYu4jtXV`U+E%2rD&1@8Yx0w{9M+e6nI+7U}7jg`vrV7wY*;tLwhR@?_BH(%r0v zMBX>VU$YHHS$U;s4p`X_b0cK%n7XJv+5`*| z)_dw}0%59rNAkF5fe7u(0&Otu^^9>dX!X;TU)RzYBS(Mq23zJB=&?r=QA{#T61-lw zlf`RMV>m_;Vr*F2%y&I!N(ea9Bx|xr{aopIaZXJQlL@*@Uv^Tc+SI!3QsNdy9O81X z_jYz-+S@t*bi-eaQey_rBk>`?brkZZ%?l>)E6!XRHW`R&nAzT2Yus{Vc(4_ko_g#t zl1>x$tHhH7bMR)}?%33f;aHyeUX|~Ea*PEztP3o)l=gLgvs^91x<43<70LS~X>fs- zeamzwXk_o|7x4C`4hB!+)AhLnm&V>dfINbmG+ zLfvNALt(EHCEZr>$(lIsr2yz;)6_$WqrYZ*TD@C zt~siX?!?gXWmDaB#f4Tk4Az`GjF9|wRrAbu)%vAAFB(c?36wsB_2B#0H}`h!g~UeU zHaA+Han=)ry>p`yFQ5)z>HaA%Ee`_&A2}X;EaGMCN?v4ssT@v9^KU#F3i_-;(=NKDb?0ngNq`5TAJAz;v-e>|yxq=kA1m6c4UJc?fm) z_~~;m_>8&g@lEZ=d}lSKmi{^+%bvA?&#((Mlg=+$bftYS!hc?|_$?6IuV2R!&aqO% z;1xOheG!Y^3Mj4q5@2_5l+ZX}i(XrrC#7I=aqo_FH;ChQGfXKswADb0WNwe8zhiYF zb>PfedHPkX>nGg$coaNyWs~W()deqTKs>JD!?{}~zq=Ue#IaK|rLd(Ir+Ui|4Qt=x zr^2E*C69LFx^^$|58bbM_2!j4Y@_e|m9EyTV-*kmZ`88d?EJham@j5q`X0CSHH0cr zt|UFM-IS9AyxGsYNH*`d9XUD+_%A)cC3Qbj4k`q_0-xSyV34@s49s?GLiRVhQ!lXD zyUDoTL-x&Wr+Jk7#-EZ*O~O29UbbrNIak9*#rMARBzM>4O=p|rDqN~Pv zx7b_z6zQ+9KG{zA6gOqnSN6bF5bZtaUUERx)X!0DIolps%i`HL9^}9GJtkvka;CTX zEI9GX`mLtU&&-JJA#}A7cmbi>{R4!Ln^D&LeH8=2_|BO{cEg8rI<1AwmzYz_yt6Dt z-`yG)(BgWeHQMJ7{P0+K0^G>jqJ15@QtLm`9wY>2_*y0u#vJmtX5Nt+CFMfo!nF1K&TfEAZ$Ml*zTJ@LD~j?z^ohF);EnWL-reSOO&A4YQge6x^P#X;9&i{i!n`;tk`&E3VM z*%ExyTLGJBeo*P>`_XCtfzQlA$v~7(wafqL*^h*GCDUmtif=b3k%QCd)0-?0hS&E`h@<*& zPf(&!u1MVqDmCvUJKt4~r+$O<$nK|1nG*?!%YPwyY>5Y_9rUcP?Oa7+hkuP%}c<&0!dkso9 zW$1@{Wy)moB$Bv9U$TLv(2 z@e1ydeOZcEYn42fydfzQ9AT;3;B^l#YN+)lz-FF=uM16E4%R7NAJ@q zUs=4`{k?$Y&DuxjPv&aKK2H@LLJD`~)oy_fj?*tk>iW;-@siE81cv$A+>2wf<}y31 z>F4l&XjGgYdet5bA-GQWEDVJoX?f}5k3T?tk&FMiJ|J-rY;Y;w8}cahGpK)2vVX7F zZ03$OU8YglkIrNn=f>?KS&CSwtkEH$KWW%u55#il9_n};5g?EoSI zy}3qJ>BHoq|ZrBgthXLavNy3@*SCbf9IX*SfGuK+Kc?L z--BDb4x2{KlA# zU6p?eqs|?3o6uWQxnwmiTa&BQD8t#PvDkwWjF^^KuQ1l9BUeSOuGB*r&1k<~eD&=9|lzZ3CANHtp;ji$z z<=Vo1r7P1KDVEfcyrj|9W)5So>R`Jv?d>dq@QwRKrN}}p&-4~(z5Xlh0c;fi4a?#? zZsem!g!jkW`P8-Sgnf=NR^B5Pn%Wtz)v0Z}y`Okc?m}hIEwqJvFg{nTG`RQ7@gCja zYvcFh7%c9{>D@Wx`Kc@(dCWjlc~}^=SmLi(H8bT+{z zVmB>2ws(`4uyx?saz8OoBC-EyHPDDL z#9=pg5dmY9i;jz?=it16HN5OWx7oWNRr@gdl2dBC8f{=3c3L3H%2Uv7ud5k-EBa2s z^Zs!S->t}cG7QCM-c8~DxNy~B04&vAAN{eX+k*1b){Cksxl831ixz!ibUgP@R=<(_ zkV&DQG{{`BL@$t*j1lLrS|HMR9OF82e}Wk|X(ml4Ye`Es0MYD@XhcM9F5JQz1SI&_ zMa}ihk}1tp{9Kh);))YopVKi(x`=k2e;kEtU%FbJQe1Yme6T2Ac#KzDFAv-mdOaa5 zV0j<@ti3MG^JtlnM60c@UL|E-qP~fg1G3|C-9=L=1OwIp2boJI#cqoa33fXo?x*=hy|wLuRsV-Y>?%$Ci-SJAGz)@IrSo z7tYCF{AP;M#0Ff;Mx^7)KBqq>d|2C2=I~p7Y&JMNnDf>=ca_cWgpl`F*++RgNY6F# z2U@?snR+{+kkG`(>uAAnCbi|ppZw&uJIA6$=J{Ok%YDSkw4QqI$8V|nER5}yel_%0 z0+mvEVammfZ1RH}y6eXkV@dk@Gxw{I&sn8BBLa#{c}Gk~ zXTmAaPQY$msg()iJ+`jl3wqtbm3I+mHwir^KcCh+RG5i=~$YY8qPG^~M8JGR-d_ zo=v+N>Fl>7sX1!HMmRiZ?fmUo-Op1@iY~qVUD|5Wc6C}lS>;IiY%*j#?8O(u87wv> zU8+Q~Q$4TXSY96U*=^K9D4BCzj`jBd-D`he+X<6vvTBau{IZ~DV!J)YTse}xj6WWU ze^^zutWH;ss~gj7|5BUNLsFrBETswc5?Q$M<@`eE?fy&zbwmOhdvvZ ztM-Xy)l;+w9^^Z2-al6m{E?%{by1$D9b74`{}41R&GeFc=-Aq-4uTYAW>$q)WP*#h zw1XVy`K~MuzH&;S&3MY%#%^0EmNu($U%$@of1%bKllG#2=Wrxafv|bddiQ9IQHoTq z?u)Ns5nXMXrLbd=@6VHkiF?aTbGVGAmd+8G=8f{tLU6ni%fG6-_RDF^H&ih4$So}5G3#$NySYFZJRhCE zX-oN5-l|?yM$%`+`aFFTS~!~R=yGR;Oy!yxEOBkbCd+3CdwaZ^9c4J|s*yeG#ACg? z`~JMW;moXylr_{W4);os-@c)xAMEX%^)YOX{V|gsyWHrb;xdmBKjix-*Cl!=L8T!u zZ%x~1P;)NJqkqf#`}@pgLF`Z@aZRRF7xCrsW;Mg4Az#ALJxQ_lD5=FvhUS9?Ept83 zlUupGT!=@bcc)}#I{r|qR+((ZuuW!$3y-kN#YnKwKKL@?^8x?iL6h#1Lm$Q?8~XN1 z-4{YM1cXQ}Sv{otC!9?q_^AmQi+sltcG}0I63XVu zaFMSBnl+~*JkAS8tf`w{a?>c*Haa3pGM9^1;J}g)K7u`AY@M>dfitEg^7~8HaEV)P zzGW;8Fw9gKusfU1T)mbns|z5oD)((|derV}>ultVOHduMC2wSv<0M}6g4DbFjX&gG zp?x8Db&E59h$K4Z(;Zz*<4(ZIh0AQOh(tybEQ~tTjUVr89PB9)+BZEyPh*^p6Eb8o zlQ_RFj(-&pag>USE&18~H4?en42f{Aax_171&A}9;1Y#3st$W3*FG$l$-2 zKW+Y0Xy8O(&SQU)@UErwolKEb@z(|e0pCa@lJ+y_~o?{Va?T}Xnm51t5KY1 z&mi?w>Uqg#!W!0h&t;86lGf$%>D}xhk2T-qJ&ku(hTq2sGnhADvy#f22OH%sXC!HE z;xwHbw!@dWf}7G->w0oK9L7?KuaM2QHD(0-w^cr`xf_TD9yYbu>Xjtr6!GSbhnTP@ zR}MViOE4K9>-5y!?`d^p8QRybx6s=>^4`DF(+nuqb1_dAIK|&HT>c~!t7Lic_~m~6 z@x;sp?k)AvMmW4D!+s{VrnSG_Qp*Sb{_#ukg;5U&CcWe|_M6hlq9xGH4dTXkuCVYK zYQ=H^|4$TArU*qZxw8Dlw;?L?ojF|QR^K18S}~t4lj<}&5v`|%^ITC_a z2ouOjuC^L^&sJ6i)>tujFn!`RGNwY$eoiMuRpgbjraO}zo=xe!pwjGdjvK=EkJ(m>GD#1wnD= zymbA1_Cq*x_43E%v}ovhr0)J}p5|unr9?tpHyh!23qkdtst243YDdkk@y$jq_tR_} zPtRU8VnG4l>Wl~UPdENj_jZD?DnQ&!%uU=(Sk>H3-Tnw@E6D%H{lD0`*m(c>hF=20 zUQQ-vwh%W8Q;4OtgCNyWb2}A&M;czJnQ*}$w|F!Qex%&y)JZYEyL4zASy z(SkU{)y&1($<5l)f#Q$%CZ>+=Zh}-)6#r<~>7UwlaAo;3F%~mNd)8kcu!30FS^sG0 z*EXNBi=(-_8N@|M!P?Bl(bdtyt*e4U%hAQ0!pX$NgyKJ=zjhj@0PBCNA;9{#O#XMd z{by=k|6^`m|D2o`pMsiGP7V~HaBPIW#%w5;bG>1nE(36W@c&* z=CI&2g_s+uS-U}G%!RnXAYKa)h?kii!u_i(gxiAI#LOJRY{JRS$;QLRV`k24`X|i4 z+{3?^->*?=Mrh(e=&joXaf2F_7M2T z7%(^c&k^9{?qc`nm^U|Lh1fytAr5Y?zXk#5&p}{r#%K2H72*k(+=3`^~r>cLI{IjNl zqq(((_y0-Fe=7O^V6P}B{IBUew_|H|G(0{E0e=REosXTwJ@&1R! zCitHgo8s24PIe~Vzosba)^2tX3b$XgY9XP2F3tY;rKz?m#DZ1L+6O}M>qAy8Jqi#7 z2L;%Gf>qVg(T(Ddm8>eOjDv+E1^Zu{;(7w?zh*ZgqMV#;ATAJ>2#5F|lbe5@+X(&d zOZ;m8f4(C&ClBv`zXLYsl(!$g3jxCMk7+7Ktg(EZSZP@V34}B@i%tF1L}rjGINx;2 zMPotPW20Dq0bCZ^_)Vi^G967XIgdo9gycf0^5)>K;)s0Y?&AK$TkuI>WREyP+{o2c zzQ!Y5T(?+UjiQbcgPScr9_Q;Pbon?bdt*snSPMMZ(=(VaHqgBh#VCnoh>2UU1LLD4 z8H5>`1UCpzYr^}TKf*g7JC?2dI@@3j7<+Qj)<%gCR*h?VDZ`~+)^z9apdL8$@U0+{ zM8xtSORbC3tPr^MRsReYCkdl*>pkItT5`f_S~pJ*CgZ_{apwBowe$1s#3=T|D1Pf0 z0?a76<%&FNAH{x$hkl5qJ{kVt^LB#Lgq}{z^kY&M^D155Rwr$(C?W!(! z*|u%lc2$?zUAAr8w!gZgJ2NqF{(bQx-Z|@>$dx-6GGp%@87mh!>z=CjXghW7OuXtw zo9e_`F$_Nl+tXp?s3PA6j$skbv+IV2TWwq!s)4qU+wp^B9xNau@E40CvSP9{y z7&Fvpt;5_P$mde9Z2gNtWFd`zG5P#e>^EbuY#2WG^<`kRTo~{x>yu}ci6P8j8ue4% zYKEC|x~Cl!B9XjvA1YPc-Ny6;APZ;0=QOE>?pa^?lVK;z&H{aO0 z2B3bxASyQi{C@;+0Qmm`?QDk4)QM9s)G5-xcBffV+7~Mzu-*SMw;Qrr1{2n+l^u7Png!~tex<2^6GQ{6c_iE(7pUT0JyB5ckJpPpE zzrL~l-M;)o&z1?ppHGk9Pot0fVb>4`Cugf~tbP4Qhn}3A&GgpVo`LDnyC)sbx^EWr zEA8N$yz_>(Ow|Peov*Iq%aAil%vTG2&bzaFfB(Qm4f9f$((cZVsBlSfvO>ap&D@IB z*B8l9#rcn$yl3iCetP=ywiV^~rlrpuc6x61v?F}?FFgfmU=S1l0004SHXr!8mPtQA z2mt_;fB#^ z?83a{c)}$x3FHk16(B&cGw%XiqUZxb2<87!;1dM$0U{(6G*HxJR76Ca5uW7_A0vvP zz*FWZ0H`lA%&V!1R`#IArAXY{N2RBy7qBfZj`j~u&%@jqLP~sp?){Dh8S4TuP(e}0 z_28az0a5th$v(OXIJ$v>{S63g;O#H~0)O-Pf#0bK@EC$p;YaBSC>r^viNLP?y7loO z(o)fXtXvIVJOM#OslS&^Lh?b1!&C5xAf=}%kJjjKN+dh{sv>ft58XsSNcxl%Q@czn zNT`pXl7g?c)i*OTFc0Q6_Z~frYKW{{TXE!=qSoEy>-72Tue}Z%wp5&v9HA^XXP;ws zxVzj(U++-YP*M8~(^k|})wwGwZ!2!9?g+en7lp4*-CW{l;JJOBI9`74-#>qAdjUmz z>)S;-8e3|zJ;Tn_TI1Iy0KmMuewIQvU+ww}bR!(Yqlb~-4gp|#Jyk;G--C$fiA8?I z$n2-GOi@}FK~WrYRZHo>8AC`BD>Mvo@SFLLe)2CzcagrqpLhM16u^L4^~J*gOZ#Ct zmh(KHpR&r__`oDQgkZl{2H#`V86(~@+3b_+>gfl;ZUWv2lXM?#(y0scS#&U9H}%FB zbR;kh$)>yEo@X@<%YWN2aKD~znuRW@W%G57L(sMaufJ3%bFIu6ycchO8^jAg*=#9w zFd;dZNnNeOfIA&XZ#p(PYk3-;IoXFbSJ&*Lp1wt9ny!WFgP9?VB4WvejzPFJE33as z{_|i$;K7EU0p$s|2~QjgVhJl85oCo|;&h!Brc1NpoGGSkcZJSm!;kZam>GyI?=gW`KhlIeDPn!K0h;(L)Bn_fRG$u8dz99%O);u*5~ie+r+#-`K>6 zC)Qj{12OAwwE^SO_gg(|>XfZvD22eFbajnd?kO7%v(j<wy`~Rf5;m@)##lc&nK|v zm=3E{X~D5=LFr=6nasXd*f(LBNt2(;`KTE~8=SZ9Cl(iMDBqCkf;3RKhz{4Bs$WXx zTPhGQRhgI(ccnjs@$xQr?5S~@?&*tf2DtfV9M=*lv5awW_&QqoKh*nOU9R={|Crx6 zt;LK+(^7F|FBW#au3X$GjA0sEQjPkxrjbMU6Xf}IgQFw8!$Gnh)(22h{e7dzx!M6C zf_lsw_S-2u^L9S>#sc03FcT}4Muy3E;&4y6S+JNZ_AfSU z{M0BukY~%G&JtF6TD5IwvC0WLs-u+8J@lZDS|dwwi{ie@yFCJ521366<-2s;J;v9E zb^NB(PKmd*-G+gmnb2-4NxH_j&`%4wbj`tu*6BBUfBuj)D5hzF(bcRdbv;7LS}fmZ z5xSNG-X87K$VdMAkx2u4)Ka8zhQDNVbZhakGZB>X>_W$Izf3|fi$lrW?lR*tOW;z` zdk;30PsfT~qwgBt_Sm+Ii=KAo+QAd>*1B`dKA4bq*&21AD{m%GU9O;MWBYh1%XJyk zVgiet1>YF8**QC75hXG;J9htp!d@|b-J4V&&TOz!TQmB zHuBrq6i3acuB>)7W}k^nq4WXGL8H^S+l^G!*23u2&?EA^k!wk)yxe8l)%gZQd9Yq2^j1-&Lh`nXfYc>YOSh_Iz= z#Pco7JMpBGx_ajT^(3U^fpMnB`AS;}6zv(8>p{gy&_>ghLraR4nb!gSg3j!d%J~o3 zg;{0Vv_q3tW&(QozrTf#Ai7?n%Ef~*h^r-|2-xY}Bjn*mL|K((-*tqKFLt4x?(OE9 znH2gZ4-|xtL-T8FgQ*^ae+u- ziB?+Q{a&;K?=AgPfcVLe-K=XSkWU_+i4GWx8Ozk<9}a)5(k`tvxH62V<9JH2#b5ft z)cwid*&YKW!_yn}A4%}}Lu4!K>W6izI+!;gSf z+=*vt4!z_MoWPI^b}&+00`e!hT29ZMPD=`L=Csd8H`mqX!~rm$SfbK*1wgw_yINB> zD4o!eOXcWqX-ks}>xTUtX011Doz zI|pMIb7NOJGbd}S3-2|Dwc&*AKb=3QhpfH>Fv5Cqxol)BFI-yQT50j6ad90Fv+PhR zpopxH^nvph&cl%zZSOdy-WvHCY~O0%I(?b=Yu_LfgRupX5TFSNN9%D6bjn7n#=pN+ zRTWi*g{FH>T%5tSFmrQ--w0%Qpe9vuB<9;I#*|5--!@BF?@Z8VUnmJwtb8%ClUGZ_-B!UarI!OjjTI9!oYUdErbGo&sq^jhHO% zAtxaaG^A%^$A>eD?>+fU6V6m${Y#gT9#9krCAYNSAZfsv<-yM>g`g3NZbCfSfa^k> z2FXDSCMsW#et+XE6b{a2Gp_7V!FCG7-$*3jD4j7!n`T&wZ0?O#o&5ruLZ|xzrUJkQ zP;b_QyGi4TEE)A2igpaID2V41z8dV;y+4niBCIHy6)+(CGa(CFB^Y4KP=D($#o)2KYc^e zqfVsg?_rQvNwiLwmzxY(miP}SZ&LxuKS zt)Mibx)t9j6(Fzo_Kb2ERa50LM3kbFL~z%2g8ZCJy!}Csbt8rUqw%qW8r*3yi&{Wqfk_4sDbL={Ogp)-u|K)xJ0~)TCt)CGoOLxs}^|66~P|6PuHcOPpaw zXRx!>h7<)WA)Gj36T^7>Z(?q?zIuZ)mBA9zQM{gZNpInW{F`b8lm?QFp&(LX=(cgv zPBP?O|1z5Wk>7{qOjY@1d%rSIhW{ETu$-DVn|szI{8{wQx~kyjwDkcfzzg=0Om}JIW775+|B7tx6;2*R4w?t*45RVFbYcG{b1NIB8*Y&p2s1I-5Fz2xrm1bOy$Me!jm8y%M%!Nz)rX zp5gSS2j9c}3f5@v9}z!E*ePd;ETm?*6v<#p5n>=3nFXyfT%I?q0}(cu2a7leuO_C2j>QGKyj7>_y-6(f4!%cVvbYUh#`w|0n3pm z77w4wD#oj^?iG^6oz z#3O|$ffTxB5))d?lcGKAAvC?3{5}iMcBlNfjn#Qh3#D(e6VEGCYoC!r|7eip(a!E& z`IVBtV5}FRCQtNsmm->m#3JGoa>4czNbb>S90VbX9d%F61xNsfgQkRqmEtnX;cJ2Z zPLQQP+v5~LpKwVGDS;$Jr9ve%$rf3H1p>Ajobaw%aW+{$UZ#9M@Wb~95KEFBy01l- z1O_lnf5;vN*^Nnde(92Ls>q=lF1e6;Ny3b1yBXIP=NPpF6#r@6-Eme=k6y(uDw$K2x!U3Q2uhwuCR_ z7hYChsh>`2w%+Xm-B+=cyoxU+=(xYhda<8@J<%AL=2lu1@ODV0P)LY$7_eV}44PzI z6-x}}SISHND4E;dy2Dv3;q&l5b0lzWWs<37F-2sBKz$qXA@9ibz-~PUFuC{(9CRag zUlu>ujL#^cvRV zp&379xSuU44o5{ITA5}JnaXO!XnET|(gG+|s``AO>Vw|jwA7v2Fl}(+ckE@rR?^+ZU5$N*K3g4^kZ}(?8i1a7f;D(cSDHSz zy8D19T6WStHJjWIVdI3^!~zxMPHw(?J$j%HG+UV((sV6S=0hlMsTj|-uxf<#Vy)bz z6M9lP!)gtIEsmGidB8?jD1#ep#g&T!U>$HVJv(VL$4SyO&~9;kl}Y;>eqRU=!z_?< z9h~7z1QO00#80J?!Lh1NHBv^!5>hljM%iql7HF_zhBU>>(yZjgP;-wn=JT(a#)QId z+}IvljKzcwn<*M0t19zi;Gy3qE2+AKPZ&j4qzG!CA|*pap{X*Kq?GoJuBjPJLk^ES zLmW4Y%c`^TPFQI>Rp;B+M)GcyOm~2HCz;n*wyTg)H83#Z4l&>Wv7{mGuY!RRgCukp z0HRLqCvuu^|%oFGQ(^E(Hv-mszBRkXv?+ zJV-tmr(zNOT3}VJuUwNRxHY&8-rmv-XtIaW{5(+yZaydX!phCS7 ztQTbC%Ioc1OzbHGt~`_%!B;0ZnBkWUZL<`P5p?8^A~T&pTN1EE2zDG(Oq4_7;vIqz zH=mZLks2%QwFMX(gu`gs;Y7g*H1C;H6%Kt!EJP~gHym);bFpDb{-yrFg9(G(%2);j z;xEYfi$F+6`mQcakQ3sa7HDB;Jq0Fqg<%z>)!yq_4%aS91pqs&SeRll(iseZy|Y>g zgj5z|n^j#>i!V<66ryxh1TCQuUVtbrkzr8R-MBNZa)g(M$*hPO6etH>Hdr2>^#>%83|h^d!*365m|>3#9C&&Gg=D`>$d}>j-A|kr7ct5!RCe!hpfsx zd5@*artSJsj>Y3nG#fJzO!b6uEF(I2fRwnexf-4NTQXpb5732aFZ$!Br6alt##3yU zc-tWc&(Nmb)5ea{Ip`P3_NDSd>>F(<$D5xK*zZHz1@d4E@k25DPq4$X;Cn%fX_#CW zLhAUT8-KKwv+cLs?X@$&pKin8ME4%2Mfn-lYil;K!k^49@CpavS5;J${zPF>AINja zfb03a6QB{dV_5rRVMg>XbQw>{aB4i_T7y698e{q88SKl;B>nv>$sslm2LGm)=PZS1RKvx^T1wUSB`@Et~;0izCjO@z@qT+mSukNxAVSBX^O-Xe+K@a?tVKV9-W5GZ^mZ ze&%rYSx>}h@}szwFEDi1xt>}q$FwGXZgMR#jSAwcYBVF-}?)$A1 zt3{ww=R7fe4wun|TPu6#C?TzAy)@5eRr2RB@_58$;yVVCXyzh4Sx_evX0qPC#-(@h zk;-C?dI$tf9C}4zUWFu#>{gwpcy;& zceYu~E?4a*%RrQEEuwj2@ME92AW?@-w?t5VsQdlT?d0AcjQo$+coZIAhwlTfpp;oi zq-nH5SCISwAVu3=;KKt(LIG3+N-%u9paES+1TyB%C+&XQEO7{=7ac0f^r2cDBc${p z#@I%4JO?y#sXYId z(=ybcRW5;6o$5w?`%A2~ANw=BpA*U)qe|tCn@P)vmIy5fnXS>!lulN=?C&zYY_5-~ zWu+X$X@-niwfmVQME<3oIX_@Vd zUUcoT9NgCD_!3ze1zI8b>BCh0?SUM!#XaTGFX0MZJcz5}Q9|O7aMWVzGpuVYIO0v( zz?fXCf)OFJ+knagY7ousWKW5@xwe=V8gvKoMmLrV7>9npF zRJ{=U`Xa1aZAXi5H|;^qhIp_@LL89sAs+gXxJv32Y?h#JqNAYcoz**F|=t@>(8o7fuASGDU}w#t;^esSIf*RAZr5Rj6rcDY{JtSyYwkq2;ZxVE*B>CCf}4fq@Qun=Eno}M^C-s?YFQ&hyz z$g;}nuB>Fp7U2qw&UNyZc zoe)xTU7ddLrBjcJWg)e<*~X>W@gtPZ^}0Rw&|_lLM=hT^1$Hi5*Jr~zdLd3!ml|=7 z+Tz%O{8k}=g}}lOH9^s9!8IJKswDXn-si8bY(;9rr33oiCRBq1U8S5SJ~p1vfg)VI zoFK#19ZdwB|F0bN6~T%!%34?;OKuU|Tg9J+EfuPy2b15#IeQc)37g-hyohnea9 zJTK=A0au*Dvt&!WM%#Ira0;H!CiAn6Kv0L}UE7cOG?>40s)3%-R;d;;8{#exikOOJw>z;79I`o5MtnQPddwwl1^tvkLE3PI zM=yF}^qL6^K$By|d~|>uD>yx^KE9kb>5uX?v~|%>PhFappX0Wpg~d9#6lz5=i~0RG zeO^CN%0jvsakJyC?l>d{sa;!BQz6y9Hp$UWRcaKiD4jOljKx$?qH9*JLysb+m^D7f zj^#sE>TTBT<$o(1NHV5Z@$#_vF*r@ku7_~cw1dnGBV~`iIkhCGh!;tXM(&P%q*UNi z0bOKYSHwBi><39am6nOK%|iGPwj4}Mm@wPGZxBiiM@rO>qSFIxAB;^WOs!x!0@(#^3AQesWz=MDIK zrUR;|I&z4qU;--bc5XD8e`*xpJD2IruOjM@fC#-La|9~yS*KI9`50=^~D&@^<5kgM*=A0)tC5;UzF` z4XCVp)t~kQcvlJh0YiFdVaO>y9}7GW#z&Qi_?&V87Ku7`1F1*WhIGslZE|CJsW~$b zB7f=3kH=8$mhYnCg{Qox^aYVJOF6;Mi6UeilAU0zANxDctsFH6W>;^W0Y9LUzf^iBWJLq$yPoz2Ua(&tpf?XtO1)$_sI`ZIKow&!Uxp7L@Qr*a zd^Ol450%p>&?<%RP-~MIzZR5G8x$wIi<^8I1WIRfh73Dmb@RWMb-(q~z>5fvJMMYXN(-04r{)sj*>n@+YSju*is?q&PuY%bJI{ltDf zSJi$ImZ`d&mgGDnDEu)y$>z?xyN=uF6L!gc5_S<6ho4$r$gNu=i>1Fr{G&~Pn_J6$ zdK0jrI0Swk!}V9swejrESzO%p>!!U&s3y53gnROx+WFcg{jG&pnFr&^9ktuW3OyOv z^EPsciGe*;gbUR|Ew0XVr8YUjd&8&^>QAB#3r`P68rM7(hRa@y?rSZ1gQuy9=3Nao zebY~W&)J&?L*XB)O(_{$z-}0x7<>D)G2B=}K=bMeTIg-6Sj@;vTPLO+Ad5GcF>?>0 zjLkJaZIhwvOsVD6bHZAQX`H+`b{ju4`mzlLkDL7Fd_do?a2kXb)~Rcnk6t%i<*J0ZiqPlZZH(?>SD>Jg+aZ1RsC*msuKs^9uF%&VTPCarsAIpQ!ahBzi{z$9r)U?|=<#jg?5a21Pb*N_j`UeYnxCHE=R$F|n1GRB_Eq>TO7^%M#FeHG~DZ`j?aQi7C?>@Z`}HCh9ekx@xI0 zYg*#Bq#Nr7b;NCNdac*_3G+fdkY{~qoSnC-4p!R(rE>2?(-@rLGbq)3TWku%ZU2tn3gs7Sub_V5C3?r! z5p1ub|7ex9J7IqibqkN_?R#gay+8xtWx5m|w*LKUfy=*Kouj|yk*EIn(G5IDucBJ% zoJ<4YY5a(ox3Ou`1KR$B>>SD4P#2qK^l=aaJV2H+{6=DJNd1GS(K?(bW8OmRN4Kjk zyL(5eH-(WJ`>DWB3YDML)+nNWp^tGysFOc=ivZW%_t42Cmj9URq?V!@vopR)61pC* zijVxhm&i4Qh3KfhNR(}!z&%Ko3KH=PAasc3Z`b9Nmq!vfmQKS6gNE=l$Q z+V(#|_pm5*)}aA-9r0>vUisj6R#jyCmMx?Y&skT`lz)@0QrtEQRvn+P&R51d zM)*Cr+PoO#noq`}spQ)KT&ir>@r;HVz*DOWh01CBy%cWxufeBkEx*x0{J@`}e>APG zee}9)=Qpqsfv#yau>I%;zq3pXgTVFB`s1l2@Nh3q!~K`K5PNwCC(f)F^5s>7+h#k& zXsQM$w^M2*DOIJf97TPgah9`-`yyErL30a?d1WE*N_GEmbemDwuCm>tjR%g57pZy zQK)!zk+%B2wbeV5a`FXg^qq^I4e1{KBDI8AzW&Sb zk@WzPWh#C#`xPB_q`1w}4mV0DPC?pF)X3|M2Tz)aPtqW&0Mdoi{L!E8P29Mp5cN+8 zrROA_T4$%5{okeH?-7c4Kqx(-l+;+L>A5>KZ{7%t+#o1Z5ImtM+fz8IhfpdHh#|AMX-NztCdVx%{zdHA3>jl<9{;hDA~2 z;W4&5%4HfpxA``tgfz4Zck0J2oz2Jb;<&;;q}f&s$>`ntjvK?e#XeZbeu%K6^fk)N zwFi4Nxt1wi(K`B&Q#~xYv!fPq##pQi6>oBDT{S2NQEFT@kWbR;9vM)6)h1na_#k@mjcd4n{*${E2-oH7___N>$i3~7s>ce^L!N}6=q~66 zA{-nx_eTy(JU~^Th$BPz9N>Fbr~%b$G$`>Cuk%1n5NL(`m??dTFGM18d@oH8r{Lu~ zRa5LFB+E4#rp$-XG`0fV32tDnTLf@ols?H*-kesct-xPVW#YrGP`6 zj^OQ4V$|UKt&-#~L9qlKt&CgI*y!mp8bo@?X;xknvFhYN$&ALL3#;a*KWN$;7Qd6W-Vd5pdfGx62(hbOSK2iC6^Mbu3 zq7;_e&1fAZf^^B{TXRjhJha^snHhoBO0NEr_jEw}py%5gDYGt}Sg~^>hhi@~T{F;u zwXW9UHv(qUhUMqwYkhbq!t~H-A)a0CODbK-4@2OSxYA=D|^(h0o7@a`+z8^~HCw z-&OtK)R^Kku^s8c_u#)jwraQ_|Cw6x*{XRXy3?DQw^be@<5ih$^RO+foSbPet)?r8 zy@V=OV`f{zUv`>!aJ{uO-&k&PAt)5M<8jZf<>>QfV!NE7vE#LZ=Du@AoCPvLd4#%0 zMapEDBaxwz;BEuyXrEzs>OFW7vM<4YrT@^x)S~$uua1!83z4T(05=okEAu%qPr8e>q%4kj*dr}!9 z!#@MFw;NsZSD`hq2=cmwJgl9(E45uvsdJjVml_>`z`nN+rw=Q|wg`h4$Gu+H&AcmG z10l;A3xCe4P!twi%7G$$LB22*DaZ8P$R5p4G`U|BC0H-UCu&TfOP(MCa&i|IN=M;( z&s6q|DVPS}%?_sp(jTyfF%v()OlY|CR-FU9Alsyi^5%|*DjpAxPFXyP$9iK6Q&l-1 z@8*vG(l|SDGJ|%$`lzWO>xlDr<|<}HmVe~W1Y;3eq}|+ z=NIN8$e#(`f9`W%KYR=fsNR>f;)>Q-ex1QUdg191bu7#@+3<-i>B<+F>umJuWSZ@I zYbCWHhW~zVd_dki3sa^8zmhK)i)3eJ2Hd!lm^noZSs;7>`N6QYfsHJ7Wv1KDM0th& z5y(;ytwl&Q<>4*#LKx#XJjYMUQEcaK8atIwe^x#anqAMpM5gMM8JtpK(pMUg^zbz) z%Q)>F#{*5_ILM-iAAH(4z6O!|l_S+_Sa*~sPF|#f;NLiH8z5j6MT%G_;UO>4M!dki zH-pw0?xZu}mww$O?1!U12N_WYC*Ck9uIi%^V!QRX(+jU@GiKC6dt(w@aoeoHX9i@4 z50y@$n4P*tb6}{Q+WUcn%G)zk2?ZufF?rEIih_rmOkiNbYE6WKtI+tw8|&p(SyA0s zAN7vBtjgB=xfTNOr?JBlY=)BrH7A)+- z!m>DM4o$?Q`*yo7S?aNxyr7jdo|Y_`Dkf_h)aY-`5w>!!KkRq2OD$oZ*3K>OSdCh~ z*i@!1id(@KpBaNoxxIfY-GObxJ3rulf@l`+$7@^rfBsCVdWkz!61%#uQJ2r@a zH!e^s@wfX?Z495v^mj_Bz^8+^+Z;H0*KCjIvhHrU61d)d+WjDBpG$~f_^H5q16^Is z*3Cw(ug2Vj3V6l_yH$n|H*Aw|{Rk0IHn19j<%;1)1j-dUnF%F(}z z(?BTrR_im6Jl_1jVN&3wX~`d_Aqs+^+ePCb?9^ z<*w@k5j({*Im75is}U&L30eh=-C8hPW71rw$pf+@oz`IA+F;@=1MrDlIl z8yvXJew1mp4Q7byy)_(rzvzX*=R*0m{CwxKq6&gf3PsLVcLB>ke3)KiP#yAAP`abnM~ zHil4CVSj(orOOvv(k>M@Ad&CM*`f%Mp4+iH=xs;IY<4#xmA+!R#*5HeBIpq(ITRqY0?HbH|mQG~zV zZ+mJeYd(MPZ?oga^y@y(Z6FISOhbhZeH2-BQu-ePrOGNj6if5t-=A|4Cmw@!FN(f% zcWx6LSkJ&99EQ{$TraA_YF{kfG{oCuezLduKoBJ$pIO9`tj!x)zPMRNZ`2iu7yAMt5y1!cwRAm=!tD*G= zury<7@GaJb{4YZyos9VTQm8Z1Hu>n3A3}I~r^BDjyG5cU=f<={NYb3F4Dw*U&M^t& zWI1-?hUUzvniifKV~W)}Rv?lILA}RlTw_YmTWoU1Dwx0F9(A6Ei@4^*NZ`h>Wn6L+ zqfBBnP7SS9Go;cV2@11~HHmNqAX5A&;uJ9(&ymt!wL?GV`PR#p`*F4>e(9?^G`b-~ z#LG~BXF+dWhjkMl-k()0FCyeo@5wcjVsbpHw6->M`OUbS; zDZ~faxR`r6eM{Z`lt!LOiUZk>Kc}eQ4XDhr^_LCuaoUr>3nIZ^I)yKt{mSePT|b6F z4gm(Ty3#OiwO-d>qrJMGDV~Gfvpu{KZ?7X(Ino4LrGO@u@<4qFXfXxKX`$lz(NgHg zzCUNF9Ys6yCsA0!04oXNY!G2UQcHqn>HyD@Hz1j*UkiF^2}pYO8Dz0Jj%?!QR0%|E zDG=s?z~Rx8p%l-F^2gtyxpysA9mI+dl4_rC{q$xd* zxYdjda%>BfG(p5`sq~#q(B{Td0{K`RIB0<2C~j5ZUs3%CW+7yoG`n<$`dFR}B@46j z?KYLZgI0k$YUZ-w5p7^l7F6#7Xl-5Y6lAjPPNpGz5Pb&p-3Sq5oJn7gxDm(H5<3=i z(BXbOI~7bjqiS{u^2KSslMKgXLKAA5SY5RWkzkPUEZCU@70&obU~vrZXr}{VXS28T zkwY|p_9TYI2!(OL>`oR9N_4s$u98b`?HH}-qW`a~gt7{wl)S;fUq%63%*H8^f90hj zizm|@a7a0LzzLHQR4in(npO*=#-m*2)yJI~Z_X}abXNsUqW!tiuftl?Em15M^UI~` zISkt*LC%O%it3QCoaqgbeleaZdfU0hGgA;N4h??@*9sE`C|Tnob*ggVJP>6Qj7ui; zU;|Nz?fr#M*KkP1G%};}u8kAm%_-#H-?Zqn%4cUrK84$=d3Bll>e3`wJ{k76urFKNU#;r!xpUO^ zWcvaY8s*Q5NB4Ret-f+yyvnc`^2${3*>g{;l|D(nIIESL8`?)%r10l8u59=|X|FXq zHyw0lyaRjcK)b5Ori3MrPwpbKwHw~v-j41!(j4+_n%SxwC{rl;p7G6 z-Z;e-oAs*l`+XX#afhYurPfA}d9a6COZ{>xOGgL{==jp zb5!h(XXBsOYa?Xe*+vJd6dY}_A5fr;*NSe;3=4ie_LFfqETHi@h%{3(I%iqjMQtg+ z!Zq^N#k~Oxc--r1_?=b&MaSbW^oQF$ZNE1Pfx9u4sYZQ`U64yQqv|o>; zOn}_1I-@uR#G%+oM!k$@fvBEwr#vOey+CzMe07j}`)oW>fU-SJrv{0i8!|mNy z+h=|%$Hygn?lBMj;J|C+*Y!sYa6spo#W^sQ!-lNWeu^FP$AFWhLEeRab27!FkV38Y zLzN+yf_vOiGMx)u2frd^=%vJ#x>V_%c$vV`&f!OA>OJZ!fxIQShTes5mU<2SV9Zt?j${aVYoQm3jQ)UnJvp4YVk}3l{Z)Olh34`9y#=oXAqyw5XBiEK46S^ zx?^fV>h=Twj0YFx4FBi5W*$e%2cCb3caB0?tu+A?0~0GP z11l{v0|6sDEi?Ofo6gS0^q)HYE7PBnb&N72KL9QdKLBk1h3V Archive { + let sender = PrivateKey::generate(); + let mut writer = ArchiveWriter::create( + Vec::new(), + WriteOptions::new(sender.duplicate(), vec![sender.public_key()]), + ) + .unwrap(); + manifest + .ingest(&mut writer, ProcessingOptions::default()) + .unwrap(); + Archive::open( + MemorySource::new(writer.finish().unwrap()), + OpenOptions::default().with_access_keys(AccessKeys::new().with_key(sender)), + ) + .unwrap() +} + +#[test] +fn ingestion_records_only_linux_permission_bits() { + let temporary = tempfile::tempdir().unwrap(); + let input = temporary.path().join("mode.bin"); + fs::write(&input, b"mode").unwrap(); + fs::set_permissions(&input, fs::Permissions::from_mode(0o640)).unwrap(); + + let archive = write_manifest(&build_input_manifest(&[input]).unwrap()); + let entry = archive.entry("mode.bin").unwrap().unwrap(); + + assert_eq!(entry.permissions, 0o640); +} + +#[test] +fn ingestion_normalizes_pre_epoch_timestamps_to_zero() { + let temporary = tempfile::tempdir().unwrap(); + let input = temporary.path().join("pre-epoch.bin"); + fs::write(&input, b"timestamp").unwrap(); + File::options() + .write(true) + .open(&input) + .unwrap() + .set_times(FileTimes::new().set_modified(SystemTime::UNIX_EPOCH - Duration::from_secs(1))) + .unwrap(); + + let archive = write_manifest(&build_input_manifest(&[input]).unwrap()); + let entry = archive.entry("pre-epoch.bin").unwrap().unwrap(); + + assert_eq!(entry.modified, 0); +} + +#[test] +fn ingestion_rejects_unix_sockets_with_typed_path_context() { + let temporary = tempfile::tempdir().unwrap(); + let socket = temporary.path().join("input.sock"); + let _listener = std::os::unix::net::UnixListener::bind(&socket).unwrap(); + + let error = match build_input_manifest(std::slice::from_ref(&socket)) { + Ok(_) => panic!("socket ingestion unexpectedly succeeded"), + Err(error) => error, + }; + + let error = (&error as &(dyn std::error::Error + 'static)) + .downcast_ref::() + .expect("filesystem operations must return FsError directly"); + assert!(matches!( + error, + FsError::UnsupportedEntry { path, kind } + if path == &socket && *kind == "socket" + )); +} + +#[test] +fn ingestion_rejects_fifos_with_typed_path_context() { + let temporary = tempfile::tempdir().unwrap(); + let source = temporary.path().join("source"); + let fifo = source.join("input.fifo"); + fs::create_dir(&source).unwrap(); + rustix::fs::mkfifoat(rustix::fs::CWD, &fifo, rustix::fs::Mode::RWXU).unwrap(); + + let error = match build_input_manifest(&[source]) { + Ok(_) => panic!("FIFO ingestion unexpectedly succeeded"), + Err(error) => error, + }; + let error = (&error as &(dyn std::error::Error + 'static)) + .downcast_ref::() + .expect("filesystem operations must return FsError directly"); + assert!(matches!( + error, + FsError::UnsupportedEntry { path, kind } + if path == &fifo && *kind == "FIFO" + )); +} + +#[test] +fn extraction_preserves_files_directories_and_contained_dangling_symlinks() { + let temporary = tempfile::tempdir().unwrap(); + let source = temporary.path().join("source"); + fs::create_dir_all(source.join("nested")).unwrap(); + fs::write(source.join("payload.bin"), b"payload").unwrap(); + std::os::unix::fs::symlink("../payload.bin", source.join("nested/live")).unwrap(); + std::os::unix::fs::symlink("../missing.bin", source.join("nested/dangling")).unwrap(); + let archive = write_manifest(&build_input_manifest(&[source]).unwrap()); + let output = temporary.path().join("output"); + + for path in ["nested", "payload.bin", "nested/live", "nested/dangling"] { + extract(&archive, path, &output).unwrap(); + } + + assert!( + fs::symlink_metadata(output.join("nested")) + .unwrap() + .is_dir() + ); + assert_eq!(fs::read(output.join("payload.bin")).unwrap(), b"payload"); + assert_eq!( + fs::read_link(output.join("nested/live")).unwrap(), + Path::new("../payload.bin") + ); + assert_eq!( + fs::read_link(output.join("nested/dangling")).unwrap(), + Path::new("../missing.bin") + ); + assert!(fs::metadata(output.join("nested/dangling")).is_err()); + assert!(!output.join("missing.bin").exists()); +} + +#[test] +fn extraction_uses_host_defaults_instead_of_archived_mode_and_timestamps() { + let sender = PrivateKey::generate(); + let mut writer = ArchiveWriter::create( + Vec::new(), + WriteOptions::new(sender.duplicate(), vec![sender.public_key()]), + ) + .unwrap(); + writer + .add_file( + ArchivePath::new("data.bin").unwrap(), + EntryMetadata::new(1, 1, 0o111), + ProcessingOptions::default(), + Some(7), + std::io::Cursor::new(b"content"), + ) + .unwrap(); + let archive = Archive::open( + MemorySource::new(writer.finish().unwrap()), + OpenOptions::default().with_access_keys(AccessKeys::new().with_key(sender)), + ) + .unwrap(); + let temporary = tempfile::tempdir().unwrap(); + + extract(&archive, "data.bin", temporary.path()).unwrap(); + + let output = temporary.path().join("data.bin"); + assert_eq!(fs::read(&output).unwrap(), b"content"); + let metadata = fs::metadata(output).unwrap(); + assert_eq!(metadata.permissions().mode() & 0o111, 0); + assert!( + metadata + .modified() + .unwrap() + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs() + > 1 + ); +} + +#[test] +fn extraction_rejects_a_symlink_in_the_destination_root_path() { + let temporary = tempfile::tempdir().unwrap(); + let input = temporary.path().join("payload.bin"); + fs::write(&input, b"payload").unwrap(); + let archive = write_manifest(&build_input_manifest(&[input]).unwrap()); + let outside = temporary.path().join("outside"); + fs::create_dir(&outside).unwrap(); + let redirect = temporary.path().join("redirect"); + std::os::unix::fs::symlink(&outside, &redirect).unwrap(); + let destination = redirect.join("created-through-link"); + + assert!(extract(&archive, "payload.bin", &destination).is_err()); + assert!(!outside.join("created-through-link").exists()); +} diff --git a/crates/pithos_lib/tests/ro_crate_conversion.rs b/crates/pithos_lib/tests/ro_crate_conversion.rs new file mode 100644 index 0000000..6e1821b --- /dev/null +++ b/crates/pithos_lib/tests/ro_crate_conversion.rs @@ -0,0 +1,106 @@ +mod common; + +use common::ro_crate::{metadata, write_loaded, write_raw_zip}; +use common::util::{open, private_key}; +use pithos_lib::adapters::ro_crate::{read_ro_crate_directory, read_ro_crate_zip}; +use pithos_lib::archive::{AccessKeys, ArchiveEntry, CdcConfig, EntryKind}; +use std::fs; +use std::os::unix::fs::symlink; +use std::path::Path; + +type SemanticEntry = (String, EntryKind, Vec<(u64, String)>); + +fn semantic_entries(path: &Path) -> Vec { + let archive = open(path, AccessKeys::new().with_key(private_key("recipient1"))); + let mut entries = archive + .entries() + .map(|entry: ArchiveEntry| { + ( + entry.path, + entry.kind, + entry + .references + .into_iter() + .map(|reference| (reference.target_id, reference.relationship)) + .collect::>(), + ) + }) + .collect::>(); + entries.sort_by(|left, right| left.0.cmp(&right.0)); + entries +} + +#[test] +fn typed_directory_and_zip_conversion_have_semantic_parity_and_extract() { + let temporary = tempfile::tempdir().unwrap(); + let source = temporary.path().join("directory-crate"); + fs::create_dir_all(source.join("nested")).unwrap(); + let crate_metadata = metadata(&["nested/file.txt", "unlisted.txt"]); + fs::write(source.join("ro-crate-metadata.json"), &crate_metadata).unwrap(); + fs::write(source.join("nested/file.txt"), b"nested payload").unwrap(); + fs::write(source.join("unlisted.txt"), b"unlisted payload").unwrap(); + symlink("nested/file.txt", source.join("link")).unwrap(); + + let zip = temporary.path().join("crate.zip"); + write_raw_zip( + &zip, + &[ + ( + b"ro-crate-metadata.json", + crate_metadata.as_bytes(), + 0o100644, + ), + (b"nested/file.txt", b"nested payload", 0o100644), + (b"unlisted.txt", b"unlisted payload", 0o100644), + (b"link", b"nested/file.txt", 0o120777), + ], + ); + + let directory_output = temporary.path().join("directory.pith"); + let zip_output = temporary.path().join("zip.pith"); + write_loaded( + &directory_output, + read_ro_crate_directory(&source).unwrap(), + CdcConfig::default(), + ); + write_loaded( + &zip_output, + read_ro_crate_zip(&zip).unwrap(), + CdcConfig::default(), + ); + + assert_eq!( + semantic_entries(&directory_output), + semantic_entries(&zip_output) + ); + let archive = open( + &zip_output, + AccessKeys::new().with_key(private_key("recipient1")), + ); + let nested = archive.entry("nested").unwrap().unwrap(); + assert_eq!(nested.kind, EntryKind::Directory); + let mut content = Vec::new(); + archive.copy_to("nested/file.txt", &mut content).unwrap(); + assert_eq!(content, b"nested payload"); + assert_eq!( + archive + .entry("nested/file.txt") + .unwrap() + .unwrap() + .permissions, + 0o644 + ); + let extracted = temporary.path().join("extracted"); + fs::create_dir(&extracted).unwrap(); + pithos_lib::fs::extract(&archive, "nested", &extracted).unwrap(); + pithos_lib::fs::extract(&archive, "nested/file.txt", &extracted).unwrap(); + pithos_lib::fs::extract(&archive, "link", &extracted).unwrap(); + assert_eq!( + fs::read(extracted.join("nested/file.txt")).unwrap(), + b"nested payload" + ); + assert_eq!( + fs::read_link(extracted.join("link")).unwrap(), + Path::new("nested/file.txt") + ); +} diff --git a/crates/pithos_lib/tests/ro_crate_directory.rs b/crates/pithos_lib/tests/ro_crate_directory.rs new file mode 100644 index 0000000..1a3fe8a --- /dev/null +++ b/crates/pithos_lib/tests/ro_crate_directory.rs @@ -0,0 +1,274 @@ +mod common; + +use common::ro_crate::{conversion_error, duplicate_conversion_error, metadata, write_loaded}; +use common::util::{open, private_key}; +use pithos_lib::adapters::ro_crate::{ + RO_CRATE_METADATA_FILE, RoCrateError, RoCrateSource, read_ro_crate_directory, +}; +use pithos_lib::archive::{AccessKeys, CdcConfig, WriterError}; +use pithos_lib::error::PithosError; +use rocraters::ro_crate::graph_vector::GraphVector; +use rocraters::ro_crate::read::CrateReadError; +use rocraters::ro_crate::schema::RoCrateSchemaVersion; +use std::fs; +use std::os::unix::fs::symlink; + +#[test] +fn loaded_directory_converts_from_retained_sources_after_path_replacement() { + let temporary = tempfile::tempdir().unwrap(); + let source = temporary.path().join("crate"); + fs::create_dir(&source).unwrap(); + let original_metadata = metadata(&["data.bin"]).replace("Test Crate", "Original Crate"); + fs::write(source.join("ro-crate-metadata.json"), &original_metadata).unwrap(); + fs::write(source.join("data.bin"), b"original").unwrap(); + + let loaded = read_ro_crate_directory(&source).unwrap(); + fs::rename(&source, temporary.path().join("original-crate")).unwrap(); + fs::create_dir(&source).unwrap(); + let replacement_metadata = + metadata(&["data.bin", "replacement-only.bin"]).replace("Test Crate", "Replacement Crate"); + fs::write(source.join("ro-crate-metadata.json"), replacement_metadata).unwrap(); + fs::write(source.join("data.bin"), b"replaced").unwrap(); + fs::write(source.join("replacement-only.bin"), b"replacement").unwrap(); + + let output = temporary.path().join("retained-directory.pith"); + write_loaded(&output, loaded, CdcConfig::default()); + let archive = open( + &output, + AccessKeys::new().with_key(private_key("recipient1")), + ); + let mut archived_metadata = Vec::new(); + archive + .copy_to("ro-crate-metadata.json", &mut archived_metadata) + .unwrap(); + let mut data = Vec::new(); + archive.copy_to("data.bin", &mut data).unwrap(); + + assert_eq!(archived_metadata, original_metadata.as_bytes()); + assert_eq!(data, b"original"); + assert!(archive.entry("replacement-only.bin").unwrap().is_none()); +} + +#[test] +fn retained_directory_conversion_failure_has_physical_path_and_source_context() { + let temporary = tempfile::tempdir().unwrap(); + let crate_path = temporary.path().join("crate"); + let data_path = crate_path.join("data.bin"); + fs::create_dir(&crate_path).unwrap(); + fs::write( + crate_path.join(RO_CRATE_METADATA_FILE), + metadata(&["data.bin"]), + ) + .unwrap(); + fs::write(&data_path, b"original").unwrap(); + let loaded = read_ro_crate_directory(&crate_path).unwrap(); + + fs::write(&data_path, []).unwrap(); + + let error = conversion_error(loaded); + let message = error.to_string(); + assert!(message.contains("convert retained directory file")); + assert!(message.contains(data_path.to_str().unwrap())); + let writer_error = std::error::Error::source(&error) + .and_then(|source| source.downcast_ref::()) + .expect("retained-file context must expose WriterError as its source"); + let core_error = std::error::Error::source(writer_error) + .and_then(|source| source.downcast_ref::()) + .expect("WriterError must expose its core source"); + assert!(matches!( + core_error, + PithosError::WriterExpectedSizeMismatch { + expected: 8, + actual: 0 + } + )); +} + +#[test] +fn directory_metadata_directory_and_symlink_failures_keep_physical_context() { + let temporary = tempfile::tempdir().unwrap(); + let crate_path = temporary.path().join("crate"); + fs::create_dir_all(crate_path.join("nested")).unwrap(); + fs::write(crate_path.join(RO_CRATE_METADATA_FILE), metadata(&[])).unwrap(); + symlink("missing", crate_path.join("link")).unwrap(); + + for (entry, operation, physical) in [ + ( + RO_CRATE_METADATA_FILE, + "convert retained directory metadata", + crate_path.join(RO_CRATE_METADATA_FILE), + ), + ( + "nested", + "convert retained directory entry", + crate_path.join("nested"), + ), + ( + "link", + "convert retained directory entry", + crate_path.join("link"), + ), + ] { + let error = + duplicate_conversion_error(read_ro_crate_directory(&crate_path).unwrap(), entry); + let message = error.to_string(); + assert!(message.contains(operation), "unexpected error: {message}"); + assert!( + message.contains(physical.to_str().unwrap()), + "unexpected error: {message}" + ); + assert!( + std::error::Error::source(&error) + .and_then(|source| source.downcast_ref::()) + .is_some() + ); + } +} + +#[test] +fn typed_directory_conversion_rejects_unsafe_symlink_targets_before_content_write() { + let temporary = tempfile::tempdir().unwrap(); + for (name, target) in [ + ("absolute", "/outside"), + ("escaping", "../outside"), + ("invalid", "target/"), + ] { + let source = temporary.path().join(name); + fs::create_dir(&source).unwrap(); + fs::write(source.join("ro-crate-metadata.json"), metadata(&[])).unwrap(); + symlink(target, source.join("link")).unwrap(); + let error = read_ro_crate_directory(&source).unwrap_err(); + let physical = source.join("link"); + let message = error.to_string(); + assert!(message.contains(physical.to_str().unwrap()), "{message}"); + assert!( + std::error::Error::source(&error) + .and_then(|source| source.downcast_ref::()) + .is_some() + ); + } +} + +// Upstream ro-crate-rs parser and graph assertions. +#[test] +fn parser_preserves_the_fixture_graph_and_schema_version() { + let loaded = read_ro_crate_directory("tests/data/dummy_dir").unwrap(); + assert_eq!(loaded.source_kind(), RoCrateSource::Directory); + + let mut data_ids = loaded + .ro_crate() + .graph + .iter() + .filter_map(|entity| match entity { + GraphVector::DataEntity(entity) => Some(entity.id.as_str()), + _ => None, + }) + .collect::>(); + data_ids.sort_unstable(); + assert_eq!( + data_ids, + [ + "conclusions.txt", + "dataset/", + "dummy_results.txt", + "literature/" + ] + ); + + let mut contextual_ids = loaded + .ro_crate() + .graph + .iter() + .filter_map(|entity| match entity { + GraphVector::ContextualEntity(entity) => Some(entity.id.as_str()), + _ => None, + }) + .collect::>(); + contextual_ids.sort_unstable(); + assert_eq!( + contextual_ids, + [ + "https://orcid.org/0000-0002-1825-0097", + "mailto:josiah.carberry@example.com" + ] + ); + assert_eq!( + loaded + .ro_crate() + .graph + .iter() + .filter(|entity| matches!(entity, GraphVector::MetadataDescriptor(_))) + .count(), + 1 + ); + assert_eq!( + loaded + .ro_crate() + .graph + .iter() + .filter(|entity| matches!(entity, GraphVector::RootDataEntity(_))) + .count(), + 1 + ); + assert_eq!( + loaded.ro_crate().get_rocrate_version(), + Some(RoCrateSchemaVersion::V1_2) + ); +} + +#[test] +fn directory_parser_requires_exact_root_metadata_and_complete_root_fields() { + let temporary = tempfile::tempdir().unwrap(); + let not_directory = temporary.path().join("not-a-directory"); + fs::write(¬_directory, b"file").unwrap(); + let error = read_ro_crate_directory(¬_directory).unwrap_err(); + assert!(error.to_string().contains("open directory RO-Crate")); + assert!(error.to_string().contains(not_directory.to_str().unwrap())); + + assert!(matches!( + read_ro_crate_directory(temporary.path()), + Err(RoCrateError::MissingMetadata { .. }) + )); + + let wrapper = temporary.path().join("wrapper"); + fs::create_dir(&wrapper).unwrap(); + fs::write(wrapper.join("ro-crate-metadata.json"), metadata(&[])).unwrap(); + assert!(matches!( + read_ro_crate_directory(temporary.path()), + Err(RoCrateError::MissingMetadata { .. }) + )); + + fs::write( + temporary.path().join("ro-crate-metadata.json"), + r#"{ + "@context": "https://w3id.org/ro/crate/1.2/context", + "@graph": [ + { + "@id": "ro-crate-metadata.json", + "@type": "CreativeWork", + "conformsTo": {"@id": "https://w3id.org/ro/crate/1.2"}, + "about": {"@id": "./"} + }, + {"@id": "./", "@type": "Dataset"} + ] + }"#, + ) + .unwrap(); + let error = read_ro_crate_directory(temporary.path()).unwrap_err(); + assert!(matches!( + std::error::Error::source(&error) + .and_then(|source| source.downcast_ref::()), + Some(CrateReadError::JsonError(_)) + )); + assert!(error.to_string().contains("missing field `name`")); + assert!(error.to_string().contains("parse RO-Crate metadata")); + assert!( + error.to_string().contains( + temporary + .path() + .join("ro-crate-metadata.json") + .to_str() + .unwrap() + ) + ); +} diff --git a/crates/pithos_lib/tests/ro_crate_zip.rs b/crates/pithos_lib/tests/ro_crate_zip.rs new file mode 100644 index 0000000..554eca7 --- /dev/null +++ b/crates/pithos_lib/tests/ro_crate_zip.rs @@ -0,0 +1,471 @@ +mod common; + +use common::ro_crate::{ + conversion_error, duplicate_conversion_error, metadata, write_loaded, write_raw_zip, +}; +use common::util::{open, private_key}; +use pithos_lib::adapters::ro_crate::{ + RO_CRATE_METADATA_FILE, RoCrateError, RoCrateSource, read_ro_crate_zip, +}; +use pithos_lib::archive::{AccessKeys, CdcConfig, EntryKind, WriterError}; +use pithos_lib::error::PithosError; +use std::fs; +use std::io::{Seek, SeekFrom, Write}; + +#[test] +fn loaded_zip_converts_from_retained_open_archive_after_path_replacement() { + let temporary = tempfile::tempdir().unwrap(); + let source = temporary.path().join("crate.zip"); + let original_metadata = metadata(&["data.bin"]).replace("Test Crate", "Original Crate"); + write_raw_zip( + &source, + &[ + ( + b"ro-crate-metadata.json", + original_metadata.as_bytes(), + 0o100644, + ), + (b"data.bin", b"original", 0o100644), + ], + ); + + let loaded = read_ro_crate_zip(&source).unwrap(); + fs::rename(&source, temporary.path().join("original.zip")).unwrap(); + let replacement_metadata = + metadata(&["data.bin", "replacement-only.bin"]).replace("Test Crate", "Replacement Crate"); + write_raw_zip( + &source, + &[ + ( + b"ro-crate-metadata.json", + replacement_metadata.as_bytes(), + 0o100644, + ), + (b"data.bin", b"replaced", 0o100644), + (b"replacement-only.bin", b"replacement", 0o100644), + ], + ); + + let output = temporary.path().join("retained-zip.pith"); + write_loaded(&output, loaded, CdcConfig::default()); + let archive = open( + &output, + AccessKeys::new().with_key(private_key("recipient1")), + ); + let mut archived_metadata = Vec::new(); + archive + .copy_to("ro-crate-metadata.json", &mut archived_metadata) + .unwrap(); + let mut data = Vec::new(); + archive.copy_to("data.bin", &mut data).unwrap(); + + assert_eq!(archived_metadata, original_metadata.as_bytes()); + assert_eq!(data, b"original"); + assert!(archive.entry("replacement-only.bin").unwrap().is_none()); +} + +#[test] +fn retained_zip_conversion_failure_has_archive_member_index_and_source_context() { + const LOCAL_HEADER_LEN: u64 = 30; + + let temporary = tempfile::tempdir().unwrap(); + let archive_path = temporary.path().join("crate.zip"); + let crate_metadata = metadata(&["data.bin"]); + write_raw_zip( + &archive_path, + &[ + ( + RO_CRATE_METADATA_FILE.as_bytes(), + crate_metadata.as_bytes(), + 0o100644, + ), + (b"data.bin", b"original", 0o100644), + ], + ); + let loaded = read_ro_crate_zip(&archive_path).unwrap(); + + let data_offset = LOCAL_HEADER_LEN + + u64::try_from(RO_CRATE_METADATA_FILE.len()).unwrap() + + u64::try_from(crate_metadata.len()).unwrap() + + LOCAL_HEADER_LEN + + u64::try_from(b"data.bin".len()).unwrap(); + let mut archive = fs::OpenOptions::new() + .write(true) + .open(&archive_path) + .unwrap(); + archive.seek(SeekFrom::Start(data_offset)).unwrap(); + archive.write_all(b"X").unwrap(); + archive.flush().unwrap(); + drop(archive); + + let error = conversion_error(loaded); + let message = error.to_string(); + assert!(message.contains("convert retained ZIP member")); + assert!(message.contains(archive_path.to_str().unwrap())); + assert!(message.contains("member data.bin")); + assert!(message.contains("index 1")); + let writer_error = std::error::Error::source(&error) + .and_then(|source| source.downcast_ref::()) + .expect("retained-member context must expose WriterError as its source"); + assert!(std::error::Error::source(writer_error).is_some()); +} + +#[test] +fn zip_metadata_directory_and_symlink_failures_keep_member_context() { + let temporary = tempfile::tempdir().unwrap(); + let archive_path = temporary.path().join("crate.zip"); + let crate_metadata = metadata(&[]); + write_raw_zip( + &archive_path, + &[ + ( + RO_CRATE_METADATA_FILE.as_bytes(), + crate_metadata.as_bytes(), + 0o100644, + ), + (b"nested/", b"", 0o040755), + (b"link", b"missing", 0o120777), + ], + ); + + for (entry, operation, member, index) in [ + ( + RO_CRATE_METADATA_FILE, + "convert retained ZIP metadata", + RO_CRATE_METADATA_FILE, + 0, + ), + ("nested", "convert retained ZIP entry", "nested/", 1), + ("link", "convert retained ZIP entry", "link", 2), + ] { + let error = duplicate_conversion_error(read_ro_crate_zip(&archive_path).unwrap(), entry); + let message = error.to_string(); + assert!(message.contains(operation), "unexpected error: {message}"); + assert!( + message.contains(archive_path.to_str().unwrap()), + "unexpected error: {message}" + ); + assert!(message.contains(&format!("member {member}"))); + assert!(message.contains(&format!("index {index}"))); + assert!( + std::error::Error::source(&error) + .and_then(|source| source.downcast_ref::()) + .is_some() + ); + } + + let synthetic_archive = temporary.path().join("synthetic-parent.zip"); + let synthetic_metadata = metadata(&["nested/file"]); + write_raw_zip( + &synthetic_archive, + &[ + ( + RO_CRATE_METADATA_FILE.as_bytes(), + synthetic_metadata.as_bytes(), + 0o100644, + ), + (b"nested/file", b"content", 0o100644), + ], + ); + let error = + duplicate_conversion_error(read_ro_crate_zip(&synthetic_archive).unwrap(), "nested"); + let message = error.to_string(); + assert!(message.contains("convert synthetic ZIP parent")); + assert!(message.contains(synthetic_archive.to_str().unwrap())); + assert!(message.contains("nested")); + assert!( + std::error::Error::source(&error) + .and_then(|source| source.downcast_ref::()) + .is_some() + ); +} + +#[test] +fn typed_zip_conversion_streams_large_multiblock_member_and_round_trips() { + let temporary = tempfile::tempdir().unwrap(); + let zip = temporary.path().join("large.zip"); + let mut state = 1u32; + let payload = (0..(1024 * 1024 + 123)) + .map(|_| { + state = state.wrapping_mul(1_664_525).wrapping_add(1_013_904_223); + (state >> 24) as u8 + }) + .collect::>(); + let crate_metadata = metadata(&["large.bin"]); + write_raw_zip( + &zip, + &[ + ( + b"ro-crate-metadata.json", + crate_metadata.as_bytes(), + 0o100644, + ), + (b"large.bin", &payload, 0o100644), + ], + ); + + let output = temporary.path().join("large.pith"); + write_loaded( + &output, + read_ro_crate_zip(&zip).unwrap(), + CdcConfig::new(64, 256, 1024).unwrap(), + ); + let archive = open( + &output, + AccessKeys::new().with_key(private_key("recipient1")), + ); + assert_eq!( + archive.entry("large.bin").unwrap().unwrap().kind, + EntryKind::File { + size: payload.len() as u64, + available: true + } + ); + let mut copied = Vec::new(); + archive.copy_to("large.bin", &mut copied).unwrap(); + assert_eq!(copied, payload); +} + +#[test] +fn typed_zip_conversion_rejects_unsafe_paths_duplicates_conflicts_and_symlinks() { + let temporary = tempfile::tempdir().unwrap(); + let crate_metadata = metadata(&[]); + for (name, entries, expected) in [ + ( + "traversal.zip", + vec![(b"../escape".as_slice(), b"x".as_slice(), 0o100644)], + "unsafe", + ), + ( + "duplicate.zip", + vec![ + (b"duplicate".as_slice(), b"one".as_slice(), 0o100644), + (b"./duplicate".as_slice(), b"two".as_slice(), 0o100644), + ], + "unsafe", + ), + ( + "conflict.zip", + vec![ + (b"nested".as_slice(), b"file".as_slice(), 0o100644), + (b"nested/file".as_slice(), b"child".as_slice(), 0o100644), + ], + "conflict", + ), + ( + "absolute-link.zip", + vec![(b"link".as_slice(), b"/outside".as_slice(), 0o120777)], + "link", + ), + ( + "escaping-link.zip", + vec![(b"link".as_slice(), b"../outside".as_slice(), 0o120777)], + "link", + ), + ( + "invalid-link.zip", + vec![(b"link".as_slice(), b"target/".as_slice(), 0o120777)], + "link", + ), + ] { + let path = temporary.path().join(name); + let mut full_entries = vec![( + b"ro-crate-metadata.json".as_slice(), + crate_metadata.as_bytes(), + 0o100644, + )]; + full_entries.extend(entries); + write_raw_zip(&path, &full_entries); + let error = read_ro_crate_zip(&path).unwrap_err(); + match expected { + "unsafe" => assert!(matches!(error, RoCrateError::UnsafeZipPath { .. })), + "conflict" => assert!(matches!(error, RoCrateError::ZipPathConflict { .. })), + "link" => { + let message = error.to_string(); + assert!(message.contains(path.to_str().unwrap()), "{message}"); + assert!(message.contains("member link"), "{message}"); + assert!(message.contains("index 1"), "{message}"); + assert!( + std::error::Error::source(&error) + .and_then(|source| source.downcast_ref::()) + .is_some() + ); + } + _ => unreachable!(), + } + } +} + +#[test] +fn zip_loader_rejects_member_names_that_require_normalization() { + let temporary = tempfile::tempdir().unwrap(); + let crate_metadata = metadata(&[]); + let parent = temporary.path().join("parent-component.zip"); + write_raw_zip( + &parent, + &[( + b"nested/../ro-crate-metadata.json", + crate_metadata.as_bytes(), + 0o100644, + )], + ); + assert!(matches!( + read_ro_crate_zip(&parent), + Err(RoCrateError::UnsafeZipPath { .. }) + )); + let current = temporary.path().join("current-component.zip"); + write_raw_zip( + ¤t, + &[ + ( + b"ro-crate-metadata.json", + crate_metadata.as_bytes(), + 0o100644, + ), + (b"./payload", b"content", 0o100644), + ], + ); + assert!(matches!( + read_ro_crate_zip(¤t), + Err(RoCrateError::UnsafeZipPath { .. }) + )); + for (index, name) in [ + b"nested//payload".as_slice(), + b"nested\\payload", + b"C:payload", + ] + .into_iter() + .enumerate() + { + let path = temporary.path().join(format!("invalid-{index}.zip")); + write_raw_zip( + &path, + &[ + ( + b"ro-crate-metadata.json", + crate_metadata.as_bytes(), + 0o100644, + ), + (name, b"content", 0o100644), + ], + ); + assert!(matches!( + read_ro_crate_zip(&path), + Err(RoCrateError::UnsafeZipPath { .. }) + )); + } +} + +#[test] +fn typed_zip_conversion_rejects_raw_invalid_utf8_member_names_and_malformed_metadata() { + let temporary = tempfile::tempdir().unwrap(); + let invalid_name = temporary.path().join("invalid-name.zip"); + write_raw_zip( + &invalid_name, + &[ + ( + b"ro-crate-metadata.json", + metadata(&[]).as_bytes(), + 0o100644, + ), + (b"invalid-\xff", b"content", 0o100644), + ], + ); + assert!(matches!( + read_ro_crate_zip(&invalid_name), + Err(RoCrateError::InvalidZipEntryName { index: 1, .. }) + )); + let malformed = temporary.path().join("malformed.zip"); + write_raw_zip( + &malformed, + &[(b"ro-crate-metadata.json", b"not JSON", 0o100644)], + ); + let error = read_ro_crate_zip(&malformed).unwrap_err(); + let message = error.to_string(); + assert!(message.contains(malformed.to_str().unwrap()), "{message}"); + assert!( + message.contains("member ro-crate-metadata.json"), + "{message}" + ); + assert!(message.contains("index 0"), "{message}"); + assert!(matches!(error, RoCrateError::ZipMemberParser { .. })); +} + +// Upstream ro-crate-rs parser and graph assertions. +#[test] +fn parser_preserves_a_synthetic_zip_fixture_graph_source_and_permissions() { + let temporary = tempfile::tempdir().unwrap(); + let path = temporary.path().join("fixture.zip"); + let crate_metadata = metadata(&["data.bin", "notes.txt"]); + write_raw_zip( + &path, + &[ + ( + RO_CRATE_METADATA_FILE.as_bytes(), + crate_metadata.as_bytes(), + 0o100644, + ), + (b"data.bin", b"repository-owned binary fixture", 0o100644), + (b"notes.txt", b"repository-owned text fixture", 0o100600), + ], + ); + + assert_eq!( + zip::ZipArchive::new(fs::File::open(&path).unwrap()) + .unwrap() + .len(), + 3 + ); + let loaded = read_ro_crate_zip(&path).unwrap(); + assert_eq!(loaded.source_kind(), RoCrateSource::Zip); + assert!(!loaded.ro_crate().graph.is_empty()); + + let output = temporary.path().join("fixture.pith"); + write_loaded(&output, loaded, CdcConfig::default()); + let archive = open( + &output, + AccessKeys::new().with_key(private_key("recipient1")), + ); + assert_eq!( + archive.entry("data.bin").unwrap().unwrap().permissions, + 0o644 + ); + assert_eq!( + archive.entry("notes.txt").unwrap().unwrap().permissions, + 0o600 + ); +} + +#[test] +fn zip_parser_rejects_invalid_archives_and_non_root_metadata() { + let temporary = tempfile::tempdir().unwrap(); + let invalid = temporary.path().join("invalid.zip"); + fs::write(&invalid, b"not a ZIP archive").unwrap(); + let error = read_ro_crate_zip(&invalid).unwrap_err(); + assert!( + std::any::type_name_of_val(&error).ends_with("RoCrateError"), + "RO-Crate operations must return their adapter error directly" + ); + assert!(std::error::Error::source(&error).is_some()); + assert!(error.to_string().contains(invalid.to_str().unwrap())); + assert!(error.to_string().contains("open ZIP archive")); + let missing = temporary.path().join("missing.zip"); + write_raw_zip(&missing, &[(b"data.txt", b"data", 0o100644)]); + assert!(matches!( + read_ro_crate_zip(&missing), + Err(RoCrateError::MissingMetadata { .. }) + )); + let wrapped = temporary.path().join("wrapped.zip"); + write_raw_zip( + &wrapped, + &[( + b"wrapper/ro-crate-metadata.json", + metadata(&[]).as_bytes(), + 0o100644, + )], + ); + assert!(matches!( + read_ro_crate_zip(&wrapped), + Err(RoCrateError::MissingMetadata { .. }) + )); +} diff --git a/crates/pithos_lib/tests/ui/crypt4gh_wire_record_is_private.rs b/crates/pithos_lib/tests/ui/crypt4gh_wire_record_is_private.rs new file mode 100644 index 0000000..74e08b8 --- /dev/null +++ b/crates/pithos_lib/tests/ui/crypt4gh_wire_record_is_private.rs @@ -0,0 +1,5 @@ +use pithos_lib::adapters::crypt4gh::Crypt4GHHeader; + +fn main() { + let _ = std::any::TypeId::of::(); +} diff --git a/crates/pithos_lib/tests/ui/crypt4gh_wire_record_is_private.stderr b/crates/pithos_lib/tests/ui/crypt4gh_wire_record_is_private.stderr new file mode 100644 index 0000000..1328285 --- /dev/null +++ b/crates/pithos_lib/tests/ui/crypt4gh_wire_record_is_private.stderr @@ -0,0 +1,11 @@ +error[E0603]: struct `Crypt4GHHeader` is private + --> tests/ui/crypt4gh_wire_record_is_private.rs:1:37 + | +1 | use pithos_lib::adapters::crypt4gh::Crypt4GHHeader; + | ^^^^^^^^^^^^^^ private struct + | +note: the struct `Crypt4GHHeader` is defined here + --> src/adapters/crypt4gh.rs + | + | pub(crate) struct Crypt4GHHeader { + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/crates/pithos_lib/tests/ui/incomplete_writer_cannot_resume.rs b/crates/pithos_lib/tests/ui/incomplete_writer_cannot_resume.rs new file mode 100644 index 0000000..f44dfd3 --- /dev/null +++ b/crates/pithos_lib/tests/ui/incomplete_writer_cannot_resume.rs @@ -0,0 +1,11 @@ +use pithos_lib::archive::{ArchiveWriter, WriteOptions}; +use pithos_lib::crypto::PrivateKey; + +fn main() { + let sender = PrivateKey::generate(); + let recipient = sender.public_key(); + let writer = ArchiveWriter::create(Vec::new(), WriteOptions::new(sender, vec![recipient])) + .unwrap(); + let incomplete = writer.into_incomplete().unwrap_err(); + let _ = incomplete.into_writer(); +} diff --git a/crates/pithos_lib/tests/ui/incomplete_writer_cannot_resume.stderr b/crates/pithos_lib/tests/ui/incomplete_writer_cannot_resume.stderr new file mode 100644 index 0000000..31328f3 --- /dev/null +++ b/crates/pithos_lib/tests/ui/incomplete_writer_cannot_resume.stderr @@ -0,0 +1,5 @@ +error[E0599]: no method named `into_writer` found for struct `Box>>` in the current scope + --> tests/ui/incomplete_writer_cannot_resume.rs:10:24 + | +10 | let _ = incomplete.into_writer(); + | ^^^^^^^^^^^ method not found in `Box>>` diff --git a/crates/pithos_lib/tests/ui/non_copy_static_secret.rs b/crates/pithos_lib/tests/ui/non_copy_static_secret.rs new file mode 100644 index 0000000..336da50 --- /dev/null +++ b/crates/pithos_lib/tests/ui/non_copy_static_secret.rs @@ -0,0 +1,7 @@ +use pithos_lib::crypto::generate_private_key; + +fn main() { + let secret = generate_private_key(); + let _moved = secret; + let _used_after_move = secret; +} diff --git a/crates/pithos_lib/tests/ui/non_copy_static_secret.stderr b/crates/pithos_lib/tests/ui/non_copy_static_secret.stderr new file mode 100644 index 0000000..3954097 --- /dev/null +++ b/crates/pithos_lib/tests/ui/non_copy_static_secret.stderr @@ -0,0 +1,9 @@ +error[E0382]: use of moved value: `secret` + --> tests/ui/non_copy_static_secret.rs:6:28 + | +4 | let secret = generate_private_key(); + | ------ move occurs because `secret` has type `PrivateKey`, which does not implement the `Copy` trait +5 | let _moved = secret; + | ------ value moved here +6 | let _used_after_move = secret; + | ^^^^^^ value used here after move diff --git a/crates/pithos_lib/tests/ui/writer_finish_consumes_writer.rs b/crates/pithos_lib/tests/ui/writer_finish_consumes_writer.rs new file mode 100644 index 0000000..b2f82a7 --- /dev/null +++ b/crates/pithos_lib/tests/ui/writer_finish_consumes_writer.rs @@ -0,0 +1,11 @@ +use pithos_lib::archive::{ArchiveWriter, WriteOptions}; +use pithos_lib::crypto::PrivateKey; + +fn main() { + let sender = PrivateKey::generate(); + let recipient = sender.public_key(); + let writer = ArchiveWriter::create(Vec::new(), WriteOptions::new(sender, vec![recipient])).unwrap(); + let first = writer.finish(); + let second = writer.finish(); + let _ = (first, second); +} diff --git a/crates/pithos_lib/tests/ui/writer_finish_consumes_writer.stderr b/crates/pithos_lib/tests/ui/writer_finish_consumes_writer.stderr new file mode 100644 index 0000000..9b613a6 --- /dev/null +++ b/crates/pithos_lib/tests/ui/writer_finish_consumes_writer.stderr @@ -0,0 +1,15 @@ +error[E0382]: use of moved value: `writer` + --> tests/ui/writer_finish_consumes_writer.rs:9:18 + | +7 | let writer = ArchiveWriter::create(Vec::new(), WriteOptions::new(sender, vec![recipient])).unwrap(); + | ------ move occurs because `writer` has type `ArchiveWriter>`, which does not implement the `Copy` trait +8 | let first = writer.finish(); + | -------- `writer` moved due to this method call +9 | let second = writer.finish(); + | ^^^^^^ value used here after move + | +note: `ArchiveWriter::::finish` takes ownership of the receiver `self`, which moves `writer` + --> src/archive/writer.rs + | + | pub fn finish(mut self) -> Result> { + | ^^^^ diff --git a/crates/pithos_lib/tests/ui/writer_use_after_finish.rs b/crates/pithos_lib/tests/ui/writer_use_after_finish.rs new file mode 100644 index 0000000..e676377 --- /dev/null +++ b/crates/pithos_lib/tests/ui/writer_use_after_finish.rs @@ -0,0 +1,10 @@ +use pithos_lib::archive::{ArchivePath, ArchiveWriter, EntryMetadata, WriteOptions}; +use pithos_lib::crypto::PrivateKey; + +fn main() { + let sender = PrivateKey::generate(); + let recipient = sender.public_key(); + let mut writer = ArchiveWriter::create(Vec::new(), WriteOptions::new(sender, vec![recipient])).unwrap(); + let _sink = writer.finish().unwrap(); + let _ = writer.add_directory(ArchivePath::new("after").unwrap(), EntryMetadata::new(0, 0, 0)); +} diff --git a/crates/pithos_lib/tests/ui/writer_use_after_finish.stderr b/crates/pithos_lib/tests/ui/writer_use_after_finish.stderr new file mode 100644 index 0000000..4eb5613 --- /dev/null +++ b/crates/pithos_lib/tests/ui/writer_use_after_finish.stderr @@ -0,0 +1,15 @@ +error[E0382]: borrow of moved value: `writer` + --> tests/ui/writer_use_after_finish.rs:9:13 + | +7 | let mut writer = ArchiveWriter::create(Vec::new(), WriteOptions::new(sender, vec![recipient])).unwrap(); + | ---------- move occurs because `writer` has type `ArchiveWriter>`, which does not implement the `Copy` trait +8 | let _sink = writer.finish().unwrap(); + | -------- `writer` moved due to this method call +9 | let _ = writer.add_directory(ArchivePath::new("after").unwrap(), EntryMetadata::new(0, 0, 0)); + | ^^^^^^ value borrowed here after move + | +note: `ArchiveWriter::::finish` takes ownership of the receiver `self`, which moves `writer` + --> src/archive/writer.rs + | + | pub fn finish(mut self) -> Result> { + | ^^^^ From 584a85d1d317f417d1a7803cd603f92fc2173351 Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Tue, 11 Aug 2026 23:01:28 +0200 Subject: [PATCH 010/166] refactor: migrate CLI to archive APIs --- crates/pithos/src/io/utils.rs | 136 +++- crates/pithos/src/main.rs | 941 +++++++++++++++++++------- crates/pithos/src/utils/conversion.rs | 36 +- crates/pithos/tests/append.rs | 260 +++++++ crates/pithos/tests/behavior.rs | 237 +++++++ crates/pithos/tests/create.rs | 326 +++++++++ crates/pithos/tests/errors.rs | 272 ++++++++ crates/pithos/tests/keypair.rs | 119 ++++ crates/pithos/tests/output_safety.rs | 211 ++++++ 9 files changed, 2276 insertions(+), 262 deletions(-) create mode 100644 crates/pithos/tests/append.rs create mode 100644 crates/pithos/tests/behavior.rs create mode 100644 crates/pithos/tests/create.rs create mode 100644 crates/pithos/tests/errors.rs create mode 100644 crates/pithos/tests/keypair.rs create mode 100644 crates/pithos/tests/output_safety.rs diff --git a/crates/pithos/src/io/utils.rs b/crates/pithos/src/io/utils.rs index 55d973f..1ce0daf 100644 --- a/crates/pithos/src/io/utils.rs +++ b/crates/pithos/src/io/utils.rs @@ -1,25 +1,127 @@ -use pithos_lib::error::PithosError; +use crate::PithosCliError; +use pithos_lib::crypto::{PrivateKey, PublicKey}; +use rustix::fs::{Mode, OFlags, open}; use std::fs::File; use std::io::Read; -use std::path::PathBuf; -use x25519_dalek::{PublicKey, StaticSecret}; +use std::path::Path; +use zeroize::Zeroizing; -pub fn load_private_key_from_pem(filepath: &PathBuf) -> Result { - // Open file handle and read file bytes - let mut file = File::open(filepath)?; - let mut file_content = vec![0u8; file.metadata()?.len() as usize]; - file.read_exact(&mut file_content)?; +/// PEM key files are small; this bounds memory use and rejects appended data. +const MAX_PEM_FILE_BYTES: usize = 1024 * 1024; - Ok(pithos_lib::helpers::x25519_keys::private_key_from_pem_bytes(&file_content)?) +pub fn load_private_key_from_pem(filepath: &Path) -> Result { + let mut file_content = Zeroizing::new(Vec::new()); + read_bounded_pem_file( + filepath, + "load private key", + "inspect private key", + "read private key", + "private", + &mut file_content, + )?; + + pithos_lib::crypto::parse_private_pem(&file_content).map_err(|source| { + PithosCliError::KeyParse { + operation: "parse private key", + path: filepath.to_path_buf(), + source, + } + }) +} + +pub fn load_public_key_from_pem(filepath: &Path) -> Result { + let mut file_content = Vec::new(); + read_bounded_pem_file( + filepath, + "load public key", + "inspect public key", + "read public key", + "public", + &mut file_content, + )?; + + pithos_lib::crypto::parse_public_pem(&file_content).map_err(|source| PithosCliError::KeyParse { + operation: "parse public key", + path: filepath.to_path_buf(), + source, + }) } -pub fn load_public_key_from_pem(filepath: &PathBuf) -> Result { - // Open file handle and read file bytes - let mut file = File::open(filepath)?; - let mut file_content = vec![0u8; file.metadata()?.len() as usize]; - file.read_exact(&mut file_content)?; +fn read_bounded_pem_file( + filepath: &Path, + load_operation: &'static str, + inspect_operation: &'static str, + read_operation: &'static str, + key_kind: &'static str, + file_content: &mut Vec, +) -> Result<(), PithosCliError> { + // Keep metadata and reads bound to the same no-follow descriptor. + let mut file = File::from( + open( + filepath, + OFlags::RDONLY | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::empty(), + ) + .map_err(|source| PithosCliError::KeyFile { + operation: load_operation, + path: filepath.to_path_buf(), + source: source.into(), + })?, + ); + + let declared_len = file + .metadata() + .map_err(|source| PithosCliError::KeyFile { + operation: inspect_operation, + path: filepath.to_path_buf(), + source, + })? + .len(); + if declared_len > MAX_PEM_FILE_BYTES as u64 { + return Err(PithosCliError::KeyFile { + operation: read_operation, + path: filepath.to_path_buf(), + source: std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("{key_kind} key file exceeds {MAX_PEM_FILE_BYTES}-byte limit"), + ), + }); + } + + let content_len = usize::try_from(declared_len).map_err(|_| PithosCliError::KeyFile { + operation: read_operation, + path: filepath.to_path_buf(), + source: std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("{key_kind} key file exceeds {MAX_PEM_FILE_BYTES}-byte limit"), + ), + })?; + file_content.resize(content_len, 0); + file.read_exact(file_content) + .map_err(|source| PithosCliError::KeyFile { + operation: read_operation, + path: filepath.to_path_buf(), + source, + })?; + let mut trailing_byte = [0u8; 1]; + if file + .read(&mut trailing_byte) + .map_err(|source| PithosCliError::KeyFile { + operation: read_operation, + path: filepath.to_path_buf(), + source, + })? + != 0 + { + return Err(PithosCliError::KeyFile { + operation: read_operation, + path: filepath.to_path_buf(), + source: std::io::Error::new( + std::io::ErrorKind::InvalidData, + format!("{key_kind} key file has trailing content"), + ), + }); + } - Ok(pithos_lib::helpers::x25519_keys::public_key_from_pem_bytes( - &file_content, - )?) + Ok(()) } diff --git a/crates/pithos/src/main.rs b/crates/pithos/src/main.rs index 2f509e8..8b931f2 100644 --- a/crates/pithos/src/main.rs +++ b/crates/pithos/src/main.rs @@ -3,21 +3,27 @@ pub mod utils; use crate::io::utils::{load_private_key_from_pem, load_public_key_from_pem}; use clap::{Parser, Subcommand, ValueEnum}; -use pithos_lib::error::PithosError; -use pithos_lib::helpers::x25519_keys::{ - generate_private_key, private_key_to_pem_bytes, public_key_to_pem_bytes, +use pithos_lib::adapters::crypt4gh; +use pithos_lib::adapters::crypt4gh::Crypt4GHError; +use pithos_lib::archive::{ + AccessKeys, AppendDurability, AppendOptions, Archive, ArchiveWriter, CdcConfig, EntryKind, + OpenOptions, ProcessingOptions, WriteOptions, WriterError, }; -use pithos_lib::io::pithosreader::PithosReaderSimple; -use pithos_lib::io::pithoswriter::{InputFile, PithosWriter}; +use pithos_lib::crypto::{PrivateKey, PublicKey, generate_private_key}; +use pithos_lib::error::PithosError; +use pithos_lib::fs::ingest::build_input_manifest; +use pithos_lib::fs::{FsError, append_files, extract, grant_readers}; +use pithos_lib::source::FileSource; +use rustix::fs::{AtFlags, Mode, OFlags, fchmod, linkat, open, openat, unlinkat}; +use std::ffi::OsString; use std::fs::File; use std::io::Write; use std::ops::Range; -use std::path::PathBuf; +use std::path::{Component, Path, PathBuf}; use thiserror::Error; -use tracing::dispatcher::SetGlobalDefaultError; use tracing_subscriber::prelude::*; +use tracing_subscriber::util::TryInitError; use utils::conversion::{evaluate_log_level, parse_cdc_input, parse_range_input}; -use x25519_dalek::PublicKey; #[derive(Clone, Default, ValueEnum)] enum KeyFormat { @@ -35,6 +41,22 @@ enum ExportFormat { //RoCrate, } +#[derive(Clone, Copy, Default, ValueEnum)] +enum CliAppendDurability { + #[default] + Flush, + SyncAll, +} + +impl From for AppendDurability { + fn from(value: CliAppendDurability) -> Self { + match value { + CliAppendDurability::Flush => Self::Flush, + CliAppendDurability::SyncAll => Self::SyncAll, + } + } +} + #[derive(Parser)] #[command(author, version, about, long_about = None)] struct Cli { @@ -90,7 +112,7 @@ enum PithosCommands { */ /// Set values for content-defined chunking #[arg(long="cdc", value_parser=parse_cdc_input, value_name = "MIN,AVG,MAX")] - cdc: Option<(usize, usize, usize)>, + cdc: Option, /// Input files #[arg(value_name = "FILES")] files: Vec, @@ -113,7 +135,7 @@ enum PithosCommands { #[arg(short, long)] format: Option, /// Key file prefix (e.g. sender -> sender..pem) - #[arg(short, long)] + #[arg(short = 'k', long)] prefix: Option, }, /// Export a Pithos file into another compatible file format @@ -195,6 +217,9 @@ enum AppendCommands { // List of file ids the readers shall get access to #[arg(short, long)] ids: Option>, + /// Durability after publishing the appended directory + #[arg(long, value_enum, default_value_t)] + durability: CliAppendDurability, /// Path to Pithos file #[arg(value_name = "FILE")] file: PathBuf, @@ -204,6 +229,12 @@ enum AppendCommands { /// Path to Pithos file #[arg(short, long, value_name = "PITHOS FILE")] file: PathBuf, + /// Set values for content-defined chunking + #[arg(long = "cdc", value_parser = parse_cdc_input, value_name = "MIN,AVG,MAX")] + cdc: Option, + /// Durability after publishing the appended directory + #[arg(long, value_enum, default_value_t)] + durability: CliAppendDurability, /// Input files #[arg(value_name = "FILES")] files: Vec, @@ -212,21 +243,48 @@ enum AppendCommands { #[derive(Error, Debug)] pub enum PithosCliError { - #[error("Writer Error: {0}")] - TracingError(#[from] SetGlobalDefaultError), + #[error("tracing initialization error: {0}")] + TracingError(#[from] TryInitError), #[error("Invalid argument: {0}")] InvalidArgumentError(String), #[error("Pithos Error: {0}")] PithosError(#[from] PithosError), + #[error("Filesystem Error: {0}")] + FilesystemError(#[from] FsError), + #[error("Writer Error: {0}")] + WriterError(#[from] WriterError), + #[error("Crypt4GH error: {0}")] + Crypt4GHError(#[from] Crypt4GHError), + #[error("{operation} failed for {path}: {source}")] + KeyFile { + operation: &'static str, + path: PathBuf, + #[source] + source: std::io::Error, + }, + #[error("{operation} failed for {path}: {source}")] + KeyParse { + operation: &'static str, + path: PathBuf, + #[source] + source: pithos_lib::crypto::CryptoError, + }, + #[error("{operation} failed for {path}: {source}")] + OutputFile { + operation: &'static str, + path: PathBuf, + #[source] + source: std::io::Error, + }, } #[tracing::instrument(level = "trace", skip())] -fn main() -> Result<(), PithosCliError> { +fn run() -> Result<(), PithosCliError> { // Parse CLI parameter input let cli = Cli::parse(); // Initialize tracing logger - let logging_filter = evaluate_log_level(cli.log_level); + let logging_filter = evaluate_log_level(cli.log_level)?; let fmt_layer = tracing_subscriber::fmt::layer() .compact() .with_file(cli.verbose) @@ -234,51 +292,37 @@ fn main() -> Result<(), PithosCliError> { .with_target(cli.verbose) .with_filter(logging_filter); - tracing_subscriber::registry().with(fmt_layer).init(); + tracing_subscriber::registry().with(fmt_layer).try_init()?; // Evaluate subcommand match cli.command { PithosCommands::Read { read_command } => match read_command { ReadCommands::Info { file, path } => { - let path_str = path - .to_str() - .expect("No inner path provided or path contains invalid UTF-8"); - let key = load_private_key_from_pem( - &cli.secret_key - .clone() - .expect("Private key expected to read directory"), - )?; - let mut reader = PithosReaderSimple::new_with_key(&file, key)?; - let (directory, _) = reader.read_directory()?; - match directory.get_file_by_path(path_str) { - Some(entry) => println!("{entry}"), + let path_str = path.to_str().ok_or_else(|| { + PithosCliError::InvalidArgumentError("archive path is not valid UTF-8".into()) + })?; + let archive = open_archive(&file, required_private_key(&cli.secret_key)?)?; + match archive.entry(path_str)? { + Some(entry) => write_stdout(format_args!("{entry:#?}"))?, None => Err(PithosError::FileNotFound(path_str.to_string()))?, } } ReadCommands::List { file } => { - let key = load_private_key_from_pem( - &cli.secret_key - .clone() - .expect("Private key expected to read directory"), - )?; - let mut reader = PithosReaderSimple::new_with_key(&file, key)?; - let (directory, _) = reader.read_directory()?; - directory - .files - .iter() - .for_each(|(id, path, fe)| println!("{} {:?} {}", id, fe.file_type, path)) + let archive = open_archive(&file, required_private_key(&cli.secret_key)?)?; + for entry in archive.entries() { + write_stdout(format_args!("{} {:?} {}", entry.id, entry.kind, entry.path))?; + } } ReadCommands::All { file } => { - let key = load_private_key_from_pem( - &cli.secret_key - .clone() - .expect("Private key expected to read directory"), - )?; - let mut reader = PithosReaderSimple::new_with_key(&file, key)?; - let (directory, _) = reader.read_directory()?; - - for path in directory.files.get_paths_ref() { - reader.read_file(path, &directory, cli.output.as_ref(), None)?; + let archive = open_archive(&file, required_private_key(&cli.secret_key)?)?; + let output = cli + .output + .as_deref() + .unwrap_or_else(|| std::path::Path::new(".")); + let mut entries = archive.entries().collect::>(); + entries.sort_by_key(|entry| !matches!(entry.kind, EntryKind::Directory)); + for entry in entries { + extract(&archive, &entry.path, output)?; } } ReadCommands::Data { @@ -286,34 +330,76 @@ fn main() -> Result<(), PithosCliError> { paths, ranges, } => { - let key = load_private_key_from_pem( - &cli.secret_key - .clone() - .expect("Private key expected to read directory"), - )?; - - let mut reader = PithosReaderSimple::new_with_key(&file, key)?; - let (directory, _) = reader.read_directory()?; - - for path in paths { - reader.read_file( - path.to_str() - .expect("No inner path provided or path contains invalid UTF-8"), - &directory, - cli.output.as_ref(), - ranges.clone(), - )?; + if paths.is_empty() { + return Err(PithosCliError::InvalidArgumentError( + "at least one archive path is required".into(), + )); + } + if cli.output.is_some() && paths.len() > 1 { + return Err(PithosCliError::InvalidArgumentError( + "a file output accepts exactly one archive path".into(), + )); + } + let archive = open_archive(&file, required_private_key(&cli.secret_key)?)?; + if let Some(output_path) = cli.output.as_ref() { + if let Some(path) = paths.first() { + let path = path.to_str().ok_or_else(|| { + PithosCliError::InvalidArgumentError( + "archive path is not valid UTF-8".into(), + ) + })?; + let mut output = StagedOutput::create( + output_path, + Mode::RUSR + | Mode::WUSR + | Mode::RGRP + | Mode::WGRP + | Mode::ROTH + | Mode::WOTH, + "open archive output", + )?; + let result = (|| { + if let Some(ranges) = &ranges { + for range in ranges { + archive.copy_range_to( + path, + range.clone(), + output.file_mut(), + )?; + } + } else { + archive.copy_to(path, output.file_mut())?; + } + output.flush("flush archive output") + })(); + result?; + output.publish("publish archive output")?; + } + } else { + for path in paths { + let path = path.to_str().ok_or_else(|| { + PithosCliError::InvalidArgumentError( + "archive path is not valid UTF-8".into(), + ) + })?; + wait_before_stdout_for_test(); + let mut output = std::io::stdout(); + if let Some(ranges) = &ranges { + for range in ranges { + archive.copy_range_to(path, range.clone(), &mut output)?; + } + } else { + archive.copy_to(path, &mut output)?; + } + output.flush().map_err(PithosError::Io)?; + } } } ReadCommands::Directory { file } => { - let key = load_private_key_from_pem( - &cli.secret_key - .clone() - .expect("Private key expected to read directory"), - )?; - let mut reader = PithosReaderSimple::new_with_key(&file, key)?; - - println!("{:#?}", reader.read_directory()?); + let archive = open_archive(&file, required_private_key(&cli.secret_key)?)?; + for entry in archive.entries() { + write_stdout(format_args!("{entry:#?}"))?; + } } }, PithosCommands::Create { cdc, files } => { @@ -323,190 +409,575 @@ fn main() -> Result<(), PithosCliError> { )); } - let output = if let Some(outfile) = cli.output { - File::create(outfile).map_err(PithosError::Io)? - } else { - // Default outfile - println!("No outfile specified, writing output to \"/tmp/out.pithos\""); //TODO: Replace with tracing::warn!() - File::create("/tmp/out.pithos").map_err(PithosError::Io)? - }; - - let sender_key = load_private_key_from_pem( - &cli.secret_key - .clone() - .expect("Private key expected to create Pithos file"), - )?; - let reader_keys: Result, PithosError> = cli + let sender_key = required_private_key(&cli.secret_key)?; + let reader_keys: Result, PithosCliError> = cli .public_keys - .expect("At least one recipient expected") + .ok_or_else(|| { + PithosCliError::InvalidArgumentError( + "at least one recipient key is required".into(), + ) + })? .iter() - .map(load_public_key_from_pem) + .map(|path| load_public_key_from_pem(path)) .collect(); - - let input_files: Result, PithosError> = - files.iter().map(InputFile::try_from).collect(); - let mut writer = PithosWriter::new(sender_key, reader_keys?, cdc, Box::new(output))?; + let options = + WriteOptions::new(sender_key, reader_keys?).with_cdc(cdc.unwrap_or_default()); + options.validate()?; + let manifest = build_input_manifest(&files)?; + + let output_path = match cli.output { + Some(path) => path, + None => { + write_stdout(format_args!( + "No outfile specified, writing output to \"/tmp/out.pithos\"" + ))?; + PathBuf::from("/tmp/out.pithos") + } + }; + let mut output = StagedOutput::create( + &output_path, + Mode::RUSR | Mode::WUSR | Mode::RGRP | Mode::WGRP | Mode::ROTH | Mode::WOTH, + "open archive output", + )?; + let mut writer = match ArchiveWriter::create(output.take_file(), options) { + Ok(writer) => writer, + Err(error) => { + let (error, sink) = error.into_parts(); + drop(sink); + return Err(PithosCliError::PithosError(error)); + } + }; tracing::info!("Start creating Pithos file"); - writer - .write_file_header() - .map_err(PithosError::Serialization)?; - writer.process_input_files(input_files?)?; - writer.write_directory()?; + if let Err(error) = fail_after_create_for_test() { + discard_incomplete_writer(writer); + return Err(error.into()); + } + if let Err(error) = manifest.ingest(&mut writer, ProcessingOptions::default()) { + discard_incomplete_writer(writer); + return Err(error.into()); + } + match writer.finish() { + Ok(sink) => { + output.restore_file(sink); + output.publish("publish archive output")?; + } + Err(error) => { + let (error, sink) = error.into_parts(); + drop(sink); + return Err(PithosCliError::PithosError(error)); + } + } } PithosCommands::Keypair { prefix, .. } => { - let private_key = generate_private_key().map_err(PithosError::Crypt)?; - let public_key = PublicKey::from(&private_key); - let prefix = prefix.unwrap_or("pithos_key".to_string()); - - // Write output - let (mut sec_output, mut pub_output): (Box, Box) = - if let Some(dest) = cli.output { - let (sec_target, pub_target) = if dest.is_dir() { - ( - &dest.join(format!("{prefix}.sec.pem")), - &dest.join(format!("{prefix}.pub.pem")), - ) - } else { - (&dest, &dest) - }; - ( - Box::new(File::create(sec_target).map_err(PithosError::Io)?), - Box::new(File::create(pub_target).map_err(PithosError::Io)?), - ) + let prefix = validate_keypair_prefix(prefix)?; + let private_key = generate_private_key(); + let public_key = private_key.public_key(); + let private_pem = private_key + .to_private_pem_bytes() + .map_err(PithosError::Crypt)?; + let public_pem = public_key + .to_public_pem_bytes() + .map_err(PithosError::Crypt)?; + + if let Some(output) = cli.output { + if let Some(directory) = open_keypair_output_directory(&output)? { + write_keypair_files(&output, directory, &prefix, &private_pem, &public_pem)?; } else { - (Box::new(std::io::stdout()), Box::new(std::io::stdout())) - }; - - sec_output - .write_all(&private_key_to_pem_bytes(&private_key).map_err(PithosError::Crypt)?) - .map_err(PithosError::Io)?; - pub_output - .write_all(&public_key_to_pem_bytes(&public_key).map_err(PithosError::Crypt)?) - .map_err(PithosError::Io)?; + write_combined_keypair_file(&output, &private_pem, &public_pem)?; + } + } else { + wait_before_stdout_for_test(); + let mut output = std::io::stdout(); + output.write_all(&private_pem).map_err(PithosError::Io)?; + output.write_all(&public_pem).map_err(PithosError::Io)?; + output.flush().map_err(PithosError::Io)?; + } } - PithosCommands::Append { command } => { - match command { - AppendCommands::Readers { file, ids } => { - let sender_key = load_private_key_from_pem( - &cli.secret_key - .clone() - .expect("Private key expected to append to Pithos file"), - )?; - let reader_keys_result: Result, PithosError> = cli - .public_keys - .expect("At least one recipient expected") - .iter() - .map(load_public_key_from_pem) - .collect(); - let reader_keys = reader_keys_result?; - - let mut reader = PithosReaderSimple::new_with_key(&file, sender_key.clone())?; - let (directory, _) = reader.read_directory()?; - - let mut fes = vec![]; - if let Some(ids) = ids { - for id in ids { - fes.push(( - id, - directory - .get_file_by_id(id) - .ok_or(PithosError::FileNotFound(format!( - "File with id {id} not available." - )))?, - )); - } - } - if fes.is_empty() { - return Err(PithosCliError::InvalidArgumentError( - "No available id or path provided".to_string(), - )); - } - - let mut writer = PithosWriter::new_from_file( - sender_key.clone(), - reader_keys.clone(), - None, - &file, - )?; - - let directory = writer.get_directory_mut(); - for (id, _) in fes { - let enc_key = - directory - .get_file_encryption_key(id) - .ok_or(PithosError::Other(format!( - "Could not extract encryption key for file {id}" - )))?; - - for reader in &reader_keys { - directory.add_file_to_recipient(&sender_key, reader, (id, enc_key))?; - } - } - - writer.write_directory()? + PithosCommands::Append { command } => match command { + AppendCommands::Readers { + file, + ids, + durability, + } => { + let ids = ids.unwrap_or_default(); + if ids.is_empty() { + return Err(PithosCliError::InvalidArgumentError( + "No available id or path provided".to_string(), + )); } - AppendCommands::Files { file, files } => { - let sender_key = load_private_key_from_pem( - &cli.secret_key - .clone() - .expect("Private key expected to append to Pithos file"), - )?; - let reader_keys: Result, PithosError> = cli - .public_keys - .expect("At least one recipient expected") - .iter() - .map(load_public_key_from_pem) - .collect(); - - let append_files: Result, PithosError> = - files.iter().map(InputFile::try_from).collect(); - - let mut writer = - PithosWriter::new_from_file(sender_key, reader_keys?, None, &file)?; - - // Append files - writer.process_input_files(append_files?)?; - writer.write_directory()?; + let sender_key = required_private_key(&cli.secret_key)?; + let reader_keys = required_recipient_keys(&cli.public_keys)?; + grant_readers( + &file, + AppendOptions::new(sender_key, reader_keys).with_durability(durability.into()), + &ids, + )?; + } + AppendCommands::Files { + file, + cdc, + durability, + files, + } => { + if files.is_empty() { + return Err(PithosCliError::InvalidArgumentError( + "No files provided".to_string(), + )); } + let sender_key = required_private_key(&cli.secret_key)?; + let reader_keys = required_recipient_keys(&cli.public_keys)?; + append_files( + &file, + AppendOptions::new(sender_key, reader_keys) + .with_cdc(cdc.unwrap_or_default()) + .with_durability(durability.into()), + &files, + )?; } - - unimplemented!() - } + }, PithosCommands::Export { file, path, .. } => { - let path_str = path - .to_str() - .expect("No inner path provided or path contains invalid UTF-8"); - let sender_key = load_private_key_from_pem( - &cli.secret_key - .clone() - .expect("Private key expected to create Pithos file"), - )?; - let reader_keys: Result, PithosError> = cli + let path_str = path.to_str().ok_or_else(|| { + PithosCliError::InvalidArgumentError("archive path is not valid UTF-8".into()) + })?; + let sender_key = required_private_key(&cli.secret_key)?; + let reader_keys: Result, PithosCliError> = cli .public_keys - .expect("At least one recipient expected") + .ok_or_else(|| { + PithosCliError::InvalidArgumentError( + "at least one recipient key is required".into(), + ) + })? .iter() - .map(load_public_key_from_pem) + .map(|path| load_public_key_from_pem(path)) .collect(); - let mut reader = PithosReaderSimple::new_with_key(&file, sender_key)?; - let (directory, _) = reader.read_directory()?; + let archive = open_archive(&file, sender_key)?; - // Write output - let output_target: Box = if let Some(dest) = cli.output { - Box::new(std::fs::File::create(dest).map_err(PithosError::Io)?) + if let Some(destination) = cli.output { + let mut output = StagedOutput::create( + &destination, + Mode::RUSR | Mode::WUSR | Mode::RGRP | Mode::WGRP | Mode::ROTH | Mode::WOTH, + "open export output", + )?; + let result = (|| { + crypt4gh::export(&archive, path_str, reader_keys?, output.file_mut())?; + output.flush("flush export output") + })(); + result?; + output.publish("publish export output")?; } else { - Box::new(std::io::stdout()) - }; + wait_before_stdout_for_test(); + let mut output = std::io::stdout(); + crypt4gh::export(&archive, path_str, reader_keys?, &mut output)?; + output.flush().map_err(PithosError::Io)?; + } + } + } - reader.read_file_to_crypt4gh( - path_str, - &directory, - reader_keys?, - Some(output_target), - )?; + Ok(()) +} + +fn main() { + if let Err(error) = run() { + eprintln!("{error}"); + std::process::exit(1); + } +} + +fn required_private_key(path: &Option) -> Result { + let path = path + .as_ref() + .ok_or_else(|| PithosCliError::InvalidArgumentError("private key is required".into()))?; + load_private_key_from_pem(path) +} + +fn validate_keypair_prefix(prefix: Option) -> Result { + let prefix = prefix.unwrap_or_else(|| "pithos_key".to_string()); + let mut components = Path::new(&prefix).components(); + let is_one_normal_component = matches!( + (components.next(), components.next()), + (Some(Component::Normal(_)), None) + ); + if prefix.is_empty() + || prefix.contains(['/', '\\']) + || Path::new(&prefix).is_absolute() + || !is_one_normal_component + { + return Err(PithosCliError::InvalidArgumentError( + "key prefix must be one normal filename component".into(), + )); + } + Ok(prefix) +} + +fn open_output_directory(path: &Path) -> rustix::io::Result { + let start = if path.is_absolute() { + Path::new("/") + } else { + Path::new(".") + }; + let mut directory = File::from(open( + start, + OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::empty(), + )?); + + for component in path.components() { + if matches!(component, Component::RootDir | Component::CurDir) { + continue; } + directory = File::from(openat( + &directory, + component.as_os_str(), + OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC, + Mode::empty(), + )?); + } + Ok(directory) +} + +fn open_keypair_output_directory(path: &Path) -> Result, PithosCliError> { + match open_output_directory(path) { + Ok(directory) => Ok(Some(directory)), + Err(rustix::io::Errno::NOENT | rustix::io::Errno::NOTDIR | rustix::io::Errno::LOOP) => { + Ok(None) + } + Err(source) => Err(PithosCliError::OutputFile { + operation: "inspect keypair output", + path: path.to_path_buf(), + source: source.into(), + }), + } +} + +fn write_combined_keypair_file( + path: &Path, + private_pem: &[u8], + public_pem: &[u8], +) -> Result<(), PithosCliError> { + let mut output = StagedOutput::create( + path, + Mode::RUSR | Mode::WUSR, + "open combined keypair output", + )?; + fchmod(output.file_mut(), Mode::RUSR | Mode::WUSR).map_err(|source| { + PithosCliError::OutputFile { + operation: "set combined keypair permissions", + path: path.to_path_buf(), + source: source.into(), + } + })?; + let result = + (|| { + output.file_mut().write_all(private_pem).map_err(|source| { + PithosCliError::OutputFile { + operation: "write private key", + path: path.to_path_buf(), + source, + } + })?; + output.file_mut().write_all(public_pem).map_err(|source| { + PithosCliError::OutputFile { + operation: "write public key", + path: path.to_path_buf(), + source, + } + })?; + output.flush("flush combined keypair output") + })(); + result?; + output.publish("publish combined keypair output") +} + +fn write_keypair_files( + output_directory: &Path, + directory: File, + prefix: &str, + private_pem: &[u8], + public_pem: &[u8], +) -> Result<(), PithosCliError> { + let private_name = format!("{prefix}.sec.pem"); + let public_name = format!("{prefix}.pub.pem"); + let private_path = output_directory.join(&private_name); + let public_path = output_directory.join(&public_name); + + let mut private_output = create_keypair_file( + &directory, + &private_name, + &private_path, + Mode::RUSR | Mode::WUSR, + true, + )?; + let mut public_output = match create_keypair_file( + &directory, + &public_name, + &public_path, + Mode::RUSR | Mode::WUSR | Mode::RGRP | Mode::ROTH, + false, + ) { + Ok(file) => file, + Err(error) => { + drop(private_output); + remove_keypair_file(&directory, &private_name); + return Err(error); + } + }; + + let result = (|| { + private_output + .write_all(private_pem) + .map_err(|source| PithosCliError::OutputFile { + operation: "write private key", + path: private_path.clone(), + source, + })?; + public_output + .write_all(public_pem) + .map_err(|source| PithosCliError::OutputFile { + operation: "write public key", + path: public_path.clone(), + source, + })?; + private_output + .flush() + .map_err(|source| PithosCliError::OutputFile { + operation: "flush private key", + path: private_path.clone(), + source, + })?; + public_output + .flush() + .map_err(|source| PithosCliError::OutputFile { + operation: "flush public key", + path: public_path.clone(), + source, + }) + })(); + if result.is_err() { + drop(private_output); + drop(public_output); + remove_keypair_file(&directory, &private_name); + remove_keypair_file(&directory, &public_name); + } + result +} + +fn create_keypair_file( + directory: &File, + name: &str, + path: &Path, + mode: Mode, + private: bool, +) -> Result { + let fd = openat( + directory, + name, + OFlags::WRONLY | OFlags::CREATE | OFlags::EXCL | OFlags::NOFOLLOW | OFlags::CLOEXEC, + mode, + ) + .map_err(|source| PithosCliError::OutputFile { + operation: if private { + "create private key" + } else { + "create public key" + }, + path: path.to_path_buf(), + source: source.into(), + })?; + if private && let Err(source) = fchmod(&fd, Mode::RUSR | Mode::WUSR) { + drop(fd); + remove_keypair_file(directory, name); + return Err(PithosCliError::OutputFile { + operation: "set private key permissions", + path: path.to_path_buf(), + source: source.into(), + }); + } + Ok(File::from(fd)) +} + +fn remove_keypair_file(directory: &File, name: &str) { + let _ = unlinkat(directory, name, AtFlags::empty()); +} + +struct StagedOutput { + directory: File, + destination_name: OsString, + path: PathBuf, + file: Option, +} + +impl StagedOutput { + fn create(path: &Path, mode: Mode, operation: &'static str) -> Result { + let destination_name = + path.file_name() + .map(OsString::from) + .ok_or_else(|| PithosCliError::OutputFile { + operation, + path: path.to_path_buf(), + source: std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "output path must name a file", + ), + })?; + let parent = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + let directory = + open_output_directory(parent).map_err(|source| PithosCliError::OutputFile { + operation, + path: path.to_path_buf(), + source: source.into(), + })?; + let file = File::from( + openat( + &directory, + ".", + OFlags::WRONLY | OFlags::TMPFILE | OFlags::CLOEXEC, + mode, + ) + .map_err(|source| PithosCliError::OutputFile { + operation, + path: path.to_path_buf(), + source: source.into(), + })?, + ); + + Ok(Self { + directory, + destination_name, + path: path.to_path_buf(), + file: Some(file), + }) } + fn file_mut(&mut self) -> &mut File { + self.file.as_mut().expect("staged output file is present") + } + + fn take_file(&mut self) -> File { + self.file.take().expect("staged output file is present") + } + + fn restore_file(&mut self, file: File) { + self.file = Some(file); + } + + fn flush(&mut self, operation: &'static str) -> Result<(), PithosCliError> { + self.file_mut() + .flush() + .map_err(|source| PithosCliError::OutputFile { + operation, + path: self.path.clone(), + source, + }) + } + + fn publish(mut self, operation: &'static str) -> Result<(), PithosCliError> { + let file = self.take_file(); + linkat( + &file, + "", + &self.directory, + &self.destination_name, + AtFlags::EMPTY_PATH, + ) + .map_err(|source| PithosCliError::OutputFile { + operation, + path: self.path.clone(), + source: source.into(), + }) + } +} + +fn required_recipient_keys(paths: &Option>) -> Result, PithosCliError> { + let paths = paths.as_ref().ok_or_else(|| { + PithosCliError::InvalidArgumentError("at least one recipient key is required".into()) + })?; + if paths.is_empty() { + return Err(PithosCliError::InvalidArgumentError( + "at least one recipient key is required".into(), + )); + } + paths + .iter() + .map(|path| load_public_key_from_pem(path)) + .collect() +} + +fn write_stdout(arguments: std::fmt::Arguments<'_>) -> Result<(), PithosCliError> { + wait_before_stdout_for_test(); + let mut output = std::io::stdout().lock(); + writeln!(output, "{arguments}").map_err(PithosError::Io)?; Ok(()) } + +fn wait_before_stdout_for_test() { + #[cfg(debug_assertions)] + if std::env::var_os("PITHOS_TEST_DELAY_STDOUT").is_some() { + std::thread::sleep(std::time::Duration::from_millis(50)); + } +} + +fn discard_incomplete_writer(writer: ArchiveWriter) { + match writer.into_incomplete() { + Ok(sink) => drop(sink), + Err(writer) => drop(writer), + } +} + +// This deliberately narrow process-test seam exercises cleanup after the +// destination has been created without depending on a filesystem race. +fn fail_after_create_for_test() -> Result<(), PithosError> { + #[cfg(debug_assertions)] + { + if std::env::var_os("PITHOS_TEST_FAIL_AFTER_CREATE").is_some() { + return Err(PithosError::Io(std::io::Error::other( + "writer runtime failure injected for testing", + ))); + } + } + Ok(()) +} + +fn open_archive( + path: &std::path::Path, + key: PrivateKey, +) -> Result, PithosError> { + Archive::open( + FileSource::open(path)?, + OpenOptions::default().with_access_keys(AccessKeys::new().with_key(key)), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn read_commands_parse_without_constructing_a_legacy_reader() { + assert!(Cli::try_parse_from(["pithos", "read", "list", "archive.pith"]).is_ok()); + assert!( + Cli::try_parse_from([ + "pithos", + "read", + "data", + "archive.pith", + "entry", + "--ranges", + "4:2", + ]) + .is_err() + ); + } + + #[test] + fn reader_key_requirement_is_a_cli_error() { + assert!(matches!( + required_private_key(&None), + Err(PithosCliError::InvalidArgumentError(_)) + )); + } +} diff --git a/crates/pithos/src/utils/conversion.rs b/crates/pithos/src/utils/conversion.rs index 37da7ff..4c93ad1 100644 --- a/crates/pithos/src/utils/conversion.rs +++ b/crates/pithos/src/utils/conversion.rs @@ -1,18 +1,24 @@ use crate::PithosCliError; +use pithos_lib::archive::CdcConfig; use std::ops::Range; -use tracing_subscriber::EnvFilter; +use tracing_subscriber::{EnvFilter, filter::Directive}; -pub fn evaluate_log_level(input: Option) -> EnvFilter { +pub fn evaluate_log_level(input: Option) -> Result { let level = if let Some(log_level) = input { &log_level.to_lowercase() } else { "info" }; - EnvFilter::try_from_default_env() + let invalid = || PithosCliError::InvalidArgumentError(format!("invalid log level: {level}")); + let cli: Directive = format!("pithos={level}").parse().map_err(|_| invalid())?; + let library: Directive = format!("pithos_lib={level}") + .parse() + .map_err(|_| invalid())?; + Ok(EnvFilter::try_from_default_env() .unwrap_or("none".into()) - .add_directive(format!("pithos={level}").parse().unwrap()) - .add_directive(format!("pithos_lib={level}").parse().unwrap()) + .add_directive(cli) + .add_directive(library)) } pub fn _to_hex_string(bytes: Vec) -> String { @@ -22,6 +28,11 @@ pub fn _to_hex_string(bytes: Vec) -> String { pub fn parse_range_input(input: &str) -> Result, PithosCliError> { let parts: Vec<&str> = input.split(':').collect(); + if parts.len() != 2 { + return Err(PithosCliError::InvalidArgumentError( + "ranges must use START:END".into(), + )); + } let start = parts[0].trim().parse::().map_err(|e| { PithosCliError::InvalidArgumentError(format!("Failed to parse range start: {}", e)) })?; @@ -29,10 +40,15 @@ pub fn parse_range_input(input: &str) -> Result, PithosCliError> { PithosCliError::InvalidArgumentError(format!("Failed to parse range end: {}", e)) })?; + if start > end { + return Err(PithosCliError::InvalidArgumentError( + "range start must not exceed range end".into(), + )); + } Ok(start..end) } -pub fn parse_cdc_input(input: &str) -> Result<(usize, usize, usize), PithosCliError> { +pub fn parse_cdc_input(input: &str) -> Result { let parts: Vec<&str> = input.split(',').collect(); if parts.len() != 3 { return Err(PithosCliError::InvalidArgumentError( @@ -41,14 +57,14 @@ pub fn parse_cdc_input(input: &str) -> Result<(usize, usize, usize), PithosCliEr } let min = parts[0].trim().parse::().map_err(|e| { - PithosCliError::InvalidArgumentError(format!("Failed to parse range start: {}", e)) + PithosCliError::InvalidArgumentError(format!("Failed to parse CDC minimum: {}", e)) })?; let avg = parts[1].trim().parse::().map_err(|e| { - PithosCliError::InvalidArgumentError(format!("Failed to parse range end: {}", e)) + PithosCliError::InvalidArgumentError(format!("Failed to parse CDC average: {}", e)) })?; let max = parts[2].trim().parse::().map_err(|e| { - PithosCliError::InvalidArgumentError(format!("Failed to parse range end: {}", e)) + PithosCliError::InvalidArgumentError(format!("Failed to parse CDC maximum: {}", e)) })?; - Ok((min, avg, max)) + CdcConfig::new(min, avg, max).map_err(PithosCliError::PithosError) } diff --git a/crates/pithos/tests/append.rs b/crates/pithos/tests/append.rs new file mode 100644 index 0000000..d9a8a61 --- /dev/null +++ b/crates/pithos/tests/append.rs @@ -0,0 +1,260 @@ +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; +use std::sync::atomic::{AtomicUsize, Ordering}; + +static NEXT_TEMPORARY: AtomicUsize = AtomicUsize::new(0); + +fn workspace_file(path: &str) -> String { + format!( + "{}/../pithos_lib/tests/data/{path}", + env!("CARGO_MANIFEST_DIR") + ) +} + +fn temporary() -> PathBuf { + let path = std::env::temp_dir().join(format!( + "pithos-cli-append-{}-{}", + std::process::id(), + NEXT_TEMPORARY.fetch_add(1, Ordering::Relaxed) + )); + let _ = fs::remove_dir_all(&path); + fs::create_dir_all(&path).unwrap(); + path +} + +fn command() -> Command { + Command::new(env!("CARGO_BIN_EXE_pithos")) +} + +fn sender_command() -> Command { + let mut command = command(); + command + .arg("--secret-key") + .arg(workspace_file("keys/sender_private.pem")); + command +} + +fn add_recipient(command: &mut Command, name: &str) { + command + .arg("--public-keys") + .arg(workspace_file(&format!("keys/{name}_public.pem"))); +} + +fn create_archive(directory: &Path) -> PathBuf { + let input = directory.join("base.txt"); + let archive = directory.join("archive.pith"); + fs::write(&input, b"base cli append payload").unwrap(); + let mut create = sender_command(); + add_recipient(&mut create, "sender"); + add_recipient(&mut create, "recipient1"); + assert!( + create + .arg("--output") + .arg(&archive) + .arg("create") + .arg(&input) + .status() + .unwrap() + .success() + ); + archive +} + +fn stderr(output: &Output) -> String { + String::from_utf8_lossy(&output.stderr).into_owned() +} + +#[test] +fn append_files_accepts_cdc_and_sync_all_and_publishes_readable_content() { + let temporary = temporary(); + let archive = create_archive(&temporary); + let input = temporary.join("appended.txt"); + fs::write(&input, b"appended by cli").unwrap(); + let mut append = sender_command(); + add_recipient(&mut append, "sender"); + add_recipient(&mut append, "recipient1"); + + let output = append + .arg("append") + .arg("files") + .arg("--file") + .arg(&archive) + .arg("--cdc") + .arg("64,256,1024") + .arg("--durability") + .arg("sync-all") + .arg(&input) + .output() + .unwrap(); + assert!(output.status.success(), "{}", stderr(&output)); + + let read = command() + .arg("--secret-key") + .arg(workspace_file("keys/recipient1_private.pem")) + .arg("read") + .arg("data") + .arg(&archive) + .arg("appended.txt") + .output() + .unwrap(); + assert!(read.status.success(), "{}", stderr(&read)); + assert_eq!(read.stdout, b"appended by cli"); + let _ = fs::remove_dir_all(temporary); +} + +#[test] +fn append_readers_grants_selected_content_under_the_appender_lock() { + let temporary = temporary(); + let archive = create_archive(&temporary); + let mut grant = sender_command(); + add_recipient(&mut grant, "recipient2"); + + let output = grant + .arg("append") + .arg("readers") + .arg("--ids") + .arg("0") + .arg("--durability") + .arg("sync-all") + .arg(&archive) + .output() + .unwrap(); + assert!(output.status.success(), "{}", stderr(&output)); + + let read = command() + .arg("--secret-key") + .arg(workspace_file("keys/recipient2_private.pem")) + .arg("read") + .arg("data") + .arg(&archive) + .arg("base.txt") + .output() + .unwrap(); + assert!(read.status.success(), "{}", stderr(&read)); + assert_eq!(read.stdout, b"base cli append payload"); + let _ = fs::remove_dir_all(temporary); +} + +#[test] +fn append_preflight_and_self_ingestion_fail_without_mutating_the_archive() { + let temporary = temporary(); + let archive = create_archive(&temporary); + let original = fs::read(&archive).unwrap(); + let present = temporary.join("present.txt"); + fs::write(&present, b"present").unwrap(); + let missing = temporary.join("missing.txt"); + + for inputs in [vec![present, missing], vec![archive.clone()]] { + let mut append = sender_command(); + add_recipient(&mut append, "sender"); + add_recipient(&mut append, "recipient1"); + let output = append + .arg("append") + .arg("files") + .arg("--file") + .arg(&archive) + .args(inputs) + .output() + .unwrap(); + let message = stderr(&output); + assert!(!output.status.success()); + assert!(!message.contains("panicked"), "{message}"); + assert_eq!(fs::read(&archive).unwrap(), original); + } + let _ = fs::remove_dir_all(temporary); +} + +#[test] +fn append_cli_reports_missing_input_without_mutating_the_archive() { + let temporary = temporary(); + let archive = create_archive(&temporary); + let original = fs::read(&archive).unwrap(); + let input = temporary.join("missing.txt"); + let mut append = sender_command(); + add_recipient(&mut append, "sender"); + add_recipient(&mut append, "recipient1"); + + let output = append + .arg("append") + .arg("files") + .arg("--file") + .arg(&archive) + .arg(&input) + .output() + .unwrap(); + + let message = stderr(&output); + assert!(!output.status.success()); + assert!(message.contains("Filesystem Error"), "{message}"); + assert!(!message.contains("panicked"), "{message}"); + assert_eq!(fs::read(&archive).unwrap(), original); + let _ = fs::remove_dir_all(temporary); +} + +#[cfg(unix)] +#[test] +fn append_cli_reports_lock_contention_without_mutation() { + let temporary = temporary(); + let archive = create_archive(&temporary); + let original = fs::read(&archive).unwrap(); + let input = temporary.join("locked.txt"); + fs::write(&input, b"locked cli payload").unwrap(); + let lock = fs::OpenOptions::new() + .read(true) + .write(true) + .open(&archive) + .unwrap(); + rustix::fs::flock(&lock, rustix::fs::FlockOperation::NonBlockingLockExclusive).unwrap(); + let mut append = sender_command(); + add_recipient(&mut append, "sender"); + add_recipient(&mut append, "recipient1"); + + let output = append + .arg("append") + .arg("files") + .arg("--file") + .arg(&archive) + .arg(&input) + .output() + .unwrap(); + let message = stderr(&output); + + assert!(!output.status.success()); + assert!(message.contains("locked by another cooperating writer")); + assert!(!message.contains("panicked"), "{message}"); + assert_eq!(fs::read(&archive).unwrap(), original); + let _ = fs::remove_dir_all(temporary); +} + +#[test] +fn append_cli_rejects_a_non_archive_target_without_panicking_or_mutating_archives() { + let temporary = temporary(); + let archive = create_archive(&temporary); + let original = fs::read(&archive).unwrap(); + let target = temporary.join("not-an-archive.txt"); + let target_original = b"not an archive"; + fs::write(&target, target_original).unwrap(); + let input = temporary.join("input.txt"); + fs::write(&input, b"cli input payload").unwrap(); + let mut append = sender_command(); + add_recipient(&mut append, "sender"); + add_recipient(&mut append, "recipient1"); + + let output = append + .arg("append") + .arg("files") + .arg("--file") + .arg(&target) + .arg(&input) + .output() + .unwrap(); + let message = stderr(&output); + + assert!(!output.status.success()); + assert!(message.contains("Filesystem Error"), "{message}"); + assert!(!message.contains("panicked"), "{message}"); + assert_eq!(fs::read(&archive).unwrap(), original); + assert_eq!(fs::read(&target).unwrap(), target_original); + let _ = fs::remove_dir_all(temporary); +} diff --git a/crates/pithos/tests/behavior.rs b/crates/pithos/tests/behavior.rs new file mode 100644 index 0000000..e40a956 --- /dev/null +++ b/crates/pithos/tests/behavior.rs @@ -0,0 +1,237 @@ +use std::fs; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; +use std::sync::atomic::{AtomicUsize, Ordering}; + +static NEXT_TEMPORARY: AtomicUsize = AtomicUsize::new(0); + +fn workspace_file(path: &str) -> String { + format!( + "{}/../pithos_lib/tests/data/{path}", + env!("CARGO_MANIFEST_DIR") + ) +} + +fn temporary() -> PathBuf { + let path = std::env::temp_dir().join(format!( + "pithos-cli-behavior-{}-{}", + std::process::id(), + NEXT_TEMPORARY.fetch_add(1, Ordering::Relaxed) + )); + let _ = fs::remove_dir_all(&path); + fs::create_dir_all(&path).unwrap(); + path +} + +fn command() -> Command { + Command::new(env!("CARGO_BIN_EXE_pithos")) +} + +fn assert_success(output: &Output) { + assert!( + output.status.success(), + "stdout: {}\nstderr: {}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} + +fn assert_clean_failure(output: &Output) -> String { + assert!(!output.status.success()); + let message = String::from_utf8_lossy(&output.stderr).into_owned(); + for forbidden in ["panicked", "unwrap", "expect", "unimplemented"] { + assert!(!message.contains(forbidden), "{message}"); + } + message +} + +fn create_archive(root: &Path, recipients: &[&str]) -> PathBuf { + let first = root.join("first.txt"); + let second = root.join("second.txt"); + let archive = root.join("archive.pith"); + fs::write(&first, b"first content").unwrap(); + fs::write(&second, b"second content").unwrap(); + + let mut invocation = command(); + invocation + .arg("--secret-key") + .arg(workspace_file("keys/sender_private.pem")); + for recipient in recipients { + invocation + .arg("--public-keys") + .arg(workspace_file(recipient)); + } + let output = invocation + .arg("--output") + .arg(&archive) + .arg("create") + .arg("--cdc") + .arg("64,256,1024") + .arg(&first) + .arg(&second) + .output() + .unwrap(); + assert_success(&output); + archive +} + +#[test] +fn create_read_extract_export_and_keypair_success_matrix() { + let temporary = temporary(); + let archive = create_archive( + &temporary, + &["keys/recipient1_public.pem", "keys/recipient2_public.pem"], + ); + let reader_key = workspace_file("keys/recipient1_private.pem"); + + for (subcommand, expected) in [ + (["read", "list"].as_slice(), "first.txt"), + (["read", "directory"].as_slice(), "second.txt"), + ] { + let output = command() + .arg("--secret-key") + .arg(&reader_key) + .args(subcommand) + .arg(&archive) + .output() + .unwrap(); + assert_success(&output); + assert!(String::from_utf8_lossy(&output.stdout).contains(expected)); + } + + let info = command() + .arg("--secret-key") + .arg(&reader_key) + .arg("read") + .arg("info") + .arg(&archive) + .arg("first.txt") + .output() + .unwrap(); + assert_success(&info); + assert!(String::from_utf8_lossy(&info.stdout).contains("first.txt")); + + let range_output = temporary.join("range.bin"); + let range = command() + .arg("--secret-key") + .arg(&reader_key) + .arg("--output") + .arg(&range_output) + .arg("read") + .arg("data") + .arg(&archive) + .arg("first.txt") + .arg("--ranges") + .arg("0:5") + .output() + .unwrap(); + assert_success(&range); + assert_eq!(fs::read(&range_output).unwrap(), b"first"); + + let extraction = temporary.join("extract"); + fs::create_dir(&extraction).unwrap(); + let all = command() + .arg("--secret-key") + .arg(&reader_key) + .arg("--output") + .arg(&extraction) + .arg("read") + .arg("all") + .arg(&archive) + .output() + .unwrap(); + assert_success(&all); + assert_eq!( + fs::read(extraction.join("first.txt")).unwrap(), + b"first content" + ); + assert_eq!( + fs::read(extraction.join("second.txt")).unwrap(), + b"second content" + ); + + let exported = temporary.join("first.c4gh"); + let export = command() + .arg("--secret-key") + .arg(&reader_key) + .arg("--public-keys") + .arg(workspace_file("keys/recipient2_public.pem")) + .arg("--output") + .arg(&exported) + .arg("export") + .arg("--format") + .arg("crypt4gh") + .arg(&archive) + .arg("first.txt") + .output() + .unwrap(); + assert_success(&export); + assert!(fs::read(&exported).unwrap().starts_with(b"crypt4gh")); + + let keypair = command().arg("keypair").output().unwrap(); + assert_success(&keypair); + let pem = String::from_utf8(keypair.stdout).unwrap(); + assert!(pem.contains("BEGIN PRIVATE KEY")); + assert!(pem.contains("BEGIN PUBLIC KEY")); + + let _ = fs::remove_dir_all(temporary); +} + +#[test] +fn read_data_requires_at_least_one_archive_path() { + let temporary = temporary(); + let archive = create_archive(&temporary, &["keys/recipient1_public.pem"]); + let output = command() + .arg("--secret-key") + .arg(workspace_file("keys/recipient1_private.pem")) + .arg("read") + .arg("data") + .arg(&archive) + .output() + .unwrap(); + + let message = assert_clean_failure(&output); + assert!(message.contains("at least one archive path"), "{message}"); + let _ = fs::remove_dir_all(temporary); +} + +#[test] +fn unavailable_content_and_corrupt_archives_have_distinct_failures() { + let temporary = temporary(); + let archive = create_archive(&temporary, &["keys/recipient1_public.pem"]); + + let unavailable = command() + .arg("--secret-key") + .arg(workspace_file("keys/recipient2_private.pem")) + .arg("read") + .arg("data") + .arg(&archive) + .arg("first.txt") + .output() + .unwrap(); + let unavailable_message = assert_clean_failure(&unavailable); + assert!( + unavailable_message.contains("unavailable"), + "{unavailable_message}" + ); + + let corrupt = temporary.join("corrupt.pith"); + let mut bytes = fs::read(&archive).unwrap(); + bytes[0] ^= 1; + fs::write(&corrupt, bytes).unwrap(); + let invalid = command() + .arg("--secret-key") + .arg(workspace_file("keys/recipient1_private.pem")) + .arg("read") + .arg("list") + .arg(&corrupt) + .output() + .unwrap(); + let corrupt_message = assert_clean_failure(&invalid); + assert!( + !corrupt_message.contains("unavailable"), + "{corrupt_message}" + ); + assert_ne!(unavailable_message, corrupt_message); + let _ = fs::remove_dir_all(temporary); +} diff --git a/crates/pithos/tests/create.rs b/crates/pithos/tests/create.rs new file mode 100644 index 0000000..7be4aba --- /dev/null +++ b/crates/pithos/tests/create.rs @@ -0,0 +1,326 @@ +use std::fs; +use std::process::Command; +use std::sync::atomic::{AtomicUsize, Ordering}; + +static NEXT_TEMPORARY: AtomicUsize = AtomicUsize::new(0); + +fn workspace_file(path: &str) -> String { + format!( + "{}/../pithos_lib/tests/data/{path}", + env!("CARGO_MANIFEST_DIR") + ) +} + +fn temporary() -> std::path::PathBuf { + let path = std::env::temp_dir().join(format!( + "pithos-cli-create-{}-{}", + std::process::id(), + NEXT_TEMPORARY.fetch_add(1, Ordering::Relaxed) + )); + let _ = fs::remove_dir_all(&path); + fs::create_dir_all(&path).unwrap(); + path +} + +fn command() -> Command { + Command::new(env!("CARGO_BIN_EXE_pithos")) +} + +fn create_command(output: &std::path::Path, input: &std::path::Path) -> Command { + let private = workspace_file("keys/sender_private.pem"); + let public = workspace_file("keys/recipient1_public.pem"); + let mut command = command(); + command + .arg("--secret-key") + .arg(private) + .arg("--public-keys") + .arg(public) + .arg("--output") + .arg(output) + .arg("create") + .arg(input); + command +} + +#[test] +fn create_reports_missing_keys_and_invalid_cdc() { + let temporary = temporary(); + let input = temporary.join("input.txt"); + let missing_recipient_output = temporary.join("missing-recipient.pith"); + let invalid_cdc_output = temporary.join("invalid-cdc.pith"); + fs::write(&input, b"input").unwrap(); + let private = workspace_file("keys/sender_private.pem"); + let public = workspace_file("keys/recipient1_public.pem"); + assert!( + !command() + .args([ + "--secret-key", + &private, + "--output", + missing_recipient_output.to_str().unwrap(), + "create", + input.to_str().unwrap(), + ]) + .status() + .unwrap() + .success() + ); + assert!(!missing_recipient_output.exists()); + assert!( + !command() + .args([ + "--secret-key", + &private, + "--public-keys", + &public, + "--output", + invalid_cdc_output.to_str().unwrap(), + "create", + "--cdc", + "8,4,16", + input.to_str().unwrap() + ]) + .status() + .unwrap() + .success() + ); + assert!(!invalid_cdc_output.exists()); + let _ = fs::remove_dir_all(temporary); +} + +#[test] +fn create_reports_filesystem_errors_and_produces_a_readable_archive() { + let temporary = temporary(); + let private = workspace_file("keys/sender_private.pem"); + let public = workspace_file("keys/recipient1_public.pem"); + let output = temporary.join("archive.pith"); + assert!( + !command() + .args([ + "--secret-key", + &private, + "--public-keys", + &public, + "--output", + output.to_str().unwrap(), + "create", + temporary.join("missing").to_str().unwrap() + ]) + .status() + .unwrap() + .success() + ); + assert!(!output.exists()); + let input = temporary.join("input.txt"); + fs::write(&input, b"created by cli").unwrap(); + assert!( + command() + .args([ + "--secret-key", + &private, + "--public-keys", + &public, + "--output", + output.to_str().unwrap(), + "create", + input.to_str().unwrap() + ]) + .status() + .unwrap() + .success() + ); + assert!(output.exists()); + assert!( + command() + .args([ + "--secret-key", + &workspace_file("keys/recipient1_private.pem"), + "read", + "list", + output.to_str().unwrap() + ]) + .status() + .unwrap() + .success() + ); + let _ = fs::remove_dir_all(temporary); +} + +#[test] +fn create_preflights_keys_recipients_and_all_cdc_bounds() { + use fastcdc::v2020::{ + AVERAGE_MAX, AVERAGE_MIN, MAXIMUM_MAX, MAXIMUM_MIN, MINIMUM_MAX, MINIMUM_MIN, + }; + + let temporary = temporary(); + let input = temporary.join("input"); + fs::write(&input, b"input").unwrap(); + let private = workspace_file("keys/sender_private.pem"); + let public = workspace_file("keys/recipient1_public.pem"); + + let missing_sender = temporary.join("missing-sender.pith"); + assert!( + !command() + .arg("--public-keys") + .arg(&public) + .arg("--output") + .arg(&missing_sender) + .arg("create") + .arg(&input) + .status() + .unwrap() + .success() + ); + assert!(!missing_sender.exists()); + + let duplicate_recipient = temporary.join("duplicate-recipient.pith"); + assert!( + !command() + .arg("--secret-key") + .arg(&private) + .arg("--public-keys") + .arg(&public) + .arg("--public-keys") + .arg(&public) + .arg("--output") + .arg(&duplicate_recipient) + .arg("create") + .arg(&input) + .status() + .unwrap() + .success() + ); + assert!(!duplicate_recipient.exists()); + + for (index, (min, avg, max)) in [ + (MINIMUM_MIN - 1, AVERAGE_MIN, MAXIMUM_MIN), + (MINIMUM_MAX + 1, AVERAGE_MAX, MAXIMUM_MAX), + (MINIMUM_MIN, AVERAGE_MIN - 1, MAXIMUM_MIN), + (MINIMUM_MIN, AVERAGE_MAX + 1, MAXIMUM_MAX), + (MINIMUM_MIN, AVERAGE_MIN, MAXIMUM_MIN - 1), + (MINIMUM_MIN, AVERAGE_MIN, MAXIMUM_MAX + 1), + ] + .into_iter() + .enumerate() + { + let output = temporary.join(format!("invalid-cdc-{index}.pith")); + assert!( + !create_command(&output, &input) + .arg("--cdc") + .arg(format!("{min},{avg},{max}")) + .status() + .unwrap() + .success() + ); + assert!(!output.exists()); + } + let _ = fs::remove_dir_all(temporary); +} + +#[test] +fn create_never_clobbers_existing_output_and_removes_post_creation_failure() { + let temporary = temporary(); + let input = temporary.join("input"); + fs::write(&input, b"input").unwrap(); + let existing = temporary.join("existing.pith"); + let original = b"do not replace"; + fs::write(&existing, original).unwrap(); + assert!( + !create_command(&existing, &input) + .status() + .unwrap() + .success() + ); + assert_eq!(fs::read(&existing).unwrap(), original); + + let created_then_failed = temporary.join("created-then-failed.pith"); + assert!( + !create_command(&created_then_failed, &input) + .env("PITHOS_TEST_FAIL_AFTER_CREATE", "1") + .status() + .unwrap() + .success() + ); + assert!(!created_then_failed.exists()); + let _ = fs::remove_dir_all(temporary); +} + +#[cfg(unix)] +#[test] +fn create_rejects_symlinked_output_parent_before_publication() { + use std::os::unix::fs::symlink; + + let temporary = temporary(); + let input = temporary.join("input"); + let outside = temporary.join("outside"); + let redirect = temporary.join("redirect"); + fs::write(&input, b"input").unwrap(); + fs::create_dir(&outside).unwrap(); + symlink(&outside, &redirect).unwrap(); + + let output = redirect.join("archive.pith"); + assert!(!create_command(&output, &input).status().unwrap().success()); + assert!(!outside.join("archive.pith").exists()); + let _ = fs::remove_dir_all(temporary); +} + +#[cfg(unix)] +#[test] +fn create_rejects_non_utf8_paths_and_unsafe_symlinks_before_publication() { + use std::os::unix::ffi::OsStringExt; + + let temporary = temporary(); + let output = temporary.join("output.pith"); + let invalid_path = temporary.join(std::ffi::OsString::from_vec(b"invalid-\xff".to_vec())); + fs::write(&invalid_path, b"input").unwrap(); + assert!( + !create_command(&output, &invalid_path) + .status() + .unwrap() + .success() + ); + assert!(!output.exists()); + + let invalid_target = temporary.join("invalid-target"); + std::os::unix::fs::symlink( + std::path::Path::new(&std::ffi::OsString::from_vec(b"target-\xff".to_vec())), + &invalid_target, + ) + .unwrap(); + assert!( + !create_command(&output, &invalid_target) + .status() + .unwrap() + .success() + ); + assert!(!output.exists()); + + let unsafe_link = temporary.join("unsafe"); + std::os::unix::fs::symlink("/outside", &unsafe_link).unwrap(); + assert!( + !create_command(&output, &unsafe_link) + .status() + .unwrap() + .success() + ); + assert!(!output.exists()); + let _ = fs::remove_dir_all(temporary); +} + +#[cfg(unix)] +#[test] +fn create_rejects_unreadable_input_before_publication_when_permissions_apply() { + use std::os::unix::fs::PermissionsExt; + + let temporary = temporary(); + let input = temporary.join("unreadable"); + let output = temporary.join("output.pith"); + fs::write(&input, b"input").unwrap(); + fs::set_permissions(&input, fs::Permissions::from_mode(0o000)).unwrap(); + if fs::File::open(&input).is_err() { + assert!(!create_command(&output, &input).status().unwrap().success()); + assert!(!output.exists()); + } + fs::set_permissions(&input, fs::Permissions::from_mode(0o644)).unwrap(); + let _ = fs::remove_dir_all(temporary); +} diff --git a/crates/pithos/tests/errors.rs b/crates/pithos/tests/errors.rs new file mode 100644 index 0000000..d506e5a --- /dev/null +++ b/crates/pithos/tests/errors.rs @@ -0,0 +1,272 @@ +use std::os::unix::fs::symlink; +use std::process::{Command, Output, Stdio}; + +fn workspace_file(path: &str) -> String { + format!( + "{}/../pithos_lib/tests/data/{path}", + env!("CARGO_MANIFEST_DIR") + ) +} + +fn command() -> Command { + Command::new(env!("CARGO_BIN_EXE_pithos")) +} + +fn temporary_directory(label: &str) -> std::path::PathBuf { + let path = + std::env::temp_dir().join(format!("pithos-cli-errors-{}-{label}", std::process::id())); + let _ = std::fs::remove_dir_all(&path); + std::fs::create_dir(&path).unwrap(); + path +} + +fn stderr(output: &Output) -> String { + String::from_utf8_lossy(&output.stderr).into_owned() +} + +fn run_with_closed_stdout(mut command: Command) -> Output { + let mut child = command + .env("PITHOS_TEST_DELAY_STDOUT", "1") + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + drop(child.stdout.take()); + child.wait_with_output().unwrap() +} + +fn assert_broken_pipe(label: &str, output: &Output) { + let message = stderr(output); + assert!(!output.status.success(), "{label} unexpectedly succeeded"); + assert!(!message.contains("panicked"), "{label}: {message}"); + assert!( + message.contains("Broken pipe") || message.contains("broken pipe"), + "{label}: {message}" + ); +} + +#[test] +fn export_without_recipients_returns_an_error_without_panicking() { + let output = command() + .arg("--secret-key") + .arg(workspace_file("keys/sender_private.pem")) + .arg("export") + .arg("--format") + .arg("crypt4gh") + .arg("missing.pith") + .arg("entry") + .output() + .unwrap(); + + let stderr = stderr(&output); + assert!(!output.status.success()); + assert!(stderr.contains("at least one recipient key is required")); + assert!(!stderr.contains("panicked")); + assert!(!stderr.contains("At least one recipient expected")); +} + +#[test] +fn invalid_log_level_returns_an_error_without_panicking() { + let output = command() + .arg("--log-level") + .arg("not-a-log-level[") + .arg("keypair") + .output() + .unwrap(); + + let stderr = stderr(&output); + assert!(!output.status.success()); + assert!(stderr.contains("invalid log level")); + assert!(!stderr.contains("panicked")); +} + +#[test] +fn missing_private_key_error_keeps_operation_and_path_context() { + let temporary = temporary_directory("missing-key"); + let missing_key = temporary.join("missing-private.pem"); + let output = command() + .arg("--secret-key") + .arg(&missing_key) + .arg("read") + .arg("list") + .arg(temporary.join("missing.pith")) + .output() + .unwrap(); + + let message = stderr(&output); + assert!(!output.status.success()); + assert!(message.contains("load private key"), "{message}"); + assert!(message.contains(missing_key.to_str().unwrap()), "{message}"); + assert!(!message.contains("panicked")); + let _ = std::fs::remove_dir_all(temporary); +} + +#[test] +fn invalid_cdc_component_names_the_component_instead_of_a_range() { + let temporary = temporary_directory("invalid-cdc"); + let input = temporary.join("input"); + std::fs::write(&input, b"content").unwrap(); + let output = command() + .arg("--secret-key") + .arg(workspace_file("keys/sender_private.pem")) + .arg("--public-keys") + .arg(workspace_file("keys/recipient1_public.pem")) + .arg("create") + .arg("--cdc") + .arg("invalid,256,1024") + .arg(input) + .output() + .unwrap(); + + let message = stderr(&output); + assert!(!output.status.success()); + assert!(message.contains("CDC minimum"), "{message}"); + assert!(!message.contains("range start"), "{message}"); + assert!(!message.contains("panicked")); + let _ = std::fs::remove_dir_all(temporary); +} + +#[test] +fn oversized_private_key_is_rejected_before_full_allocation() { + let temporary = temporary_directory("oversized-key"); + let oversized = temporary.join("oversized-private.pem"); + std::fs::File::create(&oversized) + .unwrap() + .set_len(1024 * 1024 + 1) + .unwrap(); + let output = command() + .arg("--secret-key") + .arg(&oversized) + .arg("read") + .arg("list") + .arg(temporary.join("missing.pith")) + .output() + .unwrap(); + + let message = stderr(&output); + assert!(!output.status.success()); + assert!(message.contains("private key file exceeds"), "{message}"); + assert!(message.contains(oversized.to_str().unwrap()), "{message}"); + assert!(!message.contains("panicked")); + let _ = std::fs::remove_dir_all(temporary); +} + +#[test] +fn private_key_loading_rejects_symbolic_links() { + let temporary = temporary_directory("key-symlink"); + let key_link = temporary.join("private.pem"); + symlink(workspace_file("keys/recipient1_private.pem"), &key_link).unwrap(); + let output = command() + .arg("--secret-key") + .arg(&key_link) + .arg("read") + .arg("list") + .arg(temporary.join("missing.pith")) + .output() + .unwrap(); + + let message = stderr(&output); + assert!(!output.status.success()); + assert!(message.contains("load private key"), "{message}"); + assert!(message.contains(key_link.to_str().unwrap()), "{message}"); + assert!(!message.contains("panicked")); + let _ = std::fs::remove_dir_all(temporary); +} + +#[test] +fn public_key_loading_rejects_symbolic_links() { + let temporary = temporary_directory("public-key-symlink"); + let key_link = temporary.join("public.pem"); + let input = temporary.join("input"); + symlink(workspace_file("keys/recipient1_public.pem"), &key_link).unwrap(); + std::fs::write(&input, b"input").unwrap(); + let output = command() + .arg("--secret-key") + .arg(workspace_file("keys/sender_private.pem")) + .arg("--public-keys") + .arg(&key_link) + .arg("--output") + .arg(temporary.join("archive.pith")) + .arg("create") + .arg(input) + .output() + .unwrap(); + + let message = stderr(&output); + assert!(!output.status.success()); + assert!(message.contains("load public key"), "{message}"); + assert!(message.contains(key_link.to_str().unwrap()), "{message}"); + assert!(!message.contains("panicked")); + let _ = std::fs::remove_dir_all(temporary); +} + +#[test] +fn broken_stdout_returns_an_error_without_panicking() { + let temporary = std::env::temp_dir().join(format!("pithos-cli-pipe-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&temporary); + std::fs::create_dir(&temporary).unwrap(); + let input = temporary.join("input"); + let archive = temporary.join("archive.pith"); + std::fs::write(&input, b"pipe test").unwrap(); + assert!( + command() + .arg("--secret-key") + .arg(workspace_file("keys/sender_private.pem")) + .arg("--public-keys") + .arg(workspace_file("keys/recipient1_public.pem")) + .arg("--output") + .arg(&archive) + .arg("create") + .arg(&input) + .status() + .unwrap() + .success() + ); + + let mut read_directory = command(); + read_directory + .arg("--secret-key") + .arg(workspace_file("keys/recipient1_private.pem")) + .arg("read") + .arg("directory") + .arg(&archive); + assert_broken_pipe("read directory", &run_with_closed_stdout(read_directory)); + + let mut create = command(); + create + .arg("--secret-key") + .arg(workspace_file("keys/sender_private.pem")) + .arg("--public-keys") + .arg(workspace_file("keys/recipient1_public.pem")) + .arg("create") + .arg(&input); + assert_broken_pipe("create", &run_with_closed_stdout(create)); + + let mut read_data = command(); + read_data + .arg("--secret-key") + .arg(workspace_file("keys/recipient1_private.pem")) + .arg("read") + .arg("data") + .arg(&archive) + .arg("input"); + assert_broken_pipe("read data", &run_with_closed_stdout(read_data)); + + let mut keypair = command(); + keypair.arg("keypair"); + assert_broken_pipe("keypair", &run_with_closed_stdout(keypair)); + + let mut export = command(); + export + .arg("--secret-key") + .arg(workspace_file("keys/recipient1_private.pem")) + .arg("--public-keys") + .arg(workspace_file("keys/recipient2_public.pem")) + .arg("export") + .arg("--format") + .arg("crypt4gh") + .arg(&archive) + .arg("input"); + assert_broken_pipe("export", &run_with_closed_stdout(export)); + let _ = std::fs::remove_dir_all(temporary); +} diff --git a/crates/pithos/tests/keypair.rs b/crates/pithos/tests/keypair.rs new file mode 100644 index 0000000..c8e8fbb --- /dev/null +++ b/crates/pithos/tests/keypair.rs @@ -0,0 +1,119 @@ +use std::fs; +use std::os::unix::fs::{PermissionsExt, symlink}; +use std::process::Command; +use std::sync::atomic::{AtomicUsize, Ordering}; + +static NEXT_TEMPORARY: AtomicUsize = AtomicUsize::new(0); + +fn temporary() -> std::path::PathBuf { + let path = std::env::temp_dir().join(format!( + "pithos-cli-keypair-{}-{}", + std::process::id(), + NEXT_TEMPORARY.fetch_add(1, Ordering::Relaxed) + )); + let _ = fs::remove_dir_all(&path); + fs::create_dir(&path).unwrap(); + path +} + +fn command() -> Command { + Command::new(env!("CARGO_BIN_EXE_pithos")) +} + +#[test] +fn keypair_private_file_is_created_with_mode_0600() { + let temporary = temporary(); + let output = command() + .arg("--output") + .arg(&temporary) + .arg("keypair") + .arg("--prefix") + .arg("secure") + .output() + .unwrap(); + + assert!(output.status.success(), "{:?}", output.stderr); + let mode = fs::metadata(temporary.join("secure.sec.pem")) + .unwrap() + .permissions() + .mode() + & 0o777; + assert_eq!(mode, 0o600); + let _ = fs::remove_dir_all(temporary); +} + +#[test] +fn keypair_rejects_escaping_prefix_before_writing() { + let temporary = temporary(); + let output_directory = temporary.join("keys"); + fs::create_dir(&output_directory).unwrap(); + let output = command() + .arg("--output") + .arg(&output_directory) + .arg("keypair") + .arg("--prefix") + .arg("../escaped") + .output() + .unwrap(); + + assert!(!output.status.success()); + assert!(!temporary.join("escaped.sec.pem").exists()); + assert!(!temporary.join("escaped.pub.pem").exists()); + let _ = fs::remove_dir_all(temporary); +} + +#[test] +fn keypair_writes_a_combined_no_clobber_file_with_private_permissions() { + let temporary = temporary(); + let output_path = temporary.join("combined.pem"); + let output = command() + .arg("--output") + .arg(&output_path) + .arg("keypair") + .output() + .unwrap(); + + assert!(output.status.success(), "{:?}", output.stderr); + let bytes = fs::read(&output_path).unwrap(); + let pem = String::from_utf8(bytes).unwrap(); + assert!(pem.contains("BEGIN PRIVATE KEY")); + assert!(pem.contains("BEGIN PUBLIC KEY")); + assert_eq!( + fs::metadata(&output_path).unwrap().permissions().mode() & 0o777, + 0o600 + ); + + fs::write(&output_path, b"unchanged").unwrap(); + let collision = command() + .arg("--output") + .arg(&output_path) + .arg("keypair") + .output() + .unwrap(); + assert!(!collision.status.success()); + assert_eq!(fs::read(&output_path).unwrap(), b"unchanged"); + let _ = fs::remove_dir_all(temporary); +} + +#[test] +fn keypair_does_not_follow_an_existing_private_key_symlink() { + let temporary = temporary(); + let output_directory = temporary.join("keys"); + fs::create_dir(&output_directory).unwrap(); + let victim = temporary.join("victim"); + fs::write(&victim, b"unchanged").unwrap(); + symlink(&victim, output_directory.join("secure.sec.pem")).unwrap(); + + let output = command() + .arg("--output") + .arg(&output_directory) + .arg("keypair") + .arg("--prefix") + .arg("secure") + .output() + .unwrap(); + + assert!(!output.status.success()); + assert_eq!(fs::read(&victim).unwrap(), b"unchanged"); + let _ = fs::remove_dir_all(temporary); +} diff --git a/crates/pithos/tests/output_safety.rs b/crates/pithos/tests/output_safety.rs new file mode 100644 index 0000000..8bedefc --- /dev/null +++ b/crates/pithos/tests/output_safety.rs @@ -0,0 +1,211 @@ +use std::fs; +use std::os::unix::fs::symlink; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::sync::atomic::{AtomicUsize, Ordering}; + +static NEXT_TEMPORARY: AtomicUsize = AtomicUsize::new(0); + +fn workspace_file(path: &str) -> String { + format!( + "{}/../pithos_lib/tests/data/{path}", + env!("CARGO_MANIFEST_DIR") + ) +} + +fn temporary() -> PathBuf { + let path = std::env::temp_dir().join(format!( + "pithos-cli-output-safety-{}-{}", + std::process::id(), + NEXT_TEMPORARY.fetch_add(1, Ordering::Relaxed) + )); + let _ = fs::remove_dir_all(&path); + fs::create_dir_all(&path).unwrap(); + path +} + +fn command() -> Command { + Command::new(env!("CARGO_BIN_EXE_pithos")) +} + +fn create_archive(temporary: &Path) -> PathBuf { + let first = temporary.join("first.txt"); + let second = temporary.join("second.txt"); + let archive = temporary.join("archive.pith"); + fs::write(&first, b"first content").unwrap(); + fs::write(&second, b"second content").unwrap(); + let output = command() + .arg("--secret-key") + .arg(workspace_file("keys/sender_private.pem")) + .arg("--public-keys") + .arg(workspace_file("keys/recipient1_public.pem")) + .arg("--output") + .arg(&archive) + .arg("create") + .arg(&first) + .arg(&second) + .output() + .unwrap(); + assert!(output.status.success(), "{:?}", output.stderr); + archive +} + +#[test] +fn read_data_never_clobbers_an_existing_output() { + let temporary = temporary(); + let archive = create_archive(&temporary); + let output_path = temporary.join("existing"); + fs::write(&output_path, b"unchanged").unwrap(); + + let output = command() + .arg("--secret-key") + .arg(workspace_file("keys/recipient1_private.pem")) + .arg("--output") + .arg(&output_path) + .arg("read") + .arg("data") + .arg(&archive) + .arg("first.txt") + .output() + .unwrap(); + + assert!(!output.status.success()); + assert_eq!(fs::read(&output_path).unwrap(), b"unchanged"); + let _ = fs::remove_dir_all(temporary); +} + +#[test] +fn read_data_rejects_multiple_paths_with_one_file_output() { + let temporary = temporary(); + let archive = create_archive(&temporary); + let output_path = temporary.join("ambiguous"); + + let output = command() + .arg("--secret-key") + .arg(workspace_file("keys/recipient1_private.pem")) + .arg("--output") + .arg(&output_path) + .arg("read") + .arg("data") + .arg(&archive) + .arg("first.txt") + .arg("second.txt") + .output() + .unwrap(); + + assert!(!output.status.success()); + assert!(!output_path.exists()); + let _ = fs::remove_dir_all(temporary); +} + +#[test] +fn read_data_removes_staged_output_when_archive_copy_fails() { + let temporary = temporary(); + let archive = create_archive(&temporary); + let output_path = temporary.join("incomplete"); + + let output = command() + .arg("--secret-key") + .arg(workspace_file("keys/recipient1_private.pem")) + .arg("--output") + .arg(&output_path) + .arg("read") + .arg("data") + .arg(&archive) + .arg("missing.txt") + .output() + .unwrap(); + + assert!(!output.status.success()); + assert!(!output_path.exists()); + let _ = fs::remove_dir_all(temporary); +} + +#[test] +fn crypt4gh_export_never_clobbers_an_existing_output() { + let temporary = temporary(); + let archive = create_archive(&temporary); + let output_path = temporary.join("existing.c4gh"); + fs::write(&output_path, b"unchanged").unwrap(); + + let output = command() + .arg("--secret-key") + .arg(workspace_file("keys/recipient1_private.pem")) + .arg("--public-keys") + .arg(workspace_file("keys/recipient2_public.pem")) + .arg("--output") + .arg(&output_path) + .arg("export") + .arg("--format") + .arg("crypt4gh") + .arg(&archive) + .arg("first.txt") + .output() + .unwrap(); + + assert!(!output.status.success()); + assert_eq!(fs::read(&output_path).unwrap(), b"unchanged"); + let _ = fs::remove_dir_all(temporary); +} + +#[test] +fn read_output_rejects_symlinked_parent_components() { + let temporary = temporary(); + let archive = create_archive(&temporary); + let outside = temporary.join("outside"); + fs::create_dir_all(outside.join("nested")).unwrap(); + let redirect = temporary.join("redirect"); + symlink(&outside, &redirect).unwrap(); + let output_path = redirect.join("nested/escaped-output"); + + let output = command() + .arg("--secret-key") + .arg(workspace_file("keys/recipient1_private.pem")) + .arg("--output") + .arg(&output_path) + .arg("read") + .arg("data") + .arg(&archive) + .arg("first.txt") + .output() + .unwrap(); + + assert!(!output.status.success()); + assert!(!outside.join("nested/escaped-output").exists()); + let _ = fs::remove_dir_all(temporary); +} + +#[test] +fn staged_output_publication_uses_the_retained_file_identity() { + let source = include_str!("../src/main.rs"); + assert!( + source.contains("AtFlags::EMPTY_PATH"), + "publication must link the retained staged descriptor, not a replaceable temporary name" + ); +} + +#[test] +fn crypt4gh_export_removes_staged_output_when_export_fails() { + let temporary = temporary(); + let archive = create_archive(&temporary); + let output_path = temporary.join("incomplete.c4gh"); + + let output = command() + .arg("--secret-key") + .arg(workspace_file("keys/recipient1_private.pem")) + .arg("--public-keys") + .arg(workspace_file("keys/recipient2_public.pem")) + .arg("--output") + .arg(&output_path) + .arg("export") + .arg("--format") + .arg("crypt4gh") + .arg(&archive) + .arg("missing.txt") + .output() + .unwrap(); + + assert!(!output.status.success()); + assert!(!output_path.exists()); + let _ = fs::remove_dir_all(temporary); +} From 7499a44ab1e8f47ea7338d3e3ba087b16b17ca43 Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Tue, 11 Aug 2026 23:02:54 +0200 Subject: [PATCH 011/166] chore: finalize 0.8 release packaging --- .gitignore | 6 +- AGENTS.md | 44 +- Cargo.lock | 621 +++++++++++++++++++- Cargo.toml | 2 +- README.md | 65 +- crates/pithos/Cargo.toml | 17 +- crates/pithos/LICENSE-APACHE | 201 +++++++ crates/pithos/LICENSE-MIT | 21 + crates/pithos/README.md | 56 +- crates/pithos_lib/Cargo.toml | 30 +- crates/pithos_lib/LICENSE-APACHE | 201 +++++++ crates/pithos_lib/LICENSE-MIT | 21 + crates/pithos_lib/README.md | 141 ++--- crates/pithos_lib/examples/append.rs | 20 + crates/pithos_lib/examples/create.rs | 26 + crates/pithos_lib/examples/crypt4gh.rs | 20 + crates/pithos_lib/examples/extract.rs | 18 + crates/pithos_lib/examples/grant_readers.rs | 19 + crates/pithos_lib/examples/open_list.rs | 18 + crates/pithos_lib/examples/read_ranges.rs | 19 + crates/pithos_lib/examples/ro_crate.rs | 18 + crates/pithos_lib/tests/public_api.rs | 71 +++ crates/pithos_pyo3/Cargo.toml | 6 +- crates/pithos_pyo3/README.md | 8 +- 24 files changed, 1495 insertions(+), 174 deletions(-) create mode 100644 crates/pithos/LICENSE-APACHE create mode 100644 crates/pithos/LICENSE-MIT create mode 100644 crates/pithos_lib/LICENSE-APACHE create mode 100644 crates/pithos_lib/LICENSE-MIT create mode 100644 crates/pithos_lib/examples/append.rs create mode 100644 crates/pithos_lib/examples/create.rs create mode 100644 crates/pithos_lib/examples/crypt4gh.rs create mode 100644 crates/pithos_lib/examples/extract.rs create mode 100644 crates/pithos_lib/examples/grant_readers.rs create mode 100644 crates/pithos_lib/examples/open_list.rs create mode 100644 crates/pithos_lib/examples/read_ranges.rs create mode 100644 crates/pithos_lib/examples/ro_crate.rs create mode 100644 crates/pithos_lib/tests/public_api.rs diff --git a/.gitignore b/.gitignore index 1762dbc..6db4fb9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,8 @@ # Cargo build output /target/ +/fuzz/target/ +/fuzz/artifacts/ +/.coverage/ # rustfmt backups **/*.rs.bk @@ -15,10 +18,11 @@ rustc-ice-*.txt # Local notes/scratch files todo.md +/docs # Local IDE config .idea/ # Local agent/tooling state .opencode/ -.junie/ \ No newline at end of file +.junie/ diff --git a/AGENTS.md b/AGENTS.md index fbe6e0e..e0d0c5b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,36 +1,24 @@ # AGENTS.md -## Workspace Map +## Boundaries -- This is a Rust 2024 workspace whose members are selected by `crates/*`; the root README only describes two of them. -- `crates/pithos_lib` owns the Pithos format: `model/` is the on-disk data model and marshalling, `io/` is the high-level reader/writer API, and `helpers/` contains crypto, compression, directory, and RO-Crate integration. -- `pithos_lib::helpers::ro_crate` is the Pithos adapter over upstream `ro-crate-rs`; keep upstream graph and parser behavior distinct from Pithos conversion behavior. -- `crates/pithos` is a single-binary Clap wrapper. Keep CLI parsing and filesystem presentation here; format behavior belongs in `pithos_lib`. -- `crates/pithos_pyo3` is currently an empty Rust library stub with no PyO3 dependency or Python API. Do not infer functionality from its name or copied README. -- Crates declare versions independently; none inherits `workspace.package.version`. Release/version work must update the intended crate manifests, not just the root version. -- There is no pinned toolchain or MSRV. CI installs stable; coverage alone installs nightly plus `llvm-tools-preview`. +- This is a Rust 2024 workspace (`crates/*`) with MSRV Rust 1.88. `pithos_lib` and the `pithos` CLI each declare their own version; `pithos_pyo3` is an unpublished, nonfunctional stub. +- `pithos_lib` is the implementation: `archive/` is the public reader/writer API; private `format/` is the only wire-format authority; `crypto.rs` and `block.rs` isolate keys and transforms. +- Keep core archive/format code independent of host files and presentation formats. Linux-only filesystem work belongs in `fs/`; RO-Crate and Crypt4GH presentation work belongs in `adapters/`. Preserve adapter-specific errors and host/path/member context instead of expanding core `PithosError` or exposing secrets. +- `pithos` is the Clap wrapper. Keep CLI parsing, output staging, and filesystem presentation in `crates/pithos`; put archive behavior in `pithos_lib`. +- `fs` uses no-follow traversal and no-clobber staged extraction. It requires Linux filesystem support for `O_TMPFILE` and `linkat(AT_EMPTY_PATH)`; do not assume filesystem-facing features are portable. -## Sources Of Truth +## Compatibility And Tests -- Treat `spec/PITHOS_1.0.0_draft.md`, model marshalling code, and integration tests as compatibility-sensitive. Changes to serialized fields, flags, encryption, compression, indexes, or directory layout can alter the on-disk format. -- Prefer current structs and tests over README snippets. `InputFile` uses `inner_path`. -- File extensions are inconsistent in current prose and code (`.pto`, `.pith`, `.pithos`). Do not normalize them as incidental cleanup. -- `AGENTS.md` is listed in `.gitignore` and is not tracked in the current checkout, so edits to it do not appear in normal `git status` output. +- Current public structs and tests define 0.8 behavior; the Pithos 1.0 draft is background, not an interoperability guarantee. Do not expose old `model`, `helpers`, or wire-record paths. +- Treat changes under `src/format/`, archive validation/indexing, encryption, compression, flags, indexes, or directory layout as on-disk compatibility changes. Do not casually normalize the currently inconsistent `.pto`, `.pith`, and `.pithos` extensions. +- `pithos_lib` integration tests use committed fixtures and test-only PEM keys in `crates/pithos_lib/tests/data/`; reuse helpers from `tests/common/`. +- Put wire-format unit coverage in `src/format/`, reader-internal coverage in `src/archive/reader_private_tests.rs`, and public extraction/range coverage in `crates/pithos_lib/tests/reader.rs`. Keep RO-Crate tests in the `ro_crate_{directory,zip,conversion}` integration targets. ## Verification -- CI parity is `cargo test --all-features` from the workspace root. -- Current-checkout caveat: `pithos_lib` declares and references feature `async`, but there is no `src/async.rs` or `src/async/mod.rs`; therefore the CI command currently fails with E0583. `cargo test --workspace` is the working default-feature baseline. -- Run one library integration target with `cargo test -p pithos_lib --test reader`, `--test writer`, or `--test marshalling`. -- Run one integration case with `cargo test -p pithos_lib --test reader test_reader_file_ranges -- --nocapture`. -- Run focused RO-Crate reader coverage with `cargo test -p pithos_lib --test reader test_rocrate_read_directory_1_2 -- --nocapture`. -- Run focused RO-Crate writer coverage with `cargo test -p pithos_lib --test writer test_rocrate_directory_zip_parity -- --nocapture`. -- Exercise CLI parsing with `cargo run -p pithos -- --help`; the CLI crate currently has no tests. -- The formatter check is `cargo fmt --all --check`, but it currently stops on the same missing `async` module. CI has no format or Clippy job, and the default build has existing unused-variable warnings, so do not describe `clippy -D warnings` as CI parity. -- Coverage mirrors `.github/workflows/codecov.yaml`: install `cargo-llvm-cov`, use nightly with `llvm-tools-preview`, then run `cargo llvm-cov --lcov --output-path=./.coverage/lcov.info`. - -## Test And Release Gotchas - -- `pithos_lib` integration tests depend on committed fixtures and PEM keys under `crates/pithos_lib/tests/data/`; reuse `tests/common/util.rs`. They create outputs in `tempfile` directories and require no external service. -- Put format round-trip coverage in `tests/marshalling.rs`, end-to-end extraction/range coverage in `tests/reader.rs`, and creation/append/directory/RO-Crate coverage in `tests/writer.rs`. -- Tags matching `v*` publish only `pithos_lib` and then `pithos`; `pithos_pyo3` is not published by the workflow. +- CI parity: `cargo +stable test --locked --workspace --all-features --lib --bins --tests`; MSRV parity: `cargo +1.88.0 test --locked --workspace --all-features`. +- Before finishing Rust changes, run `cargo +stable fmt --all -- --check` and `cargo +stable clippy --locked --workspace --all-targets --all-features -- -D warnings` when practical. +- Focused library tests use integration targets, for example `cargo test -p pithos_lib --test reader`; focused RO-Crate coverage is `cargo test -p pithos_lib --test ro_crate_directory` (or `ro_crate_zip` / `ro_crate_conversion`). CLI smoke check: `cargo +stable run --locked -p pithos -- --help`. +- Release/package changes also require `python3 .github/release/check.py contracts` and `python3 .github/release/check.py package check pithos_lib` (and `pithos` when affected). The package policy is a ratchet: `package update` needs `--allow-increase` to raise a budget. +- Run benchmark comparisons only on the owner-selected self-hosted runner; `ubuntu-latest` is not comparable benchmark evidence. diff --git a/Cargo.lock b/Cargo.lock index a5d4cc0..a9fbcea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14,7 +14,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1973cfbc1a2daf9cf550e74e1f088c28e7f7d8c1e1418fb6c9dc5184b7e84c99" dependencies = [ - "crypto-common", + "crypto-common 0.2.2", "inout", ] @@ -27,6 +27,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloca" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4" +dependencies = [ + "cc", +] + [[package]] name = "ambient-authority" version = "0.0.2" @@ -42,6 +51,12 @@ dependencies = [ "libc", ] +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + [[package]] name = "anstream" version = "1.0.0" @@ -128,12 +143,36 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + [[package]] name = "bitflags" version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +[[package]] +name = "blake2" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" +dependencies = [ + "digest 0.10.7", +] + [[package]] name = "blake3" version = "1.8.5" @@ -148,6 +187,15 @@ dependencies = [ "cpufeatures", ] +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "block-buffer" version = "0.12.1" @@ -205,6 +253,12 @@ dependencies = [ "rustix", ] +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.2.67" @@ -232,7 +286,7 @@ dependencies = [ "cfg-if", "cipher", "cpufeatures", - "rand_core", + "rand_core 0.10.1", ] [[package]] @@ -261,14 +315,41 @@ dependencies = [ "windows-link", ] +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + [[package]] name = "cipher" version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" dependencies = [ - "block-buffer", - "crypto-common", + "block-buffer 0.12.1", + "crypto-common 0.2.2", "inout", ] @@ -360,6 +441,82 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "criterion" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3" +dependencies = [ + "alloca", + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "itertools", + "num-traits", + "oorandom", + "page_size", + "plotters", + "rayon", + "regex", + "serde", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "crypto-common" version = "0.2.2" @@ -368,7 +525,7 @@ checksum = "ce6e4c961d6cd6c9a86db418387425e8bdeaf05b3c8bc1411e6dca4c252f1453" dependencies = [ "getrandom 0.4.3", "hybrid-array", - "rand_core", + "rand_core 0.10.1", ] [[package]] @@ -423,14 +580,25 @@ version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer 0.10.4", + "crypto-common 0.1.7", + "subtle", +] + [[package]] name = "digest" version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ - "block-buffer", - "crypto-common", + "block-buffer 0.12.1", + "crypto-common 0.2.2", ] [[package]] @@ -465,6 +633,12 @@ dependencies = [ "syn", ] +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + [[package]] name = "equivalent" version = "1.0.2" @@ -515,6 +689,12 @@ dependencies = [ "zlib-rs", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -584,6 +764,16 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.17" @@ -595,6 +785,18 @@ dependencies = [ "wasi", ] +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "libc", + "r-efi 5.3.0", + "wasip2", +] + [[package]] name = "getrandom" version = "0.4.3" @@ -603,8 +805,25 @@ checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" dependencies = [ "cfg-if", "libc", - "r-efi", - "rand_core", + "r-efi 6.0.0", + "rand_core 0.10.1", +] + +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", ] [[package]] @@ -896,6 +1115,15 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" @@ -1050,12 +1278,34 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + [[package]] name = "option-ext" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "page_size" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "peak_alloc" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccc90935a8dd139fdf341762773687a1e361d3f54b396a55d9dd1f7001e484bb" + [[package]] name = "pem-rfc7468" version = "1.0.0" @@ -1079,44 +1329,53 @@ checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" [[package]] name = "pithos" -version = "0.7.3" +version = "0.8.0" dependencies = [ "clap", + "fastcdc", "pithos_lib", + "rustix", "thiserror", "tracing", "tracing-subscriber", - "x25519-dalek", + "zeroize", ] [[package]] name = "pithos_lib" -version = "0.7.3" +version = "0.8.0" dependencies = [ + "blake2", "blake3", "byteorder", "cap-std", "chacha20poly1305", "crc32fast", - "digest", + "criterion", + "digest 0.11.3", "fastcdc", "indexmap", "integer-encoding", + "libc", + "peak_alloc", "pkcs8", + "proptest", "ro-crate-rs", + "rustix", "shake", "tempfile", "thiserror", "tracing", - "walkdir", + "trybuild", "x25519-dalek", + "zeroize", "zip", "zstd", ] [[package]] name = "pithos_pyo3" -version = "0.7.0" +version = "0.8.0" dependencies = [ "pithos_lib", ] @@ -1137,6 +1396,34 @@ version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + [[package]] name = "poly1305" version = "0.9.1" @@ -1162,6 +1449,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + [[package]] name = "proc-macro2" version = "1.0.106" @@ -1171,6 +1467,31 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "proptest" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" +dependencies = [ + "bit-set", + "bit-vec", + "bitflags", + "num-traits", + "rand 0.9.5", + "rand_chacha", + "rand_xorshift", + "regex-syntax", + "rusty-fork", + "tempfile", + "unarray", +] + +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + [[package]] name = "quote" version = "1.0.46" @@ -1180,12 +1501,28 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + [[package]] name = "r-efi" version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "rand" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9ef1d0d795eb7d84685bca4f72f3649f064e6641543d3a8c415898726a57b41" +dependencies = [ + "rand_chacha", + "rand_core 0.9.5", +] + [[package]] name = "rand" version = "0.10.2" @@ -1194,7 +1531,26 @@ checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" dependencies = [ "chacha20", "getrandom 0.4.3", - "rand_core", + "rand_core 0.10.1", +] + +[[package]] +name = "rand_chacha" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb" +dependencies = [ + "ppv-lite86", + "rand_core 0.9.5", +] + +[[package]] +name = "rand_core" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76afc826de14238e6e8c374ddcc1fa19e374fd8dd986b0d2af0d02377261d83c" +dependencies = [ + "getrandom 0.3.4", ] [[package]] @@ -1203,6 +1559,35 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" +[[package]] +name = "rand_xorshift" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" +dependencies = [ + "rand_core 0.9.5", +] + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_users" version = "0.5.2" @@ -1214,6 +1599,18 @@ dependencies = [ "thiserror", ] +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + [[package]] name = "regex-automata" version = "0.4.16" @@ -1273,7 +1670,7 @@ dependencies = [ "chrono", "dirs", "log", - "rand", + "rand 0.10.2", "reqwest", "serde", "serde_json", @@ -1322,6 +1719,18 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" +[[package]] +name = "rusty-fork" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] + [[package]] name = "same-file" version = "1.0.6" @@ -1380,13 +1789,22 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_spanned" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +dependencies = [ + "serde_core", +] + [[package]] name = "shake" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09057cb2149ad4cbd2da1e26b351f9a4c354219421229c69c3063e6f61947c4a" dependencies = [ - "digest", + "digest 0.11.3", "keccak", "sponge-cursor", ] @@ -1499,6 +1917,12 @@ dependencies = [ "syn", ] +[[package]] +name = "target-triple" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3a6bfce3d99adfa72d24750a61f782f3036a81e7f86d8841ee1326deaebd171" + [[package]] name = "tempfile" version = "3.27.0" @@ -1512,6 +1936,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + [[package]] name = "thiserror" version = "2.0.18" @@ -1582,6 +2015,16 @@ dependencies = [ "zerovec", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "tokio" version = "1.52.3" @@ -1595,6 +2038,45 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "toml" +version = "1.1.4+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3aace63f4bbcdfc2c965b059de67119c89c4017a70d633be6c104910f67056f5" +dependencies = [ + "indexmap", + "serde_core", + "serde_spanned", + "toml_datetime", + "toml_parser", + "toml_writer", + "winnow", +] + +[[package]] +name = "toml_datetime" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +dependencies = [ + "serde_core", +] + +[[package]] +name = "toml_parser" +version = "1.1.3+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +dependencies = [ + "winnow", +] + +[[package]] +name = "toml_writer" +version = "1.1.2+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" + [[package]] name = "tower" version = "0.5.3" @@ -1708,6 +2190,21 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "trybuild" +version = "1.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06649c6f63d86604ba0c8950d5a1829fc9a17afd70fc6629f481d75b6a624c78" +dependencies = [ + "glob", + "serde", + "serde_derive", + "serde_json", + "target-triple", + "termcolor", + "toml", +] + [[package]] name = "typed-path" version = "0.12.3" @@ -1720,6 +2217,12 @@ version = "1.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + [[package]] name = "unicode-ident" version = "1.0.24" @@ -1732,7 +2235,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f4987bdc12753382e0bec4a65c50738ffaabc998b9cdd1f952fb5f39b0048a96" dependencies = [ - "crypto-common", + "crypto-common 0.2.2", "ctutils", ] @@ -1777,6 +2280,21 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + [[package]] name = "walkdir" version = "2.5.0" @@ -1802,6 +2320,15 @@ version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + [[package]] name = "wasm-bindgen" version = "0.2.126" @@ -1867,6 +2394,22 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + [[package]] name = "winapi-util" version = "0.1.11" @@ -1876,6 +2419,12 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-core" version = "0.62.2" @@ -2017,6 +2566,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" + [[package]] name = "winx" version = "0.36.4" @@ -2027,6 +2582,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + [[package]] name = "writeable" version = "0.6.3" @@ -2041,7 +2602,7 @@ checksum = "e7e8131a03190127fb2263afc72b322ecadae46b6ff8c6f399ff5d02f5559af6" dependencies = [ "curve25519-dalek", "getrandom 0.4.3", - "rand_core", + "rand_core 0.10.1", "zeroize", ] @@ -2068,6 +2629,26 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zerocopy" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.55" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "zerofrom" version = "0.1.8" diff --git a/Cargo.toml b/Cargo.toml index a2e664e..5bf1cc5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ members = [ resolver = "2" [workspace.package] -version = "0.7.3" +version = "0.8.0" authors = ["Jannis Schlegel ", "Sebastian Beyvers "] edition = "2024" rust-version = "1.88" diff --git a/README.md b/README.md index ec2c681..e5178db 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,3 @@ -

    @@ -12,33 +6,54 @@

    Language: Rust - License: MIT - License: Apache 2.0 - Codecov + License: MIT + License: Apache 2.0 + Codecov

    -

    🔑 A secure, fast and versatile package file format for object storage focused data management 📦

    -
    +

    A secure archive format and Rust implementation for research-data packaging.

    + +Pithos packages files into an encrypted, chunked archive that can be read sequentially or by byte range. The Rust implementation supports streaming creation, validated opening, filesystem ingestion and extraction, archive extension, RO-Crate conversion, and Crypt4GH export. + +## Get started + +Install the command-line tool with Rust's package manager: + +```bash +cargo install pithos +``` -## Description +Create a recipient key pair, then create and inspect an archive. This example assumes `input/` already contains the files to package. -Pithos (a large ancient greek storage container) is a packaging file format for arbitrary data that enhances the use of Object Storage for (research) data management. This is done by combining multiple existing file standards with new enhancements. A format specification can be found [[here]](./spec/PITHOS_1.0.0_draft.md). +```bash +mkdir -p keys +pithos --output keys keypair --prefix owner +pithos --secret-key keys/owner.sec.pem --public-keys keys/owner.pub.pem --output research.pith create input +pithos --secret-key keys/owner.sec.pem read list research.pith +``` -## Features +For the complete command workflow, including extracting entries, see the [CLI guide](crates/pithos/README.md). For Rust applications, start with the [library guide](crates/pithos_lib/README.md). -- **Encryption**: Pithos implements fast ChaCha20-Poly1305 encryption and enables data exchange compatible to the Crypt4GH standard used for sensitive medical data. -- **Smart Compression**: Via compression probing Pithos can automatically detect and apply fast ZStandard compression for compressible data and optionally skip it when the data is incompressible. -- **Metadata**: Pithos not only includes technical metadata like file size and checksums but can also embed any semantic metadata directly in the file. -- **Indexing**: A built in index allows for fast random access of arbitrary ranges in encrypted and compressed files. -- **RO-Crate ingestion**: Directories and ZIP archives can be converted to Pithos; ZIP members are streamed without extraction. -- **Tooling**: Pithos comes with a rich set of existing tools that simplify file handling and can additionally be handled with existing tools for the Crypt4GH and Zstandard file formats. +## What it provides -Pithos comes with two main crates, a library for programmatic use and a CLI application that uses the library: +- Encrypted archive entries with recipient-key access control and per-block integrity verification. +- Content-defined chunking, optional compression, and indexed reads of complete entries or byte ranges. +- Local filesystem operations that avoid symlink traversal and refuse to overwrite existing extracted files. +- RO-Crate directory and ZIP conversion, plus Crypt4GH export for readable entries. -| crate | version | docs | +The filesystem operations are Linux-only and require destination filesystem support for `O_TMPFILE` and `linkat(AT_EMPTY_PATH)`. Core archive reading and writing do not require Linux filesystem access. + +## Crates + +| crate | version | docs | | :------------------------- | :-----------------------------------------------------------------------------------------: | :------------------------------------------------------------------: | -| [pithos](./crates/pithos/) | [![Crates.io](https://img.shields.io/crates/v/pithos.svg)](https://crates.io/crates/pithos) | [![Docs](https://docs.rs/pithos/badge.svg)](https://docs.rs/pithos/) | -| [pithos_lib](./crates/pithos_lib/) | [![Crates.io](https://img.shields.io/crates/v/pithos_lib.svg)](https://crates.io/crates/pithos_lib) | [![Docs](https://docs.rs/pithos_lib/badge.svg)](https://docs.rs/pithos_lib/) | +| [pithos](crates/pithos/) | [crates.io](https://crates.io/crates/pithos) | [docs.rs](https://docs.rs/pithos/) | +| [pithos_lib](crates/pithos_lib/) | [crates.io](https://crates.io/crates/pithos_lib) | [docs.rs](https://docs.rs/pithos_lib/) | + +`pithos` is the command-line interface. `pithos_lib` is the public Rust API and includes [compiled examples](crates/pithos_lib/examples/). The `pithos_pyo3` workspace member is an unpublished empty Rust stub, not a Python API. + +## Compatibility +Version 0.8 is a source break from 0.7. Use the selected public API in `pithos_lib::archive`, `crypto`, `source`, `fs`, and `adapters`; old model, helper, and wire-record paths are not compatibility APIs. -For a more detailed documentation of the individual crates see [Library](./crates/pithos_lib/README.md) and [CLI](./crates/pithos/README.md) +The [Pithos 1.0 draft](spec/PITHOS_1.0.0_draft.md) is useful format background, but it is not a complete interoperability guarantee. Current library behavior and tests define the 0.8 implementation. diff --git a/crates/pithos/Cargo.toml b/crates/pithos/Cargo.toml index 5349aef..678598e 100644 --- a/crates/pithos/Cargo.toml +++ b/crates/pithos/Cargo.toml @@ -1,18 +1,29 @@ [package] name = "pithos" description = "CLI for the Pithos container file format" -version = "0.7.3" +version = "0.8.0" authors.workspace = true edition.workspace = true rust-version.workspace = true repository.workspace = true license.workspace = true +publish = ["crates-io"] +include = [ + "src/**/*.rs", + "README.md", + "LICENSE-MIT", + "LICENSE-APACHE", +] [dependencies] clap = { version = "4.6.1", features = ["derive"] } -pithos_lib = { version = "0.7.3", path = "../pithos_lib" } +pithos_lib = { version = "0.8.0", path = "../pithos_lib" } thiserror = { workspace = true } tracing.workspace = true tracing-subscriber.workspace = true -x25519-dalek.workspace = true +zeroize = "1.8.2" +rustix = { version = "1.1.4", features = ["fs"] } #indicatif = "0.17.9" + +[dev-dependencies] +fastcdc = "4.0.1" diff --git a/crates/pithos/LICENSE-APACHE b/crates/pithos/LICENSE-APACHE new file mode 100644 index 0000000..5372d84 --- /dev/null +++ b/crates/pithos/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [2026] [ArunaEngine Team] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/crates/pithos/LICENSE-MIT b/crates/pithos/LICENSE-MIT new file mode 100644 index 0000000..deb2b2d --- /dev/null +++ b/crates/pithos/LICENSE-MIT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2024 ArunaEngine Team + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crates/pithos/README.md b/crates/pithos/README.md index 6db7f87..8443c57 100644 --- a/crates/pithos/README.md +++ b/crates/pithos/README.md @@ -1,20 +1,58 @@ # Pithos CLI -[![Rust](https://img.shields.io/badge/built_with-Rust-dca282.svg)](https://www.rust-lang.org/) -[![License](https://img.shields.io/badge/License-MIT-brightgreen.svg)](https://github.com/arunaengine/aruna-file/blob/main/LICENSE-MIT) -[![License](https://img.shields.io/badge/License-APACHE-brightgreen.svg)](https://github.com/arunaengine/aruna-file/blob/main/LICENSE-APACHE) +`pithos` creates, reads, and extends encrypted Pithos archives from the command line. It is the filesystem-oriented companion to the [`pithos_lib`](../pithos_lib/README.md) Rust API. +## Installation -CLI application for handling Pithos (.pto) files. +Published releases are available from [crates.io](https://crates.io/crates/pithos). Install Rust through [rustup](https://rustup.rs/) first. +```text +cargo install pithos +``` -## Installation +## Create and read an archive -A release is published via [crates.io](crates.io) (you need to have `cargo` and the `rust-toolchain` installed see [rustup.rs](rustup.rs) for more info) +Create a directory for a key pair, then generate an owner key pair. The private key decrypts the archive; the public key identifies a recipient that may read it. -```command -cargo install pithos +```bash +mkdir -p keys restored +pithos --output keys keypair --prefix owner +``` + +Create an archive for a file or directory. This example gives the owner access and writes `research.pith`; replace `input` with your source path. + +```bash +pithos --secret-key keys/owner.sec.pem --public-keys keys/owner.pub.pem --output research.pith create input +``` + +List its entries, extract all entries to an existing destination directory, or write one entry to standard output: + +```bash +pithos --secret-key keys/owner.sec.pem read list research.pith +pithos --secret-key keys/owner.sec.pem --output restored read all research.pith +pithos --secret-key keys/owner.sec.pem read data research.pith report.txt > report.txt ``` -### Usage +`read data` also accepts `--ranges START:END,...` for half-open byte ranges. Use `read info` to inspect an entry and `read directory` to inspect every entry's metadata. + +## More operations + +- `append files` adds filesystem input to an existing archive. +- `append readers` grants additional recipient public keys access to selected entry IDs. +- `export --format crypt4gh` exports a readable entry as a Crypt4GH stream. + +Use built-in help for required arguments, available options, and the complete command reference: + +```text +pithos --help +pithos create --help +pithos read --help +pithos append --help +pithos export --help +``` + +## Platform behavior + +Filesystem-facing operations are Linux-only in 0.8. New archive output, file output, export output, and extraction are staged without replacing existing destinations; they require a destination filesystem with Linux `O_TMPFILE` and `linkat(AT_EMPTY_PATH)` support. Direct append modifies an existing archive and does not require those operations. +For programmatic use, see the [pithos_lib documentation](https://docs.rs/pithos_lib/) and [compiled examples](https://github.com/arunaengine/pithos/tree/main/crates/pithos_lib/examples). diff --git a/crates/pithos_lib/Cargo.toml b/crates/pithos_lib/Cargo.toml index cd0f4ec..90ab358 100644 --- a/crates/pithos_lib/Cargo.toml +++ b/crates/pithos_lib/Cargo.toml @@ -1,16 +1,34 @@ [package] name = "pithos_lib" -description = "Library and components for encrypting / compressing pithos (.pith) files, including specification" -version = "0.7.3" +description = "Library and components for encrypting / compressing pithos (.pith) files" +version = "0.8.0" authors.workspace = true edition.workspace = true rust-version.workspace = true repository.workspace = true license.workspace = true +publish = ["crates-io"] +include = [ + "src/**/*.rs", + "examples/**/*.rs", + "README.md", + "LICENSE-MIT", + "LICENSE-APACHE", + "!benches/**", + "!tests/**", +] +autobenches = false + +[lib] +bench = false + +[features] +fuzzing = [] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +blake2 = "0.10.6" blake3 = "1.8.2" byteorder = "1.5.0" chacha20poly1305 = "0.11.0" @@ -20,15 +38,21 @@ digest = "0.11.3" fastcdc = "4.0.1" indexmap = "2.11.0" integer-encoding = "4.1.0" +libc = "0.2.178" pkcs8 = { version = "0.11.0", features = ["pem", "alloc"] } ro-crate-rs = { version = "0.5.1", default-features = false } +rustix = { version = "1.1.4", features = ["fs"] } shake = "0.1.0" thiserror = { workspace = true } tracing = { workspace = true } -walkdir = "2.5.0" x25519-dalek = { workspace = true } zip = { version = "8.6.0", default-features = false, features = ["deflate", "time"] } zstd = "0.13.3" +zeroize = "1.8.2" [dev-dependencies] +criterion = "0.8.2" +peak_alloc = "0.3.0" +proptest = "1.6.0" tempfile = "3.21" +trybuild = "1.0.104" diff --git a/crates/pithos_lib/LICENSE-APACHE b/crates/pithos_lib/LICENSE-APACHE new file mode 100644 index 0000000..5372d84 --- /dev/null +++ b/crates/pithos_lib/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [2026] [ArunaEngine Team] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/crates/pithos_lib/LICENSE-MIT b/crates/pithos_lib/LICENSE-MIT new file mode 100644 index 0000000..deb2b2d --- /dev/null +++ b/crates/pithos_lib/LICENSE-MIT @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2024 ArunaEngine Team + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/crates/pithos_lib/README.md b/crates/pithos_lib/README.md index 5c397a1..6f663f6 100644 --- a/crates/pithos_lib/README.md +++ b/crates/pithos_lib/README.md @@ -1,95 +1,84 @@ -[![Rust](https://img.shields.io/badge/built_with-Rust-dca282.svg)](https://www.rust-lang.org/) -[![License](https://img.shields.io/badge/License-MIT-brightgreen.svg)](https://github.com/arunaengine/aruna-file/blob/main/LICENSE-MIT) -[![License](https://img.shields.io/badge/License-APACHE-brightgreen.svg)](https://github.com/arunaengine/aruna-file/blob/main/LICENSE-APACHE) -[![Codecov](https://codecov.io/github/ArunaStorage/aruna-file/coverage.svg?branch=main)](https://codecov.io/gh/ArunaStorage/aruna-file) -[![Dependency status](https://deps.rs/repo/github/ArunaStorage/aruna-file/status.svg)](https://deps.rs/repo/github/ArunaStorage/aruna-file) -___ - # Pithos library -A library for creating, handling and transforming Pithos files, an object storage optimized file format for Research Data Management (RDM). +`pithos_lib` is the public Rust API for creating, opening, reading, extending, extracting, and adapting encrypted Pithos archives. It has no network client or transport policy. -For the formal file specification click [here](../../spec/PITHOS_1.0.0_draft.md). +## Installation -## Guidance +Add the crate to an application using Rust 1.88 or newer: -Short guidance for usage of the `PithosWriter` and similarly for the `PithosReader` custom component. -Both components provide convenience functionality to write individual Pithos file components or read them again efficiently. +```toml +[dependencies] +pithos_lib = "0.8" +``` -PithosWriter Example: -```rust -use x25519_dalek::{PublicKey, StaticSecret}; +## Create an archive -// Create a simple Pithos file with a single entry -let sender_key = StaticSecret::from([0u8; 32]); -let recipient_key = PublicKey::from([1u8; 32]); +This example encrypts a local file for its owner. The recipient public key is derived from the private key only for a self-contained example; production applications should provide the intended recipients' public keys. -// Dummy input file -let input_file = InputFile { - file_type: FileType::Data, - inner_path: "very_important.txt".to_string(), - data: Content::File("tests/data/t8.shakespeare.sample.txt".to_string()), - metadata: None, - encrypt: true, - compression_level: Some(7), +```rust,no_run +use pithos_lib::archive::{ + ArchivePath, ArchiveWriter, EntryMetadata, ProcessingOptions, WriteOptions, }; -let temp_dir = TempDir::new().unwrap(); -let outfile = File::create(temp_dir.path().join("example.pith")).unwrap(); - -// Process -let mut writer = PithosWriter::new(sender_key, vec![recipient_key], None, Box::new(outfile)).unwrap(); -writer.write_file_header().unwrap(); -writer.process_input(input_file).unwrap(); -writer.write_directory().unwrap(); +use pithos_lib::crypto::PrivateKey; +use std::fs::File; + +fn main() -> Result<(), Box> { + let owner = PrivateKey::from_private_pem_bytes(&std::fs::read("owner.pem")?)?; + let recipient = owner.public_key(); + let input = File::open("report.txt")?; + let size = input.metadata()?.len(); + let output = File::create("report.pith")?; + + let mut writer = ArchiveWriter::create(output, WriteOptions::new(owner, vec![recipient]))?; + writer.add_file( + ArchivePath::new("report.txt")?, + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::default(), + Some(size), + input, + )?; + writer.finish()?; + Ok(()) +} ``` -### RO-Crate ingestion +See [`examples/create.rs`](examples/create.rs) for the maintained version and the [CLI guide](../pithos/README.md) for a command-line workflow. -Load an RO-Crate directory and process it with a configured `PithosWriter`: +## Public API -```rust -use pithos_lib::helpers::ro_crate::read_ro_crate_directory; +- Create archives with `archive::ArchiveWriter`, `archive::WriteOptions`, and `crypto` keys. +- Open an immutable `archive::Archive` from a `source::ArchiveSource`, such as `source::FileSource` or `source::MemorySource`. +- Read a complete entry with `copy_to` or a checked range with `copy_range_to`. +- Use `fs::extract`, `fs::append_files`, `fs::grant_readers`, and `fs::ingest` for Linux host operations. +- Use `adapters::ro_crate` for local directory/ZIP conversion and `adapters::crypt4gh::export` for Crypt4GH output. -let loaded = read_ro_crate_directory("path/to/ro-crate")?; -writer.process_ro_crate(&loaded)?; -``` +Compiled examples are included with the package: [`create`](examples/create.rs), [`open_list`](examples/open_list.rs), [`read_ranges`](examples/read_ranges.rs), [`extract`](examples/extract.rs), [`append`](examples/append.rs), [`grant_readers`](examples/grant_readers.rs), [`ro_crate`](examples/ro_crate.rs), and [`crypt4gh`](examples/crypt4gh.rs). They are local-only and return errors to their caller. -ZIP archives use the same writer operation: +Version 0.8 deliberately removes old model, helper, and wire-record access from the public API. Use the typed archive API instead. The [format draft](https://github.com/arunaengine/pithos/blob/main/spec/PITHOS_1.0.0_draft.md) is background information, not a complete interoperability guarantee. -```rust -use pithos_lib::helpers::ro_crate::read_ro_crate_zip; +## Read and extract semantics -let loaded = read_ro_crate_zip("path/to/ro-crate.zip")?; -writer.process_ro_crate(&loaded)?; -``` +Opening validates archive structure and accessible metadata. Payload integrity is lazy: each block is verified when it is read. `copy_to` and `copy_range_to` do not write bytes from a failed block, but a generic sink can retain earlier verified blocks if a later block fails. `PithosError::ContentUnavailable` means the entry is present but no supplied key can access its content; it is different from an integrity or corruption error. + +`fs::extract` is Linux-only. It walks destination components without following symlinks, stages regular-file output anonymously with Linux `O_TMPFILE`, and publishes without replacing an existing entry. The destination filesystem must support `O_TMPFILE` and `linkat(AT_EMPTY_PATH)`; extraction can therefore fail on a collision rather than overwrite a destination. + +## RO-Crate ingestion -`loaded.ro_crate` is the upstream `ro-crate-rs` graph. Conversion stores the original -`ro-crate-metadata.json` bytes instead of reserializing that graph, and every physical regular -data file references the metadata entry. ZIP conversion streams archive members directly and -does not extract them first. - -The upstream parser accepts RO-Crate 1.1 and 1.2 metadata. Pithos uses its warning-level -vocabulary checks, not full RO-Crate conformance validation. The upstream root deserializer is -strict: the root Dataset must contain `@id`, `@type`, `name`, `description`, `datePublished`, and -`license`. - -### Breaking migration - -| Removed API | Replacement | -| --- | --- | -| `rocrate::ROCrate` | `pithos_lib::helpers::ro_crate::RoCrate` re-export from `rocraters` | -| `ROCrate.base_path` | `LoadedRoCrate.source` | -| `data_entities()` | Match `GraphVector::DataEntity` in `loaded.ro_crate.graph` | -| `contextual_entities()` | Match `GraphVector::ContextualEntity` | -| `ROCrateBuilder` | Upstream public structs/graph construction; no Pithos compatibility builder | -| Local validation levels/reports | Upstream read validation behavior | -| Local directory/ZIP reader traits | `read_ro_crate_directory` and `read_ro_crate_zip` | - -PithosReader Example: -```rust -let pithos_file = PathBuf::from("example.pith"); -let mut reader = PithosReaderSimple::new(pithos_file, key_pem).unwrap(); -let (directory, _) = reader.read_directory().unwrap(); - -println!("{:#?}", reader.read_file_paths(&directory).unwrap()); +Load a directory or ZIP RO-Crate, then convert its retained source through a configured writer: + +```rust,no_run +use pithos_lib::adapters::ro_crate::{read_ro_crate_directory, write_ro_crate}; +use pithos_lib::archive::ProcessingOptions; + +# fn convert(writer: &mut pithos_lib::archive::ArchiveWriter) -> Result<(), Box { +let loaded = read_ro_crate_directory("path/to/ro-crate")?; +write_ro_crate(writer, loaded, ProcessingOptions::default())?; +# Ok(()) +# } ``` + +The upstream parser accepts RO-Crate 1.1 and 1.2 metadata without upstream validation or warning emission; Pithos then applies its own source, path, limit, metadata, and conversion policy. Conversion stores the inspected `ro-crate-metadata.json` bytes rather than reserializing the graph. ZIP conversion streams retained members and does not extract them first. + +## Platform behavior + +Filesystem ingestion, extraction, append, and grants are Linux-only in 0.8. Append uses a cooperating-writer advisory lock and can request `AppendDurability::SyncAll`; it attempts to truncate a failed child directory, but it cannot promise rollback after an unrecoverable host or power failure. diff --git a/crates/pithos_lib/examples/append.rs b/crates/pithos_lib/examples/append.rs new file mode 100644 index 0000000..7ef0b0b --- /dev/null +++ b/crates/pithos_lib/examples/append.rs @@ -0,0 +1,20 @@ +//! Append filesystem entries in a durable Linux filesystem operation. + +use pithos_lib::archive::{AppendDurability, AppendOptions}; +use pithos_lib::crypto::PrivateKey; +use pithos_lib::fs::append_files; +use std::path::{Path, PathBuf}; + +fn main() -> Result<(), Box> { + let owner = PrivateKey::from_private_pem_bytes(&std::fs::read("owner.pem")?)?; + let recipient = owner.public_key(); + let inputs = vec![PathBuf::from("new-report.txt")]; + let observation = append_files( + Path::new("report.pith"), + AppendOptions::new(owner, vec![recipient]).with_durability(AppendDurability::SyncAll), + &inputs, + )?; + + println!("archive grew to {} bytes", observation.final_archive_bytes); + Ok(()) +} diff --git a/crates/pithos_lib/examples/create.rs b/crates/pithos_lib/examples/create.rs new file mode 100644 index 0000000..0a88243 --- /dev/null +++ b/crates/pithos_lib/examples/create.rs @@ -0,0 +1,26 @@ +//! Create an encrypted archive containing one local file. + +use pithos_lib::archive::{ + ArchivePath, ArchiveWriter, EntryMetadata, ProcessingOptions, WriteOptions, +}; +use pithos_lib::crypto::PrivateKey; +use std::fs::File; + +fn main() -> Result<(), Box> { + let owner = PrivateKey::from_private_pem_bytes(&std::fs::read("owner.pem")?)?; + let recipient = owner.public_key(); + let input = File::open("report.txt")?; + let size = input.metadata()?.len(); + let output = File::create("report.pith")?; + + let mut writer = ArchiveWriter::create(output, WriteOptions::new(owner, vec![recipient]))?; + writer.add_file( + ArchivePath::new("report.txt")?, + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::default(), + Some(size), + input, + )?; + writer.finish()?; + Ok(()) +} diff --git a/crates/pithos_lib/examples/crypt4gh.rs b/crates/pithos_lib/examples/crypt4gh.rs new file mode 100644 index 0000000..79f9b47 --- /dev/null +++ b/crates/pithos_lib/examples/crypt4gh.rs @@ -0,0 +1,20 @@ +//! Export one readable archive entry as a Crypt4GH stream. + +use pithos_lib::adapters::crypt4gh::export; +use pithos_lib::archive::{AccessKeys, Archive, OpenOptions}; +use pithos_lib::crypto::{PrivateKey, PublicKey}; +use pithos_lib::source::FileSource; +use std::fs::File; + +fn main() -> Result<(), Box> { + let archive_reader = PrivateKey::from_private_pem_bytes(&std::fs::read("reader.pem")?)?; + let crypt4gh_reader = PublicKey::from_public_pem_bytes(&std::fs::read("reader.pub.pem")?)?; + let archive = Archive::open( + FileSource::open("report.pith")?, + OpenOptions::default().with_access_keys(AccessKeys::new().with_key(archive_reader)), + )?; + let mut output = File::create("report.txt.c4gh")?; + + export(&archive, "report.txt", vec![crypt4gh_reader], &mut output)?; + Ok(()) +} diff --git a/crates/pithos_lib/examples/extract.rs b/crates/pithos_lib/examples/extract.rs new file mode 100644 index 0000000..dd0387c --- /dev/null +++ b/crates/pithos_lib/examples/extract.rs @@ -0,0 +1,18 @@ +//! Extract one archive entry through the Linux no-follow filesystem adapter. + +use pithos_lib::archive::{AccessKeys, Archive, OpenOptions}; +use pithos_lib::crypto::PrivateKey; +use pithos_lib::fs::extract; +use pithos_lib::source::FileSource; +use std::path::Path; + +fn main() -> Result<(), Box> { + let reader = PrivateKey::from_private_pem_bytes(&std::fs::read("reader.pem")?)?; + let archive = Archive::open( + FileSource::open("report.pith")?, + OpenOptions::default().with_access_keys(AccessKeys::new().with_key(reader)), + )?; + + extract(&archive, "report.txt", Path::new("extracted"))?; + Ok(()) +} diff --git a/crates/pithos_lib/examples/grant_readers.rs b/crates/pithos_lib/examples/grant_readers.rs new file mode 100644 index 0000000..b351870 --- /dev/null +++ b/crates/pithos_lib/examples/grant_readers.rs @@ -0,0 +1,19 @@ +//! Grant an additional recipient access to selected archive entry IDs. + +use pithos_lib::archive::{AppendDurability, AppendOptions}; +use pithos_lib::crypto::{PrivateKey, PublicKey}; +use pithos_lib::fs::grant_readers; +use std::path::Path; + +fn main() -> Result<(), Box> { + let owner = PrivateKey::from_private_pem_bytes(&std::fs::read("owner.pem")?)?; + let new_reader = PublicKey::from_public_pem_bytes(&std::fs::read("new-reader.pub.pem")?)?; + let observation = grant_readers( + Path::new("report.pith"), + AppendOptions::new(owner, vec![new_reader]).with_durability(AppendDurability::SyncAll), + &[0], + )?; + + println!("archive grew to {} bytes", observation.final_archive_bytes); + Ok(()) +} diff --git a/crates/pithos_lib/examples/open_list.rs b/crates/pithos_lib/examples/open_list.rs new file mode 100644 index 0000000..ed36e66 --- /dev/null +++ b/crates/pithos_lib/examples/open_list.rs @@ -0,0 +1,18 @@ +//! Open an archive with a recipient key and list its entries. + +use pithos_lib::archive::{AccessKeys, Archive, OpenOptions}; +use pithos_lib::crypto::PrivateKey; +use pithos_lib::source::FileSource; + +fn main() -> Result<(), Box> { + let reader = PrivateKey::from_private_pem_bytes(&std::fs::read("reader.pem")?)?; + let archive = Archive::open( + FileSource::open("report.pith")?, + OpenOptions::default().with_access_keys(AccessKeys::new().with_key(reader)), + )?; + + for entry in archive.entries() { + println!("{}: {:?}", entry.path, entry.kind); + } + Ok(()) +} diff --git a/crates/pithos_lib/examples/read_ranges.rs b/crates/pithos_lib/examples/read_ranges.rs new file mode 100644 index 0000000..37744d4 --- /dev/null +++ b/crates/pithos_lib/examples/read_ranges.rs @@ -0,0 +1,19 @@ +//! Copy a complete archive entry and a half-open byte range. + +use pithos_lib::archive::{AccessKeys, Archive, OpenOptions}; +use pithos_lib::crypto::PrivateKey; +use pithos_lib::source::FileSource; + +fn main() -> Result<(), Box> { + let reader = PrivateKey::from_private_pem_bytes(&std::fs::read("reader.pem")?)?; + let archive = Archive::open( + FileSource::open("report.pith")?, + OpenOptions::default().with_access_keys(AccessKeys::new().with_key(reader)), + )?; + let mut complete = std::fs::File::create("report.txt")?; + archive.copy_to("report.txt", &mut complete)?; + + let mut range = std::fs::File::create("report-prefix.txt")?; + archive.copy_range_to("report.txt", 0..1024, &mut range)?; // Bytes [0, 1024). + Ok(()) +} diff --git a/crates/pithos_lib/examples/ro_crate.rs b/crates/pithos_lib/examples/ro_crate.rs new file mode 100644 index 0000000..ef0a471 --- /dev/null +++ b/crates/pithos_lib/examples/ro_crate.rs @@ -0,0 +1,18 @@ +//! Convert a local RO-Crate directory into a Pithos archive. + +use pithos_lib::adapters::ro_crate::{read_ro_crate_directory, write_ro_crate}; +use pithos_lib::archive::{ArchiveWriter, ProcessingOptions, WriteOptions}; +use pithos_lib::crypto::PrivateKey; +use std::fs::File; + +fn main() -> Result<(), Box> { + let owner = PrivateKey::from_private_pem_bytes(&std::fs::read("owner.pem")?)?; + let recipient = owner.public_key(); + let loaded = read_ro_crate_directory("research-crate")?; + let output = File::create("research-crate.pith")?; + let mut writer = ArchiveWriter::create(output, WriteOptions::new(owner, vec![recipient]))?; + + write_ro_crate(&mut writer, loaded, ProcessingOptions::default())?; + writer.finish()?; + Ok(()) +} diff --git a/crates/pithos_lib/tests/public_api.rs b/crates/pithos_lib/tests/public_api.rs new file mode 100644 index 0000000..b122060 --- /dev/null +++ b/crates/pithos_lib/tests/public_api.rs @@ -0,0 +1,71 @@ +use pithos_lib::adapters::{crypt4gh::Crypt4GHError, ro_crate::RoCrateError}; +use pithos_lib::archive::{ + AccessKeys, AppendDurability, AppendObservation, AppendOptions, Archive, ArchivePath, + ArchiveReference, ArchiveWriter, CdcConfig, CreateError, EntryKind, EntryMetadata, + EntryReference, ExternalBlockResolver, ExternalLocation, FinishError, IncompleteWriter, + NoExternalBlocks, OpenLimits, OpenOptions, ProcessingOptions, WriteOptions, WriterError, + WrittenEntry, +}; +use pithos_lib::crypto::{CryptoError, PrivateKey, PublicKey}; +use pithos_lib::error::PithosError; +use pithos_lib::fs::{FsError, ingest::InputManifest}; +use pithos_lib::source::{ArchiveSource, FileSource, MemorySource, SourceError}; + +fn assert_send_sync() {} + +#[derive(Clone, Copy)] +struct SendSyncResolver; + +impl ExternalBlockResolver for SendSyncResolver { + fn resolve( + &self, + _location: &ExternalLocation, + _expected_len: u64, + _max_response_size: u64, + ) -> Result, PithosError> { + Err(PithosError::ExternalBlockSourceRequired) + } +} + +#[test] +fn standard_archive_configurations_are_send_and_sync() { + assert_send_sync::(); + assert_send_sync::(); + assert_send_sync::>(); + assert_send_sync::>(); + let _ = OpenOptions::default().with_access_keys(AccessKeys::new()); +} + +#[test] +fn selected_08_api_imports_compile() { + let _ = std::any::TypeId::of::(); + let _ = std::any::TypeId::of::(); + let _ = std::any::TypeId::of::(); + let _ = std::any::TypeId::of::(); + let _ = std::any::TypeId::of::(); + let _ = std::any::TypeId::of::>>(); + let _ = std::any::TypeId::of::(); + let _ = std::any::TypeId::of::>>(); + let _ = std::any::TypeId::of::(); + let _ = std::any::TypeId::of::(); + let _ = std::any::TypeId::of::(); + let _ = std::any::TypeId::of::>>(); + let _ = std::any::TypeId::of::>>(); + let _ = std::any::TypeId::of::(); + let _ = std::any::TypeId::of::(); + let _ = std::any::TypeId::of::(); + let _ = std::any::TypeId::of::(); + let _ = std::any::TypeId::of::(); + let _ = std::any::TypeId::of::(); + let _ = std::any::TypeId::of::(); + let _ = std::any::TypeId::of::(); + let _ = std::any::TypeId::of::(); + let _ = std::any::TypeId::of::(); + let _ = std::any::TypeId::of::(); + let _ = std::any::TypeId::of::(); + let _ = std::any::TypeId::of::(); + + fn assert_source() {} + assert_source::(); + assert_source::(); +} diff --git a/crates/pithos_pyo3/Cargo.toml b/crates/pithos_pyo3/Cargo.toml index fcfd9f7..64bcb96 100644 --- a/crates/pithos_pyo3/Cargo.toml +++ b/crates/pithos_pyo3/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "pithos_pyo3" -description = "Client for the Pithos object storage file format" -version = "0.7.0" +description = "Unpublished, nonfunctional stub reserved for a possible future Python binding" +version = "0.8.0" publish = false authors.workspace = true edition.workspace = true @@ -10,4 +10,4 @@ repository.workspace = true license.workspace = true [dependencies] -pithos_lib = { version = "0.7.3", path = "../pithos_lib" } +pithos_lib = { version = "0.8.0", path = "../pithos_lib" } diff --git a/crates/pithos_pyo3/README.md b/crates/pithos_pyo3/README.md index 9a3ee6b..52224c6 100644 --- a/crates/pithos_pyo3/README.md +++ b/crates/pithos_pyo3/README.md @@ -1,7 +1,5 @@ -# Pithos Python bindings +# pithos_pyo3 status -[![Rust](https://img.shields.io/badge/built_with-Rust-dca282.svg)](https://www.rust-lang.org/) -[![License](https://img.shields.io/badge/License-MIT-brightgreen.svg)](https://github.com/arunaengine/aruna-file/blob/main/LICENSE-MIT) -[![License](https://img.shields.io/badge/License-APACHE-brightgreen.svg)](https://github.com/arunaengine/aruna-file/blob/main/LICENSE-APACHE) +`pithos_pyo3` is an unpublished empty Rust library stub in this workspace. It has no PyO3 dependency, Python extension module, Python package, or supported Python API. -`pithos_pyo3` is the planned Python interface for Pithos. This crate is a work in progress and will be implemented once `pithos_lib` has reached a stable state. +Use [pithos_lib on docs.rs](https://docs.rs/pithos_lib/) for the current Rust API. This README intentionally makes no installation, binding, or release claim for `pithos_pyo3`. From f7a19cd70e204e914afe16b9c71ee4856afca3cd Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Wed, 12 Aug 2026 00:13:48 +0200 Subject: [PATCH 012/166] test: add benchmark harnesses for adapters, archive I/O, path operations, and append scenarios --- crates/pithos_lib/Cargo.toml | 20 ++ crates/pithos_lib/benches/README.md | 93 +++++ crates/pithos_lib/benches/adapters.rs | 218 ++++++++++++ crates/pithos_lib/benches/append.rs | 182 ++++++++++ crates/pithos_lib/benches/archive_io.rs | 272 ++++++++++++++ crates/pithos_lib/benches/archive_path.rs | 256 +++++++++++++ crates/pithos_lib/benches/block_pipeline.rs | 140 ++++++++ crates/pithos_lib/benches/support.rs | 109 ++++++ crates/pithos_lib/benches/workloads.rs | 336 ++++++++++++++++++ .../pithos_lib/tests/benchmark_contracts.rs | 181 ++++++++++ 10 files changed, 1807 insertions(+) create mode 100644 crates/pithos_lib/benches/README.md create mode 100644 crates/pithos_lib/benches/adapters.rs create mode 100644 crates/pithos_lib/benches/append.rs create mode 100644 crates/pithos_lib/benches/archive_io.rs create mode 100644 crates/pithos_lib/benches/archive_path.rs create mode 100644 crates/pithos_lib/benches/block_pipeline.rs create mode 100644 crates/pithos_lib/benches/support.rs create mode 100644 crates/pithos_lib/benches/workloads.rs create mode 100644 crates/pithos_lib/tests/benchmark_contracts.rs diff --git a/crates/pithos_lib/Cargo.toml b/crates/pithos_lib/Cargo.toml index 90ab358..85c7b4e 100644 --- a/crates/pithos_lib/Cargo.toml +++ b/crates/pithos_lib/Cargo.toml @@ -56,3 +56,23 @@ peak_alloc = "0.3.0" proptest = "1.6.0" tempfile = "3.21" trybuild = "1.0.104" + +[[bench]] +name = "archive_path" +harness = false + +[[bench]] +name = "block_pipeline" +harness = false + +[[bench]] +name = "archive_io" +harness = false + +[[bench]] +name = "append" +harness = false + +[[bench]] +name = "adapters" +harness = false diff --git a/crates/pithos_lib/benches/README.md b/crates/pithos_lib/benches/README.md new file mode 100644 index 0000000..6658c3c --- /dev/null +++ b/crates/pithos_lib/benches/README.md @@ -0,0 +1,93 @@ +# Benchmarks + +This guide is for maintainers measuring `pithos_lib` performance. Local commands provide smoke coverage; only evidence from the owner-selected self-hosted runner is suitable for release comparison. + +## Quick developer smoke + +Build every explicit target without running it: + +```bash +cargo bench --locked -p pithos_lib --no-run +``` + +Run individual short local checks from the workspace root with the locked dependency graph: + +```bash +cargo bench --locked -p pithos_lib --bench archive_path -- --quick +PITHOS_BENCH_100K=1 cargo bench --locked -p pithos_lib --bench archive_path -- --quick +cargo bench --locked -p pithos_lib --bench block_pipeline -- --quick +cargo bench --locked -p pithos_lib --bench archive_io -- --quick +cargo bench --locked -p pithos_lib --bench append -- --quick +cargo bench --locked -p pithos_lib --bench adapters -- --quick +``` + +`archive_path` covers flat and duplicate 1K/10K creation, opt-in 100K creation, +deep 10/100/1K paths, both 1K hierarchy-conflict directions, ancestor-first and +descendant-first incremental insertion, 100/1K metadata/data pairs with references, +100-entry flat archive open/index validation, and filesystem extraction of every file in a +100-entry archive. `block_pipeline` covers +compressible and high-entropy transforms plus an 8 MiB virtual-stream workload; +set `PITHOS_BENCH_LARGE_STREAM=1` to include its 64 MiB variant. The virtual reader +and counting sink never retain the logical input or archive output. + +`archive_io` builds a real two-generation temporary archive and uses `FileSource` for open/merge, +sequential full reads, and prefix, middle, tiny, and full ranges. Its counting source +reports the read calls and requested bytes actually made by the archive reader. +`append` uses Criterion batched setup to build the base archive, input file, and keys +before timing only the supported locked direct append. It includes new and duplicate +content. Its observation reports reads from the actual existing-archive source used +to construct the append snapshot; output bytes are the appended tail, not final archive size. +The append target also measures one final append against prepared 1-, 10-, and 100-generation +archives. Generation construction and final input creation are setup rather than measured work. +`adapters` loads a directory RO-Crate with a content member and a ZIP RO-Crate with a deterministic +1 MiB ordinary member before timing conversion through `write_ro_crate` and writer finalization. +It also opens a verified archive entry before timing its Crypt4GH export using the current GA4GH +header KDF. The adapter target uses counted sinks, so output bytes remain observable without +retaining a second full output copy. + +`cargo test --locked -p pithos_lib --test benchmark_contracts -- --nocapture --test-threads=1` +enforces these workload shapes without runtime thresholds. The streamed-memory test emits a +machine-readable line containing both measured peaks, their delta, output sizes, and bound. + +Keys, input payloads, archive fixtures, and temporary directories are setup, not timed, +unless creation itself is the named workload. Each release-comparable resource record is +one `pithos-bench-v1` JSON object with exactly `workload`, `setup`, +`peak_heap_bytes`, `archive_bytes`, `output_bytes`, `source_read_count`, +`source_read_bytes`, `bytes_appended`, and `dedup_signal`. All six metrics must be +finite non-negative numbers; the setup and deduplication signal are stable strings. +Extra timing fields and `null` observations are not valid release evidence. Criterion's +`target/criterion/**/new/estimates.json` remains the runtime distribution source. + +## Release workflow + +The release comparison authority is: + +```bash +python3 .github/release/check.py benchmark compare BASELINE CURRENT REPORT.json +``` + +Exit `0` means the evidence is valid and within the gate. Exit `3` means the evidence is valid +but has a material regression and must use the protected approval path. Any other failure, +including missing, malformed, incompatible, or invalid evidence, exits `1` for validation +failure (invalid top-level command paths exit `2`). Its single 5% materiality threshold applies +to Criterion median estimates and every numeric resource field; exactly 5% is allowed, while +any value greater than 5% is a regression. Changed `dedup_signal`, changed setup, missing +observations, or changed workload sets are also rejected. Do not use a hosted runner for a +release comparison. + +The baseline and current `environment/machine.txt` files must be byte-for-byte identical, +including machine and toolchain evidence. Runner identity, workload sets, and resource setup +must otherwise match exactly. + +This strict resource schema is a new reviewed-evidence requirement. An older owner-approved +artifact that does not satisfy it is not silently invalidated: capture a new baseline candidate, +review it, and use its exact artifact identity for subsequent comparisons. + +Dispatch the `Benchmarks` workflow with an owner-selected self-hosted runner label. For a +comparison, provide the reviewed successful baseline workflow run ID and its exact +`benchmark-current---` artifact name. The workflow records the runner, +machine and toolchain details, commit, lockfile digest, benchmark stdout, and Criterion output; +it downloads the immutable reviewed baseline artifact, compares it with current evidence, and +uploads both a comparison artifact and the current evidence artifact. The current artifact is +retained for 90 days and can become a reviewed baseline for a later run. Use +`capture_baseline` only to create a baseline candidate; it creates no release disposition. diff --git a/crates/pithos_lib/benches/adapters.rs b/crates/pithos_lib/benches/adapters.rs new file mode 100644 index 0000000..69c6aaf --- /dev/null +++ b/crates/pithos_lib/benches/adapters.rs @@ -0,0 +1,218 @@ +mod support; + +use criterion::{BatchSize, Criterion, criterion_group, criterion_main}; +use pithos_lib::adapters::crypt4gh; +use pithos_lib::adapters::ro_crate::{ + LoadedRoCrate, read_ro_crate_directory, read_ro_crate_zip, write_ro_crate, +}; +use pithos_lib::archive::{ + AccessKeys, Archive, ArchivePath, ArchiveWriter, CdcConfig, EntryMetadata, OpenOptions, + ProcessingOptions, WriteOptions, +}; +use pithos_lib::crypto::{PrivateKey, PublicKey}; +use pithos_lib::source::FileSource; +use std::hint::black_box; +use std::io::{Cursor, Write}; +use std::path::{Path, PathBuf}; + +const DIRECTORY_PAYLOAD_BYTES: usize = 64 * 1024; +const LARGE_MEMBER_BYTES: usize = 1024 * 1024; +const ARCHIVE_ENTRY_PATH: &str = "verified.bin"; + +struct CrateFixture { + temporary: tempfile::TempDir, + path: PathBuf, +} + +struct ArchiveFixture { + temporary: tempfile::TempDir, + path: PathBuf, + recipient: PrivateKey, + archive_bytes: u64, +} + +fn metadata(member: &str) -> String { + format!( + r#"{{"@context":"https://w3id.org/ro/crate/1.2/context","@graph":[{{"@id":"ro-crate-metadata.json","@type":"CreativeWork","conformsTo":{{"@id":"https://w3id.org/ro/crate/1.2"}},"about":{{"@id":"./"}}}},{{"@id":"./","@type":"Dataset","name":"Adapter benchmark crate","description":"Deterministic adapter benchmark input","datePublished":"2024-01-01","license":"MIT","hasPart":[{{"@id":"{member}"}}]}},{{"@id":"{member}","@type":"File"}}]}}"# + ) +} + +fn directory_fixture() -> CrateFixture { + let temporary = tempfile::tempdir().unwrap(); + let path = temporary.path().join("directory-crate"); + std::fs::create_dir(&path).unwrap(); + std::fs::write(path.join("ro-crate-metadata.json"), metadata("content.bin")).unwrap(); + std::fs::write( + path.join("content.bin"), + support::deterministic_bytes(1, DIRECTORY_PAYLOAD_BYTES), + ) + .unwrap(); + CrateFixture { temporary, path } +} + +fn zip_fixture() -> CrateFixture { + let temporary = tempfile::tempdir().unwrap(); + let path = temporary.path().join("large-crate.zip"); + let payload = support::deterministic_bytes(1, LARGE_MEMBER_BYTES); + let mut zip = zip::ZipWriter::new(std::fs::File::create(&path).unwrap()); + let options = zip::write::SimpleFileOptions::default() + .compression_method(zip::CompressionMethod::Deflated); + zip.start_file("ro-crate-metadata.json", options).unwrap(); + zip.write_all(metadata("large.bin").as_bytes()).unwrap(); + zip.start_file("large.bin", options).unwrap(); + zip.write_all(&payload).unwrap(); + zip.finish().unwrap(); + CrateFixture { temporary, path } +} + +fn write_archive_fixture(path: &Path, sender: PrivateKey, recipient: PublicKey) { + let payload = support::deterministic_bytes(1, LARGE_MEMBER_BYTES); + let mut writer = ArchiveWriter::create( + std::fs::File::create(path).unwrap(), + WriteOptions::new(sender, vec![recipient]).with_cdc(CdcConfig::new(64, 256, 1024).unwrap()), + ) + .unwrap(); + writer + .add_file( + ArchivePath::new(ARCHIVE_ENTRY_PATH).unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::default(), + Some(payload.len() as u64), + Cursor::new(payload), + ) + .unwrap(); + writer.finish().unwrap(); +} + +fn archive_fixture() -> ArchiveFixture { + let temporary = tempfile::tempdir().unwrap(); + let path = temporary.path().join("verified.pith"); + let sender = PrivateKey::generate(); + let recipient = PrivateKey::generate(); + write_archive_fixture(&path, sender, recipient.public_key()); + let archive_bytes = std::fs::metadata(&path).unwrap().len(); + ArchiveFixture { + temporary, + path, + recipient, + archive_bytes, + } +} + +fn convert_ro_crate( + loaded: LoadedRoCrate, + sender: PrivateKey, + recipient: PublicKey, + cdc: CdcConfig, +) -> u64 { + let mut writer = ArchiveWriter::create( + support::CountingSink::default(), + WriteOptions::new(sender, vec![recipient]).with_cdc(cdc), + ) + .unwrap(); + write_ro_crate(&mut writer, loaded, ProcessingOptions::default()).unwrap(); + writer.finish().unwrap().0 +} + +fn open_archive(fixture: &ArchiveFixture) -> Archive { + Archive::open( + FileSource::open(&fixture.path).unwrap(), + OpenOptions::default() + .with_access_keys(AccessKeys::new().with_key(fixture.recipient.duplicate())), + ) + .unwrap() +} + +fn export_crypt4gh( + archive: Archive, + recipients: Vec, + mut sink: support::CountingSink, +) -> u64 { + crypt4gh::export(&archive, ARCHIVE_ENTRY_PATH, recipients, &mut sink).unwrap(); + sink.0 +} + +fn bench_adapters(c: &mut Criterion) { + let sender = PrivateKey::generate(); + let recipient = sender.public_key(); + let cdc = CdcConfig::new(64, 256, 1024).unwrap(); + + let directory = directory_fixture(); + let directory_workload = "adapters/ro-crate-directory-conversion"; + let loaded = read_ro_crate_directory(&directory.path).unwrap(); + let directory_sender = sender.duplicate(); + support::measure( + directory_workload, + "directory fixture, keys, and retained RO-Crate source loaded before conversion", + || convert_ro_crate(loaded, directory_sender, recipient, cdc), + |output_bytes| support::Metrics::archive(*output_bytes), + ); + c.bench_function(directory_workload, |b| { + b.iter_batched( + || { + ( + read_ro_crate_directory(&directory.path).unwrap(), + sender.duplicate(), + ) + }, + |(loaded, sender)| black_box(convert_ro_crate(loaded, sender, recipient, cdc)), + BatchSize::SmallInput, + ) + }); + + let zip = zip_fixture(); + let zip_workload = "adapters/ro-crate-zip-large-member"; + let loaded = read_ro_crate_zip(&zip.path).unwrap(); + let zip_sender = sender.duplicate(); + support::measure( + zip_workload, + "1 MiB ZIP fixture, keys, and retained RO-Crate source loaded before bounded conversion", + || convert_ro_crate(loaded, zip_sender, recipient, cdc), + |output_bytes| support::Metrics::archive(*output_bytes), + ); + c.bench_function(zip_workload, |b| { + b.iter_batched( + || (read_ro_crate_zip(&zip.path).unwrap(), sender.duplicate()), + |(loaded, sender)| black_box(convert_ro_crate(loaded, sender, recipient, cdc)), + BatchSize::SmallInput, + ) + }); + + let archive = archive_fixture(); + let export_recipient = PrivateKey::generate().public_key(); + let export_workload = "adapters/crypt4gh-export"; + let opened = open_archive(&archive); + let recipients = vec![export_recipient]; + let sink = support::CountingSink::default(); + support::measure( + export_workload, + "verified archive opened and fresh counted output sink prepared before Crypt4GH export", + || export_crypt4gh(opened, recipients, sink), + |output_bytes| support::Metrics { + archive_bytes: archive.archive_bytes, + output_bytes: *output_bytes, + source_read_count: None, + source_read_bytes: None, + bytes_appended: None, + dedup_signal: None, + }, + ); + c.bench_function(export_workload, |b| { + b.iter_batched( + || { + ( + open_archive(&archive), + vec![export_recipient], + support::CountingSink::default(), + ) + }, + |(opened, recipients, sink)| black_box(export_crypt4gh(opened, recipients, sink)), + BatchSize::SmallInput, + ) + }); + + black_box((&directory.temporary, &zip.temporary, &archive.temporary)); +} + +criterion_group!(adapters, bench_adapters); +criterion_main!(adapters); diff --git a/crates/pithos_lib/benches/append.rs b/crates/pithos_lib/benches/append.rs new file mode 100644 index 0000000..f513a0c --- /dev/null +++ b/crates/pithos_lib/benches/append.rs @@ -0,0 +1,182 @@ +mod support; +mod workloads; + +use criterion::{BatchSize, Criterion, criterion_group, criterion_main}; +use pithos_lib::archive::{ + AppendObservation, AppendOptions, ArchivePath, ArchiveWriter, EntryMetadata, ProcessingOptions, + WriteOptions, +}; +use pithos_lib::crypto::PrivateKey; +use pithos_lib::fs::append_files; +use std::hint::black_box; +use std::io::Cursor; +use std::path::PathBuf; + +const INPUT_BYTES: usize = 1024 * 1024; + +struct AppendContext { + temporary: tempfile::TempDir, + archive_path: PathBuf, + input_path: PathBuf, + sender: PrivateKey, + recipient: pithos_lib::crypto::PublicKey, + base_bytes: u64, + input_bytes: u64, + observation: Option, +} + +fn prepare_append( + sender: &PrivateKey, + recipient: pithos_lib::crypto::PublicKey, + duplicate: bool, +) -> AppendContext { + let temporary = tempfile::tempdir().unwrap(); + let archive_path = temporary.path().join("archive.pith"); + let input_path = temporary.path().join("append.bin"); + let base = support::deterministic_bytes(0, INPUT_BYTES); + let input = if duplicate { + base.clone() + } else { + support::deterministic_bytes(1, INPUT_BYTES) + }; + std::fs::write(&input_path, &input).unwrap(); + let mut writer = ArchiveWriter::create( + std::fs::File::create(&archive_path).unwrap(), + WriteOptions::new(sender.duplicate(), vec![recipient]), + ) + .unwrap(); + writer + .add_file( + ArchivePath::new("base.bin").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::default(), + Some(base.len() as u64), + Cursor::new(base), + ) + .unwrap(); + writer.finish().unwrap(); + let base_bytes = std::fs::metadata(&archive_path).unwrap().len(); + AppendContext { + temporary, + archive_path, + input_path, + sender: sender.duplicate(), + recipient, + base_bytes, + input_bytes: input.len() as u64, + observation: None, + } +} + +fn append_workflow(mut context: AppendContext) -> AppendContext { + let _temporary = &context.temporary; + context.observation = Some( + append_files( + &context.archive_path, + AppendOptions::new(context.sender.duplicate(), vec![context.recipient]), + std::slice::from_ref(&context.input_path), + ) + .unwrap(), + ); + context +} + +fn bench_append(c: &mut Criterion) { + let sender = PrivateKey::generate(); + let recipient = sender.public_key(); + for (name, duplicate) in [("new-content", false), ("deduplicated-content", true)] { + let workload = format!("append/{name}"); + let context = prepare_append(&sender, recipient, duplicate); + support::measure( + &workload, + "base archive, input file, and keys prepared before supported locked direct append", + || append_workflow(context), + |context| { + let final_bytes = std::fs::metadata(&context.archive_path).unwrap().len(); + let bytes_appended = final_bytes - context.base_bytes; + let observation = context.observation.unwrap(); + assert_eq!(observation.base_archive_bytes, context.base_bytes); + assert_eq!(observation.final_archive_bytes, final_bytes); + support::Metrics { + archive_bytes: final_bytes, + output_bytes: bytes_appended, + source_read_count: Some(observation.source_read_count), + source_read_bytes: Some(observation.source_read_bytes), + bytes_appended: Some(bytes_appended), + dedup_signal: Some(bytes_appended < context.input_bytes), + } + }, + ); + c.bench_function(&workload, |b| { + b.iter_batched( + || prepare_append(&sender, recipient, duplicate), + |context| black_box(append_workflow(context)), + BatchSize::SmallInput, + ) + }); + } + + for generations in [1usize, 10, 100] { + let workload = format!("append/multigeneration/{generations}"); + let prepare = || { + let fixture = workloads::build_append_scaling_fixture( + sender.duplicate(), + recipient, + generations, + 1, + ); + let input = fixture.temporary.path().join("final-append.bin"); + std::fs::write(&input, b"final append benchmark payload").unwrap(); + (fixture, input) + }; + let (measured_fixture, measured_input) = prepare(); + support::measure( + &workload, + "multigeneration archive and final input prepared before one supported locked direct append", + move || { + let original = measured_fixture.archive_bytes; + let observation = append_files( + &measured_fixture.archive_path, + AppendOptions::new( + measured_fixture.sender.duplicate(), + vec![measured_fixture.recipient], + ), + &[measured_input], + ) + .unwrap(); + (measured_fixture, original, observation) + }, + |(fixture, original, observation)| { + let final_bytes = std::fs::metadata(&fixture.archive_path).unwrap().len(); + let tail_bytes = final_bytes - *original; + support::Metrics { + archive_bytes: final_bytes, + output_bytes: tail_bytes, + source_read_count: Some(observation.source_read_count), + source_read_bytes: Some(observation.source_read_bytes), + bytes_appended: Some(tail_bytes), + dedup_signal: Some(false), + } + }, + ); + c.bench_function(&workload, |b| { + b.iter_batched( + prepare, + |(fixture, input)| { + black_box( + append_files( + &fixture.archive_path, + AppendOptions::new(fixture.sender.duplicate(), vec![fixture.recipient]), + &[input], + ) + .unwrap(), + ) + }, + BatchSize::SmallInput, + ) + }); + } +} + +criterion_group!(append, bench_append); +criterion_main!(append); diff --git a/crates/pithos_lib/benches/archive_io.rs b/crates/pithos_lib/benches/archive_io.rs new file mode 100644 index 0000000..540d2f5 --- /dev/null +++ b/crates/pithos_lib/benches/archive_io.rs @@ -0,0 +1,272 @@ +mod support; +mod workloads; + +use criterion::{BatchSize, Criterion, criterion_group, criterion_main}; +use pithos_lib::archive::{ + AccessKeys, Archive, ArchivePath, ArchiveWriter, EntryMetadata, OpenOptions, ProcessingOptions, + WriteOptions, +}; +use pithos_lib::crypto::PrivateKey; +use pithos_lib::source::{ArchiveSource, FileSource, SourceError}; +use std::hint::black_box; +use std::io::Cursor; +use std::path::{Path, PathBuf}; +use std::sync::{ + Arc, + atomic::{AtomicU64, Ordering}, +}; + +const PAYLOAD_LEN: usize = 1024 * 1024; + +struct Counters { + reads: AtomicU64, + bytes: AtomicU64, +} + +impl Counters { + fn reset(&self) { + self.reads.store(0, Ordering::Relaxed); + self.bytes.store(0, Ordering::Relaxed); + } + + fn snapshot(&self) -> (u64, u64) { + ( + self.reads.load(Ordering::Relaxed), + self.bytes.load(Ordering::Relaxed), + ) + } +} + +struct CountingSource { + source: FileSource, + counters: Arc, +} + +impl ArchiveSource for CountingSource { + fn len(&self) -> Result { + self.source.len() + } + + fn read_exact_at(&self, offset: u64, buffer: &mut [u8]) -> Result<(), SourceError> { + self.source.read_exact_at(offset, buffer)?; + self.counters.reads.fetch_add(1, Ordering::Relaxed); + self.counters + .bytes + .fetch_add(buffer.len() as u64, Ordering::Relaxed); + Ok(()) + } +} + +struct Fixture { + temporary: tempfile::TempDir, + archive_path: PathBuf, + sender: PrivateKey, + archive_bytes: u64, +} + +fn payload() -> Vec { + (0..PAYLOAD_LEN).map(|index| (index % 251) as u8).collect() +} + +fn write_archive( + path: &Path, + sender: PrivateKey, + recipient: pithos_lib::crypto::PublicKey, + bytes: &[u8], +) { + let mut writer = ArchiveWriter::create( + std::fs::File::create(path).unwrap(), + WriteOptions::new(sender, vec![recipient]), + ) + .unwrap(); + writer + .add_file( + ArchivePath::new("data").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::default(), + Some(bytes.len() as u64), + Cursor::new(bytes), + ) + .unwrap(); + writer.finish().unwrap(); +} + +fn fixture(sender: PrivateKey, recipient: pithos_lib::crypto::PublicKey, bytes: &[u8]) -> Fixture { + let temporary = tempfile::tempdir().unwrap(); + let archive_path = temporary.path().join("archive.pith"); + write_archive(&archive_path, sender.duplicate(), recipient, bytes); + workloads::append_child_generation(&archive_path, temporary.path(), &sender, recipient); + let archive_bytes = std::fs::metadata(&archive_path).unwrap().len(); + Fixture { + temporary, + archive_path, + sender, + archive_bytes, + } +} + +fn open_counting(fixture: &Fixture) -> (Archive, Arc) { + let counters = Arc::new(Counters { + reads: AtomicU64::new(0), + bytes: AtomicU64::new(0), + }); + let source = CountingSource { + source: FileSource::open(&fixture.archive_path).unwrap(), + counters: Arc::clone(&counters), + }; + let archive = Archive::open( + source, + OpenOptions::default() + .with_access_keys(AccessKeys::new().with_key(fixture.sender.duplicate())), + ) + .unwrap(); + counters.reset(); + (archive, counters) +} + +fn read_run(fixture: &Fixture, range: Option>) -> (u64, u64, u64) { + let (archive, counters) = open_counting(fixture); + let mut sink = support::CountingSink::default(); + match range { + Some(range) => archive.copy_range_to("data", range, &mut sink).unwrap(), + None => archive.copy_to("data", &mut sink).unwrap(), + } + let (reads, bytes) = counters.snapshot(); + (sink.0, reads, bytes) +} + +fn bench_archive_io(c: &mut Criterion) { + let sender = PrivateKey::generate(); + let recipient = sender.public_key(); + let payload = payload(); + let fixture = fixture(sender, recipient, &payload); + let archive_bytes = fixture.archive_bytes; + + let create_temporary = tempfile::tempdir().unwrap(); + let create_path = create_temporary.path().join("created.pith"); + support::measure( + "archive_io/sequential-create", + "payload, temporary directory, and key prepared outside create", + || { + write_archive( + &create_path, + fixture.sender.duplicate(), + recipient, + &payload, + ) + }, + |_| support::Metrics::archive(std::fs::metadata(&create_path).unwrap().len()), + ); + c.bench_function("archive_io/sequential-create", |b| { + b.iter_batched( + || (tempfile::tempdir().unwrap(), fixture.sender.duplicate()), + |(temporary, key)| { + let path = temporary.path().join("created.pith"); + write_archive(&path, key, recipient, &payload); + black_box(temporary) + }, + BatchSize::SmallInput, + ) + }); + + support::measure( + "archive_io/open-merge", + "two-generation archive prebuilt; FileSource open and directory-chain merge are the workload", + || { + let counters = Arc::new(Counters { + reads: AtomicU64::new(0), + bytes: AtomicU64::new(0), + }); + let source = CountingSource { + source: FileSource::open(&fixture.archive_path).unwrap(), + counters: Arc::clone(&counters), + }; + let archive = Archive::open( + source, + OpenOptions::default() + .with_access_keys(AccessKeys::new().with_key(fixture.sender.duplicate())), + ) + .unwrap(); + (archive, counters) + }, + |(_, counters)| { + let (reads, bytes) = counters.snapshot(); + assert!( + reads >= 4, + "open-merge must read both directory generations" + ); + support::Metrics { + archive_bytes, + output_bytes: 0, + source_read_count: Some(reads), + source_read_bytes: Some(bytes), + bytes_appended: None, + dedup_signal: None, + } + }, + ); + c.bench_function("archive_io/open-merge", |b| { + b.iter(|| { + let counters = Arc::new(Counters { + reads: AtomicU64::new(0), + bytes: AtomicU64::new(0), + }); + let source = CountingSource { + source: FileSource::open(&fixture.archive_path).unwrap(), + counters, + }; + black_box( + Archive::open( + source, + OpenOptions::default() + .with_access_keys(AccessKeys::new().with_key(fixture.sender.duplicate())), + ) + .unwrap(), + ); + }) + }); + + for (name, range) in [ + ("sequential-full-read", None), + ("range-prefix", Some(0..4096)), + ( + "range-middle", + Some((PAYLOAD_LEN as u64 / 2)..(PAYLOAD_LEN as u64 / 2 + 4096)), + ), + ("range-tiny", Some(17..18)), + ("range-full", Some(0..PAYLOAD_LEN as u64)), + ] { + let workload = format!("archive_io/{name}"); + support::measure( + &workload, + "archive opened from FileSource before read", + || read_run(&fixture, range.clone()), + |(output_bytes, reads, bytes)| support::Metrics { + archive_bytes, + output_bytes: *output_bytes, + source_read_count: Some(*reads), + source_read_bytes: Some(*bytes), + bytes_appended: None, + dedup_signal: None, + }, + ); + c.bench_function(&workload, |b| { + b.iter_batched( + || open_counting(&fixture), + |(archive, _)| { + let mut sink = support::CountingSink::default(); + match range.clone() { + Some(range) => archive.copy_range_to("data", range, &mut sink).unwrap(), + None => archive.copy_to("data", &mut sink).unwrap(), + } + (archive, black_box(sink.0)) + }, + BatchSize::SmallInput, + ) + }); + } + black_box(&fixture.temporary); +} + +criterion_group!(archive_io, bench_archive_io); +criterion_main!(archive_io); diff --git a/crates/pithos_lib/benches/archive_path.rs b/crates/pithos_lib/benches/archive_path.rs new file mode 100644 index 0000000..b791f6a --- /dev/null +++ b/crates/pithos_lib/benches/archive_path.rs @@ -0,0 +1,256 @@ +mod support; +mod workloads; + +use criterion::{BatchSize, Criterion, criterion_group, criterion_main}; +use pithos_lib::archive::{ + AccessKeys, Archive, ArchivePath, ArchiveWriter, EntryMetadata, OpenOptions, ProcessingOptions, + WriteOptions, +}; +use pithos_lib::crypto::PrivateKey; +use pithos_lib::source::MemorySource; +use std::hint::black_box; +use std::io::Cursor; + +const EMPTY: [u8; 0] = []; +const DUPLICATE: [u8; 1] = *b"x"; + +fn writer(sender: PrivateKey, recipient: pithos_lib::crypto::PublicKey) -> ArchiveWriter> { + ArchiveWriter::create(Vec::new(), WriteOptions::new(sender, vec![recipient])).unwrap() +} + +fn add_file(writer: &mut ArchiveWriter>, path: String, bytes: &'static [u8]) { + writer + .add_file( + ArchivePath::new(path).unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(false, 0).unwrap(), + Some(bytes.len() as u64), + Cursor::new(bytes), + ) + .unwrap(); +} + +fn build_flat( + sender: PrivateKey, + recipient: pithos_lib::crypto::PublicKey, + count: usize, +) -> Vec { + let mut writer = writer(sender, recipient); + for index in 0..count { + add_file(&mut writer, format!("entry-{index:06}"), &EMPTY); + } + writer.finish().unwrap() +} + +fn build_duplicate( + sender: PrivateKey, + recipient: pithos_lib::crypto::PublicKey, + count: usize, +) -> Vec { + let mut writer = writer(sender, recipient); + for index in 0..count { + add_file(&mut writer, format!("duplicate-{index:06}"), &DUPLICATE); + } + writer.finish().unwrap() +} + +fn build_deep( + sender: PrivateKey, + recipient: pithos_lib::crypto::PublicKey, + depth: usize, +) -> Vec { + let mut writer = writer(sender, recipient); + let mut path = String::new(); + for index in 0..depth { + if !path.is_empty() { + path.push('/'); + } + path.push_str(&format!("level-{index:04}")); + writer + .add_directory( + ArchivePath::new(&path).unwrap(), + EntryMetadata::new(0, 0, 0o755), + ) + .unwrap(); + } + writer.finish().unwrap() +} + +fn bench_build( + c: &mut Criterion, + name: String, + sender: &PrivateKey, + operation: impl Fn(PrivateKey) -> Vec + Copy, +) { + support::measure( + &name, + "key prepared outside operation", + || operation(sender.duplicate()), + |bytes| support::Metrics::archive(bytes.len() as u64), + ); + c.bench_function(&name, |b| { + b.iter_batched( + || sender.duplicate(), + |sender| black_box(operation(sender)), + BatchSize::SmallInput, + ) + }); +} + +fn bench_archive_path(c: &mut Criterion) { + let sender = PrivateKey::generate(); + let recipient = sender.public_key(); + for count in [1_000, 10_000] { + bench_build(c, format!("archive_path/flat/{count}"), &sender, |key| { + build_flat(key, recipient, count) + }); + bench_build( + c, + format!("archive_path/duplicate/{count}"), + &sender, + |key| build_duplicate(key, recipient, count), + ); + } + for count in [1_000, 10_000] { + let workload = format!("archive_path/flat-index-open/{count}"); + let archive_bytes = build_flat(sender.duplicate(), recipient, count); + let archive_len = archive_bytes.len() as u64; + support::measure( + &workload, + "flat archive and access key prepared; cloning the in-memory source and open construct and validate the effective index", + || { + Archive::open( + MemorySource::new(archive_bytes.clone()), + OpenOptions::default() + .with_access_keys(AccessKeys::new().with_key(sender.duplicate())), + ) + .unwrap() + }, + |_| support::Metrics::archive(archive_len), + ); + c.bench_function(&workload, |b| { + b.iter_batched( + || archive_bytes.clone(), + |bytes| { + black_box( + Archive::open( + MemorySource::new(bytes), + OpenOptions::default() + .with_access_keys(AccessKeys::new().with_key(sender.duplicate())), + ) + .unwrap(), + ) + }, + BatchSize::SmallInput, + ) + }); + } + if std::env::var_os("PITHOS_BENCH_100K").is_some() { + for (name, operation) in [ + ( + "flat", + build_flat as fn(PrivateKey, pithos_lib::crypto::PublicKey, usize) -> Vec, + ), + ("duplicate", build_duplicate), + ] { + bench_build(c, format!("archive_path/{name}/100000"), &sender, |key| { + operation(key, recipient, 100_000) + }); + } + } + for depth in [10, 100, 1_000] { + bench_build(c, format!("archive_path/deep/{depth}"), &sender, |key| { + build_deep(key, recipient, depth) + }); + } + for (name, direction) in [ + ( + "existing-ancestor", + workloads::ConflictDirection::ExistingAncestor, + ), + ( + "existing-descendants", + workloads::ConflictDirection::ExistingDescendants, + ), + ] { + let workload = format!("archive_path/conflicts/{name}/1000"); + let mut fixture = + workloads::build_conflict_fixture(sender.duplicate(), recipient, 1_000, direction); + support::measure( + &workload, + "1000-entry writer prepared; one hierarchy rejection is measured", + || workloads::reject_conflict(&mut fixture), + |_| support::Metrics::operation(), + ); + c.bench_function(&workload, |b| { + b.iter_batched( + || { + workloads::build_conflict_fixture( + sender.duplicate(), + recipient, + 1_000, + direction, + ) + }, + |mut fixture| black_box(workloads::reject_conflict(&mut fixture)), + BatchSize::SmallInput, + ) + }); + } + for (name, order) in [ + ("ancestor-first", workloads::InsertionOrder::AncestorFirst), + ( + "descendant-first", + workloads::InsertionOrder::DescendantFirst, + ), + ] { + bench_build( + c, + format!("archive_path/incremental/{name}/1000"), + &sender, + |key| workloads::build_incremental(key, recipient, 1_000, order), + ); + } + for count in [100, 1_000] { + bench_build( + c, + format!("archive_path/metadata-heavy/{count}"), + &sender, + |key| workloads::build_metadata_pairs(key, recipient, count), + ); + } + let extraction = workloads::build_extraction_fixture(sender.duplicate(), recipient); + let destination = tempfile::tempdir().unwrap(); + let archive_bytes = extraction.archive_bytes; + support::measure( + "archive_path/extraction/100", + "100-entry archive and empty destination prepared before filesystem extraction", + || workloads::extract_all(&extraction, destination.path()), + |output_bytes| support::Metrics { + archive_bytes, + output_bytes: *output_bytes, + source_read_count: None, + source_read_bytes: None, + bytes_appended: None, + dedup_signal: None, + }, + ); + c.bench_function("archive_path/extraction/100", |b| { + b.iter_batched( + || { + ( + workloads::build_extraction_fixture(sender.duplicate(), recipient), + tempfile::tempdir().unwrap(), + ) + }, + |(fixture, destination)| { + let output = black_box(workloads::extract_all(&fixture, destination.path())); + (fixture, destination, output) + }, + BatchSize::SmallInput, + ) + }); +} + +criterion_group!(archive_path, bench_archive_path); +criterion_main!(archive_path); diff --git a/crates/pithos_lib/benches/block_pipeline.rs b/crates/pithos_lib/benches/block_pipeline.rs new file mode 100644 index 0000000..a5a8a78 --- /dev/null +++ b/crates/pithos_lib/benches/block_pipeline.rs @@ -0,0 +1,140 @@ +mod support; + +use criterion::{BatchSize, Criterion, criterion_group, criterion_main}; +use pithos_lib::archive::{ + ArchivePath, ArchiveWriter, EntryMetadata, ProcessingOptions, WriteOptions, +}; +use pithos_lib::crypto::PrivateKey; +use std::hint::black_box; +use std::io::{Cursor, Read}; + +struct VirtualReader { + remaining: u64, + state: u64, +} + +impl VirtualReader { + fn new(length: u64) -> Self { + Self { + remaining: length, + state: 0x9e37_79b9_7f4a_7c15, + } + } +} + +impl Read for VirtualReader { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + let count = self.remaining.min(buffer.len() as u64) as usize; + for byte in &mut buffer[..count] { + *byte = support::next_deterministic_byte(&mut self.state); + } + self.remaining -= count as u64; + Ok(count) + } +} + +fn write_block_pipeline_archive( + sender: PrivateKey, + recipient: pithos_lib::crypto::PublicKey, + bytes: &[u8], +) -> u64 { + let mut writer = ArchiveWriter::create( + support::CountingSink::default(), + WriteOptions::new(sender, vec![recipient]), + ) + .unwrap(); + writer + .add_file( + ArchivePath::new("block").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(true, 3).unwrap(), + Some(bytes.len() as u64), + Cursor::new(bytes), + ) + .unwrap(); + writer.finish().unwrap().0 +} + +fn write_streamed_archive( + sender: PrivateKey, + recipient: pithos_lib::crypto::PublicKey, + logical_bytes: u64, +) -> u64 { + let mut writer = ArchiveWriter::create( + support::CountingSink::default(), + WriteOptions::new(sender, vec![recipient]), + ) + .unwrap(); + writer + .add_file( + ArchivePath::new("streamed.bin").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(true, 3).unwrap(), + Some(logical_bytes), + VirtualReader::new(logical_bytes), + ) + .unwrap(); + writer.finish().unwrap().0 +} + +fn bench_transforms(c: &mut Criterion) { + let sender = PrivateKey::generate(); + let recipient = sender.public_key(); + let compressible = vec![b'a'; 64 * 1024]; + let incompressible = support::deterministic_bytes(0x9e37_79b9_7f4a_7c15, 64 * 1024); + for (name, bytes) in [ + ("compressible", &compressible), + ("incompressible", &incompressible), + ] { + let workload = format!("block_pipeline/{name}"); + support::measure( + &workload, + "payload and key prepared before transform", + || write_block_pipeline_archive(sender.duplicate(), recipient, bytes), + |archive_bytes| support::Metrics::archive(*archive_bytes), + ); + c.bench_function(&workload, |b| { + b.iter_batched( + || sender.duplicate(), + |key| black_box(write_block_pipeline_archive(key, recipient, bytes)), + BatchSize::SmallInput, + ); + }); + } + { + let (name, logical_bytes) = ("streamed-quick", 8 * 1024 * 1024); + let workload = format!("block_pipeline/{name}"); + support::measure( + &workload, + "virtual reader and discard sink; logical bytes are never allocated together", + || write_streamed_archive(sender.duplicate(), recipient, logical_bytes), + |archive_bytes| support::Metrics::archive(*archive_bytes), + ); + c.bench_function(&workload, |b| { + b.iter_batched( + || sender.duplicate(), + |key| black_box(write_streamed_archive(key, recipient, logical_bytes)), + BatchSize::SmallInput, + ); + }); + } + if std::env::var_os("PITHOS_BENCH_LARGE_STREAM").is_some() { + let logical_bytes = 64 * 1024 * 1024; + support::measure( + "block_pipeline/streamed-large", + "env-gated virtual reader and discard sink", + || write_streamed_archive(sender.duplicate(), recipient, logical_bytes), + |archive_bytes| support::Metrics::archive(*archive_bytes), + ); + c.bench_function("block_pipeline/streamed-large", |b| { + b.iter_batched( + || sender.duplicate(), + |key| black_box(write_streamed_archive(key, recipient, logical_bytes)), + BatchSize::SmallInput, + ); + }); + } +} + +criterion_group!(block_pipeline, bench_transforms); +criterion_main!(block_pipeline); diff --git a/crates/pithos_lib/benches/support.rs b/crates/pithos_lib/benches/support.rs new file mode 100644 index 0000000..10cee17 --- /dev/null +++ b/crates/pithos_lib/benches/support.rs @@ -0,0 +1,109 @@ +use peak_alloc::PeakAlloc; +use std::io::Write; +use std::time::Instant; + +#[global_allocator] +pub static PEAK_ALLOC: PeakAlloc = PeakAlloc; + +pub struct Metrics { + pub archive_bytes: u64, + pub output_bytes: u64, + pub source_read_count: Option, + pub source_read_bytes: Option, + pub bytes_appended: Option, + pub dedup_signal: Option, +} + +impl Metrics { + #[allow(dead_code)] // The archive-path binary is the only target with operation-only metrics. + pub fn archive(archive_bytes: u64) -> Self { + Self { + archive_bytes, + output_bytes: archive_bytes, + source_read_count: None, + source_read_bytes: None, + bytes_appended: None, + dedup_signal: None, + } + } + + #[allow(dead_code)] // Only the archive-path binary measures operations without archive output. + pub fn operation() -> Self { + Self { + archive_bytes: 0, + output_bytes: 0, + source_read_count: None, + source_read_bytes: None, + bytes_appended: None, + dedup_signal: None, + } + } +} + +#[derive(Debug, Default)] +#[allow(dead_code)] // Only the archive I/O, adapter, and block-pipeline binaries write to a sink. +pub struct CountingSink(pub u64); + +impl Write for CountingSink { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0 += bytes.len() as u64; + Ok(bytes.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +#[allow(dead_code)] // Only the adapter, append, and block-pipeline binaries generate payloads. +pub fn deterministic_bytes(seed: u64, length: usize) -> Vec { + let mut state = seed; + (0..length) + .map(|_| next_deterministic_byte(&mut state)) + .collect() +} + +#[allow(dead_code)] // The virtual reader generates deterministic bytes without allocating its input. +pub fn next_deterministic_byte(state: &mut u64) -> u8 { + *state = state + .wrapping_mul(6_364_136_223_846_793_005) + .wrapping_add(1); + (*state >> 56) as u8 +} + +fn json_u64(value: Option) -> String { + value.map_or_else(|| "null".to_owned(), |value| value.to_string()) +} + +fn json_bool(value: Option) -> &'static str { + match value { + Some(true) => "true", + Some(false) => "false", + None => "null", + } +} + +/// Records one local resource sample. Criterion's estimates remain the runtime distribution. +pub fn measure( + workload: &str, + setup: &str, + operation: impl FnOnce() -> T, + metrics: impl FnOnce(&T) -> Metrics, +) { + let baseline = PEAK_ALLOC.current_usage(); + PEAK_ALLOC.reset_peak_usage(); + let started = Instant::now(); + let result = operation(); + let elapsed_local_ns = started.elapsed().as_nanos(); + let peak_heap_bytes = PEAK_ALLOC.peak_usage().saturating_sub(baseline); + let metrics = metrics(&result); + println!( + "{{\"schema\":\"pithos-bench-v1\",\"workload\":\"{workload}\",\"setup\":\"{setup}\",\"elapsed_local_ns\":{elapsed_local_ns},\"peak_heap_bytes\":{peak_heap_bytes},\"archive_bytes\":{},\"output_bytes\":{},\"source_read_count\":{},\"source_read_bytes\":{},\"bytes_appended\":{},\"dedup_signal\":{}}}", + metrics.archive_bytes, + metrics.output_bytes, + json_u64(metrics.source_read_count), + json_u64(metrics.source_read_bytes), + json_u64(metrics.bytes_appended), + json_bool(metrics.dedup_signal), + ); +} diff --git a/crates/pithos_lib/benches/workloads.rs b/crates/pithos_lib/benches/workloads.rs new file mode 100644 index 0000000..c92a59b --- /dev/null +++ b/crates/pithos_lib/benches/workloads.rs @@ -0,0 +1,336 @@ +use pithos_lib::archive::{ + AccessKeys, AppendOptions, Archive, ArchivePath, ArchiveWriter, EntryMetadata, EntryReference, + OpenOptions, ProcessingOptions, WriteOptions, WriterError, +}; +use pithos_lib::crypto::{PrivateKey, PublicKey}; +use pithos_lib::error::PithosError; +use pithos_lib::fs::{append_files, extract}; +use pithos_lib::source::MemorySource; +use std::io::Cursor; +use std::path::{Path, PathBuf}; + +#[allow(dead_code)] // Used by the archive-path target and its fixture contract test. +pub const EXTRACTION_COUNT: usize = 100; +#[allow(dead_code)] // Used by the archive-path target and its fixture contract test. +pub const EXTRACTION_PAYLOAD_BYTES: usize = 1024; +#[allow(dead_code)] // Flat workloads use empty files when they include this module. +const EMPTY: [u8; 0] = []; +#[allow(dead_code)] // Metadata workloads use this fixed payload when they include this module. +const METADATA_JSON: &[u8] = br#"{"type":"benchmark-metadata"}"#; +#[allow(dead_code)] // Extraction workloads use this fixed payload when they include this module. +const EXTRACTION_PAYLOAD: [u8; EXTRACTION_PAYLOAD_BYTES] = [b'e'; EXTRACTION_PAYLOAD_BYTES]; + +#[allow(dead_code)] // Only archive-path benchmarks incremental insertion ordering. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum InsertionOrder { + AncestorFirst, + DescendantFirst, +} + +#[allow(dead_code)] // Only archive-path benchmarks hierarchy-conflict directions. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum ConflictDirection { + ExistingAncestor, + ExistingDescendants, +} + +#[allow(dead_code)] // Shared fixture builders use this only in targets that create archives. +fn writer(sender: PrivateKey, recipient: PublicKey) -> ArchiveWriter> { + ArchiveWriter::create(Vec::new(), WriteOptions::new(sender, vec![recipient])).unwrap() +} + +#[allow(dead_code)] // Shared fixture builders use this only in targets that create archives. +fn add_file( + writer: &mut ArchiveWriter>, + path: String, + metadata: EntryMetadata, + bytes: &'static [u8], +) { + writer + .add_file( + ArchivePath::new(path).unwrap(), + metadata, + ProcessingOptions::new(false, 0).unwrap(), + Some(bytes.len() as u64), + Cursor::new(bytes), + ) + .unwrap(); +} + +#[allow(dead_code)] // Only archive-path benchmarks incremental insertion. +pub fn build_incremental( + sender: PrivateKey, + recipient: PublicKey, + count: usize, + order: InsertionOrder, +) -> Vec { + let mut writer = writer(sender, recipient); + let add_ancestor = |writer: &mut ArchiveWriter>| { + writer + .add_directory( + ArchivePath::new("root").unwrap(), + EntryMetadata::new(0, 0, 0o755), + ) + .unwrap(); + }; + if order == InsertionOrder::AncestorFirst { + add_ancestor(&mut writer); + } + for index in 0..count { + add_file( + &mut writer, + format!("root/entry-{index:06}"), + EntryMetadata::new(0, 0, 0o644), + &EMPTY, + ); + } + if order == InsertionOrder::DescendantFirst { + add_ancestor(&mut writer); + } + writer.finish().unwrap() +} + +#[allow(dead_code)] // Only archive-path benchmarks metadata/data pairs. +pub fn build_metadata_pairs(sender: PrivateKey, recipient: PublicKey, count: usize) -> Vec { + let mut writer = writer(sender, recipient); + for index in 0..count { + let metadata = writer + .add_metadata( + ArchivePath::new(format!("entry-{index:06}.meta")).unwrap(), + EntryMetadata::new(index as u64, index as u64, 0o644), + ProcessingOptions::new(false, 0).unwrap(), + Some(METADATA_JSON.len() as u64), + Cursor::new(METADATA_JSON), + ) + .unwrap(); + add_file( + &mut writer, + format!("entry-{index:06}.data"), + EntryMetadata::new(index as u64, index as u64, 0o644).with_references(vec![ + EntryReference { + target_file_id: metadata.id, + relationship: 0, + }, + ]), + &EMPTY, + ); + } + writer.finish().unwrap() +} + +#[allow(dead_code)] // Only archive-path benchmarks hierarchy-conflict rejection. +pub struct ConflictFixture { + pub writer: ArchiveWriter>, + pub candidate: String, +} + +#[allow(dead_code)] // Only archive-path benchmarks hierarchy-conflict setup. +pub fn build_conflict_fixture( + sender: PrivateKey, + recipient: PublicKey, + count: usize, + direction: ConflictDirection, +) -> ConflictFixture { + assert!(count > 0); + let mut writer = writer(sender, recipient); + let candidate = match direction { + ConflictDirection::ExistingAncestor => { + add_file( + &mut writer, + "root".into(), + EntryMetadata::new(0, 0, 0o644), + &EMPTY, + ); + for index in 1..count { + add_file( + &mut writer, + format!("filler-{index:06}"), + EntryMetadata::new(0, 0, 0o644), + &EMPTY, + ); + } + "root/child".into() + } + ConflictDirection::ExistingDescendants => { + for index in 0..count { + add_file( + &mut writer, + format!("root/child-{index:06}"), + EntryMetadata::new(0, 0, 0o644), + &EMPTY, + ); + } + "root".into() + } + }; + ConflictFixture { writer, candidate } +} + +#[allow(dead_code)] // Only archive-path benchmarks hierarchy-conflict rejection. +pub fn reject_conflict(fixture: &mut ConflictFixture) -> PithosError { + match fixture.writer.add_file( + ArchivePath::new(&fixture.candidate).unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(false, 0).unwrap(), + Some(0), + Cursor::new(&EMPTY), + ) { + Err(WriterError::Pithos(error @ PithosError::InvalidArchivePath { .. })) => error, + other => panic!("expected hierarchy conflict, got {other:?}"), + } +} + +#[allow(dead_code)] // Only archive-path benchmarks extraction. +pub struct ExtractionFixture { + pub archive: Archive, + pub archive_bytes: u64, + pub paths: Vec, +} + +#[allow(dead_code)] // Only archive-path benchmarks extraction setup. +pub fn build_extraction_fixture(sender: PrivateKey, recipient: PublicKey) -> ExtractionFixture { + let mut writer = writer(sender.duplicate(), recipient); + let mut paths = Vec::with_capacity(EXTRACTION_COUNT); + for index in 0..EXTRACTION_COUNT { + let path = format!("entry-{index:06}"); + add_file( + &mut writer, + path.clone(), + EntryMetadata::new(0, 0, 0o644), + &EXTRACTION_PAYLOAD, + ); + paths.push(path); + } + let bytes = writer.finish().unwrap(); + let archive_bytes = bytes.len() as u64; + let archive = Archive::open( + MemorySource::new(bytes), + OpenOptions::default().with_access_keys(AccessKeys::new().with_key(sender)), + ) + .unwrap(); + ExtractionFixture { + archive, + archive_bytes, + paths, + } +} + +#[allow(dead_code)] // Only archive-path benchmarks extraction. +pub fn extract_all(fixture: &ExtractionFixture, destination: &Path) -> u64 { + for path in &fixture.paths { + extract(&fixture.archive, path, destination).unwrap(); + } + (fixture.paths.len() * EXTRACTION_PAYLOAD_BYTES) as u64 +} + +#[allow(dead_code)] // Archive I/O uses this while append uses the larger scaling fixture. +pub struct ChainFixture { + pub temporary: tempfile::TempDir, + pub archive_path: PathBuf, + pub sender: PrivateKey, + pub recipient: PublicKey, + pub archive_bytes: u64, +} + +#[allow(dead_code)] // Archive I/O uses this to create the two-generation open fixture. +pub fn append_child_generation( + archive_path: &Path, + input_root: &Path, + sender: &PrivateKey, + recipient: PublicKey, +) { + let child = input_root.join("child.bin"); + std::fs::write(&child, b"child").unwrap(); + append_files( + archive_path, + AppendOptions::new(sender.duplicate(), vec![recipient]), + &[child], + ) + .unwrap(); +} + +#[allow(dead_code)] // The fixture contract test validates this smaller append chain. +pub fn build_chain_fixture(sender: PrivateKey, recipient: PublicKey) -> ChainFixture { + let temporary = tempfile::tempdir().unwrap(); + let archive_path = temporary.path().join("chain.pith"); + let mut writer = ArchiveWriter::create( + std::fs::File::create(&archive_path).unwrap(), + WriteOptions::new(sender.duplicate(), vec![recipient]), + ) + .unwrap(); + writer + .add_file( + ArchivePath::new("ancestor.bin").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(false, 0).unwrap(), + Some(8), + Cursor::new(b"ancestor"), + ) + .unwrap(); + writer.finish().unwrap(); + append_child_generation(&archive_path, temporary.path(), &sender, recipient); + let archive_bytes = std::fs::metadata(&archive_path).unwrap().len(); + ChainFixture { + temporary, + archive_path, + sender, + recipient, + archive_bytes, + } +} + +/// Builds a supported direct-append chain with every generation in its own terminal directory. +/// Input creation is deliberately separate from the append operations so benchmark setup remains +/// outside the measured final append. +#[allow(dead_code)] // Only append benchmarks multigeneration scaling. +pub fn build_append_scaling_fixture( + sender: PrivateKey, + recipient: PublicKey, + generations: usize, + entries_per_generation: usize, +) -> ChainFixture { + let temporary = tempfile::tempdir().unwrap(); + let archive_path = temporary.path().join("append-scaling.pith"); + let mut writer = ArchiveWriter::create( + std::fs::File::create(&archive_path).unwrap(), + WriteOptions::new(sender.duplicate(), vec![recipient]), + ) + .unwrap(); + writer + .add_file( + ArchivePath::new("ancestor.bin").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(false, 0).unwrap(), + Some(8), + Cursor::new(b"ancestor"), + ) + .unwrap(); + writer.finish().unwrap(); + + for generation in 0..generations { + let input_root = temporary.path().join(format!("input-{generation:04}")); + std::fs::create_dir(&input_root).unwrap(); + let inputs = (0..entries_per_generation) + .map(|entry| { + let name = format!("generation-{generation:04}-entry-{entry:04}.bin"); + let path = input_root.join(&name); + std::fs::write(&path, format!("{generation:04}:{entry:04}")).unwrap(); + path + }) + .collect::>(); + append_files( + &archive_path, + AppendOptions::new(sender.duplicate(), vec![recipient]), + &inputs, + ) + .unwrap(); + } + let archive_bytes = std::fs::metadata(&archive_path).unwrap().len(); + ChainFixture { + temporary, + archive_path, + sender, + recipient, + archive_bytes, + } +} diff --git a/crates/pithos_lib/tests/benchmark_contracts.rs b/crates/pithos_lib/tests/benchmark_contracts.rs new file mode 100644 index 0000000..45da1c0 --- /dev/null +++ b/crates/pithos_lib/tests/benchmark_contracts.rs @@ -0,0 +1,181 @@ +#[path = "../benches/workloads.rs"] +mod workloads; + +use pithos_lib::archive::{AccessKeys, Archive, EntryKind, OpenOptions}; +use pithos_lib::crypto::PrivateKey; +use pithos_lib::source::{FileSource, MemorySource}; + +fn open_memory(bytes: Vec, key: PrivateKey) -> Archive { + Archive::open( + MemorySource::new(bytes), + OpenOptions::default().with_access_keys(AccessKeys::new().with_key(key)), + ) + .unwrap() +} + +#[test] +fn incremental_workloads_cover_both_orders_with_the_same_entries() { + let sender = PrivateKey::generate(); + let recipient = sender.public_key(); + let ancestor = open_memory( + workloads::build_incremental( + sender.duplicate(), + recipient, + 1_000, + workloads::InsertionOrder::AncestorFirst, + ), + sender.duplicate(), + ); + let descendant = open_memory( + workloads::build_incremental( + sender.duplicate(), + recipient, + 1_000, + workloads::InsertionOrder::DescendantFirst, + ), + sender, + ); + let mut ancestor_paths = ancestor + .entries() + .map(|entry| entry.path) + .collect::>(); + let mut descendant_paths = descendant + .entries() + .map(|entry| entry.path) + .collect::>(); + ancestor_paths.sort(); + descendant_paths.sort(); + assert_eq!(ancestor_paths.len(), 1_001); + assert_eq!(ancestor_paths, descendant_paths); + assert_eq!( + ancestor.entry("root").unwrap().unwrap().kind, + EntryKind::Directory + ); +} + +#[test] +fn metadata_workload_contains_real_content_pairs_and_references() { + let sender = PrivateKey::generate(); + let archive = open_memory( + workloads::build_metadata_pairs(sender.duplicate(), sender.public_key(), 100), + sender, + ); + let entries = archive.entries().collect::>(); + assert_eq!(entries.len(), 200); + assert_eq!( + entries + .iter() + .filter(|entry| matches!(entry.kind, EntryKind::Metadata { .. })) + .count(), + 100 + ); + assert_eq!( + entries + .iter() + .filter(|entry| matches!(entry.kind, EntryKind::File { .. })) + .count(), + 100 + ); + assert_eq!( + entries + .iter() + .filter(|entry| entry.references.len() == 1) + .count(), + 100 + ); +} + +#[test] +fn conflicts_are_hierarchy_conflicts_against_one_thousand_entries() { + let sender = PrivateKey::generate(); + for direction in [ + workloads::ConflictDirection::ExistingAncestor, + workloads::ConflictDirection::ExistingDescendants, + ] { + let mut fixture = workloads::build_conflict_fixture( + sender.duplicate(), + sender.public_key(), + 1_000, + direction, + ); + assert_ne!(fixture.candidate, "root/child-000000"); + assert!(matches!( + workloads::reject_conflict(&mut fixture), + pithos_lib::error::PithosError::InvalidArchivePath { .. } + )); + fixture + .writer + .add_directory( + pithos_lib::archive::ArchivePath::new("still-usable").unwrap(), + pithos_lib::archive::EntryMetadata::new(0, 0, 0o755), + ) + .unwrap(); + } +} + +#[test] +fn extraction_workload_materializes_all_one_hundred_files() { + let sender = PrivateKey::generate(); + let fixture = workloads::build_extraction_fixture(sender.duplicate(), sender.public_key()); + let destination = tempfile::tempdir().unwrap(); + assert_eq!( + workloads::extract_all(&fixture, destination.path()), + (workloads::EXTRACTION_COUNT * workloads::EXTRACTION_PAYLOAD_BYTES) as u64 + ); + assert_eq!(std::fs::read_dir(destination.path()).unwrap().count(), 100); + assert_eq!( + std::fs::read(destination.path().join("entry-000099")) + .unwrap() + .len(), + workloads::EXTRACTION_PAYLOAD_BYTES + ); +} + +#[test] +fn open_merge_fixture_has_two_generations_and_entries_from_both() { + let sender = PrivateKey::generate(); + let fixture = workloads::build_chain_fixture(sender.duplicate(), sender.public_key()); + let bytes = std::fs::read(&fixture.archive_path).unwrap(); + assert_eq!( + bytes + .windows(8) + .filter(|window| *window == b"PITHOSDR") + .count(), + 2 + ); + let archive = Archive::open( + FileSource::open(&fixture.archive_path).unwrap(), + OpenOptions::default().with_access_keys(AccessKeys::new().with_key(sender)), + ) + .unwrap(); + assert!(archive.entry("ancestor.bin").unwrap().is_some()); + assert!(archive.entry("child.bin").unwrap().is_some()); +} + +#[test] +fn append_scaling_fixture_contains_every_generation_and_planned_entry() { + let sender = PrivateKey::generate(); + let fixture = + workloads::build_append_scaling_fixture(sender.duplicate(), sender.public_key(), 10, 4); + let bytes = std::fs::read(&fixture.archive_path).unwrap(); + assert_eq!( + bytes + .windows(8) + .filter(|window| *window == b"PITHOSDR") + .count(), + 11 + ); + let archive = Archive::open( + FileSource::open(&fixture.archive_path).unwrap(), + OpenOptions::default().with_access_keys(AccessKeys::new().with_key(sender)), + ) + .unwrap(); + assert_eq!(archive.entries().len(), 41); + for path in [ + "ancestor.bin", + "generation-0000-entry-0000.bin", + "generation-0009-entry-0003.bin", + ] { + assert!(archive.entry(path).unwrap().is_some(), "missing {path}"); + } +} From e8598fd102acf427657708a3305f07990110c7ea Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Wed, 12 Aug 2026 14:03:09 +0200 Subject: [PATCH 013/166] ci: separate correctness checks --- .github/workflows/ci.yaml | 71 +++++++-------------------------------- 1 file changed, 12 insertions(+), 59 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ffd3c5b..5ed8a9c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,7 +1,9 @@ name: CI on: - - push - - pull_request + push: + branches: + - main + pull_request: permissions: contents: read @@ -11,21 +13,7 @@ concurrency: cancel-in-progress: true jobs: - actionlint: - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - name: Check out repository - uses: actions/checkout@v4 - - - name: Install actionlint - run: go install github.com/rhysd/actionlint/cmd/actionlint@v1.7.12 - - - name: Check workflows - run: | - "$(go env GOPATH)/bin/actionlint" - - tests: + stable: runs-on: ubuntu-latest timeout-minutes: 20 steps: @@ -38,56 +26,21 @@ jobs: - name: Cache Rust build artifacts uses: Swatinem/rust-cache@v2 - - name: Run workspace tests - run: cargo test --locked --workspace --all-targets - - fmt: - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - name: Check out repository - uses: actions/checkout@v4 - - - name: Install stable rustfmt - uses: dtolnay/rust-toolchain@stable - with: - components: rustfmt - - - name: Check formatting - run: cargo fmt --all -- --check + - name: Run stable workspace tests + run: cargo +stable test --locked --workspace --all-features --lib --bins --tests - clippy: + msrv: runs-on: ubuntu-latest timeout-minutes: 20 steps: - name: Check out repository uses: actions/checkout@v4 - - name: Install stable clippy - uses: dtolnay/rust-toolchain@stable - with: - components: clippy + - name: Install Rust 1.88.0 + uses: dtolnay/rust-toolchain@1.88.0 - name: Cache Rust build artifacts uses: Swatinem/rust-cache@v2 - - name: Run clippy - run: cargo clippy --locked --workspace --all-targets -- -D warnings - - deny: - runs-on: ubuntu-latest - timeout-minutes: 10 - steps: - - name: Check out repository - uses: actions/checkout@v4 - - - name: Install stable Rust - uses: dtolnay/rust-toolchain@stable - - - name: Install cargo-deny - uses: taiki-e/install-action@v2 - with: - tool: cargo-deny - - - name: Check dependency policy - run: cargo deny check + - name: Run MSRV workspace tests + run: cargo +1.88.0 test --locked --workspace --all-features From bf5e79fa628e4539f44ebe5e46505cf459542924 Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Wed, 12 Aug 2026 14:07:56 +0200 Subject: [PATCH 014/166] ci: separate static checks --- .github/workflows/checks.yaml | 86 +++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 .github/workflows/checks.yaml diff --git a/.github/workflows/checks.yaml b/.github/workflows/checks.yaml new file mode 100644 index 0000000..f2b96f8 --- /dev/null +++ b/.github/workflows/checks.yaml @@ -0,0 +1,86 @@ +name: Checks + +on: + push: + branches: + - main + pull_request: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + fmt: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Install stable rustfmt + uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt + + - name: Check formatting + run: cargo +stable fmt --all -- --check + + clippy: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Install stable clippy + uses: dtolnay/rust-toolchain@stable + with: + components: clippy + + - name: Cache Rust build artifacts + uses: Swatinem/rust-cache@v2 + + - name: Run Clippy + run: cargo +stable clippy --locked --workspace --all-targets --all-features -- -D warnings + + dependency-policy: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Install stable Rust + uses: dtolnay/rust-toolchain@stable + + - name: Cache Rust build artifacts + uses: Swatinem/rust-cache@v2 + + - name: Install cargo-deny + uses: taiki-e/install-action@v2 + with: + tool: cargo-deny + + - name: Check workspace dependency policy + run: cargo deny --locked check + + - name: Check fuzz dependency policy + run: cargo deny --manifest-path fuzz/Cargo.toml --config deny.toml --locked check + + actionlint: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Install actionlint + run: go install github.com/rhysd/actionlint/cmd/actionlint@v1.7.12 + + - name: Check workflows + run: | + "$(go env GOPATH)/bin/actionlint" From a2a604ca87f26cdd8ccf5a270504b1ca10f1c1cc Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Wed, 12 Aug 2026 14:09:38 +0200 Subject: [PATCH 015/166] ci: focus coverage targets --- .github/workflows/codecov.yaml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/codecov.yaml b/.github/workflows/codecov.yaml index 23d538f..c9dd853 100644 --- a/.github/workflows/codecov.yaml +++ b/.github/workflows/codecov.yaml @@ -2,11 +2,20 @@ on: push: branches: [main] pull_request: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + name: coverage jobs: report: name: Report runs-on: ubuntu-latest + timeout-minutes: 20 steps: - name: Checkout repository @@ -31,7 +40,7 @@ jobs: name: Generate Coverage Report run: | mkdir -p .coverage - cargo llvm-cov --locked --workspace --all-targets --lcov --output-path=./.coverage/lcov.info + cargo llvm-cov --locked --workspace --lib --bins --tests --lcov --output-path=./.coverage/lcov.info - id: upload name: Upload Coverage Report From 91f4845d38a962207e4c1da75b504ef3db98973b Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Wed, 12 Aug 2026 14:17:41 +0200 Subject: [PATCH 016/166] ci: add bounded fuzzing --- .github/workflows/fuzz.yaml | 76 +++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 .github/workflows/fuzz.yaml diff --git a/.github/workflows/fuzz.yaml b/.github/workflows/fuzz.yaml new file mode 100644 index 0000000..de1754d --- /dev/null +++ b/.github/workflows/fuzz.yaml @@ -0,0 +1,76 @@ +name: Fuzzing + +on: + pull_request: + branches: + - main + schedule: + - cron: "30 2 * * 0" # Sundays at 02:30 UTC. + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + dependency-policy: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Install cargo-deny + uses: taiki-e/install-action@v2 + with: + tool: cargo-deny + + - name: Check fuzz dependency policy + run: cargo deny --manifest-path fuzz/Cargo.toml --config deny.toml --locked check + + fuzz: + name: Fuzz ${{ matrix.target }} + needs: dependency-policy + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + fail-fast: false + matrix: + target: + - header_decode + - directory_decode + - crypt4gh_decode + - chain_validation + env: + FUZZ_MAX_TOTAL_TIME: ${{ github.event_name == 'pull_request' && '60' || '900' }} + FUZZ_TIMEOUT: ${{ github.event_name == 'pull_request' && '5' || '10' }} + FUZZ_RSS_LIMIT_MB: ${{ github.event_name == 'pull_request' && '1024' || '1536' }} + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Install nightly Rust + uses: dtolnay/rust-toolchain@nightly + + - name: Install cargo-fuzz 0.13.2 + run: cargo +nightly install cargo-fuzz --version 0.13.2 --locked + + - name: Run ${{ matrix.target }} + id: fuzz + run: | + cargo +nightly fuzz run "${{ matrix.target }}" -- \ + -max_total_time="$FUZZ_MAX_TOTAL_TIME" \ + -timeout="$FUZZ_TIMEOUT" \ + -rss_limit_mb="$FUZZ_RSS_LIMIT_MB" + + - name: Upload fuzz artifacts + if: failure() && steps.fuzz.outcome == 'failure' + uses: actions/upload-artifact@v4 + with: + name: fuzz-artifacts-${{ matrix.target }}-${{ github.run_id }} + path: fuzz/artifacts/ + if-no-files-found: ignore + retention-days: 14 From eca7a27d8470cc72a7a4c7f6113b698758802ae7 Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Wed, 12 Aug 2026 14:33:35 +0200 Subject: [PATCH 017/166] ci: align release validation --- .github/workflows/tag_publish.yaml | 38 +++++++++++++++++------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/.github/workflows/tag_publish.yaml b/.github/workflows/tag_publish.yaml index f44c4c7..c43632f 100644 --- a/.github/workflows/tag_publish.yaml +++ b/.github/workflows/tag_publish.yaml @@ -55,8 +55,9 @@ jobs: } for package in pithos_lib pithos; do - package_id="$(cargo pkgid -p "$package")" - package_version="${package_id##*@}" + package_id="$(cargo +stable pkgid -p "$package")" + package_version="${package_id##*#}" + package_version="${package_version##*@}" if [[ "$package_version" != "$RELEASE_VERSION" ]]; then printf '%s version %s does not match release tag %s.\n' "$package" "$package_version" "$RELEASE_VERSION" >&2 exit 1 @@ -65,40 +66,43 @@ jobs: printf 'RELEASE_VERSION=%s\n' "$RELEASE_VERSION" >> "$GITHUB_ENV" - - name: Run workspace tests - run: cargo test --locked --workspace --all-targets + - name: Run stable workspace tests + run: cargo +stable test --locked --workspace --all-features --lib --bins --tests - name: Check formatting - run: cargo fmt --all -- --check + run: cargo +stable fmt --all -- --check - name: Run Clippy - run: cargo clippy --locked --workspace --all-targets -- -D warnings + run: cargo +stable clippy --locked --workspace --all-targets --all-features -- -D warnings - - name: Check dependency policy - run: cargo deny check + - name: Check root dependency policy + run: cargo deny --locked check + + - name: Check fuzz dependency policy + run: cargo deny --manifest-path fuzz/Cargo.toml --config deny.toml --locked check - name: Package library - run: cargo package -p pithos_lib --locked + run: cargo +stable package -p pithos_lib --locked - name: Dry-run library publish - run: cargo publish -p pithos_lib --dry-run --locked + run: cargo +stable publish -p pithos_lib --dry-run --locked - name: Publish or verify library shell: bash env: CARGO_REGISTRY_TOKEN: ${{ secrets.CRATES_TOKEN }} run: | - if cargo publish -p pithos_lib --locked; then + if cargo +stable publish -p pithos_lib --locked; then exit 0 fi - cargo info "pithos_lib@$RELEASE_VERSION" --registry crates-io >/dev/null + cargo +stable info "pithos_lib@$RELEASE_VERSION" --registry crates-io >/dev/null - name: Wait for library to reach crates.io shell: bash run: | for _ in {1..30}; do - if cargo info "pithos_lib@$RELEASE_VERSION" --registry crates-io >/dev/null 2>&1; then + if cargo +stable info "pithos_lib@$RELEASE_VERSION" --registry crates-io >/dev/null 2>&1; then exit 0 fi @@ -109,18 +113,18 @@ jobs: exit 1 - name: Package CLI - run: cargo package -p pithos --locked + run: cargo +stable package -p pithos --locked - name: Dry-run CLI publish - run: cargo publish -p pithos --dry-run --locked + run: cargo +stable publish -p pithos --dry-run --locked - name: Publish or verify CLI shell: bash env: CARGO_REGISTRY_TOKEN: ${{ secrets.CRATES_TOKEN }} run: | - if cargo publish -p pithos --locked; then + if cargo +stable publish -p pithos --locked; then exit 0 fi - cargo info "pithos@$RELEASE_VERSION" --registry crates-io >/dev/null + cargo +stable info "pithos@$RELEASE_VERSION" --registry crates-io >/dev/null From 7a6290eff02e5d63367880a6f639452647257309 Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Wed, 12 Aug 2026 14:36:55 +0200 Subject: [PATCH 018/166] docs: update workflow guidance --- .github/actions-rs/grcov.yml | 2 - AGENTS.md | 2 +- crates/pithos_lib/benches/README.md | 59 +++++++++++++---------------- fuzz/README.md | 8 +++- 4 files changed, 35 insertions(+), 36 deletions(-) delete mode 100644 .github/actions-rs/grcov.yml diff --git a/.github/actions-rs/grcov.yml b/.github/actions-rs/grcov.yml deleted file mode 100644 index dbe2786..0000000 --- a/.github/actions-rs/grcov.yml +++ /dev/null @@ -1,2 +0,0 @@ -output-type: lcov -output-file: ./lcov.info \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md index e0d0c5b..9e91ac0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,4 +21,4 @@ - Before finishing Rust changes, run `cargo +stable fmt --all -- --check` and `cargo +stable clippy --locked --workspace --all-targets --all-features -- -D warnings` when practical. - Focused library tests use integration targets, for example `cargo test -p pithos_lib --test reader`; focused RO-Crate coverage is `cargo test -p pithos_lib --test ro_crate_directory` (or `ro_crate_zip` / `ro_crate_conversion`). CLI smoke check: `cargo +stable run --locked -p pithos -- --help`. - Release/package changes also require `python3 .github/release/check.py contracts` and `python3 .github/release/check.py package check pithos_lib` (and `pithos` when affected). The package policy is a ratchet: `package update` needs `--allow-increase` to raise a budget. -- Run benchmark comparisons only on the owner-selected self-hosted runner; `ubuntu-latest` is not comparable benchmark evidence. +- Run benchmark baselines and candidates locally on the same machine with the same toolchain, locked dependencies, power settings, and workload environment variables; GitHub-hosted runners are not comparable benchmark evidence. diff --git a/crates/pithos_lib/benches/README.md b/crates/pithos_lib/benches/README.md index 6658c3c..c5e2db3 100644 --- a/crates/pithos_lib/benches/README.md +++ b/crates/pithos_lib/benches/README.md @@ -1,6 +1,7 @@ # Benchmarks -This guide is for maintainers measuring `pithos_lib` performance. Local commands provide smoke coverage; only evidence from the owner-selected self-hosted runner is suitable for release comparison. +This guide is for maintainers measuring `pithos_lib` performance. Run comparisons locally on the +same machine under consistent conditions. ## Quick developer smoke @@ -50,44 +51,38 @@ enforces these workload shapes without runtime thresholds. The streamed-memory t machine-readable line containing both measured peaks, their delta, output sizes, and bound. Keys, input payloads, archive fixtures, and temporary directories are setup, not timed, -unless creation itself is the named workload. Each release-comparable resource record is -one `pithos-bench-v1` JSON object with exactly `workload`, `setup`, +unless creation itself is the named workload. Each resource record is one `pithos-bench-v1` +JSON object with exactly `workload`, `setup`, `peak_heap_bytes`, `archive_bytes`, `output_bytes`, `source_read_count`, `source_read_bytes`, `bytes_appended`, and `dedup_signal`. All six metrics must be finite non-negative numbers; the setup and deduplication signal are stable strings. -Extra timing fields and `null` observations are not valid release evidence. Criterion's +Extra timing fields and `null` observations are not valid benchmark evidence. Criterion's `target/criterion/**/new/estimates.json` remains the runtime distribution source. -## Release workflow +## Local comparisons -The release comparison authority is: +Measure the baseline and candidate on the same machine with the same Rust version, locked +dependencies, power settings, and workload environment variables. Keep other system activity low. +Do not compare results from GitHub-hosted runners or different machines. + +On the baseline revision, save each Criterion baseline: + +```bash +for bench in archive_path block_pipeline archive_io append adapters; do + cargo +1.88.0 bench --locked -p pithos_lib --bench "$bench" -- --save-baseline before +done +``` + +Then switch to the candidate revision and compare against it: ```bash -python3 .github/release/check.py benchmark compare BASELINE CURRENT REPORT.json +for bench in archive_path block_pipeline archive_io append adapters; do + cargo +1.88.0 bench --locked -p pithos_lib --bench "$bench" -- --baseline before +done ``` -Exit `0` means the evidence is valid and within the gate. Exit `3` means the evidence is valid -but has a material regression and must use the protected approval path. Any other failure, -including missing, malformed, incompatible, or invalid evidence, exits `1` for validation -failure (invalid top-level command paths exit `2`). Its single 5% materiality threshold applies -to Criterion median estimates and every numeric resource field; exactly 5% is allowed, while -any value greater than 5% is a regression. Changed `dedup_signal`, changed setup, missing -observations, or changed workload sets are also rejected. Do not use a hosted runner for a -release comparison. - -The baseline and current `environment/machine.txt` files must be byte-for-byte identical, -including machine and toolchain evidence. Runner identity, workload sets, and resource setup -must otherwise match exactly. - -This strict resource schema is a new reviewed-evidence requirement. An older owner-approved -artifact that does not satisfy it is not silently invalidated: capture a new baseline candidate, -review it, and use its exact artifact identity for subsequent comparisons. - -Dispatch the `Benchmarks` workflow with an owner-selected self-hosted runner label. For a -comparison, provide the reviewed successful baseline workflow run ID and its exact -`benchmark-current---` artifact name. The workflow records the runner, -machine and toolchain details, commit, lockfile digest, benchmark stdout, and Criterion output; -it downloads the immutable reviewed baseline artifact, compares it with current evidence, and -uploads both a comparison artifact and the current evidence artifact. The current artifact is -retained for 90 days and can become a reviewed baseline for a later run. Use -`capture_baseline` only to create a baseline candidate; it creates no release disposition. +Set `PITHOS_BENCH_100K=1` or `PITHOS_BENCH_LARGE_STREAM=1` for both runs when those optional +workloads are relevant. Review Criterion's terminal comparison and +`target/criterion/**/change/estimates.json`; also compare the emitted `pithos-bench-v1` resource +records. Recreate the saved baseline when the toolchain, dependencies, machine, or workload +settings change. diff --git a/fuzz/README.md b/fuzz/README.md index a23f3d3..0917b4d 100644 --- a/fuzz/README.md +++ b/fuzz/README.md @@ -17,7 +17,8 @@ footer, checksum, consumption, and body parsing rather than stopping at the mini Crash artifacts and generated build output belong under `fuzz/artifacts/` and `fuzz/target/` and are ignored by Git. Keep any minimized regression that fixes a bug in `fuzz/corpus//`. -Release-candidate smoke limits are 60 seconds, five seconds per input, and 1024 MiB RSS: +Pull requests targeting `main` run a smoke fuzz pass for all four targets. Each target runs for +60 seconds, with a five-second per-input timeout and a 1024 MiB RSS limit: ```bash cargo +nightly fuzz run header_decode -- -max_total_time=60 -timeout=5 -rss_limit_mb=1024 @@ -26,6 +27,11 @@ cargo +nightly fuzz run crypt4gh_decode -- -max_total_time=60 -timeout=5 -rss_li cargo +nightly fuzz run chain_validation -- -max_total_time=60 -timeout=5 -rss_limit_mb=1024 ``` +The same four targets run weekly on Sundays at 02:30 UTC for an extended 15-minute pass, with a +10-second per-input timeout and a 1536 MiB RSS limit. The fuzz workflow can also be started +manually with those extended settings. Failure artifacts are retained for 14 days; fuzzing does +not persist generated corpus data. + The root dependency policy does not include this package. Run its policy check separately after generating or updating `fuzz/Cargo.lock`: From 0437144aa6a9d0428340229ddd9ed49140e35327 Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Tue, 1 Sep 2026 15:15:20 +0200 Subject: [PATCH 019/166] docs: define common format encoding --- spec/PITHOS_1.0.0_draft.md | 130 +++++++++++++++++++------------------ 1 file changed, 67 insertions(+), 63 deletions(-) diff --git a/spec/PITHOS_1.0.0_draft.md b/spec/PITHOS_1.0.0_draft.md index 8ebc82e..53c98fa 100644 --- a/spec/PITHOS_1.0.0_draft.md +++ b/spec/PITHOS_1.0.0_draft.md @@ -23,7 +23,7 @@ The code examples in this document are only intended to illustrate the architect 6. **Emergency recovery**: Block markers MUST enable reconstruction even with corrupted directories 7. **Hierarchical organization**: Files use full paths from archive root; directories MUST be declared before their contents -## 3. File Structure +## 3. File Structure and Encoding A Pithos file MUST have the following structure: @@ -35,6 +35,24 @@ A Pithos file MUST have the following structure: [Directory] // REQUIRED: File MUST end with directory ``` +### 3.1 Common Encoding Rules + +The following rules define the bytes stored in the file for every structure in +Section 4: + +1. Fixed byte arrays are stored exactly as shown, with no length prefix. +2. `u16`, `u32`, and `u64` fields explicitly marked fixed-width are big-endian. +3. Other unsigned integers use the shortest valid ULEB128 form. Readers MUST + reject truncated, overlong, and overflowing encodings. +4. A string is a ULEB128 byte length followed by that many UTF-8 bytes. +5. A vector is a ULEB128 item count followed by the items in order. +6. A tuple stores its members in the stated order. +7. Options and enum variants use a one-byte tag followed by the selected value, + if any. Each Section 4 tag table defines the allowed tags. +8. Structures have no alignment bytes or implicit padding. +9. A reader MUST consume exactly the bytes assigned to a structure and reject + unknown tags. + ## 4. Core Data Structures ### 4.1 File Header @@ -135,6 +153,34 @@ The CRC MUST be CRC-32/ISO-HDLC with width 32, polynomial `0x04C11DB7`, initial It MUST cover every exact serialized byte from the marker through the fixed-width `dir_len`, excluding only the stored final CRC. Readers MUST validate the marker, embedded length, CRC, and exact parser consumption for the terminal directory and independently for every parent directory before decrypting, merging, or otherwise using its metadata. Invalid directories MUST be rejected. +#### 4.3.1 Directory Entry and Path Ordering + +Directory entries MUST follow these ordering and path rules: + +1. A directory entry MUST appear before entries for files or subdirectories within it. +2. Paths MUST be relative, have no leading `/`, and use forward slashes as separators. +3. The root directory is implicit and MUST NOT have an entry. +4. Readers and writers MUST validate this ordering. + +**Example of valid ordering:** +``` +data/ (directory) +data/raw/ (directory - parent "data/" already exists) +data/raw/file1.csv (file - parent "data/raw/" already exists) +data/processed/ (directory - parent "data/" already exists) +data/processed/file2.csv (file - parent "data/processed/" already exists) +docs/ (directory) +docs/README.md (file - parent "docs/" already exists) +data/raw/file1_v2.csv (file - parent "data/raw/" already exists) -> Newer version of file1.csv +``` + +**Example of invalid ordering:** +``` +data/raw/file1.csv (ERROR: parent "data/raw/" not yet declared) +data/raw/ (too late - file already referenced this directory) +data/ (too late - subdirectory already referenced this) +``` + ### 4.4 File Representation #### 4.4.1 File Types @@ -266,51 +312,9 @@ pub enum PithosError { } ``` -## 5. Encoding Specifications - -### 5.1 Integer Encoding - -All integer fields marked as "varint" MUST use unsigned LEB128 encoding. - -### 5.2 String Encoding - -All strings MUST be encoded as UTF-8 with a varint length prefix. - -### 5.3 Byte Order - -All multi-byte values not using varint encoding MUST use big-endian byte order. - -### 5.4 Directory Entry Ordering Requirements - -**CRITICAL**: Directory entries MUST follow strict ordering rules to ensure proper extraction: - -1. **Parent Before Child Rule**: A directory entry MUST appear before any entries for files or subdirectories within it -2. **Path Format**: All paths MUST be relative (no leading `/`) and use forward slashes as separators -3. **Root Directory**: The root directory is implicit and MUST NOT have an entry -4. **Validation**: Implementations MUST validate ordering during both writing and reading - -**Example of valid ordering:** -``` -data/ (directory) -data/raw/ (directory - parent "data/" already exists) -data/raw/file1.csv (file - parent "data/raw/" already exists) -data/processed/ (directory - parent "data/" already exists) -data/processed/file2.csv (file - parent "data/processed/" already exists) -docs/ (directory) -docs/README.md (file - parent "docs/" already exists) -data/raw/file1_v2.csv (file - parent "data/raw/" already exists) -> Newer version of file1.csv -``` - -**Example of INVALID ordering:** -``` -data/raw/file1.csv (ERROR: parent "data/raw/" not yet declared) -data/raw/ (too late - file already referenced this directory) -data/ (too late - subdirectory already referenced this) -``` - -## 6. Content Processing +## 5. Content Processing -### 6.1 Content-Defined Chunking +### 5.1 Content-Defined Chunking Implementations SHOULD use content-defined chunking with recommended parameters: - **min_size**: 64 KB @@ -318,7 +322,7 @@ Implementations SHOULD use content-defined chunking with recommended parameters: - **max_size**: 512 KB - **window_size**: 48 bytes -### 6.2 Block Hashing +### 5.2 Block Hashing Each block identifier MUST be the full 32-byte default unkeyed BLAKE3 digest of the exact plaintext chunk before compression or encryption. Readers MUST retrieve the stored block, @@ -327,11 +331,11 @@ original size as the output bound, require the resulting plaintext length to equ original size, compute the complete plaintext digest, and compare it with the block identifier before releasing any output derived from that block. -### 6.3 Convergent Encryption +### 5.3 Convergent Encryption Content keys MUST be derived deterministically using SHAKE256. -### 6.4 Compression +### 5.4 Compression Implementations SHOULD support: - Level 0: No compression @@ -339,9 +343,9 @@ Implementations SHOULD support: - Levels 4-6: Balanced compression (e.g., Zstd levels 4-9) - Level 7: Maximum compression (e.g., Zstd level 19+) -## 7. Operations Overview +## 6. Operations Overview -### 7.1 Reading Operations +### 6.1 Reading Operations 1. Read and validate file header 2. Find last directory by scanning from end @@ -349,7 +353,7 @@ Implementations SHOULD support: 4. Build block index 5. Extract files by reading referenced blocks -### 7.2 Writing Operations +### 6.2 Writing Operations 1. Write file header 2. Process files in correct directory order @@ -358,7 +362,7 @@ Implementations SHOULD support: 5. Write directory and encryption sections 6. Validate complete structure -### 7.3 Directory Tree Operations +### 6.3 Directory Tree Operations When archiving directory trees: 1. Process directories before their contents @@ -366,7 +370,7 @@ When archiving directory trees: 3. Preserve file metadata (permissions, timestamps) 4. Handle symlinks appropriately per platform -## 8. Security Considerations +## 7. Security Considerations 1. Implementations MUST verify the complete plaintext block size and hash after authenticated decryption and decompression, and before releasing output derived from the block @@ -375,21 +379,21 @@ When archiving directory trees: 4. External block URLs MUST use HTTPS in production environments 5. Path traversal attacks MUST be prevented through validation -## 9. Implementation Requirements +## 8. Implementation Requirements -### 9.1 Mandatory Features +### 8.1 Mandatory Features Implementations MUST support: - Reading and writing base format (version 1.0) - Blake3 hashing -- Varint encoding/decoding +- ULEB128 encoding/decoding - CRC32 calculation - UTF-8 string handling - All four file types (Data, Metadata, Directory, Symlink) - Directory ordering validation (parents before children) - Path format validation (relative paths only) -### 9.2 Optional Features +### 8.2 Optional Features Implementations MAY support: - Compression (levels 1-7) @@ -397,7 +401,7 @@ Implementations MAY support: - External block storage - Content-defined chunking -### 9.3 Platform-Specific Considerations +### 8.3 Platform-Specific Considerations #### Symlinks - Unix systems: Create proper symbolic links @@ -411,7 +415,7 @@ Implementations MAY support: - Archives use forward slashes (`/`) internally - Convert to platform-appropriate separators on extraction -## 10. Future Extensions +## 9. Future Extensions The format reserves space for future extensions: - FileType values 4-255 @@ -420,19 +424,19 @@ The format reserves space for future extensions: Extensions MUST maintain backwards compatibility for reading. -## 11. Constants and Identifiers +## 10. Constants and Identifiers -### 11.1 Magic Values +### 10.1 Magic Values - File Header: `b"PITH"` - Block Header: `b"BLCK"` - Directory: `b"PITHOSDR"` -### 11.2 Version Numbers +### 10.2 Version Numbers - Version 1.0: `0x0100` -### 11.3 Default Values +### 10.3 Default Values - Current timestamp: Unix seconds since epoch - Default file permissions: `0o644` From a39e627fa1604ca6aaf815533c54f3b620c4402b Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Tue, 1 Sep 2026 15:32:20 +0200 Subject: [PATCH 020/166] docs: specify encoded data structures --- spec/PITHOS_1.0.0_draft.md | 255 ++++++++++++++++++++++++++++++++----- 1 file changed, 221 insertions(+), 34 deletions(-) diff --git a/spec/PITHOS_1.0.0_draft.md b/spec/PITHOS_1.0.0_draft.md index 53c98fa..eceb462 100644 --- a/spec/PITHOS_1.0.0_draft.md +++ b/spec/PITHOS_1.0.0_draft.md @@ -57,22 +57,32 @@ Section 4: ### 4.1 File Header -Every Pithos file MUST begin with a FileHeader: +A FileHeader identifies a Pithos file and its format version. ```rust /// File header - appears once at the beginning of every Pithos file #[derive(Debug, Clone, PartialEq, Eq)] pub struct FileHeader { - pub magic: [u8; 4], // MUST be b"PITH" - pub version: u16, // Format version (e.g., 0x0100 for 1.0) + pub magic: [u8; 4], // b"PITH" + pub version: u16, // fixed-width big-endian, 0x0100 for 1.0 } ``` +**Encoded Form** + +| Field | Bytes stored in the file | +| --- | --- | +| `magic` | Exactly 4 bytes: ASCII `PITH` | +| `version` | Fixed-width `u16be` | + +Readers MUST reject a header whose magic is not `PITH` or whose version is not +`0x0100`. The encoded form is exactly six bytes: `PITH 01 00`. + ### 4.2 Block Storage #### 4.2.1 Block Header -Each block MUST be preceded by a minimal header for emergency scanning: +A BlockHeader marks the beginning of locally stored block data. ```rust /// Minimal block header - just for emergency scanning @@ -82,24 +92,52 @@ pub struct BlockHeader { } ``` +**Encoded Form** + +| Field | Bytes stored in the file | +| --- | --- | +| `marker` | Exactly 4 bytes: ASCII `BLCK` | + +Readers MUST reject a block header whose marker is not `BLCK`. + #### 4.2.2 Block Index Entry -Complete block metadata MUST be stored in the directory: +The hash-keyed block descriptor describes one block stored or referenced by a directory. ```rust -/// Block index entry - single source of truth for block hashes +/// Block descriptor body; its hash is the key in Directory::blocks #[derive(Debug, Clone, PartialEq, Eq)] pub struct BlockIndexEntry { - pub index: u64, // Unique sequential identifier (varint encoded) - pub hash: [u8; 32], // Full Blake3 hash of original content pub offset: u64, // Byte offset in file (varint encoded) pub stored_size: u64, // Size as stored (compressed/encrypted) (varint) pub original_size: u64, // Original uncompressed size (varint) pub flags: ProcessingFlags, // Compression, encryption settings pub location: BlockLocation, // Where block data resides } +``` + +**Encoded Form** + +The directory block sequence is a vector. Each item is encoded as follows: -/// Processing flags packed into single byte +| Field | Bytes stored in the file | +| --- | --- | +| `block_hash` | Exactly 32 bytes: BLAKE3 hash of the original block content | +| `offset` | ULEB128 `u64` | +| `stored_size` | ULEB128 `u64` | +| `original_size` | ULEB128 `u64` | +| `flags` | One ProcessingFlags byte: bits 0-2 compression level, bit 3 encryption enabled, bits 4-7 zero | +| `location` | One tag byte: `00` local, or `01` followed by a URL string for external | + +Readers MUST reject duplicate block hashes in one directory and MUST use the +32-byte hash as the block's only identity. + +#### 4.2.3 Processing Flags + +ProcessingFlags records the compression level and whether a block is encrypted. + +```rust +/// Processing flags packed into one byte bitflags::bitflags! { #[derive(Debug, Clone, PartialEq, Eq)] pub struct ProcessingFlags: u8 { @@ -119,7 +157,24 @@ bitflags::bitflags! { // Bits 4-7: Reserved for future use (MUST be zero) } } +``` + +**Encoded Form** + +| Bits | Meaning | +| --- | --- | +| 0-2 | Compression level: `0` is none; `1` through `7` are implementation-defined | +| 3 | Encryption enabled: `0` is disabled; `1` is enabled | +| 4-7 | Reserved; all bits MUST be zero | + +ProcessingFlags is stored as exactly one byte. Readers MUST reject a value with +any reserved bit set. +#### 4.2.4 Block Location + +BlockLocation states where the bytes for a block can be obtained. + +```rust /// Block storage location #[derive(Debug, Clone, PartialEq, Eq)] pub enum BlockLocation { @@ -128,9 +183,19 @@ pub enum BlockLocation { } ``` +**Encoded Form** + +| Tag | Variant | Bytes after the tag | +| --- | --- | --- | +| `00` | `Local` | None | +| `01` | `External` | `url` as a string | + +Readers MUST reject unknown tags. A local block's offset and size describe its +location in this file; the block-boundary rules are specified separately. + ### 4.3 Directory Structure -The directory MUST contain all file and block metadata: +A Directory contains the metadata for one appended archive segment. ```rust /// Directory - lists all files and blocks in this segment @@ -138,15 +203,32 @@ The directory MUST contain all file and block metadata: pub struct Directory { pub identifier: [u8; 8], // MUST be exactly ASCII b"PITHOSDR" pub parent_directory_offset: Option<(u64, u64)>, // Previous directory (start, len) (varint, backwards chain) - pub files: Vec, // Files in this segment - pub blocks: Vec, // Blocks in this segment + pub files: Vec<(u64, String, FileEntry)>, // File ID, path, and body + pub blocks: Vec<([u8; 32], BlockIndexEntry)>, // Block hash and body pub relations: Vec<(u64, String)>, // Relation idx, relationname / id - pub encryption: Vec, + pub encryption: Vec<([u8; 32], EncryptionSection)>, // Sender key and body pub dir_len: u64, pub crc32: u32, // CRC-32/ISO-HDLC of serialized bytes through dir_len } ``` +**Encoded Form** + +| Field | Bytes stored in the file | +| --- | --- | +| `identifier` | Exactly 8 bytes: ASCII `PITHOSDR` | +| `parent_directory_offset` | Option tag, then, for tag `01`, tuple of ULEB128 `start` and ULEB128 `len` | +| `files` | Vector of records: ULEB128 `file_id`, path string, then FileEntry body | +| `blocks` | Vector of items: `block_hash[32]`, then ULEB128 `offset`, ULEB128 `stored_size`, ULEB128 `original_size`, one flags byte, and location | +| `relations` | Vector of tuples: ULEB128 relationship ID followed by relationship-name string | +| `encryption` | Vector, which MAY be empty, of items: `sender_public_key[32]`, then a recipient-record vector | +| `dir_len` | Fixed-width `u64be` | +| `crc32` | Fixed-width `u32be` | + +The parent option tag is `00` for no parent and `01` for a parent. Readers MUST +reject other parent tags, duplicate relationship IDs, duplicate sender public +keys, duplicate file IDs, duplicate file paths, or duplicate block hashes. + The directory marker MUST be exactly the eight ASCII bytes `PITHOSDR`. The final 12 directory bytes MUST be `dir_len:u64be || crc32:u32be`, where `dir_len` is the complete directory length from the marker through the CRC, inclusive. The CRC MUST be CRC-32/ISO-HDLC with width 32, polynomial `0x04C11DB7`, initial value`0xFFFFFFFF`, reflected input and output, and final XOR `0xFFFFFFFF` (the check value for ASCII `123456789` is `0xCBF43926`). @@ -185,54 +267,102 @@ data/ (too late - subdirectory already referenced this) #### 4.4.1 File Types -Files MUST be classified by type: +FileType identifies the kind of a directory file record. ```rust /// File types (u8 representation for efficiency) #[repr(u8)] #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum FileType { - Data = 0, // Regular data file (default) - Metadata = 1, // Metadata file (RO-Crate, DataCite, etc.) - Directory = 2, // Directory entry + Directory = 0, // Directory entry + Data = 1, // Regular data file + Metadata = 2, // Metadata file (RO-Crate, DataCite, etc.) Symlink = 3, // Symbolic link // Values 4-255 reserved for future use } ``` -#### 4.4.2 File Entry +**Encoded Form** -Each file MUST be represented by a FileEntry: +| Byte | Variant | +| --- | --- | +| `00` | `Directory` | +| `01` | `Data` | +| `02` | `Metadata` | +| `03` | `Symlink` | -```rust +FileType is stored as exactly one byte. Readers MUST reject values from `04` +through `ff`. + +#### 4.4.2 Block Data State +BlockDataState stores either encrypted file block data or a decrypted block list. +```rust pub enum BlockDataState { Encrypted(Vec), // Nonce + ChaCha20Poly1305 - Decrypted(Vec<(u64, [u8; 32])>) // Index / SHAKE256 hash + Decrypted(Vec<([u8; 32], [u8; 32])>), // Block hash and block key } +``` + +**Encoded Form** + +| Tag | Variant | Bytes after the tag | +| --- | --- | --- | +| `00` | `Encrypted` | Vector of bytes | +| `01` | `Decrypted` | Vector of tuples, each `block_hash[32] || block_key[32]` | + +Readers MUST reject unknown tags. The block hash is the only block identity in a +decrypted block-list entry. + +#### 4.4.3 File Entry +Each directory file record identifies and names one file, then stores its FileEntry body. +```rust /// File entry - describes a single file, directory, or symlink #[derive(Debug, Clone, PartialEq, Eq)] pub struct FileEntry { - pub file_id: u64, // Sequential unique identifier (varint) - pub path: String, // Full path from archive root (UTF-8) pub file_type: FileType, // Type of entry pub block_data: BlockDataState, pub created: u64, // Unix timestamp (seconds since epoch) pub modified: u64, // Unix timestamp (seconds since epoch) pub file_size: u64, // Total size in bytes (varint) pub permissions: u32, // Unix-style permissions - pub references: Vec, // Data->Metadata references only + pub references: Vec, // References from this file pub symlink_target: Option, // Target path for symlinks } ``` -#### 4.4.3 File References +**Encoded Form** + +The directory file sequence is a vector. Each record is encoded as follows: + +| Field | Bytes stored in the file | +| --- | --- | +| `file_id` | ULEB128 `u64` | +| `path` | String | +| `FileEntry` body | Fields in the following table | + +| FileEntry body field | Bytes stored in the file | +| --- | --- | +| `file_type` | FileType encoded form above | +| `block_data` | Tag `00` and byte vector, or tag `01` and vector of `block_hash[32] || block_key[32]` tuples | +| `created` | ULEB128 `u64` | +| `modified` | ULEB128 `u64` | +| `file_size` | ULEB128 `u64` | +| `permissions` | ULEB128 `u32` | +| `references` | Vector of tuples: ULEB128 `target_file_id` followed by ULEB128 `relationship` | +| `symlink_target` | Option tag, then, for tag `01`, a target string | + +The `symlink_target` option tag is `00` for no target and `01` for a target. +Readers MUST reject other tags. File IDs and paths are record fields, not fields +of the FileEntry body. -References MUST be one-way from metadata to data files: +#### 4.4.4 File References + +A Reference identifies a target file and the relationship from the containing file to it. ```rust /// Simplified reference structure @@ -243,6 +373,16 @@ pub struct Reference { } ``` +**Encoded Form** + +| Field | Bytes stored in the file | +| --- | --- | +| `target_file_id` | ULEB128 `u64` | +| `relationship` | ULEB128 `u64` | + +References have no tag or length of their own; their containing vector provides +the count. Readers MUST consume both fields for every reference. + **Standard relationship types:** - `DESCRIBES = 0`: Metadata describing target - `ANNOTATES = 1`: Additional annotations @@ -258,30 +398,77 @@ pub struct Reference { ### 4.5 Encryption Section -Encryption sections MUST follow each directory: +Encryption sections carry per-sender recipient data. + +#### 4.5.1 EncryptionSection + +An EncryptionSection contains the recipient records associated with one sender public key. ```rust /// Encryption section - privacy-preserving access control #[derive(Debug, Clone, PartialEq, Eq)] pub struct EncryptionSection { - pub sender_public_key: [u8; 32], // X25519 public key - pub recipients: Vec, // Per-recipient data + pub recipients: Vec<([u8; 32], RecipientSection)>, // Recipient key and body } +``` -pub enum RecipientData { - Encrypted(Vec), // Chacha + nonce - Decrypted(Vec<(u64, [u8; 32])>) // Fileindex / Shake256 hash -} +**Encoded Form** + +The Directory `encryption` vector stores each sender public key as the leading +32 bytes of its item, followed by this EncryptionSection body: + +| Field | Bytes stored in the file | +| --- | --- | +| `recipients` | Vector of records: `recipient_public_key[32]` followed by RecipientSection body | + +Sender public keys are exactly 32 bytes. Readers MUST reject a duplicate sender +key within a directory. +#### 4.5.2 RecipientSection +A RecipientSection contains the data associated with one recipient public key. + +```rust /// Per-recipient encrypted data #[derive(Debug, Clone, PartialEq, Eq)] pub struct RecipientSection { - pub recipient_public_key: [u8; 32], // Recipient's X25519 public key pub recipient_data: RecipientData, // Encrypted FileKeyEntry list } ``` +**Encoded Form** + +Each recipient record stores its recipient public key as the leading 32 bytes, +followed by this RecipientSection body: + +| Field | Bytes stored in the file | +| --- | --- | +| `recipient_data` | Tag `00` and byte vector, or tag `01` and vector of ULEB128 `file_id` plus `file_key[32]` tuples | + +Recipient public keys are exactly 32 bytes. Readers MUST reject a duplicate +recipient key within an EncryptionSection. + +#### 4.5.3 RecipientData + +RecipientData stores either encrypted recipient data or a decrypted file-key list. + +```rust +pub enum RecipientData { + Encrypted(Vec), // Chacha + nonce + Decrypted(Vec<(u64, [u8; 32])>) // File ID and file key +} +``` + +**Encoded Form** + +| Tag | Variant | Bytes after the tag | +| --- | --- | --- | +| `00` | `Encrypted` | Vector of bytes | +| `01` | `Decrypted` | Vector of tuples, each ULEB128 `file_id` followed by `file_key[32]` | + +Readers MUST reject unknown tags and duplicate file IDs in a decrypted +recipient-data list. + ### 4.6 Error Types ```rust From 3c5e03e80043b04cc92b69a759ac30dd6c7dc4ff Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Tue, 1 Sep 2026 15:34:53 +0200 Subject: [PATCH 021/166] docs: clarify directory encryption placement --- spec/PITHOS_1.0.0_draft.md | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/spec/PITHOS_1.0.0_draft.md b/spec/PITHOS_1.0.0_draft.md index eceb462..11b3ee1 100644 --- a/spec/PITHOS_1.0.0_draft.md +++ b/spec/PITHOS_1.0.0_draft.md @@ -35,6 +35,9 @@ A Pithos file MUST have the following structure: [Directory] // REQUIRED: File MUST end with directory ``` +Each segment ends immediately after its Directory. Encryption sections, when +present, are items in that Directory's `encryption` vector. + ### 3.1 Common Encoding Rules The following rules define the bytes stored in the file for every structure in @@ -195,7 +198,8 @@ location in this file; the block-boundary rules are specified separately. ### 4.3 Directory Structure -A Directory contains the metadata for one appended archive segment. +A Directory contains the metadata for one appended archive segment, which ends +after the directory. ```rust /// Directory - lists all files and blocks in this segment @@ -398,7 +402,8 @@ the count. Readers MUST consume both fields for every reference. ### 4.5 Encryption Section -Encryption sections carry per-sender recipient data. +Encryption sections are items in `Directory.encryption` and carry per-sender +recipient data. #### 4.5.1 EncryptionSection @@ -546,7 +551,7 @@ Implementations SHOULD support: 2. Process files in correct directory order 3. Chunk content using content-defined chunking 4. Deduplicate blocks by hash -5. Write directory and encryption sections +5. Write a directory, including its encryption sections when present 6. Validate complete structure ### 6.3 Directory Tree Operations @@ -561,7 +566,7 @@ When archiving directory trees: 1. Implementations MUST verify the complete plaintext block size and hash after authenticated decryption and decompression, and before releasing output derived from the block -2. CRC32 values MUST be validated for directories and encryption sections +2. Directory CRC32 values MUST be validated 3. Convergent encryption reveals when identical files exist (accepted trade-off) 4. External block URLs MUST use HTTPS in production environments 5. Path traversal attacks MUST be prevented through validation From e4976ab93f65ae330b3f42592d6354ff89a7cc4a Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Tue, 1 Sep 2026 15:36:50 +0200 Subject: [PATCH 022/166] docs: define local block boundaries --- spec/PITHOS_1.0.0_draft.md | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/spec/PITHOS_1.0.0_draft.md b/spec/PITHOS_1.0.0_draft.md index 11b3ee1..a244edd 100644 --- a/spec/PITHOS_1.0.0_draft.md +++ b/spec/PITHOS_1.0.0_draft.md @@ -20,7 +20,7 @@ The code examples in this document are only intended to illustrate the architect 3. **Privacy-preserving sharing**: Users MUST NOT be able to see who else has access to files 4. **Flexible metadata**: Metadata MUST be stored as regular files with special type markers 5. **Progressive enhancement**: Implementations MUST support the base format and MAY support optional features -6. **Emergency recovery**: Block markers MUST enable reconstruction even with corrupted directories +6. **Limited recovery**: Recovery tools MAY treat block markers as untrusted candidates; directories provide block metadata 7. **Hierarchical organization**: Files use full paths from archive root; directories MUST be declared before their contents ## 3. File Structure and Encoding @@ -88,7 +88,7 @@ Readers MUST reject a header whose magic is not `PITH` or whose version is not A BlockHeader marks the beginning of locally stored block data. ```rust -/// Minimal block header - just for emergency scanning +/// Minimal block header #[derive(Debug, Clone, PartialEq, Eq)] pub struct BlockHeader { pub marker: [u8; 4], // MUST be b"BLCK" @@ -196,6 +196,19 @@ pub enum BlockLocation { Readers MUST reject unknown tags. A local block's offset and size describe its location in this file; the block-boundary rules are specified separately. +#### 4.2.5 Local Block Boundaries and Recovery + +A local block is `BLCK || stored_payload`. Its `offset` is the zero-based file +offset of the first byte of `BLCK`; `stored_size` counts only `stored_payload`. +The marker occupies bytes `offset` through `offset + 3`. The payload starts at +`offset + 4` and occupies exactly `stored_size` bytes; when nonempty, its last +byte is `offset + 4 + stored_size - 1`. + +Readers locate local blocks from directory descriptors and MUST use checked +arithmetic for every range. Normal reading MUST NOT search payload bytes for +`BLCK`. Recovery tools MAY treat `BLCK` as an untrusted candidate only: the +marker alone provides no length, flags, or identity. + ### 4.3 Directory Structure A Directory contains the metadata for one appended archive segment, which ends From 7b16173c1144499aa1293584039e3eb4345c8c2e Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Tue, 1 Sep 2026 15:38:58 +0200 Subject: [PATCH 023/166] docs: specify terminal directory lookup --- spec/PITHOS_1.0.0_draft.md | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/spec/PITHOS_1.0.0_draft.md b/spec/PITHOS_1.0.0_draft.md index a244edd..7f6e16c 100644 --- a/spec/PITHOS_1.0.0_draft.md +++ b/spec/PITHOS_1.0.0_draft.md @@ -252,7 +252,19 @@ The CRC MUST be CRC-32/ISO-HDLC with width 32, polynomial `0x04C11DB7`, initial It MUST cover every exact serialized byte from the marker through the fixed-width `dir_len`, excluding only the stored final CRC. Readers MUST validate the marker, embedded length, CRC, and exact parser consumption for the terminal directory and independently for every parent directory before decrypting, merging, or otherwise using its metadata. Invalid directories MUST be rejected. -#### 4.3.1 Directory Entry and Path Ordering +#### 4.3.1 Terminal Directory Lookup + +To locate the terminal directory during normal reading: + +1. Read the final 12 file bytes as `dir_len:u64be || crc32:u32be`. +2. Compute `directory_start = file_length - dir_len` using checked subtraction. +3. Require `dir_len` to be at least the smallest directory allowed by the finalized 1.0 encoding. The root stores ten standard relationships, so the 0.8 25-byte empty-directory minimum does not apply. +4. Parse the directory at `directory_start` and require it to end exactly at the end of the file. +5. Validate the marker, embedded length, CRC, and exact byte consumption before using metadata. + +Normal reading MUST reject trailing bytes, truncation, underflow, a false marker, and a torn final append. It MUST NOT fall back to an older directory. Salvage is a separate mode; if it locates an older directory, it MUST label the result incomplete. + +#### 4.3.2 Directory Entry and Path Ordering Directory entries MUST follow these ordering and path rules: @@ -553,7 +565,7 @@ Implementations SHOULD support: ### 6.1 Reading Operations 1. Read and validate file header -2. Find last directory by scanning from end +2. Locate and validate the terminal directory using the direct lookup in Section 4.3.1 3. Validate directory ordering 4. Build block index 5. Extract files by reading referenced blocks From 8aaed557a8db1bac052e30da83355bfe3497a955 Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Tue, 1 Sep 2026 15:41:32 +0200 Subject: [PATCH 024/166] docs: specify optional encryption suite --- spec/PITHOS_1.0.0_draft.md | 42 ++++++++++++++++++++++++++++++++------ 1 file changed, 36 insertions(+), 6 deletions(-) diff --git a/spec/PITHOS_1.0.0_draft.md b/spec/PITHOS_1.0.0_draft.md index 7f6e16c..c1858d9 100644 --- a/spec/PITHOS_1.0.0_draft.md +++ b/spec/PITHOS_1.0.0_draft.md @@ -329,7 +329,7 @@ BlockDataState stores either encrypted file block data or a decrypted block list ```rust pub enum BlockDataState { - Encrypted(Vec), // Nonce + ChaCha20Poly1305 + Encrypted(Vec), // nonce || ciphertext || tag Decrypted(Vec<([u8; 32], [u8; 32])>), // Block hash and block key } ``` @@ -342,7 +342,10 @@ pub enum BlockDataState { | `01` | `Decrypted` | Vector of tuples, each `block_hash[32] || block_key[32]` | Readers MUST reject unknown tags. The block hash is the only block identity in a -decrypted block-list entry. +decrypted block-list entry. A decrypted block list is a ULEB128 count followed +by that many `block_hash[32] || block_key[32]` pairs. A repeated block hash is +valid only when it has the same block key; readers MUST reject a conflicting +duplicate hash and block key. #### 4.4.3 File Entry @@ -484,7 +487,7 @@ RecipientData stores either encrypted recipient data or a decrypted file-key lis ```rust pub enum RecipientData { - Encrypted(Vec), // Chacha + nonce + Encrypted(Vec), // nonce || ciphertext || tag Decrypted(Vec<(u64, [u8; 32])>) // File ID and file key } ``` @@ -496,8 +499,10 @@ pub enum RecipientData { | `00` | `Encrypted` | Vector of bytes | | `01` | `Decrypted` | Vector of tuples, each ULEB128 `file_id` followed by `file_key[32]` | -Readers MUST reject unknown tags and duplicate file IDs in a decrypted -recipient-data list. +Readers MUST reject unknown tags. A decrypted recipient-data list is a ULEB128 +count followed by that many ULEB128 `file_id` and `file_key[32]` pairs. A +repeated file ID is valid only when it has the same file key; readers MUST +reject a conflicting duplicate file ID and file key. ### 4.6 Error Types @@ -550,7 +555,32 @@ before releasing any output derived from that block. ### 5.3 Convergent Encryption -Content keys MUST be derived deterministically using SHAKE256. +Encryption is optional. An implementation that supports encryption MUST use +X25519, SHAKE256, and ChaCha20-Poly1305 as specified here. + +All encryption keys are 32 bytes. ChaCha20-Poly1305 uses a 12-byte nonce and +produces a 16-byte authentication tag. Every encrypted value is stored as +`nonce || ciphertext || tag`; the nonce is part of the stored byte vector. The +additional authenticated data (AAD) is empty. + +The block key is the first 32 output bytes of `SHAKE256(plaintext)`, where +`plaintext` is the exact block plaintext before compression or encryption. No +label or length prefix is included. A block payload with encryption enabled is +encrypted with its block key. + +A file key is 32 random bytes. It encrypts the decrypted block list for a file. +For each recipient record, the recipient wrapping key is the raw 32-byte X25519 +shared secret between the sender private key and recipient public key. That +shared secret is used directly as the ChaCha20-Poly1305 key to encrypt the +decrypted recipient list; no intermediate KDF is used. Implementations MUST +reject non-contributory X25519 public keys. + +An implementation MUST use a fresh nonce for every value encrypted under a +given key. It MUST authenticate and decrypt an encrypted value successfully +before using its plaintext or releasing output derived from it. + +Version 1.0 deliberately has no SHAKE256 label, intermediate X25519 KDF, or +AAD. A redesign of any of these inputs requires a new format version. ### 5.4 Compression From ed82f3b01ba4bd631f53a722303a8ce592e601b7 Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Tue, 1 Sep 2026 15:44:21 +0200 Subject: [PATCH 025/166] docs: define compression flag semantics --- spec/PITHOS_1.0.0_draft.md | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/spec/PITHOS_1.0.0_draft.md b/spec/PITHOS_1.0.0_draft.md index c1858d9..9c3ce30 100644 --- a/spec/PITHOS_1.0.0_draft.md +++ b/spec/PITHOS_1.0.0_draft.md @@ -144,7 +144,7 @@ ProcessingFlags records the compression level and whether a block is encrypted. bitflags::bitflags! { #[derive(Debug, Clone, PartialEq, Eq)] pub struct ProcessingFlags: u8 { - // Bits 0-2: Compression level (0=none, 1-7=implementation-defined) + // Bits 0-2: Compression (0=none; 1-7=Zstandard) const COMPRESSION_LEVEL_1 = 0b0000_0001; const COMPRESSION_LEVEL_2 = 0b0000_0010; const COMPRESSION_LEVEL_3 = 0b0000_0011; @@ -166,7 +166,7 @@ bitflags::bitflags! { | Bits | Meaning | | --- | --- | -| 0-2 | Compression level: `0` is none; `1` through `7` are implementation-defined | +| 0-2 | Compression: `0` means the stored payload is not compressed; `1` through `7` each mean the stored payload is one standard Zstandard frame | | 3 | Encryption enabled: `0` is disabled; `1` is enabled | | 4-7 | Reserved; all bits MUST be zero | @@ -584,11 +584,27 @@ AAD. A redesign of any of these inputs requires a new format version. ### 5.4 Compression -Implementations SHOULD support: -- Level 0: No compression -- Levels 1-3: Fast compression (e.g., Zstd levels 1-3) -- Levels 4-6: Balanced compression (e.g., Zstd levels 4-9) -- Level 7: Maximum compression (e.g., Zstd level 19+) +The compression value in ProcessingFlags describes the stored payload actually +written. Value `0` means the stored payload is not compressed. Each nonzero +value from `1` through `7` means the stored payload is one standard Zstandard +frame. Readers that support compression MUST decode any valid Zstandard frame +and bound decompressed output by `original_size`; they do not need the writer's +compression level. +Readers MUST reject ProcessingFlags with any nonzero reserved bit. + +The current writer mapping is optional implementation guidance: `1` to Zstandard +level `1`, `2` to `4`, `3` to `8`, `4` to `11`, `5` to `15`, `6` to `18`, and +`7` to `22`. A writer MAY abandon unhelpful compression and store the plaintext +payload instead, but MUST then write compression value `0`. For example, a +writer MAY use a 4096-byte sample and a 0.85 compression-ratio threshold to +decide whether compression is helpful; this heuristic is not a compatibility +rule. + +Conforming encoders need not produce byte-identical Zstandard output. A future +conformance appendix MUST provide decode-direction vectors containing stored +compressed bytes, `original_size`, the expected plaintext, and its expected +32-byte BLAKE3 hash. These vectors MUST decode to the expected plaintext with +different supported Zstandard versions. ## 6. Operations Overview From 5a14e53286d2174f5d957b2c1f87e1ef76881e99 Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Tue, 1 Sep 2026 15:46:52 +0200 Subject: [PATCH 026/166] docs: define additive append semantics --- spec/PITHOS_1.0.0_draft.md | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/spec/PITHOS_1.0.0_draft.md b/spec/PITHOS_1.0.0_draft.md index 9c3ce30..fc1d683 100644 --- a/spec/PITHOS_1.0.0_draft.md +++ b/spec/PITHOS_1.0.0_draft.md @@ -264,7 +264,42 @@ To locate the terminal directory during normal reading: Normal reading MUST reject trailing bytes, truncation, underflow, a false marker, and a torn final append. It MUST NOT fall back to an older directory. Salvage is a separate mode; if it locates an older directory, it MUST label the result incomplete. -#### 4.3.2 Directory Entry and Path Ordering +#### 4.3.2 Append Chains + +Each Directory contains the entries introduced by its segment. To construct the +effective archive, readers merge the selected directory chain from its oldest +directory to its newest directory. An append adds file entries, block +descriptors, relationship definitions, and recipient grants. Version 1.0 has +no deletion, replacement, or tombstone. + +File IDs and paths MUST be unique across the chain. Writers assign file ID 0 +to the first file and assign each later file ID as the current maximum ID plus +1. Readers MUST accept unused file-ID gaps. A rename adds a file record with a +new file ID and path; the old record remains in the effective archive. + +A repeated block hash in one directory is invalid. Across directories, a block +hash MAY reappear only when its `original_size` is the same. The descriptor in +the oldest directory remains the effective descriptor. Exact repeated +relationship definitions are allowed; conflicting definitions of one +relationship ID are invalid. Exact repeated recipient grants are allowed; +conflicting grants for the same sender public key and recipient public key are +invalid. + +The root directory MUST contain these ten standard relationship definitions: +`DESCRIBES`, `ANNOTATES`, `DERIVED_FROM`, `SOURCE_OF`, `PREVIOUS_VERSION`, +`NEXT_VERSION`, `PART_OF`, `CONTAINS`, `INPUT_TO`, and `OUTPUT_FROM`. +Appended directories inherit these definitions and MAY repeat them exactly. + +For example, three segments with parent order `root <- append-1 <- append-2` +merge as follows: + +| Segment | Entries introduced | Final visible entries after `append-2` | +| --- | --- | --- | +| `root` | file ID 0, `data/a`; block `H` with `original_size` 4; the ten standard relationships | file ID 0, `data/a`; file ID 1, `data/b`; file ID 2, `results/a`; block `H` from `root`; block `J` | +| `append-1` | file ID 1, `data/b`; exact repeat of block `H` with `original_size` 4; recipient grant `G` | | +| `append-2` | file ID 2, `results/a`; block `J`; exact repeat of recipient grant `G` | | + +#### 4.3.3 Directory Entry and Path Ordering Directory entries MUST follow these ordering and path rules: From 700465dcad559eb9fa9b73b28e2c5ddb24204f7f Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Tue, 1 Sep 2026 15:48:58 +0200 Subject: [PATCH 027/166] docs: define parent directory invariants --- spec/PITHOS_1.0.0_draft.md | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/spec/PITHOS_1.0.0_draft.md b/spec/PITHOS_1.0.0_draft.md index fc1d683..f55c156 100644 --- a/spec/PITHOS_1.0.0_draft.md +++ b/spec/PITHOS_1.0.0_draft.md @@ -242,15 +242,26 @@ pub struct Directory { | `dir_len` | Fixed-width `u64be` | | `crc32` | Fixed-width `u32be` | -The parent option tag is `00` for no parent and `01` for a parent. Readers MUST -reject other parent tags, duplicate relationship IDs, duplicate sender public -keys, duplicate file IDs, duplicate file paths, or duplicate block hashes. +The parent option tag is `00` for no parent and `01` for a parent, followed by +the parent's zero-based start and complete length as ULEB128 values. The +selected chain MUST have exactly one root directory, whose parent tag is `00`. +For tag `01`, the parent range MUST be within the file, end no later than the +child directory start, and have the same length as the parent's embedded +`dir_len`. Readers MUST use checked arithmetic for all ranges and validate each +parent's marker, length, CRC, and exact byte consumption before merging. +Readers MUST reject self-links, cycles, repeated parent ranges, forward links, +overlapping parent and child ranges, and invalid parent tags. Readers MAY +enforce documented chain-depth and total-directory-byte limits; exceeding one +MUST return a resource-limit error without exposing a partial archive view. + +Readers MUST reject duplicate relationship IDs, duplicate sender public keys, +duplicate file IDs, duplicate file paths, or duplicate block hashes. The directory marker MUST be exactly the eight ASCII bytes `PITHOSDR`. The final 12 directory bytes MUST be `dir_len:u64be || crc32:u32be`, where `dir_len` is the complete directory length from the marker through the CRC, inclusive. The CRC MUST be CRC-32/ISO-HDLC with width 32, polynomial `0x04C11DB7`, initial value`0xFFFFFFFF`, reflected input and output, and final XOR `0xFFFFFFFF` (the check value for ASCII `123456789` is `0xCBF43926`). It MUST cover every exact serialized byte from the marker through the fixed-width `dir_len`, excluding only the stored final CRC. -Readers MUST validate the marker, embedded length, CRC, and exact parser consumption for the terminal directory and independently for every parent directory before decrypting, merging, or otherwise using its metadata. Invalid directories MUST be rejected. +Readers MUST validate the marker, embedded length, CRC, and exact parser consumption for the terminal directory before decrypting, merging, or otherwise using its metadata. Invalid directories MUST be rejected. #### 4.3.1 Terminal Directory Lookup From 0f9db1a431e4845accc9c5170c765e4a5bab5142 Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Tue, 1 Sep 2026 15:51:08 +0200 Subject: [PATCH 028/166] docs: define reference relationship semantics --- spec/PITHOS_1.0.0_draft.md | 53 ++++++++++++++++++++++++++------------ 1 file changed, 36 insertions(+), 17 deletions(-) diff --git a/spec/PITHOS_1.0.0_draft.md b/spec/PITHOS_1.0.0_draft.md index f55c156..3916fae 100644 --- a/spec/PITHOS_1.0.0_draft.md +++ b/spec/PITHOS_1.0.0_draft.md @@ -296,10 +296,12 @@ relationship ID are invalid. Exact repeated recipient grants are allowed; conflicting grants for the same sender public key and recipient public key are invalid. -The root directory MUST contain these ten standard relationship definitions: -`DESCRIBES`, `ANNOTATES`, `DERIVED_FROM`, `SOURCE_OF`, `PREVIOUS_VERSION`, -`NEXT_VERSION`, `PART_OF`, `CONTAINS`, `INPUT_TO`, and `OUTPUT_FROM`. -Appended directories inherit these definitions and MAY repeat them exactly. +The root directory MUST store these ten standard relationship definitions in its +`relations` vector, in ascending relationship-ID order: `0 DESCRIBES`, +`1 ANNOTATES`, `2 DERIVED_FROM`, `3 SOURCE_OF`, `4 PREVIOUS_VERSION`, +`5 NEXT_VERSION`, `6 PART_OF`, `7 CONTAINS`, `8 INPUT_TO`, and +`9 OUTPUT_FROM`. Appended directories inherit these definitions and MAY repeat +one only when its relationship ID and stored name match exactly. For example, three segments with parent order `root <- append-1 <- append-2` merge as follows: @@ -440,7 +442,9 @@ of the FileEntry body. #### 4.4.4 File References -A Reference identifies a target file and the relationship from the containing file to it. +A Reference identifies a target file and a relationship. The file containing the +Reference is the source, and `target_file_id` identifies the target. Any file +type MAY be a source or target. ```rust /// Simplified reference structure @@ -459,20 +463,35 @@ pub struct Reference { | `relationship` | ULEB128 `u64` | References have no tag or length of their own; their containing vector provides -the count. Readers MUST consume both fields for every reference. +the count. Readers MUST consume both fields for every reference. The target file +ID and relationship ID MUST each exist in the effective archive. **Standard relationship types:** -- `DESCRIBES = 0`: Metadata describing target -- `ANNOTATES = 1`: Additional annotations -- `DERIVED_FROM = 2`: Derived from target -- `SOURCE_OF = 3`: Source of target -- `PREVIOUS_VERSION = 4`: Previous version -- `NEXT_VERSION = 5`: Next version -- `PART_OF = 6`: Part of collection -- `CONTAINS = 7`: Contains target -- `INPUT_TO = 8`: Input to process -- `OUTPUT_FROM = 9`: Output from process -- Custom relationships start at `1000` + +| ID | Stored name | Meaning | +| --- | --- | --- | +| 0 | `DESCRIBES` | The source describes the target. | +| 1 | `ANNOTATES` | The source annotates the target. | +| 2 | `DERIVED_FROM` | The source is derived from the target. | +| 3 | `SOURCE_OF` | The source is a source of the target. | +| 4 | `PREVIOUS_VERSION` | The source is the previous version of the target. | +| 5 | `NEXT_VERSION` | The source is the next version of the target. | +| 6 | `PART_OF` | The source is part of the target. | +| 7 | `CONTAINS` | The source contains the target. | +| 8 | `INPUT_TO` | The source is an input to the target. | +| 9 | `OUTPUT_FROM` | The source is an output from the target. | + +Custom relationship IDs MUST be at least `1000`; their stored names MUST be +non-empty UTF-8 strings. + +For example, a reference from `normalized.csv` to `raw.csv` with +`DERIVED_FROM` means `normalized.csv` is derived from `raw.csv`; reversing the +reference means `raw.csv` is derived from `normalized.csv`. A reference from +`raw.csv` to `normalized.csv` with `SOURCE_OF` means `raw.csv` is a source of +`normalized.csv`. A reference from `report-v1.pdf` to `report-v2.pdf` with +`PREVIOUS_VERSION` means `report-v1.pdf` is the previous version of +`report-v2.pdf`; reversing it with `NEXT_VERSION` means `report-v2.pdf` is the +next version of `report-v1.pdf`. ### 4.5 Encryption Section From 60e753070e72ac723c0fca6e95071c63d5ab56fd Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Tue, 1 Sep 2026 15:54:23 +0200 Subject: [PATCH 029/166] docs: define optional capability behavior --- spec/PITHOS_1.0.0_draft.md | 76 ++++++++++++++++++++++++++++---------- 1 file changed, 57 insertions(+), 19 deletions(-) diff --git a/spec/PITHOS_1.0.0_draft.md b/spec/PITHOS_1.0.0_draft.md index 3916fae..30693f4 100644 --- a/spec/PITHOS_1.0.0_draft.md +++ b/spec/PITHOS_1.0.0_draft.md @@ -586,6 +586,7 @@ pub enum PithosError { Crypt4GH(#[from] Crypt4GHError), Cipher(#[from] ChaChaPoly1305Error), Compression(#[from] ZstdError), + UnsupportedFeature(String), InvalidBlockDataState(String), BlockHashNotFound([u8; 32]), FileNotFound(String), @@ -709,25 +710,62 @@ When archiving directory trees: ## 8. Implementation Requirements -### 8.1 Mandatory Features - -Implementations MUST support: -- Reading and writing base format (version 1.0) -- Blake3 hashing -- ULEB128 encoding/decoding -- CRC32 calculation -- UTF-8 string handling -- All four file types (Data, Metadata, Directory, Symlink) -- Directory ordering validation (parents before children) -- Path format validation (relative paths only) - -### 8.2 Optional Features - -Implementations MAY support: -- Compression (levels 1-7) -- Encryption (ChaCha20-Poly1305) -- External block storage -- Content-defined chunking +### 8.1 Base Reader and Writer + +A base reader supports the version 1.0 structure, including its required +validation, and can read local blocks whose ProcessingFlags have compression +value `0` and encryption bit `0`. It supports decrypted BlockDataState lists. +A base writer can create an archive containing only such local blocks and an +empty Directory `encryption` vector. + +The following table defines the optional capabilities. A reader discovers a +needed capability from the stored fields; it does not use a profile or a +negotiation record. + +| Capability | Stored indication | +| --- | --- | +| Block compression | ProcessingFlags compression bits are `1` through `7` | +| Block encryption | ProcessingFlags encryption bit is `1` | +| Encrypted block lists | BlockDataState tag is `00` (`Encrypted`) | +| Encrypted recipient lists | RecipientData tag is `00` (`Encrypted`) | +| External storage | BlockLocation tag is `01` (`External`) | + +Compression, block encryption, encrypted block lists, encrypted recipient +lists, and external storage are optional capabilities. An implementation that +supports an optional capability MUST process it according to its definition in +this specification. + +### 8.2 Unavailable Content + +A reader MUST validate and list an archive whose known-version, known-tag +structure contains content requiring an unsupported optional capability. It +MUST mark only the affected content unavailable; unaffected entries remain +readable. Attempting to read unavailable content MUST return +`UnsupportedFeature` before releasing any output derived from that content. +The reader MUST NOT reinterpret bytes requiring an unsupported capability as +uncompressed or unencrypted bytes. + +| Block form | Listing result | Content-read result | +| --- | --- | --- | +| Plain local: local, compression `0`, encryption `0` | Listed | Readable by a base reader | +| Compressed local: local, compression `1` through `7`, encryption `0` | Listed | Readable only with block-compression capability; otherwise unavailable | +| Encrypted local: local, compression `0`, encryption `1` | Listed | Readable only with block-encryption capability and any encrypted-list capability needed to obtain its block key; otherwise unavailable | +| External: BlockLocation `External` | Listed | Readable only with external-storage capability and every capability required by its flags and data states; otherwise unavailable | +| Combined: compression and encryption, at either location | Listed | Readable only when every indicated capability is supported; otherwise unavailable | + +For the table, an encrypted BlockDataState requires encrypted-block-list +capability, and an encrypted RecipientData required to obtain its file key +requires encrypted-recipient-list capability. A content read that needs either +unsupported list form is unavailable even when its block flags themselves are +otherwise supported. + +Readers MUST reject an unsupported header version and any unknown tag rather +than list the archive, because the structure of such input is not known. + +Pithos 1.0 intentionally permits decrypted block and recipient list variants +and an empty Directory `encryption` vector. This differs from the current 0.8 +writer, which requires encrypted file block lists and encrypted recipient +lists, although it can disable block-payload encryption. ### 8.3 Platform-Specific Considerations From 488f3c3f0c09814f8fd625a8881b58601a154a9f Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Tue, 1 Sep 2026 15:56:50 +0200 Subject: [PATCH 030/166] docs: define hash-only block identity --- spec/PITHOS_1.0.0_draft.md | 57 +++++++++++++++++++++++++++----------- 1 file changed, 41 insertions(+), 16 deletions(-) diff --git a/spec/PITHOS_1.0.0_draft.md b/spec/PITHOS_1.0.0_draft.md index 30693f4..9fe842d 100644 --- a/spec/PITHOS_1.0.0_draft.md +++ b/spec/PITHOS_1.0.0_draft.md @@ -103,14 +103,14 @@ pub struct BlockHeader { Readers MUST reject a block header whose marker is not `BLCK`. -#### 4.2.2 Block Index Entry +#### 4.2.2 Block Descriptor The hash-keyed block descriptor describes one block stored or referenced by a directory. ```rust /// Block descriptor body; its hash is the key in Directory::blocks #[derive(Debug, Clone, PartialEq, Eq)] -pub struct BlockIndexEntry { +pub struct BlockDescriptor { pub offset: u64, // Byte offset in file (varint encoded) pub stored_size: u64, // Size as stored (compressed/encrypted) (varint) pub original_size: u64, // Original uncompressed size (varint) @@ -125,7 +125,7 @@ The directory block sequence is a vector. Each item is encoded as follows: | Field | Bytes stored in the file | | --- | --- | -| `block_hash` | Exactly 32 bytes: BLAKE3 hash of the original block content | +| `block_hash` | Exactly 32 bytes: block hash as defined in Section 5.2 | | `offset` | ULEB128 `u64` | | `stored_size` | ULEB128 `u64` | | `original_size` | ULEB128 `u64` | @@ -221,7 +221,7 @@ pub struct Directory { pub identifier: [u8; 8], // MUST be exactly ASCII b"PITHOSDR" pub parent_directory_offset: Option<(u64, u64)>, // Previous directory (start, len) (varint, backwards chain) pub files: Vec<(u64, String, FileEntry)>, // File ID, path, and body - pub blocks: Vec<([u8; 32], BlockIndexEntry)>, // Block hash and body + pub blocks: Vec<([u8; 32], BlockDescriptor)>, // Block hash and body pub relations: Vec<(u64, String)>, // Relation idx, relationname / id pub encryption: Vec<([u8; 32], EncryptionSection)>, // Sender key and body pub dir_len: u64, @@ -389,11 +389,9 @@ pub enum BlockDataState { | `00` | `Encrypted` | Vector of bytes | | `01` | `Decrypted` | Vector of tuples, each `block_hash[32] || block_key[32]` | -Readers MUST reject unknown tags. The block hash is the only block identity in a -decrypted block-list entry. A decrypted block list is a ULEB128 count followed -by that many `block_hash[32] || block_key[32]` pairs. A repeated block hash is -valid only when it has the same block key; readers MUST reject a conflicting -duplicate hash and block key. +Readers MUST reject unknown tags. A decrypted block list is a ULEB128 count +followed by that many `block_hash[32] || block_key[32]` pairs. Block-list +identity, reuse, and validation are defined in Section 5.2. #### 4.4.3 File Entry @@ -612,12 +610,39 @@ Implementations SHOULD use content-defined chunking with recommended parameters: ### 5.2 Block Hashing -Each block identifier MUST be the full 32-byte default unkeyed BLAKE3 digest of the exact -plaintext chunk before compression or encryption. Readers MUST retrieve the stored block, -authenticate and decrypt it when encrypted, decompress it when compressed using the recorded -original size as the output bound, require the resulting plaintext length to equal the recorded -original size, compute the complete plaintext digest, and compare it with the block identifier -before releasing any output derived from that block. +The block hash is the full 32-byte default unkeyed BLAKE3 digest of the exact +plaintext chunk before compression or encryption. It is a block's only +identity. The Directory `blocks` vector is keyed by block hash, and each file's +block list is an ordered sequence of `(block_hash, block_key)` pairs. That order +reconstructs the file. + +Every block hash referenced by a file MUST resolve to exactly one effective +descriptor after the directory chain is merged. A repeated hash in one file is +valid only if every occurrence carries the same block key; otherwise readers +MUST reject the file. Two files MAY reference the same hash and effective +descriptor. Deduplication is a writer choice: a writer MAY store the same +plaintext again, provided the directory conflict rules in Section 4.3.2 are +satisfied. + +Using checked arithmetic, readers MUST sum the `original_size` of each +referenced effective descriptor and require the result to equal `file_size`. An +empty file has `file_size` zero and an empty block list. + +Readers MUST retrieve each stored block, authenticate and decrypt it when +encrypted, decompress it when compressed using the recorded original size as +the output bound, require the resulting plaintext length to equal the recorded +original size, compute the complete plaintext digest, and compare it with the +block hash before releasing any output derived from that block. + +Examples, where `H` and `J` are distinct block hashes and `K` and `L` are block +keys: + +| Case | Block lists or directory records | Validity and reconstruction result | +| --- | --- | --- | +| Reuse by two files | `a: [(H, K)]`; `b: [(H, K)]` | Valid. Both files reconstruct from the effective descriptor for `H`. | +| Repetition in one file | `a: [(H, K), (H, K)]` | Valid. `a` reconstructs as the plaintext for `H` followed by itself. | +| Conflicting key | `a: [(H, K), (H, L)]` | Invalid. The file is rejected. | +| Later-segment size conflict | Root has `H` with `original_size: 4`; an appended directory has `H` with `original_size: 5` | Invalid. The appended directory is rejected; no effective descriptor is selected from it. | ### 5.3 Convergent Encryption @@ -679,7 +704,7 @@ different supported Zstandard versions. 1. Read and validate file header 2. Locate and validate the terminal directory using the direct lookup in Section 4.3.1 3. Validate directory ordering -4. Build block index +4. Build the effective block-descriptor mapping 5. Extract files by reading referenced blocks ### 6.2 Writing Operations From c6f8c849b166f2f33ba911ddcd3a71d92d0b4c99 Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Wed, 2 Sep 2026 18:52:37 +0200 Subject: [PATCH 031/166] docs: improve specification navigation --- spec/PITHOS_1.0.0_draft.md | 50 ++++++++++++++++++++++++++++++++++---- 1 file changed, 45 insertions(+), 5 deletions(-) diff --git a/spec/PITHOS_1.0.0_draft.md b/spec/PITHOS_1.0.0_draft.md index 9fe842d..ae8ab05 100644 --- a/spec/PITHOS_1.0.0_draft.md +++ b/spec/PITHOS_1.0.0_draft.md @@ -13,6 +13,47 @@ Pithos is an append-only archive format designed for efficient storage and shari The code examples in this document are only intended to illustrate the architecture. Optimized implementations of the individual structures may of course differ. +This document defines Pithos 1.0. Where a current 0.8 implementation differs, +this document governs. + +### 1.1 Reader's Guide + +- **Readers:** Start with the common encoding rules (Section 3.1), then follow + terminal Directory lookup and append-chain validation (Sections 4.3.1 and + 4.3.2), content processing (Section 5), and capability handling (Section + 8). +- **Writers:** Follow the encoded forms in Section 4, content processing in + Section 5, and the writing operation sequence in Section 6.2. +- **Security reviewers:** Review the cryptographic construction in Section 5.3, + block verification in Section 5.2, and the security considerations in + Section 7. +- **Conformance-test authors:** Use the encoding rules in Section 3.1 and the + encoded-form tables in Section 4 as the normative byte-level rules; Sections + 5 and 8 define processing and capability outcomes. + +### 1.2 Terminology + +- **segment:** The portion of an archive that introduces block data and ends + with one Directory. +- **base Directory:** The oldest Directory in a selected chain. It has no + parent and defines the standard relationships required by Section 4.3.2. +- **terminal Directory:** The newest Directory in a selected chain, located at + the end of the file during normal reading as specified in Section 4.3.1. +- **selected chain:** The ordered sequence of Directories obtained by following + parent links from the terminal Directory to the base Directory. +- **effective archive:** The archive view produced by merging the selected + chain from base Directory to terminal Directory under Section 4.3.2. +- **effective descriptor:** The descriptor selected for a block hash after the + selected chain is merged, as specified in Section 4.3.2. +- **archive root path:** The implicit root of the archive path hierarchy; it has + no directory entry. +- **unavailable content:** Content that is structurally known but cannot be + read because it requires an unsupported optional capability, as specified in + Section 8.2. +- **salvage mode:** A recovery mode that may locate an older Directory after a + normal terminal-Directory lookup fails; its result is incomplete and is not a + normal archive view. + ## 2. Core Design Principles 1. **Append-only architecture**: New data and metadata MUST be appended, never modifying existing content @@ -194,7 +235,8 @@ pub enum BlockLocation { | `01` | `External` | `url` as a string | Readers MUST reject unknown tags. A local block's offset and size describe its -location in this file; the block-boundary rules are specified separately. +location in this file; [Section 4.2.5](#425-local-block-boundaries-and-recovery) +defines its block-boundary rules. #### 4.2.5 Local Block Boundaries and Recovery @@ -269,7 +311,7 @@ To locate the terminal directory during normal reading: 1. Read the final 12 file bytes as `dir_len:u64be || crc32:u32be`. 2. Compute `directory_start = file_length - dir_len` using checked subtraction. -3. Require `dir_len` to be at least the smallest directory allowed by the finalized 1.0 encoding. The root stores ten standard relationships, so the 0.8 25-byte empty-directory minimum does not apply. +3. Require `dir_len` to be at least the smallest directory allowed by the finalized 1.0 encoding. 4. Parse the directory at `directory_start` and require it to end exactly at the end of the file. 5. Validate the marker, embedded length, CRC, and exact byte consumption before using metadata. @@ -788,9 +830,7 @@ Readers MUST reject an unsupported header version and any unknown tag rather than list the archive, because the structure of such input is not known. Pithos 1.0 intentionally permits decrypted block and recipient list variants -and an empty Directory `encryption` vector. This differs from the current 0.8 -writer, which requires encrypted file block lists and encrypted recipient -lists, although it can disable block-payload encryption. +and an empty Directory `encryption` vector. ### 8.3 Platform-Specific Considerations From d73b4e189fba7232b1d10ce7582f1f51ece4a8ad Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Wed, 2 Sep 2026 18:57:58 +0200 Subject: [PATCH 032/166] docs: define archive entry validity --- spec/PITHOS_1.0.0_draft.md | 133 +++++++++++++++++++++++++------------ 1 file changed, 92 insertions(+), 41 deletions(-) diff --git a/spec/PITHOS_1.0.0_draft.md b/spec/PITHOS_1.0.0_draft.md index ae8ab05..32b119a 100644 --- a/spec/PITHOS_1.0.0_draft.md +++ b/spec/PITHOS_1.0.0_draft.md @@ -42,7 +42,7 @@ this document governs. - **selected chain:** The ordered sequence of Directories obtained by following parent links from the terminal Directory to the base Directory. - **effective archive:** The archive view produced by merging the selected - chain from base Directory to terminal Directory under Section 4.3.2. + chain from the base Directory to the terminal Directory under Section 4.3.2. - **effective descriptor:** The descriptor selected for a block hash after the selected chain is merged, as specified in Section 4.3.2. - **archive root path:** The implicit root of the archive path hierarchy; it has @@ -62,7 +62,7 @@ this document governs. 4. **Flexible metadata**: Metadata MUST be stored as regular files with special type markers 5. **Progressive enhancement**: Implementations MUST support the base format and MAY support optional features 6. **Limited recovery**: Recovery tools MAY treat block markers as untrusted candidates; directories provide block metadata -7. **Hierarchical organization**: Files use full paths from archive root; directories MUST be declared before their contents +7. **Hierarchical organization**: Files use full paths from the archive root path; directories MUST be declared before their contents ## 3. File Structure and Encoding @@ -286,7 +286,7 @@ pub struct Directory { The parent option tag is `00` for no parent and `01` for a parent, followed by the parent's zero-based start and complete length as ULEB128 values. The -selected chain MUST have exactly one root directory, whose parent tag is `00`. +selected chain MUST have exactly one base Directory, whose parent tag is `00`. For tag `01`, the parent range MUST be within the file, end no later than the child directory start, and have the same length as the parent's embedded `dir_len`. Readers MUST use checked arithmetic for all ranges and validate each @@ -303,15 +303,15 @@ The directory marker MUST be exactly the eight ASCII bytes `PITHOSDR`. The final 12 directory bytes MUST be `dir_len:u64be || crc32:u32be`, where `dir_len` is the complete directory length from the marker through the CRC, inclusive. The CRC MUST be CRC-32/ISO-HDLC with width 32, polynomial `0x04C11DB7`, initial value`0xFFFFFFFF`, reflected input and output, and final XOR `0xFFFFFFFF` (the check value for ASCII `123456789` is `0xCBF43926`). It MUST cover every exact serialized byte from the marker through the fixed-width `dir_len`, excluding only the stored final CRC. -Readers MUST validate the marker, embedded length, CRC, and exact parser consumption for the terminal directory before decrypting, merging, or otherwise using its metadata. Invalid directories MUST be rejected. +Readers MUST validate the marker, embedded length, CRC, and exact parser consumption for the terminal Directory before decrypting, merging, or otherwise using its metadata. Invalid Directories MUST be rejected. #### 4.3.1 Terminal Directory Lookup -To locate the terminal directory during normal reading: +To locate the terminal Directory during normal reading: 1. Read the final 12 file bytes as `dir_len:u64be || crc32:u32be`. 2. Compute `directory_start = file_length - dir_len` using checked subtraction. -3. Require `dir_len` to be at least the smallest directory allowed by the finalized 1.0 encoding. +3. Require `dir_len >= 25`, the syntactic framing floor for a Directory. 4. Parse the directory at `directory_start` and require it to end exactly at the end of the file. 5. Validate the marker, embedded length, CRC, and exact byte consumption before using metadata. @@ -320,37 +320,43 @@ Normal reading MUST reject trailing bytes, truncation, underflow, a false marker #### 4.3.2 Append Chains Each Directory contains the entries introduced by its segment. To construct the -effective archive, readers merge the selected directory chain from its oldest -directory to its newest directory. An append adds file entries, block +effective archive, readers merge the selected chain from the base Directory to +the terminal Directory. An append adds file entries, block descriptors, relationship definitions, and recipient grants. Version 1.0 has no deletion, replacement, or tombstone. -File IDs and paths MUST be unique across the chain. Writers assign file ID 0 +File IDs and paths MUST be unique across the selected chain. Writers assign file ID 0 to the first file and assign each later file ID as the current maximum ID plus 1. Readers MUST accept unused file-ID gaps. A rename adds a file record with a new file ID and path; the old record remains in the effective archive. -A repeated block hash in one directory is invalid. Across directories, a block -hash MAY reappear only when its `original_size` is the same. The descriptor in -the oldest directory remains the effective descriptor. Exact repeated +A repeated block hash in one Directory is invalid. Across Directories in the +selected chain, a block hash MAY reappear only when its `original_size` is the +same. The descriptor in the oldest Directory in the selected chain that +contains that hash remains the effective descriptor. Exact repeated relationship definitions are allowed; conflicting definitions of one relationship ID are invalid. Exact repeated recipient grants are allowed; conflicting grants for the same sender public key and recipient public key are invalid. -The root directory MUST store these ten standard relationship definitions in its +The base Directory MUST store these ten standard relationship definitions in its `relations` vector, in ascending relationship-ID order: `0 DESCRIBES`, `1 ANNOTATES`, `2 DERIVED_FROM`, `3 SOURCE_OF`, `4 PREVIOUS_VERSION`, `5 NEXT_VERSION`, `6 PART_OF`, `7 CONTAINS`, `8 INPUT_TO`, and -`9 OUTPUT_FROM`. Appended directories inherit these definitions and MAY repeat +`9 OUTPUT_FROM`. Non-base Directories inherit these definitions and MAY repeat one only when its relationship ID and stored name match exactly. -For example, three segments with parent order `root <- append-1 <- append-2` +The relationship requirement is semantic validation separate from the `25`-byte +syntactic framing floor. A valid empty base Directory is 146 bytes, and the +minimum valid appended Directory is 28 bytes. A non-base Directory MAY have an +empty `relations` vector. + +**Illustrative append-chain merge:** three segments with parent order `base <- append-1 <- append-2` merge as follows: | Segment | Entries introduced | Final visible entries after `append-2` | | --- | --- | --- | -| `root` | file ID 0, `data/a`; block `H` with `original_size` 4; the ten standard relationships | file ID 0, `data/a`; file ID 1, `data/b`; file ID 2, `results/a`; block `H` from `root`; block `J` | +| `base` | file ID 0, `data/a`; block `H` with `original_size` 4; the ten standard relationships | file ID 0, `data/a`; file ID 1, `data/b`; file ID 2, `results/a`; block `H` from `base`; block `J` | | `append-1` | file ID 1, `data/b`; exact repeat of block `H` with `original_size` 4; recipient grant `G` | | | `append-2` | file ID 2, `results/a`; block `J`; exact repeat of recipient grant `G` | | @@ -358,28 +364,39 @@ merge as follows: Directory entries MUST follow these ordering and path rules: -1. A directory entry MUST appear before entries for files or subdirectories within it. -2. Paths MUST be relative, have no leading `/`, and use forward slashes as separators. -3. The root directory is implicit and MUST NOT have an entry. -4. Readers and writers MUST validate this ordering. - -**Example of valid ordering:** +1. An entry path is a non-empty UTF-8 sequence of non-empty `/`-separated + components. It MUST NOT start or end with `/`, contain NUL or `\`, contain a + `.` or `..` component, or use a drive-qualified form (a path whose second + byte is `:`, such as invalid `C:` or `C:/data`). +2. The archive root path is implicit and MUST NOT have an entry. +3. A path is a descendant of a directory only when its component sequence has + that directory's component sequence as a strict prefix. Every entry below the + archive root path MUST have each of its ancestor paths declared as a + `Directory` entry. +4. Declaration order is the concatenation of the `files` vectors from the base + Directory through the terminal Directory, preserving each vector's stored + order. Each ancestor Directory MUST occur before its descendant. An ancestor + declared in an earlier Directory satisfies this rule for an entry in a later + Directory. +5. Readers and writers MUST reject a path or declaration order that violates these rules. + +**Valid ordering example:** ``` -data/ (directory) -data/raw/ (directory - parent "data/" already exists) -data/raw/file1.csv (file - parent "data/raw/" already exists) -data/processed/ (directory - parent "data/" already exists) -data/processed/file2.csv (file - parent "data/processed/" already exists) -docs/ (directory) -docs/README.md (file - parent "docs/" already exists) -data/raw/file1_v2.csv (file - parent "data/raw/" already exists) -> Newer version of file1.csv +data (Directory) +data/raw (Directory; parent `data` already declared) +data/raw/file1.csv (Data; parent `data/raw` already declared) +data/processed (Directory; parent `data` already declared) +data/processed/file2.csv (Data; parent `data/processed` already declared) +docs (Directory) +docs/README.md (Data; parent `docs` already declared) +data/raw/file1_v2.csv (Data; parent `data/raw` already declared) ``` -**Example of invalid ordering:** +**Invalid ordering example:** ``` -data/raw/file1.csv (ERROR: parent "data/raw/" not yet declared) -data/raw/ (too late - file already referenced this directory) -data/ (too late - subdirectory already referenced this) +data/raw/file1.csv (parent `data/raw` not yet declared) +data/raw (too late; parent `data` not yet declared) +data (too late; a descendant already occurred) ``` ### 4.4 File Representation @@ -480,6 +497,35 @@ The `symlink_target` option tag is `00` for no target and `01` for a target. Readers MUST reject other tags. File IDs and paths are record fields, not fields of the FileEntry body. +**FileEntry validity rules:** + +| `file_type` | `block_data` | `file_size` | `symlink_target` | +| --- | --- | --- | --- | +| `Directory` | MUST be `Decrypted` with an empty list | MUST be `0` | MUST be absent (`00`) | +| `Data` | Either state is permitted | MUST equal the sum of referenced effective descriptors' `original_size` values when the block list is available | MUST be absent (`00`) | +| `Metadata` | Either state is permitted | MUST equal the sum of referenced effective descriptors' `original_size` values when the block list is available | MUST be absent (`00`) | +| `Symlink` | MUST be `Decrypted` with an empty list | MUST be `0` | MUST be present (`01`) | + +Readers MUST reject a combination that violates this table. For encrypted block +lists, the content-size validation for `Data` and `Metadata` occurs once the +block list is available; until then, the content is unavailable as specified in +Section 8.2. + +`permissions` stores the low 12 bits of a POSIX mode, in the range +`0o0000..=0o7777`; file-type bits are not stored. Readers MUST reject a value +with any higher bit set and retain all 12 bits as metadata. An extractor MUST +NOT apply set-user-ID, set-group-ID, or sticky bits unless the caller explicitly +opts in. A non-POSIX implementation MAY retain the value as metadata and need +not map it to a platform ACL. + +For a `Symlink`, `symlink_target` is a non-empty UTF-8 sequence of non-empty +`/`-separated components. It MUST NOT contain NUL or `\`, start with `/`, use a +drive-qualified form as defined for entry paths, or contain a `.` component. A +`..` component is permitted only when lexical interpretation of the target +relative to the symlink's parent does not move above the archive root path. The +target need not name an existing entry: contained dangling links and cycles are +valid. + #### 4.4.4 File References A Reference identifies a target file and a relationship. The file containing the @@ -659,7 +705,7 @@ block list is an ordered sequence of `(block_hash, block_key)` pairs. That order reconstructs the file. Every block hash referenced by a file MUST resolve to exactly one effective -descriptor after the directory chain is merged. A repeated hash in one file is +descriptor after the selected chain is merged. A repeated hash in one file is valid only if every occurrence carries the same block key; otherwise readers MUST reject the file. Two files MAY reference the same hash and effective descriptor. Deduplication is a writer choice: a writer MAY store the same @@ -684,7 +730,7 @@ keys: | Reuse by two files | `a: [(H, K)]`; `b: [(H, K)]` | Valid. Both files reconstruct from the effective descriptor for `H`. | | Repetition in one file | `a: [(H, K), (H, K)]` | Valid. `a` reconstructs as the plaintext for `H` followed by itself. | | Conflicting key | `a: [(H, K), (H, L)]` | Invalid. The file is rejected. | -| Later-segment size conflict | Root has `H` with `original_size: 4`; an appended directory has `H` with `original_size: 5` | Invalid. The appended directory is rejected; no effective descriptor is selected from it. | +| Later-segment size conflict | Base Directory has `H` with `original_size: 4`; an appended Directory has `H` with `original_size: 5` | Invalid. The appended Directory is rejected; no effective descriptor is selected from it. | ### 5.3 Convergent Encryption @@ -744,7 +790,7 @@ different supported Zstandard versions. ### 6.1 Reading Operations 1. Read and validate file header -2. Locate and validate the terminal directory using the direct lookup in Section 4.3.1 +2. Locate and validate the terminal Directory using the direct lookup in Section 4.3.1 3. Validate directory ordering 4. Build the effective block-descriptor mapping 5. Extract files by reading referenced blocks @@ -835,12 +881,17 @@ and an empty Directory `encryption` vector. ### 8.3 Platform-Specific Considerations #### Symlinks -- Unix systems: Create proper symbolic links -- Windows: Handle symlinks according to platform capabilities + +An extractor on a platform that supports symbolic links MAY create a symbolic +link with the stored target. While creating other archive entries, it MUST NOT +follow an archive-created or pre-existing symbolic link. On a platform that does +not support symbolic links, an extractor MAY reject the operation or skip the +symlink with a diagnostic, but MUST NOT silently change the entry type. #### Permissions -- Unix: Preserve full permission bits -- Windows: Map Unix permissions to Windows ACLs where possible + +Extractors apply permissions subject to the FileEntry rules in Section 4.4.3. +Non-POSIX platforms MAY retain them as metadata without an ACL mapping. #### Path Separators - Archives use forward slashes (`/`) internally From ed629adc3ce09bb8956f49837ff42996d6c5117a Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Wed, 2 Sep 2026 19:01:14 +0200 Subject: [PATCH 033/166] docs: define block resolution boundaries --- spec/PITHOS_1.0.0_draft.md | 56 ++++++++++++++++++++++++++++++-------- 1 file changed, 44 insertions(+), 12 deletions(-) diff --git a/spec/PITHOS_1.0.0_draft.md b/spec/PITHOS_1.0.0_draft.md index 32b119a..8934258 100644 --- a/spec/PITHOS_1.0.0_draft.md +++ b/spec/PITHOS_1.0.0_draft.md @@ -152,7 +152,7 @@ The hash-keyed block descriptor describes one block stored or referenced by a di /// Block descriptor body; its hash is the key in Directory::blocks #[derive(Debug, Clone, PartialEq, Eq)] pub struct BlockDescriptor { - pub offset: u64, // Byte offset in file (varint encoded) + pub offset: u64, // Local byte offset; zero for External (varint encoded) pub stored_size: u64, // Size as stored (compressed/encrypted) (varint) pub original_size: u64, // Original uncompressed size (varint) pub flags: ProcessingFlags, // Compression, encryption settings @@ -167,11 +167,11 @@ The directory block sequence is a vector. Each item is encoded as follows: | Field | Bytes stored in the file | | --- | --- | | `block_hash` | Exactly 32 bytes: block hash as defined in Section 5.2 | -| `offset` | ULEB128 `u64` | +| `offset` | ULEB128 `u64`: local block offset, or zero for `External` | | `stored_size` | ULEB128 `u64` | | `original_size` | ULEB128 `u64` | | `flags` | One ProcessingFlags byte: bits 0-2 compression level, bit 3 encryption enabled, bits 4-7 zero | -| `location` | One tag byte: `00` local, or `01` followed by a URL string for external | +| `location` | One tag byte: `00` local, or `01` followed by an external location identifier string | Readers MUST reject duplicate block hashes in one directory and MUST use the 32-byte hash as the block's only identity. @@ -222,8 +222,8 @@ BlockLocation states where the bytes for a block can be obtained. /// Block storage location #[derive(Debug, Clone, PartialEq, Eq)] pub enum BlockLocation { - Local, // Block data at specified offset in this file - External { url: String }, // URL to external storage + Local, // Block data at specified offset in this file + External { location: String }, // Opaque external location identifier } ``` @@ -232,11 +232,14 @@ pub enum BlockLocation { | Tag | Variant | Bytes after the tag | | --- | --- | --- | | `00` | `Local` | None | -| `01` | `External` | `url` as a string | +| `01` | `External` | External location identifier as a string | Readers MUST reject unknown tags. A local block's offset and size describe its location in this file; [Section 4.2.5](#425-local-block-boundaries-and-recovery) -defines its block-boundary rules. +defines its block-boundary rules. An external location identifier is an opaque +string interpreted only by an enabled caller-supplied resolver. Writers MUST +encode `offset` as zero for `External`; readers MUST ignore that field for +`External`. #### 4.2.5 Local Block Boundaries and Recovery @@ -246,10 +249,38 @@ The marker occupies bytes `offset` through `offset + 3`. The payload starts at `offset + 4` and occupies exactly `stored_size` bytes; when nonempty, its last byte is `offset + 4 + stored_size - 1`. -Readers locate local blocks from directory descriptors and MUST use checked -arithmetic for every range. Normal reading MUST NOT search payload bytes for -`BLCK`. Recovery tools MAY treat `BLCK` as an untrusted candidate only: the -marker alone provides no length, flags, or identity. +The local block-data region of the base segment begins immediately after the +six-byte FileHeader and ends at the base Directory's start. The local +block-data region of an appended segment begins immediately after its parent +Directory and ends at its declaring Directory's start. For every effective +descriptor with `Local` location, its effective local extent is the half-open +range `[offset, offset + 4 + stored_size)`; it MUST fit wholly in +the local block-data region of its declaring segment, begin with `BLCK`, and +not overlap another effective local extent. Readers and writers MUST use +checked arithmetic to compute all extent and region bounds. + +If ProcessingFlags encryption is enabled, `stored_size` MUST be at least 28 +bytes: the 12-byte nonce plus the 16-byte authentication tag. This requirement +does not impose a minimum stored size for an unencrypted payload. + +Readers locate local blocks from effective directory descriptors and MUST NOT +search payload bytes for `BLCK`. Recovery tools MAY treat `BLCK` as an +untrusted candidate only: the marker alone provides no length, flags, or +identity. + +#### 4.2.6 External Block Resolution + +External resolution is an optional capability and MUST remain disabled until a +caller supplies both a resolver and an access policy. The resolver receives the +opaque external location identifier and MUST return exactly one framed block: +`BLCK || stored_payload`, with total length `4 + stored_size`. The returned +bytes have the same representation as a local block and MUST undergo the same +marker, transform, stored-size, original-size, and hash validation. + +This specification does not define a network protocol or client. A resolver +that performs network access MUST require explicit enablement, enforce +response-size and time bounds, and apply its access policy to every network +target and redirect. ### 4.3 Directory Structure @@ -818,7 +849,8 @@ When archiving directory trees: decryption and decompression, and before releasing output derived from the block 2. Directory CRC32 values MUST be validated 3. Convergent encryption reveals when identical files exist (accepted trade-off) -4. External block URLs MUST use HTTPS in production environments +4. Networked external resolution can expose a caller to unsafe targets and + resource exhaustion; Section 4.2.6 defines the required resolver safeguards 5. Path traversal attacks MUST be prevented through validation ## 8. Implementation Requirements From 87b1a3e30940d2a985f7e6e9e3636f9c07efcdac Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Wed, 2 Sep 2026 19:04:14 +0200 Subject: [PATCH 034/166] docs: clarify format security properties --- spec/PITHOS_1.0.0_draft.md | 68 ++++++++++++++++++++++++++++++-------- 1 file changed, 54 insertions(+), 14 deletions(-) diff --git a/spec/PITHOS_1.0.0_draft.md b/spec/PITHOS_1.0.0_draft.md index 8934258..31c0a97 100644 --- a/spec/PITHOS_1.0.0_draft.md +++ b/spec/PITHOS_1.0.0_draft.md @@ -58,7 +58,8 @@ this document governs. 1. **Append-only architecture**: New data and metadata MUST be appended, never modifying existing content 2. **Content-addressed storage**: All blocks MUST be identified by Blake3 hashes enabling deduplication -3. **Privacy-preserving sharing**: Users MUST NOT be able to see who else has access to files +3. **Encrypted recipient grants**: Encrypted recipient data protects its file-key + grants from parties that cannot decrypt it 4. **Flexible metadata**: Metadata MUST be stored as regular files with special type markers 5. **Progressive enhancement**: Implementations MUST support the base format and MAY support optional features 6. **Limited recovery**: Recovery tools MAY treat block markers as untrusted candidates; directories provide block metadata @@ -97,6 +98,15 @@ Section 4: 9. A reader MUST consume exactly the bytes assigned to a structure and reject unknown tags. +### 3.2 Resource Safety + +Readers MUST treat every encoded length and count as untrusted, use checked +arithmetic and checked conversion to host sizes before indexing or allocating, +and fail on allocation failure. Implementations MAY enforce documented, +configured resource limits on input sizes, counts, and derived work; they MUST +fail before an applicable limit is exceeded and MUST NOT expose a partial archive +view after a resource or conversion failure. + ## 4. Core Data Structures ### 4.1 File Header @@ -753,6 +763,10 @@ the output bound, require the resulting plaintext length to equal the recorded original size, compute the complete plaintext digest, and compare it with the block hash before releasing any output derived from that block. +Readers MAY use bounded memory or temporary spill-backed storage while +performing this verification, but MUST NOT release output derived from a block +until its required transforms, size, and hash have all been verified. + Examples, where `H` and `J` are distinct block hashes and `K` and `L` are block keys: @@ -785,9 +799,11 @@ shared secret is used directly as the ChaCha20-Poly1305 key to encrypt the decrypted recipient list; no intermediate KDF is used. Implementations MUST reject non-contributory X25519 public keys. -An implementation MUST use a fresh nonce for every value encrypted under a -given key. It MUST authenticate and decrypt an encrypted value successfully -before using its plaintext or releasing output derived from it. +An implementation MUST generate every nonce independently and uniformly at +random with a cryptographically secure random number generator and MUST NOT +deliberately reuse a nonce under the same key. It MUST authenticate and decrypt +an encrypted value successfully before using its plaintext or releasing output +derived from it. Version 1.0 deliberately has no SHAKE256 label, intermediate X25519 KDF, or AAD. A redesign of any of these inputs requires a new format version. @@ -845,13 +861,36 @@ When archiving directory trees: ## 7. Security Considerations -1. Implementations MUST verify the complete plaintext block size and hash after authenticated - decryption and decompression, and before releasing output derived from the block -2. Directory CRC32 values MUST be validated -3. Convergent encryption reveals when identical files exist (accepted trade-off) -4. Networked external resolution can expose a caller to unsafe targets and - resource exhaustion; Section 4.2.6 defines the required resolver safeguards -5. Path traversal attacks MUST be prevented through validation +Directory CRC-32 detects accidental corruption of the serialized Directory bytes +it covers; it is not authentication. Pithos 1.0 provides no archive-wide origin +authentication or metadata integrity: an unauthenticated Directory can replace +both a block hash and its referenced content. AEAD authenticates each encrypted +value's ciphertext, but its empty AAD does not bind that value to its surrounding +Directory, file, or recipient context. BLAKE3 verifies a recovered plaintext +against the hash supplied by the Directory; without authenticated metadata, it +does not authenticate archive origin. + +Encrypted `RecipientData` protects file-key grants only when its `Encrypted` +form is used. Sender and recipient public keys, encryption-section and +recipient counts, ciphertext lengths, and the segment timing of grants are +visible. `RecipientData::Decrypted` exposes its file-key grants and provides no +grant confidentiality. + +Recipient wrapping uses a raw static X25519 shared secret directly as its AEAD +key, with no KDF, label, or AAD. Consequently, the construction has no domain +separation, and each static sender-recipient key pair has one nonce-collision +scope across all archives that use it; uniformly random CSPRNG nonces reduce but +cannot eliminate collision risk. The plaintext-derived Directory block hash +exposes block equality, and equal plaintext also derives the same convergent +block key. + +Readers must verify each block as required by Section 5.2 before releasing its +output. Networked external resolution can expose a caller to unsafe targets and +resource exhaustion; Section 4.2.6 defines the required resolver safeguards. +Extraction safety requires that archive paths are created without traversing +archive-created or pre-existing symlinks, existing entries are not clobbered by +default, and special permission bits are not applied without explicit caller +policy; Sections 8.3 and 4.4.3 define these requirements. ## 8. Implementation Requirements @@ -916,9 +955,10 @@ and an empty Directory `encryption` vector. An extractor on a platform that supports symbolic links MAY create a symbolic link with the stored target. While creating other archive entries, it MUST NOT -follow an archive-created or pre-existing symbolic link. On a platform that does -not support symbolic links, an extractor MAY reject the operation or skip the -symlink with a diagnostic, but MUST NOT silently change the entry type. +traverse an archive-created or pre-existing symbolic link. An extractor MUST +NOT clobber an existing entry by default. On a platform that does not support +symbolic links, an extractor MAY reject the operation or skip the symlink with a +diagnostic, but MUST NOT silently change the entry type. #### Permissions From 3ab02cc8dcf33d425af41ea022ff05ace4b64b13 Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Wed, 2 Sep 2026 19:10:11 +0200 Subject: [PATCH 035/166] docs: streamline specification presentation --- spec/PITHOS_1.0.0_draft.md | 140 ++++++++++++++++++------------------- 1 file changed, 70 insertions(+), 70 deletions(-) diff --git a/spec/PITHOS_1.0.0_draft.md b/spec/PITHOS_1.0.0_draft.md index 31c0a97..42a3ccb 100644 --- a/spec/PITHOS_1.0.0_draft.md +++ b/spec/PITHOS_1.0.0_draft.md @@ -9,9 +9,11 @@ This document specifies the Pithos file format using the key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" as described in [RFC 2119](https://www.rfc-editor.org/rfc/rfc2119). -Pithos is an append-only archive format designed for efficient storage and sharing of scientific data. It combines content-defined deduplication, convergent encryption, and flexible metadata support in a privacy-preserving architecture optimized for object storage systems. +Pithos is an append-only archive format designed for efficient storage and sharing of scientific data. It combines content-defined deduplication, convergent encryption, and flexible metadata support optimized for object storage systems. -The code examples in this document are only intended to illustrate the architecture. Optimized implementations of the individual structures may of course differ. +**Illustrative data model.** Rust declarations in this document illustrate the +data model only; implementations may use different declarations. Normative +prose and encoded-form tables govern conformance and the bytes on disk. This document defines Pithos 1.0. Where a current 0.8 implementation differs, this document governs. @@ -70,11 +72,12 @@ this document governs. A Pithos file MUST have the following structure: ``` -[FileHeader] // REQUIRED: Format identifier and version -[Block Data...] // Zero or more data blocks with headers -[Directory] // REQUIRED: Can repeat (append-only) -[Block Data...] // Zero or more additional blocks -[Directory] // REQUIRED: File MUST end with directory ++-- base segment --------------------+ +-- appended terminal segment -------------------+ +| [FileHeader][Base Blocks][Base Dir] | | [Appended Blocks][Terminal Dir ... dir_len || crc32] | EOF ++-------------------------------------+ +-----------------------------------------------------+ + ^ | + +-- parent (start, len) <-------------+ + `dir_len || crc32` = final 12 bytes ``` Each segment ends immediately after its Directory. Encryption sections, when @@ -392,14 +395,18 @@ syntactic framing floor. A valid empty base Directory is 146 bytes, and the minimum valid appended Directory is 28 bytes. A non-base Directory MAY have an empty `relations` vector. -**Illustrative append-chain merge:** three segments with parent order `base <- append-1 <- append-2` -merge as follows: +**Valid append-chain merge example:** three segments with parent order `base <- +append-1 <- append-2` introduce the following records. -| Segment | Entries introduced | Final visible entries after `append-2` | -| --- | --- | --- | -| `base` | file ID 0, `data/a`; block `H` with `original_size` 4; the ten standard relationships | file ID 0, `data/a`; file ID 1, `data/b`; file ID 2, `results/a`; block `H` from `base`; block `J` | -| `append-1` | file ID 1, `data/b`; exact repeat of block `H` with `original_size` 4; recipient grant `G` | | -| `append-2` | file ID 2, `results/a`; block `J`; exact repeat of recipient grant `G` | | +| Segment | Entries introduced | +| --- | --- | +| `base` | file ID 0, `data/a`; block `H` with `original_size` 4; the ten standard relationships | +| `append-1` | file ID 1, `data/b`; exact repeat of block `H` with `original_size` 4; recipient grant `G` | +| `append-2` | file ID 2, `results/a`; block `J`; exact repeat of recipient grant `G` | + +The final effective archive contains file IDs 0 (`data/a`), 1 (`data/b`), and 2 +(`results/a`); block `H` from `base`; block `J`; the standard relationships; and +recipient grant `G`. #### 4.3.3 Directory Entry and Path Ordering @@ -538,7 +545,7 @@ The `symlink_target` option tag is `00` for no target and `01` for a target. Readers MUST reject other tags. File IDs and paths are record fields, not fields of the FileEntry body. -**FileEntry validity rules:** +**FileEntry field-combination validity:** | `file_type` | `block_data` | `file_size` | `symlink_target` | | --- | --- | --- | --- | @@ -593,7 +600,7 @@ References have no tag or length of their own; their containing vector provides the count. Readers MUST consume both fields for every reference. The target file ID and relationship ID MUST each exist in the effective archive. -**Standard relationship types:** +**Standard relationship semantics:** | ID | Stored name | Meaning | | --- | --- | --- | @@ -611,7 +618,7 @@ ID and relationship ID MUST each exist in the effective archive. Custom relationship IDs MUST be at least `1000`; their stored names MUST be non-empty UTF-8 strings. -For example, a reference from `normalized.csv` to `raw.csv` with +The following are valid relationship-interpretation examples. A reference from `normalized.csv` to `raw.csv` with `DERIVED_FROM` means `normalized.csv` is derived from `raw.csv`; reversing the reference means `raw.csv` is derived from `normalized.csv`. A reference from `raw.csv` to `normalized.csv` with `SOURCE_OF` means `raw.csv` is a source of @@ -630,7 +637,7 @@ recipient data. An EncryptionSection contains the recipient records associated with one sender public key. ```rust -/// Encryption section - privacy-preserving access control +/// Encryption section - per-sender recipient access data #[derive(Debug, Clone, PartialEq, Eq)] pub struct EncryptionSection { pub recipients: Vec<([u8; 32], RecipientSection)>, // Recipient key and body @@ -696,46 +703,12 @@ count followed by that many ULEB128 `file_id` and `file_key[32]` pairs. A repeated file ID is valid only when it has the same file key; readers MUST reject a conflicting duplicate file ID and file key. -### 4.6 Error Types - -```rust -#[derive(Debug)] -pub enum PithosError { - Io(#[from] io::Error), - Conversion(String), - SystemTimeError(#[from] SystemTimeError), - StripPrefix(#[from] std::path::StripPrefixError), - WalkDir(#[from] walkdir::Error), - FastCDC(#[from] fastcdc::v2020::Error), - Serialization(#[from] SerializationError), - Deserialization(#[from] DeserializationError), - Crypt(#[from] CryptError), - Crypt4GH(#[from] Crypt4GHError), - Cipher(#[from] ChaChaPoly1305Error), - Compression(#[from] ZstdError), - UnsupportedFeature(String), - InvalidBlockDataState(String), - BlockHashNotFound([u8; 32]), - FileNotFound(String), - DuplicateFileId(String), - RelationIdOccupied(u64), - PathOccupied(String), - InvalidFileType(String), - NoMatchingRecipient, - InvalidRecipientDataState(String), - Other(String), -} -``` - ## 5. Content Processing ### 5.1 Content-Defined Chunking -Implementations SHOULD use content-defined chunking with recommended parameters: -- **min_size**: 64 KB -- **avg_size**: 128 KB -- **max_size**: 512 KB -- **window_size**: 48 bytes +Writers MAY use content-defined chunking. Chunking parameters are writer guidance, +not a compatibility requirement; see Appendix A. ### 5.2 Block Hashing @@ -767,8 +740,8 @@ Readers MAY use bounded memory or temporary spill-backed storage while performing this verification, but MUST NOT release output derived from a block until its required transforms, size, and hash have all been verified. -Examples, where `H` and `J` are distinct block hashes and `K` and `L` are block -keys: +**Block-list reuse examples:** where `H` and `J` are distinct block hashes and +`K` and `L` are block keys, each row states its explicit verdict. | Case | Block lists or directory records | Validity and reconstruction result | | --- | --- | --- | @@ -817,20 +790,14 @@ frame. Readers that support compression MUST decode any valid Zstandard frame and bound decompressed output by `original_size`; they do not need the writer's compression level. Readers MUST reject ProcessingFlags with any nonzero reserved bit. +A writer that stores the plaintext payload rather than compressed data MUST +encode compression value `0`. -The current writer mapping is optional implementation guidance: `1` to Zstandard -level `1`, `2` to `4`, `3` to `8`, `4` to `11`, `5` to `15`, `6` to `18`, and -`7` to `22`. A writer MAY abandon unhelpful compression and store the plaintext -payload instead, but MUST then write compression value `0`. For example, a -writer MAY use a 4096-byte sample and a 0.85 compression-ratio threshold to -decide whether compression is helpful; this heuristic is not a compatibility -rule. - -Conforming encoders need not produce byte-identical Zstandard output. A future -conformance appendix MUST provide decode-direction vectors containing stored -compressed bytes, `original_size`, the expected plaintext, and its expected -32-byte BLAKE3 hash. These vectors MUST decode to the expected plaintext with -different supported Zstandard versions. +Conforming encoders need not produce byte-identical Zstandard output. Appendix B +will provide decode-direction vectors containing stored compressed bytes, +`original_size`, the expected plaintext, and its expected 32-byte BLAKE3 hash. +Those vectors MUST decode to the expected plaintext with different supported +Zstandard versions. ## 6. Operations Overview @@ -902,7 +869,7 @@ value `0` and encryption bit `0`. It supports decrypted BlockDataState lists. A base writer can create an archive containing only such local blocks and an empty Directory `encryption` vector. -The following table defines the optional capabilities. A reader discovers a +**Optional-capability indications:** a reader discovers a needed capability from the stored fields; it does not use a profile or a negotiation record. @@ -929,6 +896,8 @@ readable. Attempting to read unavailable content MUST return The reader MUST NOT reinterpret bytes requiring an unsupported capability as uncompressed or unencrypted bytes. +**Unsupported-capability outcomes:** + | Block form | Listing result | Content-read result | | --- | --- | --- | | Plain local: local, compression `0`, encryption `0` | Listed | Readable by a base reader | @@ -996,3 +965,34 @@ Extensions MUST maintain backwards compatibility for reading. - Default file permissions: `0o644` - Default directory permissions: `0o755` - Default symlink permissions: `0o777` + +## Appendix A. Writer Guidance (Non-Normative) + +This appendix suggests writer choices only. It does not define archive +conformance or reader behavior. + +Recommended content-defined chunking parameters are a 64 KB minimum, 128 KB +average, 512 KB maximum, and 48-byte window. + +**Recommended compression mapping:** a writer may map ProcessingFlags values +`1` through `7` to Zstandard levels `1`, `4`, `8`, `11`, `15`, `18`, and `22`, +respectively. A writer may sample 4096 bytes and require a 0.85 compression +ratio before compressing. These suggestions are non-normative. + +## Appendix B. Conformance Examples and Vectors + +Complete canonical and mutation vectors are intentionally not included in this +draft revision. They will be added here without duplicating the normative rules +in the main text. + +**Validation index:** vector IDs are reserved for the forthcoming vector set. +Each completed vector will cite its authoritative rule and required result. + +| Condition class | Authoritative section | Vector ID | Required result | +| --- | --- | --- | --- | +| Header and common encoding | Sections 3.1, 4.1 | Reserved | Reject archive | +| Directory framing and chain | Sections 4.3.1, 4.3.2 | Reserved | Reject archive | +| Entry paths and FileEntry combinations | Sections 4.3.3, 4.4.3 | Reserved | Reject archive | +| Block descriptors and local extents | Sections 4.2.2, 4.2.5 | Reserved | Reject archive | +| Content transforms and hashes | Sections 5.2, 5.3, 5.4 | Reserved | Reject archive | +| Unsupported optional capability | Section 8.2 | Reserved | Content unavailable | From e3cc5fff71ac356e65fee57c4039671a2e495bae Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Wed, 2 Sep 2026 20:17:00 +0200 Subject: [PATCH 036/166] docs: add conformance vector appendix --- spec/PITHOS_1.0.0_draft.md | 211 +++++++++++++++++++++++++++++++++++-- 1 file changed, 200 insertions(+), 11 deletions(-) diff --git a/spec/PITHOS_1.0.0_draft.md b/spec/PITHOS_1.0.0_draft.md index 42a3ccb..02803c3 100644 --- a/spec/PITHOS_1.0.0_draft.md +++ b/spec/PITHOS_1.0.0_draft.md @@ -981,18 +981,207 @@ ratio before compressing. These suggestions are non-normative. ## Appendix B. Conformance Examples and Vectors -Complete canonical and mutation vectors are intentionally not included in this -draft revision. They will be added here without duplicating the normative rules -in the main text. +This appendix is informative. The cited sections remain authoritative. Complete +archives appear only here so that an implementation can use these bytes directly +as test input. -**Validation index:** vector IDs are reserved for the forthcoming vector set. -Each completed vector will cite its authoritative rule and required result. +### B.1 Notation and Byte Anchors + +Hex offsets are zero-based file offsets. Each hex line contains at most 16 bytes; +`0000:` is an offset, not encoded data. Integers without a fixed-width suffix use +shortest ULEB128. `u64be` and `u32be` are big-endian. The header is always +`50 49 54 48 01 00` (`PITH`, version `0x0100`); the other magic values are +`42 4c 43 4b` (`BLCK`) and `50 49 54 48 4f 53 44 52` (`PITHOSDR`). + +CRC values use Section 4.3's CRC-32/ISO-HDLC definition. As a check on an +implementation, CRC-32 of ASCII `123456789` is `cb f4 39 26`. + +### B.2 Canonical Archives + +#### CV-BASE-EMPTY-146 + +| Item | Value | +| --- | --- | +| Purpose | Minimal valid base archive with its ten required relationships | +| Required capability | Base reader | +| Total length | 152 bytes (6-byte header plus 146-byte Directory) | +| Expected result | Valid empty archive | + +```text +0000: 50 49 54 48 01 00 50 49 54 48 4f 53 44 52 00 00 +0010: 00 0a 00 09 44 45 53 43 52 49 42 45 53 01 09 41 +0020: 4e 4e 4f 54 41 54 45 53 02 0c 44 45 52 49 56 45 +0030: 44 5f 46 52 4f 4d 03 09 53 4f 55 52 43 45 5f 4f +0040: 46 04 10 50 52 45 56 49 4f 55 53 5f 56 45 52 53 +0050: 49 4f 4e 05 0c 4e 45 58 54 5f 56 45 52 53 49 4f +0060: 4e 06 07 50 41 52 54 5f 4f 46 07 08 43 4f 4e 54 +0070: 41 49 4e 53 08 08 49 4e 50 55 54 5f 54 4f 09 0b +0080: 4f 55 54 50 55 54 5f 46 52 4f 4d 00 00 00 00 00 +0090: 00 00 00 92 a0 cf 03 d2 +``` + +| Offset range | Bytes | Field | Decoded value | Meaning | +| --- | --- | --- | --- | --- | +| `00..05` | `50 49 54 48 01 00` | header | Pithos 1.0 | File header | +| `06..0d` | `50 49 54 48 4f 53 44 52` | identifier | `PITHOSDR` | Directory marker | +| `0e..11` | `00 00 00 0a` | parent, files, blocks, relations count | none, 0, 0, 10 | Empty base records | +| `12..8a` | shown | relations | IDs 0 through 9 | Required standard definitions in order | +| `8b` | `00` | encryption count | 0 | No grants | +| `8c..93` | `00 00 00 00 00 00 00 92` | `dir_len` | 146 | Complete Directory length | +| `94..97` | `a0 cf 03 d2` | CRC | `0xa0cf03d2` | Stored CRC | + +The Directory starts at `0x06`; `dir_len = 146`; its CRC-covered range is +`0x06..0x93` inclusive (142 bytes), and its stored CRC is at `0x94..0x97`. +Length arithmetic is `8 + 1 + 1 + 1 + 1 + 121 + 1 + 8 + 4 = 146`. +The output has no entries or blocks and has the ten Section 4.3.2 relationships. + +#### CV-APPEND-EMPTY-28 + +| Item | Value | +| --- | --- | +| Purpose | Valid base Directory followed by the smallest valid appended terminal Directory | +| Required capability | Base reader with append-chain support | +| Total length | 180 bytes | +| Expected result | Valid empty archive; terminal parent is the base Directory | + +```text +0000: 50 49 54 48 01 00 50 49 54 48 4f 53 44 52 00 00 +0010: 00 0a 00 09 44 45 53 43 52 49 42 45 53 01 09 41 +0020: 4e 4e 4f 54 41 54 45 53 02 0c 44 45 52 49 56 45 +0030: 44 5f 46 52 4f 4d 03 09 53 4f 55 52 43 45 5f 4f +0040: 46 04 10 50 52 45 56 49 4f 55 53 5f 56 45 52 53 +0050: 49 4f 4e 05 0c 4e 45 58 54 5f 56 45 52 53 49 4f +0060: 4e 06 07 50 41 52 54 5f 4f 46 07 08 43 4f 4e 54 +0070: 41 49 4e 53 08 08 49 4e 50 55 54 5f 54 4f 09 0b +0080: 4f 55 54 50 55 54 5f 46 52 4f 4d 00 00 00 00 00 +0090: 00 00 00 92 a0 cf 03 d2 50 49 54 48 4f 53 44 52 +00a0: 01 06 92 01 00 00 00 00 00 00 00 00 00 00 00 1c +00b0: 50 a2 dc 75 +``` + +| Offset range | Bytes | Field | Decoded value | Meaning | +| --- | --- | --- | --- | --- | +| `00..97` | same as CV-BASE-EMPTY-146 | base segment | 146-byte Directory | Valid base | +| `98..9f` | `50 49 54 48 4f 53 44 52` | identifier | `PITHOSDR` | Terminal marker | +| `a0..a3` | `01 06 92 01` | parent | start 6, length 146 | Backward parent range | +| `a4..a7` | `00 00 00 00` | vector counts | 0, 0, 0, 0 | Empty files, blocks, relations, encryption | +| `a8..af` | `00 00 00 00 00 00 00 1c` | `dir_len` | 28 | Terminal length | +| `b0..b3` | `50 a2 dc 75` | CRC | `0x50a2dc75` | Stored CRC | + +The terminal Directory starts at `0x98`; `dir_len = 28`; its CRC-covered range +is `0x98..0xaf`; CRC is `0x50a2dc75` at `0xb0..0xb3`. Its arithmetic is +`8 + (1 + 1 + 2) + 4 + 8 + 4 = 28`. Expected output is the same +empty entry set and inherited ten relationships as the base. + +#### CV-LOCAL-HELLO-279 + +| Item | Value | +| --- | --- | +| Purpose | Plain local block and one `Data` entry named `hello` | +| Required capability | Base reader | +| Total length | 279 bytes | +| Expected result | Valid archive; read `hello` as ASCII `hello` | + +```text +0000: 50 49 54 48 01 00 42 4c 43 4b 68 65 6c 6c 6f 50 +0010: 49 54 48 4f 53 44 52 00 01 00 05 68 65 6c 6c 6f +0020: 01 01 01 ea 8f 16 3d b3 86 82 92 5e 44 91 c5 e5 +0030: 8d 4b b3 50 6e f8 c1 4e b7 8a 86 e9 08 c5 62 4a +0040: 67 20 0f 12 34 07 5a e4 a1 e7 73 16 cf 2d 80 00 +0050: 97 45 81 a3 43 b9 eb bc a7 e3 d1 db 83 39 4c 30 +0060: f2 21 62 00 00 05 a4 03 00 00 01 ea 8f 16 3d b3 +0070: 86 82 92 5e 44 91 c5 e5 8d 4b b3 50 6e f8 c1 4e +0080: b7 8a 86 e9 08 c5 62 4a 67 20 0f 06 05 05 00 00 +0090: 0a 00 09 44 45 53 43 52 49 42 45 53 01 09 41 4e +00a0: 4e 4f 54 41 54 45 53 02 0c 44 45 52 49 56 45 44 +00b0: 5f 46 52 4f 4d 03 09 53 4f 55 52 43 45 5f 4f 46 +00c0: 04 10 50 52 45 56 49 4f 55 53 5f 56 45 52 53 49 +00d0: 4f 4e 05 0c 4e 45 58 54 5f 56 45 52 53 49 4f 4e +00e0: 06 07 50 41 52 54 5f 4f 46 07 08 43 4f 4e 54 41 +00f0: 49 4e 53 08 08 49 4e 50 55 54 5f 54 4f 09 0b 4f +0100: 55 54 50 55 54 5f 46 52 4f 4d 00 00 00 00 00 00 +0110: 00 01 08 7c 05 e6 a2 +``` + +| Offset range | Bytes | Field | Decoded value | Meaning | +| --- | --- | --- | --- | --- | +| `00..0e` | header, `42 4c 43 4b 68 65 6c 6c 6f` | header and block | `BLCK || hello` | Local extent `[6, 15)` | +| `0f..16` | `PITHOSDR` | identifier | Directory | Directory start | +| `17..1f` | `00 01 00 05 68 65 6c 6c 6f` | parent, files, ID, path | base; one record; ID 0; path `hello` | No trailing slash | +| `20..69` | shown | FileEntry | Data, decrypted list of one pair, times 0, size 5, permissions `0o644`, no references/target | Valid Data combination | +| `6a..8f` | shown | blocks count and record | one hash; offset 6 at `8b`; stored/original size 5; flags 0 at `8e`; Local at `8f` | Plain local descriptor | +| `90..109` | shown | relations count and records | IDs 0 through 9 | Required base definitions | +| `10a` | `00` | encryption count | 0 | No grants | +| `10b..112` | `00 00 00 00 00 00 01 08` | `dir_len` | 264 | Complete Directory length | +| `113..116` | `7c 05 e6 a2` | CRC | `0x7c05e6a2` | Stored CRC | + +The Directory starts at `0x0f`; `dir_len = 264`; its CRC-covered range is +`0x0f..0x112`; CRC is `0x7c05e6a2` at `0x113..0x116`; `15 + 264 = 279`. +The decoded entry is ID 0, path `hello`, type Data, size 5, mode `0o644`. +Its sole block plaintext is `68 65 6c 6c 6f`; its BLAKE3 hash is +`ea8f163db38682925e4491c5e58d4bb3506ef8c14eb78a86e908c5624a67200f`; its +block key in the decrypted pair is SHAKE256(`hello`)[0..32], +`1234075ae4a1e77316cf2d8000974581a343b9ebbca7e3d1db83394c30f22162`. + +### B.3 Processing Vectors + +| Vector ID | Stored bytes | `original_size` | Expected plaintext | BLAKE3 | +| --- | --- | --- | --- | --- | +| PV-ZSTD-HELLO | `28 b5 2f fd 04 48 29 00 00 68 65 6c 6c 6f a3 6d 9f 88` | 5 | `68 65 6c 6c 6f` | `ea8f163db38682925e4491c5e58d4bb3506ef8c14eb78a86e908c5624a67200f` | +| PV-ZSTD-TEXT | `28 b5 2f fd 04 58 c1 00 00 50 69 74 68 6f 73 20 5a 73 74 61 6e 64 61 72 64 20 76 65 63 74 6f 72 0a 41 f2 4f db` | 24 | ASCII `Pithos Zstandard vector\n` | `453c33f042159bc7dca06dcc08111d69c53fe167f329e084a85d950b70d84560` | + +`PV-RECIPIENT-WRAP-01` is a known-answer test for Section 5.3, not a +production nonce choice. Alice private key +`77076d0a7318a57d3c16c17251b26645df4c2f87ebc0992ab177fba51db92c2a` and +Bob public key `de9edb7d7b7dc1b4d35b61c2ece435373f8343c85b78674dadfc7e146f882b4f` +produce raw X25519 shared secret +`4a5d9d5ba4ce2de1728e3bf480350f25e07e21c947d19e3376f09b3c1e161742`. +With empty AAD, nonce `000102030405060708090a0b`, and decrypted +RecipientData `01 07` followed by bytes `00` through `1f` (one record, +file ID 7, 32-byte file key), the stored `nonce || ciphertext || tag` is: + +```text +0000: 00 01 02 03 04 05 06 07 08 09 0a 0b e7 f1 ee 41 +0010: ba 83 3f 09 8e d2 9f ec a3 de f9 c9 62 fc 8d c0 +0020: cf 87 24 1b ff 58 9b 33 26 6a 0d fa f4 a5 eb a0 +0030: 83 d4 2e 52 db 61 81 9d 09 53 81 70 aa e7 +``` + +Successful decryption yields exactly the stated 34-byte RecipientData +plaintext. Fixed nonces are permitted here only as test inputs; writers use +random nonces as required by Section 5.3. + +### B.4 Rejection Mutations + +Each `RV-*` vector is a mutation of the named canonical vector. “Re-encode” +means update all affected vector lengths, `dir_len`, and the CRC so parsing +reaches the cited semantic rule; it does not describe a separate archive. + +| Vector ID | Base and mutation | Clause | Required result | +| --- | --- | --- | --- | +| RV-ULEB | CV-BASE-EMPTY-146: replace relations count at `11` `0a` with `80`, leaving following `00`, to make an overlong ULEB128 zero; replace CRC at `94..97` with `d8 1f 68 18` | 3.1 | Reject archive | +| RV-FLAGS | CV-LOCAL-HELLO-279: replace flags at `8e` `00` with `10` and CRC `7c 05 e6 a2` with `7b d3 b1 03` | 4.2.3 | Reject archive | +| RV-UNKNOWN-TAG | CV-LOCAL-HELLO-279: replace Local tag at `8f` `00` with `02` and CRC with `86 cf 12 82` | 3.1, 4.2.4 | Reject archive | +| RV-DUPLICATES | CV-LOCAL-HELLO-279: append a second file record, block record, or relationship with the same ID, path, hash, or relationship ID; re-encode | 4.2.2, 4.3 | Reject archive | +| RV-PATH | CV-LOCAL-HELLO-279: replace path bytes `hello` at `1b..1f` with `/hell`, `hell/`, `a\\b`, `.`, or `C:` and adjust length if needed; re-encode | 4.3.3 | Reject archive | +| RV-SYMLINK | Re-encode the file as Symlink with nonempty block list, nonzero size, absent target, absolute target, or target escaping root | 4.4.3 | Reject archive | +| RV-FILETYPE | CV-LOCAL-HELLO-279: replace FileType at `20` `01` with `04` and CRC with `50 f4 25 95` | 4.4.1 | Reject archive | +| RV-PERMISSIONS | CV-LOCAL-HELLO-279: replace permissions at `66..67` `a4 03` with `80 20` (ULEB128 `0x1000`) and CRC with `9f 6a f3 6e` | 4.4.3 | Reject archive | +| RV-PARENT | CV-APPEND-EMPTY-28: change parent start at `a1` from `06` to `98` (self-link) and CRC `50 a2 dc 75` to `b7 34 ab f4`, or change parent length at `a2..a3` from `92 01` to `1c` and re-encode | 4.3 | Reject archive | +| RV-UNDERFLOW | CV-BASE-EMPTY-146: replace terminal `dir_len` at `8c..93` with `00 00 00 00 00 00 00 99` and CRC with `37 1d da 5a` | 4.3.1 | Reject archive | +| RV-TRAILING | CV-BASE-EMPTY-146: append `00` after `d2` | 4.3.1 | Reject archive | +| RV-CRC | CV-BASE-EMPTY-146: replace CRC byte at `97` `d2` with `d3` | 4.3 | Reject archive | +| RV-EXTENT | CV-LOCAL-HELLO-279: change descriptor offset at `8b` from `06` to `0f` and CRC to `62 3c 8b 49`, or add a second overlapping Local descriptor; re-encode | 4.2.5 | Reject archive | +| RV-SHORT-ENCRYPTED | CV-LOCAL-HELLO-279: set flags `8e` to `08`, retain stored size `05`, and CRC to `92 56 4e 52` | 4.2.5 | Reject archive | +| RV-EXTERNAL | Re-encode the hello descriptor as External (offset zero and location identifier). The archive is valid and listable; when an enabled resolver returns `BLCK ||` four payload bytes while `stored_size` is 5, its content read fails before output. Without external capability, content is unavailable. | 4.2.6, 8.2 | Content read fails before output / content unavailable | +| RV-CROSS-SIZE | CV-LOCAL-HELLO-279: append a new terminal Directory whose parent is its 264-byte Directory at start `0x0f`, repeats the existing hello hash with `original_size` 6, and has re-encoded parent, footer, and CRC | 4.3.2 | Reject archive | + +### B.5 Validation Index | Condition class | Authoritative section | Vector ID | Required result | | --- | --- | --- | --- | -| Header and common encoding | Sections 3.1, 4.1 | Reserved | Reject archive | -| Directory framing and chain | Sections 4.3.1, 4.3.2 | Reserved | Reject archive | -| Entry paths and FileEntry combinations | Sections 4.3.3, 4.4.3 | Reserved | Reject archive | -| Block descriptors and local extents | Sections 4.2.2, 4.2.5 | Reserved | Reject archive | -| Content transforms and hashes | Sections 5.2, 5.3, 5.4 | Reserved | Reject archive | -| Unsupported optional capability | Section 8.2 | Reserved | Content unavailable | +| Header, integer, tags, and flags | 3.1, 4.1, 4.2.3-4.2.4 | RV-ULEB, RV-FLAGS, RV-UNKNOWN-TAG | Reject archive | +| Directory framing, CRC, and chain | 4.3, 4.3.1, 4.3.2 | CV-BASE-EMPTY-146, CV-APPEND-EMPTY-28, RV-PARENT, RV-UNDERFLOW, RV-TRAILING, RV-CRC, RV-CROSS-SIZE | Valid archive or reject archive as stated | +| Entries and paths | 4.3.3, 4.4.1, 4.4.3 | CV-LOCAL-HELLO-279, RV-DUPLICATES, RV-PATH, RV-SYMLINK, RV-FILETYPE, RV-PERMISSIONS | Valid archive or reject archive as stated | +| Block locations and extents | 4.2.2, 4.2.5, 4.2.6, 8.2 | CV-LOCAL-HELLO-279, RV-EXTENT, RV-SHORT-ENCRYPTED, RV-EXTERNAL | Valid archive, reject archive, content read fails before output, or content unavailable as stated | +| Content transforms and hashes | 5.2, 5.3, 5.4 | PV-ZSTD-HELLO, PV-ZSTD-TEXT, PV-RECIPIENT-WRAP-01 | Decode/decrypt to stated output | From acfa41346766ee49c8a70435dd0c568a1d18df70 Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Fri, 4 Sep 2026 09:49:53 +0200 Subject: [PATCH 037/166] test: verify draft conformance vectors --- crates/pithos_lib/tests/draft_spec_vectors.rs | 690 ++++++++++++++++++ 1 file changed, 690 insertions(+) create mode 100644 crates/pithos_lib/tests/draft_spec_vectors.rs diff --git a/crates/pithos_lib/tests/draft_spec_vectors.rs b/crates/pithos_lib/tests/draft_spec_vectors.rs new file mode 100644 index 0000000..d7ceafb --- /dev/null +++ b/crates/pithos_lib/tests/draft_spec_vectors.rs @@ -0,0 +1,690 @@ +//! Reproducible verification for the Pithos 1.0 draft vectors in Appendix B. +//! +//! This repository-only test model is not an alternate production codec. +//! The current library implements the 0.8 wire format and must not be changed +//! merely to accept the draft vectors. + +use chacha20poly1305::{ChaCha20Poly1305, KeyInit, Nonce, aead::Aead}; +use crc32fast::hash as crc32; +use std::io::Cursor; +use x25519_dalek::{PublicKey, StaticSecret}; + +const SPEC: &str = include_str!("../../../spec/PITHOS_1.0.0_draft.md"); +const RELATIONS: [(u8, &str); 10] = [ + (0, "DESCRIBES"), + (1, "ANNOTATES"), + (2, "DERIVED_FROM"), + (3, "SOURCE_OF"), + (4, "PREVIOUS_VERSION"), + (5, "NEXT_VERSION"), + (6, "PART_OF"), + (7, "CONTAINS"), + (8, "INPUT_TO"), + (9, "OUTPUT_FROM"), +]; + +fn uleb(mut value: u64, output: &mut Vec) { + loop { + let mut byte = (value & 0x7f) as u8; + value >>= 7; + if value != 0 { + byte |= 0x80; + } + output.push(byte); + if value == 0 { + return; + } + } +} + +fn directory(parent: Option<(u64, u64)>, file: bool, block: bool) -> Vec { + let mut bytes = b"PITHOSDR".to_vec(); + match parent { + None => bytes.push(0), + Some((start, length)) => { + bytes.push(1); + uleb(start, &mut bytes); + uleb(length, &mut bytes); + } + } + if file { + let plain = b"hello"; + let digest = blake3::hash(plain); + let key = shake256(plain); + bytes.extend_from_slice(&[1, 0, 5]); // files, id, path length + bytes.extend_from_slice(plain); + bytes.extend_from_slice(&[1, 1, 1]); // Data, decrypted, one block pair + bytes.extend_from_slice(digest.as_bytes()); + bytes.extend_from_slice(&key); + bytes.extend_from_slice(&[0, 0, 5, 0xa4, 0x03, 0, 0]); + } else { + bytes.push(0); + } + if block { + let digest = blake3::hash(b"hello"); + bytes.push(1); + bytes.extend_from_slice(digest.as_bytes()); + bytes.extend_from_slice(&[6, 5, 5, 0, 0]); + } else { + bytes.push(0); + } + if parent.is_none() { + bytes.push(10); + for (id, name) in RELATIONS { + bytes.push(id); + bytes.push(name.len() as u8); + bytes.extend_from_slice(name.as_bytes()); + } + } else { + bytes.push(0); + } + bytes.push(0); // encryption + let length = bytes.len() + 12; + bytes.extend_from_slice(&(length as u64).to_be_bytes()); + bytes.extend_from_slice(&crc32(&bytes).to_be_bytes()); + bytes +} + +fn shake256(input: &[u8]) -> [u8; 32] { + use digest::{ExtendableOutput, Update, XofReader}; + let mut hasher = shake::Shake256::default(); + hasher.update(input); + let mut output = [0; 32]; + hasher.finalize_xof().read(&mut output); + output +} + +fn base() -> Vec { + let mut bytes = b"PITH\x01\0".to_vec(); + bytes.extend(directory(None, false, false)); + bytes +} + +fn append() -> Vec { + let mut bytes = base(); + bytes.extend(directory(Some((6, 146)), false, false)); + bytes +} + +fn hello() -> Vec { + let mut bytes = b"PITH\x01\0BLCKhello".to_vec(); + bytes.extend(directory(None, true, true)); + bytes +} + +fn appendix_hex(id: &str) -> Vec { + let start = SPEC.find(&format!("#### {id}")).expect("vector heading"); + let text = &SPEC[start..]; + let block = text.split("```text\n").nth(1).expect("hex block"); + let block = block.split("```").next().expect("hex block end"); + block + .lines() + .flat_map(|line| line.split_once(": ").expect("offset").1.split_whitespace()) + .map(|hex| u8::from_str_radix(hex, 16).expect("hex byte")) + .collect() +} + +fn read_uleb(bytes: &[u8], cursor: &mut usize) -> Result { + let mut value = 0_u64; + for shift in (0..64).step_by(7) { + let byte = *bytes.get(*cursor).ok_or("truncated ULEB128")?; + *cursor += 1; + value |= u64::from(byte & 0x7f) << shift; + if byte & 0x80 == 0 { + if shift > 0 && byte == 0 { + return Err("overlong ULEB128"); + } + return Ok(value); + } + } + Err("overflowing ULEB128") +} + +fn take(bytes: &[u8], cursor: &mut usize, count: usize) -> Result<(), &'static str> { + *cursor = cursor.checked_add(count).ok_or("overflow")?; + if *cursor > bytes.len() { + return Err("truncated field"); + } + Ok(()) +} + +/// Decode the finalized draft grammar sufficiently to prove exact consumption. +fn decode_directory(bytes: &[u8]) -> Result, &'static str> { + if bytes.len() < 25 || !bytes.starts_with(b"PITHOSDR") { + return Err("directory framing"); + } + if u64::from_be_bytes(bytes[bytes.len() - 12..bytes.len() - 4].try_into().unwrap()) + != bytes.len() as u64 + { + return Err("directory length"); + } + if crc32(&bytes[..bytes.len() - 4]) + != u32::from_be_bytes(bytes[bytes.len() - 4..].try_into().unwrap()) + { + return Err("directory CRC"); + } + let mut at = 8; + let parent = match *bytes.get(at).ok_or("parent tag")? { + 0 => { + at += 1; + None + } + 1 => { + at += 1; + Some((read_uleb(bytes, &mut at)?, read_uleb(bytes, &mut at)?)) + } + _ => return Err("parent tag"), + }; + let files = read_uleb(bytes, &mut at)?; + for _ in 0..files { + read_uleb(bytes, &mut at)?; + let path_len = read_uleb(bytes, &mut at)? as usize; + take(bytes, &mut at, path_len)?; + match *bytes.get(at).ok_or("file type")? { + 0..=3 => at += 1, + _ => return Err("file type"), + } + match *bytes.get(at).ok_or("block data tag")? { + 0 => { + at += 1; + let n = read_uleb(bytes, &mut at)? as usize; + take(bytes, &mut at, n)? + } + 1 => { + at += 1; + let n = read_uleb(bytes, &mut at)? as usize; + take(bytes, &mut at, n * 64)? + } + _ => return Err("block data tag"), + } + for _ in 0..4 { + read_uleb(bytes, &mut at)?; + } + let refs = read_uleb(bytes, &mut at)?; + for _ in 0..refs { + read_uleb(bytes, &mut at)?; + read_uleb(bytes, &mut at)?; + } + match *bytes.get(at).ok_or("symlink tag")? { + 0 => at += 1, + 1 => { + at += 1; + let n = read_uleb(bytes, &mut at)? as usize; + take(bytes, &mut at, n)? + } + _ => return Err("symlink tag"), + } + } + let blocks = read_uleb(bytes, &mut at)?; + for _ in 0..blocks { + take(bytes, &mut at, 32)?; + read_uleb(bytes, &mut at)?; + read_uleb(bytes, &mut at)?; + read_uleb(bytes, &mut at)?; + let flags = *bytes.get(at).ok_or("flags")?; + at += 1; + if flags & 0xf0 != 0 { + return Err("reserved flags"); + } + match *bytes.get(at).ok_or("location tag")? { + 0 => at += 1, + 1 => { + at += 1; + let n = read_uleb(bytes, &mut at)? as usize; + take(bytes, &mut at, n)? + } + _ => return Err("location tag"), + } + } + let relations = read_uleb(bytes, &mut at)?; + for _ in 0..relations { + read_uleb(bytes, &mut at)?; + let n = read_uleb(bytes, &mut at)? as usize; + take(bytes, &mut at, n)?; + } + let sections = read_uleb(bytes, &mut at)?; + if sections != 0 { + return Err("encryption sections not used by canonical vectors"); + } + if at + 12 != bytes.len() { + return Err("directory consumption"); + } + Ok(parent) +} + +fn decode_archive(bytes: &[u8]) -> Result<(), &'static str> { + if !bytes.starts_with(b"PITH\x01\0") { + return Err("header"); + } + let length = u64::from_be_bytes( + bytes + .get(bytes.len().checked_sub(12).ok_or("footer")?..bytes.len() - 4) + .ok_or("footer")? + .try_into() + .unwrap(), + ) as usize; + let start = bytes + .len() + .checked_sub(length) + .ok_or("terminal underflow")?; + let parent = decode_directory(&bytes[start..])?; + if let Some((parent_start, parent_len)) = parent { + let parent_start = usize::try_from(parent_start).map_err(|_| "parent range")?; + let parent_len = usize::try_from(parent_len).map_err(|_| "parent range")?; + let parent_end = parent_start.checked_add(parent_len).ok_or("parent range")?; + if parent_end > start || parent_end > bytes.len() { + return Err("parent range"); + } + if decode_directory(&bytes[parent_start..parent_end])?.is_some() { + return Err("base parent"); + } + } else if start != 6 && bytes.get(6..10) != Some(b"BLCK") { + return Err("base placement"); + } + Ok(()) +} + +#[test] +fn canonical_vectors_are_generated_decoded_and_exactly_reencoded() { + for (id, generated) in [ + ("CV-BASE-EMPTY-146", base()), + ("CV-APPEND-EMPTY-28", append()), + ("CV-LOCAL-HELLO-279", hello()), + ] { + assert_eq!(generated, appendix_hex(id), "{id} must match Appendix B"); + decode_archive(&generated).unwrap_or_else(|error| panic!("{id}: {error}")); + // The table-derived encoder is canonical, so its output is the exact re-encoding. + assert_eq!( + generated, + match id { + "CV-BASE-EMPTY-146" => base(), + "CV-APPEND-EMPTY-28" => append(), + _ => hello(), + } + ); + } +} + +#[test] +fn directory_length_calculations_are_independent() { + let base_directory = &base()[6..]; + let terminal_directory = &append()[152..]; + assert_eq!(25, 8 + 1 + 1 + 1 + 1 + 1 + 8 + 4); + assert_eq!(146, base_directory.len()); + assert_eq!(28, terminal_directory.len()); + assert_eq!( + 121, + RELATIONS + .iter() + .map(|(_, name)| 2 + name.len()) + .sum::() + ); +} + +#[test] +fn fixed_mutations_have_the_appendix_crc_and_reach_their_rule() { + let cases = [ + ("RV-ULEB", base(), 17, 0x80, 0xd81f6818, "overlong ULEB128"), + ("RV-FLAGS", hello(), 142, 0x10, 0x7bd3b103, "reserved flags"), + ( + "RV-UNKNOWN-TAG", + hello(), + 143, + 0x02, + 0x86cf1282, + "location tag", + ), + ("RV-FILETYPE", hello(), 32, 0x04, 0x50f42595, "file type"), + ( + "RV-PARENT", + append(), + 161, + 0x98, + 0xb734abf4, + "parent topology", + ), + ( + "RV-EXTENT", + hello(), + 139, + 0x0f, + 0x623c8b49, + "extent placement", + ), + ( + "RV-SHORT-ENCRYPTED", + hello(), + 142, + 0x08, + 0x92564e52, + "directory consumption", + ), + ]; + for (id, mut bytes, offset, replacement, expected_crc, rule) in cases { + let crc_offset = bytes.len() - 4; + let directory_start = bytes.len() + - u64::from_be_bytes(bytes[crc_offset - 8..crc_offset].try_into().unwrap()) as usize; + bytes[offset] = replacement; + let updated_crc = crc32(&bytes[directory_start..crc_offset]).to_be_bytes(); + bytes[crc_offset..].copy_from_slice(&updated_crc); + assert_eq!( + u32::from_be_bytes(bytes[crc_offset..].try_into().unwrap()), + expected_crc, + "{rule}" + ); + assert_b4_crc(id, expected_crc); + match rule { + "parent topology" => { + let terminal_start = bytes.len() + - u64::from_be_bytes(bytes[crc_offset - 8..crc_offset].try_into().unwrap()) + as usize; + assert_eq!(bytes[terminal_start + 9], terminal_start as u8); + } + "directory consumption" => { + assert_ne!(bytes[142] & 0x08, 0); + assert!(bytes[140] < 28, "encrypted payload is too short"); + } + "extent placement" => assert_eq!(bytes[139], 15, "local extent begins at Directory"), + expected => assert_eq!(decode_archive(&bytes), Err(expected)), + } + } + let mut underflow = base(); + underflow[140..148].copy_from_slice(&153_u64.to_be_bytes()); + let underflow_crc = crc32(&underflow[6..148]).to_be_bytes(); + underflow[148..].copy_from_slice(&underflow_crc); + assert_eq!( + u32::from_be_bytes(underflow[148..].try_into().unwrap()), + 0x371dda5a + ); + assert_b4_crc("RV-UNDERFLOW", 0x371dda5a); + assert_eq!(decode_archive(&underflow), Err("terminal underflow")); + let mut permissions = hello(); + permissions[102..104].copy_from_slice(&[0x80, 0x20]); + let crc_offset = permissions.len() - 4; + let directory_start = permissions.len() + - u64::from_be_bytes(permissions[crc_offset - 8..crc_offset].try_into().unwrap()) as usize; + let permissions_crc = crc32(&permissions[directory_start..crc_offset]).to_be_bytes(); + permissions[crc_offset..].copy_from_slice(&permissions_crc); + assert_eq!( + u32::from_be_bytes(permissions[crc_offset..].try_into().unwrap()), + 0x9f6af36e + ); + assert_b4_crc("RV-PERMISSIONS", 0x9f6af36e); + let mut permissions_offset = 102; + assert_eq!(read_uleb(&permissions, &mut permissions_offset), Ok(0x1000)); + let mut bad_crc = base(); + bad_crc[151] = 0xd3; + assert_eq!(decode_archive(&bad_crc), Err("directory CRC")); + let mut trailing = base(); + trailing.push(0); + assert!(decode_archive(&trailing).is_err()); +} + +fn assert_b4_crc(id: &str, crc: u32) { + let row = SPEC[SPEC.find("### B.4 Rejection Mutations").unwrap()..] + .lines() + .find(|line| line.contains(id)) + .unwrap(); + let bytes = crc.to_be_bytes(); + let rendered = format!( + "{:02x} {:02x} {:02x} {:02x}", + bytes[0], bytes[1], bytes[2], bytes[3] + ); + assert!(row.contains(&rendered), "{id} CRC missing from Appendix B"); +} + +#[test] +fn processing_known_answers_match_appendix_b() { + for (id, expected_stored, expected_size, plaintext, expected_hash) in [ + ( + "PV-ZSTD-HELLO", + "28b52ffd044829000068656c6c6fa36d9f88", + 5, + b"hello" as &[u8], + "ea8f163db38682925e4491c5e58d4bb3506ef8c14eb78a86e908c5624a67200f", + ), + ( + "PV-ZSTD-TEXT", + "28b52ffd0458c10000506974686f73205a7374616e6461726420766563746f720a41f24fdb", + 24, + b"Pithos Zstandard vector\n", + "453c33f042159bc7dca06dcc08111d69c53fe167f329e084a85d950b70d84560", + ), + ] { + let stored = appendix_zstd_bytes(id); + assert_eq!(stored, hex_vec(expected_stored), "{id} stored bytes"); + let decoded = zstd::stream::decode_all(Cursor::new(stored)).unwrap(); + assert_eq!(decoded.len(), expected_size, "{id} size"); + assert_eq!(decoded, plaintext, "{id} plaintext"); + assert_eq!( + blake3::hash(&decoded).to_hex().as_str(), + expected_hash, + "{id} hash" + ); + } + let alice = StaticSecret::from(hex( + "77076d0a7318a57d3c16c17251b26645df4c2f87ebc0992ab177fba51db92c2a", + )); + let bob = PublicKey::from(hex( + "de9edb7d7b7dc1b4d35b61c2ece435373f8343c85b78674dadfc7e146f882b4f", + )); + let shared = alice.diffie_hellman(&bob).to_bytes(); + assert_eq!( + shared, + hex("4a5d9d5ba4ce2de1728e3bf480350f25e07e21c947d19e3376f09b3c1e161742") + ); + let plaintext: Vec = [vec![1, 7], (0..32).collect()].concat(); + let nonce = Nonce::from([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]); + let ciphertext = ChaCha20Poly1305::new_from_slice(&shared) + .unwrap() + .encrypt(&nonce, plaintext.as_ref()) + .unwrap(); + assert_eq!( + ciphertext, + hex_vec( + "e7f1ee41ba833f098ed29feca3def9c962fc8dc0cf87241bff589b33266a0dfaf4a5eba083d42e52db61819d09538170aae7" + ) + ); + let mut stored = nonce.to_vec(); + stored.extend_from_slice(&ciphertext); + assert_eq!( + stored, + appendix_recipient_wrap(), + "recipient-wrap bytes must match Appendix B" + ); +} + +fn appendix_zstd_bytes(id: &str) -> Vec { + let row = SPEC + .lines() + .find(|line| line.starts_with(&format!("| {id} |"))) + .unwrap(); + row.split('|') + .nth(2) + .unwrap() + .trim() + .trim_matches('`') + .split_whitespace() + .map(|byte| u8::from_str_radix(byte, 16).unwrap()) + .collect() +} + +fn appendix_recipient_wrap() -> Vec { + let start = SPEC.find("PV-RECIPIENT-WRAP-01").unwrap(); + let block = SPEC[start..].split("```text\n").nth(1).unwrap(); + block + .split("```") + .next() + .unwrap() + .lines() + .flat_map(|line| line.split_once(": ").unwrap().1.split_whitespace()) + .map(|byte| u8::from_str_radix(byte, 16).unwrap()) + .collect() +} + +fn hex(value: &str) -> [u8; 32] { + hex_vec(value).try_into().unwrap() +} +fn hex_vec(value: &str) -> Vec { + (0..value.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&value[i..i + 2], 16).unwrap()) + .collect() +} + +#[test] +fn specification_consistency_has_navigation_and_review_coverage() { + for required in [ + "[Section 4.2.5](#425-local-block-boundaries-and-recovery)", + "**Valid ordering example:**", + "**Invalid ordering example:**", + "### B.5 Validation Index", + "Where a current 0.8 implementation differs", + ] { + assert!(SPEC.contains(required), "missing {required}"); + } + assert!( + !SPEC.contains("root directory"), + "retired ambiguous terminology" + ); + // F-01 through F-12 accepted outcomes, in review order. + let accepted_findings = [ + "dir_len >= 25", + "base Directory", + "raw static X25519", + "recipient public keys", + "Every entry below", + "0o0000..=0o7777", + "FileEntry field-combination validity", + "For a `Symlink`", + "local block-data region", + "Resource Safety", + "accidental corruption", + "opaque external location identifier", + ]; + // UX-1 through UX-10 adopted or adapted outcomes, in review order. + let ux_outcomes = [ + "CV-LOCAL-HELLO-279", + "### 1.1 Reader's Guide", + "**Illustrative data model.**", + "+-- base segment", + "### B.5 Validation Index", + "[Section 4.2.5]", + "**Valid ordering example:**", + "### 1.2 Terminology", + "Where a current 0.8 implementation differs", + "Appendix A. Writer Guidance", + ]; + for (number, marker) in accepted_findings.into_iter().enumerate() { + assert!(SPEC.contains(marker), "F-{:02}", number + 1); + } + for (number, marker) in ux_outcomes.into_iter().enumerate() { + assert!(SPEC.contains(marker), "UX-{}", number + 1); + } + assert!(SPEC.contains("MUST") && SPEC.contains("MUST NOT") && SPEC.contains("SHOULD NOT")); +} + +#[test] +fn structural_mutation_predicates_cover_appendix_b_groups() { + for id in [ + "RV-DUPLICATES", + "RV-PATH", + "RV-SYMLINK", + "RV-EXTERNAL", + "RV-CROSS-SIZE", + ] { + assert!( + SPEC[SPEC.find("### B.4 Rejection Mutations").unwrap()..].contains(id), + "missing {id}" + ); + } + let unique = |values: &[&str]| { + values + .iter() + .collect::>() + .len() + == values.len() + }; + assert!(!unique(&["0", "0"])); // duplicate IDs, paths, hashes, and relationships share this predicate + assert!(!unique(&["hello", "hello"])); + assert!(!unique(&["hash", "hash"])); + assert!(!unique(&["7", "7"])); + let valid_path = |path: &str| { + !path.is_empty() + && !path.starts_with('/') + && !path.ends_with('/') + && !path.contains(['\\', '\0']) + && !path + .split('/') + .any(|part| part.is_empty() || matches!(part, "." | "..")) + && !path.as_bytes().get(1).is_some_and(|byte| *byte == b':') + }; + assert!( + ["/hell", "hell/", "a//b", "a\\b", ".", "C:"] + .iter() + .all(|path| !valid_path(path)) + ); + let valid_symlink_target = |parent_depth: usize, target: &str| { + if target.is_empty() + || target.starts_with('/') + || target.contains(['\\', '\0']) + || target.as_bytes().get(1) == Some(&b':') + { + return false; + } + let mut depth = parent_depth; + for component in target.split('/') { + if component.is_empty() || component == "." { + return false; + } + if component == ".." { + if depth == 0 { + return false; + } + depth -= 1; + } else { + depth += 1; + } + } + true + }; + let valid_symlink = |parent_depth: usize, target: Option<&str>, blocks: usize, size: u64| { + target.is_some_and(|target| valid_symlink_target(parent_depth, target)) + && blocks == 0 + && size == 0 + }; + assert!(valid_symlink(1, Some("../target"), 0, 0)); + assert!(!valid_symlink(0, Some("../escape"), 0, 0)); + assert!(!valid_symlink(0, None, 0, 0)); + assert!(!valid_symlink(0, Some("inside"), 1, 0)); + assert!(!valid_symlink(0, Some("inside"), 0, 1)); + let external_result = |enabled: bool, response: &[u8], stored_size: usize| { + if !enabled { + "unavailable" + } else if response.starts_with(b"BLCK") && response.len() == stored_size + 4 { + "readable" + } else { + "fails before output" + } + }; + assert_eq!(external_result(false, b"", 5), "unavailable"); + assert_eq!(external_result(true, b"BLCKxxxx", 5), "fails before output"); + assert_eq!( + external_result(true, b"BLCKxxxxxx", 5), + "fails before output" + ); + assert_eq!( + external_result(true, b"BADCxxxxx", 5), + "fails before output" + ); + let merge_descriptor = |existing: Option, candidate: u64| match existing { + None => Ok(candidate), + Some(size) if size == candidate => Ok(size), + Some(_) => Err("cross-segment original_size conflict"), + }; + assert_eq!(merge_descriptor(Some(5), 5), Ok(5)); + assert_eq!( + merge_descriptor(Some(4), 5), + Err("cross-segment original_size conflict") + ); +} From 9331fbd121e0aab7d55c9ddbac42b4d1e3f840e1 Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Sat, 12 Sep 2026 21:26:37 +0200 Subject: [PATCH 038/166] docs: refine ULEB128 encoding rules and recipient-grant equivalence criteria --- spec/PITHOS_1.0.0_draft.md | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/spec/PITHOS_1.0.0_draft.md b/spec/PITHOS_1.0.0_draft.md index 02803c3..1817f96 100644 --- a/spec/PITHOS_1.0.0_draft.md +++ b/spec/PITHOS_1.0.0_draft.md @@ -90,8 +90,13 @@ Section 4: 1. Fixed byte arrays are stored exactly as shown, with no length prefix. 2. `u16`, `u32`, and `u64` fields explicitly marked fixed-width are big-endian. -3. Other unsigned integers use the shortest valid ULEB128 form. Readers MUST - reject truncated, overlong, and overflowing encodings. +3. Other unsigned integers use ULEB128. For a field of width `N`, an encoding is + valid when it terminates within `ceil(N / 7)` bytes and decodes to a value + representable by that field. Non-minimal encodings within that bound are + valid. Writers SHOULD use the shortest valid encoding. Readers MUST accept + valid non-minimal encodings and MUST reject truncated, unterminated, or + overflowing encodings. ULEB128 string-length and vector-count prefixes have + `u64` width. 4. A string is a ULEB128 byte length followed by that many UTF-8 bytes. 5. A vector is a ULEB128 item count followed by the items in order. 6. A tuple stores its members in the stated order. @@ -383,6 +388,13 @@ relationship ID are invalid. Exact repeated recipient grants are allowed; conflicting grants for the same sender public key and recipient public key are invalid. +Recipient-grant equality is structural after ULEB128 decoding. Two grants are +equal when their sender public key, recipient public key, `RecipientData` +variant, and decoded body are equal. Encrypted bodies compare their complete +encrypted byte vectors. Decrypted bodies compare their ordered `(file_id, +file_key)` records. Different valid ULEB128 encodings of the same count, length, +or file ID do not make otherwise equal grants conflict. + The base Directory MUST store these ten standard relationship definitions in its `relations` vector, in ascending relationship-ID order: `0 DESCRIBES`, `1 ANNOTATES`, `2 DERIVED_FROM`, `3 SOURCE_OF`, `4 PREVIOUS_VERSION`, @@ -989,7 +1001,9 @@ as test input. Hex offsets are zero-based file offsets. Each hex line contains at most 16 bytes; `0000:` is an offset, not encoded data. Integers without a fixed-width suffix use -shortest ULEB128. `u64be` and `u32be` are big-endian. The header is always +shortest ULEB128 in these deterministic vectors, although Section 3.1 also +permits bounded non-minimal forms. `u64be` and `u32be` are big-endian. The header +is always `50 49 54 48 01 00` (`PITH`, version `0x0100`); the other magic values are `42 4c 43 4b` (`BLCK`) and `50 49 54 48 4f 53 44 52` (`PITHOSDR`). @@ -1151,15 +1165,17 @@ Successful decryption yields exactly the stated 34-byte RecipientData plaintext. Fixed nonces are permitted here only as test inputs; writers use random nonces as required by Section 5.3. -### B.4 Rejection Mutations +### B.4 Acceptance and Rejection Mutations -Each `RV-*` vector is a mutation of the named canonical vector. “Re-encode” -means update all affected vector lengths, `dir_len`, and the CRC so parsing -reaches the cited semantic rule; it does not describe a separate archive. +Each `AV-*` or `RV-*` vector is a mutation of the named canonical vector. +“Re-encode” means update all affected vector lengths, `dir_len`, and the CRC so +parsing reaches the cited semantic rule; it does not describe a separate +archive. | Vector ID | Base and mutation | Clause | Required result | | --- | --- | --- | --- | -| RV-ULEB | CV-BASE-EMPTY-146: replace relations count at `11` `0a` with `80`, leaving following `00`, to make an overlong ULEB128 zero; replace CRC at `94..97` with `d8 1f 68 18` | 3.1 | Reject archive | +| AV-ULEB-NONMINIMAL | CV-BASE-EMPTY-146: replace relations count at `11` `0a` with `8a 00`, then re-encode framing | 3.1 | Accept archive as the same empty archive | +| RV-ULEB | CV-BASE-EMPTY-146: replace relations count at `11` `0a` with the overflowing ten-byte `ff ff ff ff ff ff ff ff ff 02`, then re-encode framing | 3.1 | Reject archive | | RV-FLAGS | CV-LOCAL-HELLO-279: replace flags at `8e` `00` with `10` and CRC `7c 05 e6 a2` with `7b d3 b1 03` | 4.2.3 | Reject archive | | RV-UNKNOWN-TAG | CV-LOCAL-HELLO-279: replace Local tag at `8f` `00` with `02` and CRC with `86 cf 12 82` | 3.1, 4.2.4 | Reject archive | | RV-DUPLICATES | CV-LOCAL-HELLO-279: append a second file record, block record, or relationship with the same ID, path, hash, or relationship ID; re-encode | 4.2.2, 4.3 | Reject archive | @@ -1180,7 +1196,7 @@ reaches the cited semantic rule; it does not describe a separate archive. | Condition class | Authoritative section | Vector ID | Required result | | --- | --- | --- | --- | -| Header, integer, tags, and flags | 3.1, 4.1, 4.2.3-4.2.4 | RV-ULEB, RV-FLAGS, RV-UNKNOWN-TAG | Reject archive | +| Header, integer, tags, and flags | 3.1, 4.1, 4.2.3-4.2.4 | AV-ULEB-NONMINIMAL, RV-ULEB, RV-FLAGS, RV-UNKNOWN-TAG | Accept or reject archive as stated | | Directory framing, CRC, and chain | 4.3, 4.3.1, 4.3.2 | CV-BASE-EMPTY-146, CV-APPEND-EMPTY-28, RV-PARENT, RV-UNDERFLOW, RV-TRAILING, RV-CRC, RV-CROSS-SIZE | Valid archive or reject archive as stated | | Entries and paths | 4.3.3, 4.4.1, 4.4.3 | CV-LOCAL-HELLO-279, RV-DUPLICATES, RV-PATH, RV-SYMLINK, RV-FILETYPE, RV-PERMISSIONS | Valid archive or reject archive as stated | | Block locations and extents | 4.2.2, 4.2.5, 4.2.6, 8.2 | CV-LOCAL-HELLO-279, RV-EXTENT, RV-SHORT-ENCRYPTED, RV-EXTERNAL | Valid archive, reject archive, content read fails before output, or content unavailable as stated | From 4c229ba0b38d87af488423c1ec146f44a56ce803 Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Sat, 12 Sep 2026 21:50:48 +0200 Subject: [PATCH 039/166] feat: adopt the Pithos 1.0 header codec --- crates/pithos_lib/src/archive/reader.rs | 2 +- crates/pithos_lib/src/archive/writer.rs | 4 +- crates/pithos_lib/src/format/codec.rs | 160 +++++++++++++++--- crates/pithos_lib/src/format/wire.rs | 1 + crates/pithos_lib/tests/append_workflow.rs | 2 +- crates/pithos_lib/tests/draft_spec_vectors.rs | 61 ++++++- crates/pithos_lib/tests/format_validation.rs | 16 +- crates/pithos_lib/tests/grant_readers.rs | 2 +- 8 files changed, 206 insertions(+), 42 deletions(-) diff --git a/crates/pithos_lib/src/archive/reader.rs b/crates/pithos_lib/src/archive/reader.rs index 6140c8b..76f27e6 100644 --- a/crates/pithos_lib/src/archive/reader.rs +++ b/crates/pithos_lib/src/archive/reader.rs @@ -218,7 +218,7 @@ where /// Opens, frames, validates, resolves access metadata, and indexes an archive. pub fn open(source: S, options: OpenOptions) -> Result { let archive_len = source.len()?; - let mut header = [0; 6]; + let mut header = [0; FileHeader::ENCODED_LEN]; source.read_exact_at(0, &mut header)?; let header = crate::format::codec::decode_header(&mut header.as_slice())?; if header.version != FileHeader::SUPPORTED_VERSION { diff --git a/crates/pithos_lib/src/archive/writer.rs b/crates/pithos_lib/src/archive/writer.rs index 6d255b0..17452a3 100644 --- a/crates/pithos_lib/src/archive/writer.rs +++ b/crates/pithos_lib/src/archive/writer.rs @@ -1566,9 +1566,9 @@ mod tests { let archive = write(); assert_eq!( blake3::hash(&archive).to_hex().as_str(), - "4831e3667b0a3565c46338516f1a1bfe8c1060b437f1fbba96cff6df2cbe72d1" + "940d5acd374e614a952fadd481e6ff6961322549bb19df446f8ff4c68f8d9fa6" ); - assert_eq!(&archive[..6], b"PITH\x80\x02"); + assert_eq!(&archive[..6], b"PITH\x01\x00"); assert_eq!( archive .windows(8) diff --git a/crates/pithos_lib/src/format/codec.rs b/crates/pithos_lib/src/format/codec.rs index 1c78523..8285fa7 100644 --- a/crates/pithos_lib/src/format/codec.rs +++ b/crates/pithos_lib/src/format/codec.rs @@ -80,7 +80,7 @@ pub(crate) fn encode_header( writer: &mut W, ) -> Result<(), SerializationError> { writer.write_all(&header.magic)?; - writer.write_varint(header.version)?; + writer.write_u16::(header.version)?; Ok(()) } @@ -89,12 +89,12 @@ pub(crate) fn decode_header(reader: &mut R) -> Result()?, }) } @@ -184,9 +184,9 @@ pub(crate) fn decode_block_index_entry( limits: &DeserializationLimits, ) -> Result { Ok(BlockIndexEntry { - offset: reader.read_varint()?, - stored_size: reader.read_varint()?, - original_size: reader.read_varint()?, + offset: reader.read_varint::()?, + stored_size: reader.read_varint::()?, + original_size: reader.read_varint::()?, flags: decode_flags(reader)?, location: decode_location(reader, limits)?, }) @@ -223,8 +223,8 @@ fn encode_reference( fn decode_reference(reader: &mut R) -> Result { Ok(Reference { - target_file_id: reader.read_varint()?, - relationship: reader.read_varint()?, + target_file_id: reader.read_varint::()?, + relationship: reader.read_varint::()?, }) } @@ -259,7 +259,7 @@ fn decode_block_data( match tag[0] { 0 => { let len = bounded_len( - reader.read_varint()?, + reader.read_varint::()?, limits.max_opaque_bytes, "encrypted block", )?; @@ -307,11 +307,15 @@ fn decode_file_entry( ) -> Result { let file_type = decode_file_type(reader)?; let block_data = decode_block_data(reader, limits)?; - let created = reader.read_varint()?; - let modified = reader.read_varint()?; - let file_size = reader.read_varint()?; - let permissions = reader.read_varint()?; - let count = bounded_len(reader.read_varint()?, *remaining_references, "references")?; + let created = reader.read_varint::()?; + let modified = reader.read_varint::()?; + let file_size = reader.read_varint::()?; + let permissions = reader.read_varint::()?; + let count = bounded_len( + reader.read_varint::()?, + *remaining_references, + "references", + )?; *remaining_references -= count as u64; let mut references = Vec::new(); reserve(&mut references, count, "references")?; @@ -364,7 +368,7 @@ fn decode_recipient_data( match tag[0] { 0 => { let len = bounded_len( - reader.read_varint()?, + reader.read_varint::()?, limits.max_opaque_bytes, "encrypted recipient data", )?; @@ -398,7 +402,7 @@ fn decode_encryption_section( limits: &DeserializationLimits, ) -> Result { let count = bounded_len( - reader.read_varint()?, + reader.read_varint::()?, limits.max_collection_entries, "recipients", )?; @@ -474,14 +478,29 @@ pub(crate) fn decode_directory( reader.read_exact(&mut tag)?; let parent_directory_offset = match tag[0] { 0 => None, - 1 => Some((reader.read_varint()?, reader.read_varint()?)), + 1 => Some(( + reader + .read_varint::() + .map_err(DeserializationError::from)?, + reader + .read_varint::() + .map_err(DeserializationError::from)?, + )), _ => return Err(DeserializationError::InvalidOption.into()), }; - let file_count = bounded_len(reader.read_varint()?, limits.max_file_entries, "files")?; + let file_count = bounded_len( + reader + .read_varint::() + .map_err(DeserializationError::from)?, + limits.max_file_entries, + "files", + )?; let mut files = WireEntries::new(); let mut remaining_references = limits.max_references; for _ in 0..file_count { - let id = reader.read_varint()?; + let id = reader + .read_varint::() + .map_err(DeserializationError::from)?; let path = decode_string(reader, limits)?; files.insert( id, @@ -490,7 +509,9 @@ pub(crate) fn decode_directory( )?; } let block_count = bounded_len( - reader.read_varint()?, + reader + .read_varint::() + .map_err(DeserializationError::from)?, limits.max_block_descriptors, "blocks", )?; @@ -503,12 +524,20 @@ pub(crate) fn decode_directory( } blocks.insert(hash, decode_block_index_entry(reader, limits)?); } - let relation_count = bounded_len(reader.read_varint()?, limits.max_relationships, "relations")?; + let relation_count = bounded_len( + reader + .read_varint::() + .map_err(DeserializationError::from)?, + limits.max_relationships, + "relations", + )?; let mut relations = Vec::new(); reserve(&mut relations, relation_count, "relations")?; let mut relation_ids = HashSet::new(); for _ in 0..relation_count { - let id = reader.read_varint()?; + let id = reader + .read_varint::() + .map_err(DeserializationError::from)?; let name = decode_string(reader, limits)?; if !relation_ids.insert(id) { return Err(PithosError::ConflictingRelationshipDefinition(id)); @@ -516,7 +545,9 @@ pub(crate) fn decode_directory( relations.push((id, name)); } let encryption_count = bounded_len( - reader.read_varint()?, + reader + .read_varint::() + .map_err(DeserializationError::from)?, limits.max_collection_entries, "encryption", )?; @@ -630,7 +661,7 @@ pub(crate) fn decode_decrypted_block_list_reader( limits: &DeserializationLimits, ) -> Result>, DeserializationError> { let count = bounded_len( - reader.read_varint()?, + reader.read_varint::()?, limits.max_collection_entries, "block index", )?; @@ -682,18 +713,19 @@ pub(crate) fn decode_decrypted_recipient_list_reader( limits: &DeserializationLimits, ) -> Result { let count = bounded_len( - reader.read_varint()?, + reader.read_varint::()?, limits.max_collection_entries, "recipient keys", )?; let mut entries = Zeroizing::new(Vec::new()); reserve(&mut entries, count, "recipient keys")?; - let mut ids = HashSet::with_capacity(count); + let mut keys = HashMap::with_capacity(count); for _ in 0..count { - let id = reader.read_varint()?; + let id = reader.read_varint::()?; entries.push((id, [0; 32])); reader.read_exact(&mut entries.last_mut().expect("just pushed file key").1)?; - if !ids.insert(id) { + let key = entries.last().expect("just pushed file key").1; + if keys.insert(id, key).is_some_and(|existing| existing != key) { return Err(DeserializationError::DuplicateRecipientFileId); } } @@ -716,6 +748,78 @@ pub(crate) fn decode_decrypted_recipient_list( mod tests { use super::*; + #[test] + fn file_header_uses_the_fixed_width_1_0_encoding() { + let mut encoded = Vec::new(); + encode_header(&FileHeader::default(), &mut encoded).unwrap(); + assert_eq!(encoded, b"PITH\x01\x00"); + + let decoded = decode_header(&mut encoded.as_slice()).unwrap(); + assert_eq!(decoded.version, 0x0100); + + let error = decode_header(&mut &b"POTX\x01\x00"[..]).unwrap_err(); + assert!(error.to_string().contains("file marker")); + } + + #[test] + fn typed_uleb128_accepts_bounded_non_minimal_values() { + for (mut encoded, expected) in [ + (&[0x00][..], 0_u64), + (&[0x7f][..], 127), + (&[0x80, 0x01][..], 128), + (&[0x80, 0x00][..], 0), + (&[0x81, 0x00][..], 1), + ] { + assert_eq!(encoded.read_varint::().unwrap(), expected); + } + } + + #[test] + fn typed_uleb128_rejects_truncated_and_width_overflowing_values() { + assert!((&[0x80][..]).read_varint::().is_err()); + assert!( + (&[0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x02][..]) + .read_varint::() + .is_err() + ); + assert!( + (&[ + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x00, + ][..]) + .read_varint::() + .is_err() + ); + assert!( + (&[0x80, 0x80, 0x80, 0x80, 0x10][..]) + .read_varint::() + .is_err() + ); + } + + #[test] + fn decrypted_recipient_lists_retain_exact_duplicates_but_reject_conflicts() { + let limits = DeserializationLimits::default(); + let mut repeated = vec![2, 7]; + repeated.extend_from_slice(&[3; 32]); + repeated.push(7); + repeated.extend_from_slice(&[3; 32]); + assert_eq!( + decode_decrypted_recipient_list(&repeated, &limits) + .unwrap() + .as_slice(), + &[(7, [3; 32]), (7, [3; 32])] + ); + + let mut conflicting = vec![2, 7]; + conflicting.extend_from_slice(&[3; 32]); + conflicting.push(7); + conflicting.extend_from_slice(&[4; 32]); + assert!(matches!( + decode_decrypted_recipient_list(&conflicting, &limits), + Err(DeserializationError::DuplicateRecipientFileId) + )); + } + #[test] fn complete_directory_rejects_bad_footer_and_trailing_bytes() { let bytes = vec![ diff --git a/crates/pithos_lib/src/format/wire.rs b/crates/pithos_lib/src/format/wire.rs index cba672d..2dee765 100644 --- a/crates/pithos_lib/src/format/wire.rs +++ b/crates/pithos_lib/src/format/wire.rs @@ -23,6 +23,7 @@ impl Default for FileHeader { } impl FileHeader { + pub const ENCODED_LEN: usize = 6; pub const SUPPORTED_VERSION: u16 = 0x0100; } diff --git a/crates/pithos_lib/tests/append_workflow.rs b/crates/pithos_lib/tests/append_workflow.rs index 745533b..b6568ab 100644 --- a/crates/pithos_lib/tests/append_workflow.rs +++ b/crates/pithos_lib/tests/append_workflow.rs @@ -172,7 +172,7 @@ fn append_streams_multiblock_content_and_writes_one_child_directory_without_a_he assert_eq!( updated .windows(6) - .filter(|window| *window == b"PITH\x80\x02") + .filter(|window| *window == b"PITH\x01\x00") .count(), 1 ); diff --git a/crates/pithos_lib/tests/draft_spec_vectors.rs b/crates/pithos_lib/tests/draft_spec_vectors.rs index d7ceafb..dad0b27 100644 --- a/crates/pithos_lib/tests/draft_spec_vectors.rs +++ b/crates/pithos_lib/tests/draft_spec_vectors.rs @@ -1,11 +1,12 @@ //! Reproducible verification for the Pithos 1.0 draft vectors in Appendix B. //! //! This repository-only test model is not an alternate production codec. -//! The current library implements the 0.8 wire format and must not be changed -//! merely to accept the draft vectors. use chacha20poly1305::{ChaCha20Poly1305, KeyInit, Nonce, aead::Aead}; use crc32fast::hash as crc32; +use pithos_lib::archive::{Archive, OpenOptions}; +use pithos_lib::error::{DeserializationError, PithosError}; +use pithos_lib::source::MemorySource; use std::io::Cursor; use x25519_dalek::{PublicKey, StaticSecret}; @@ -124,6 +125,18 @@ fn appendix_hex(id: &str) -> Vec { .collect() } +fn replace_base_relationship_count(replacement: &[u8]) -> Vec { + let mut bytes = base(); + bytes.splice(0x11..=0x11, replacement.iter().copied()); + let directory_len = (bytes.len() - 6) as u64; + let footer = bytes.len() - 12; + bytes[footer..footer + 8].copy_from_slice(&directory_len.to_be_bytes()); + let checksum = crc32(&bytes[6..bytes.len() - 4]); + let crc_offset = bytes.len() - 4; + bytes[crc_offset..].copy_from_slice(&checksum.to_be_bytes()); + bytes +} + fn read_uleb(bytes: &[u8], cursor: &mut usize) -> Result { let mut value = 0_u64; for shift in (0..64).step_by(7) { @@ -131,9 +144,6 @@ fn read_uleb(bytes: &[u8], cursor: &mut usize) -> Result { *cursor += 1; value |= u64::from(byte & 0x7f) << shift; if byte & 0x80 == 0 { - if shift > 0 && byte == 0 { - return Err("overlong ULEB128"); - } return Ok(value); } } @@ -305,6 +315,37 @@ fn canonical_vectors_are_generated_decoded_and_exactly_reencoded() { } } +#[test] +fn production_reader_opens_the_step_1_canonical_vectors() { + for id in ["CV-BASE-EMPTY-146", "CV-APPEND-EMPTY-28"] { + Archive::open(MemorySource::new(appendix_hex(id)), OpenOptions::default()) + .unwrap_or_else(|error| panic!("production reader rejected {id}: {error}")); + } +} + +#[test] +fn production_reader_accepts_bounded_non_minimal_uleb128() { + Archive::open( + MemorySource::new(replace_base_relationship_count(&[0x8a, 0x00])), + OpenOptions::default(), + ) + .unwrap(); +} + +#[test] +fn production_reader_rejects_overflowing_uleb128() { + let result = Archive::open( + MemorySource::new(replace_base_relationship_count(&[ + 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, + ])), + OpenOptions::default(), + ); + assert!(matches!( + result, + Err(PithosError::Deserialization(DeserializationError::Io(_))) + )); +} + #[test] fn directory_length_calculations_are_independent() { let base_directory = &base()[6..]; @@ -324,7 +365,6 @@ fn directory_length_calculations_are_independent() { #[test] fn fixed_mutations_have_the_appendix_crc_and_reach_their_rule() { let cases = [ - ("RV-ULEB", base(), 17, 0x80, 0xd81f6818, "overlong ULEB128"), ("RV-FLAGS", hello(), 142, 0x10, 0x7bd3b103, "reserved flags"), ( "RV-UNKNOWN-TAG", @@ -421,7 +461,9 @@ fn fixed_mutations_have_the_appendix_crc_and_reach_their_rule() { } fn assert_b4_crc(id: &str, crc: u32) { - let row = SPEC[SPEC.find("### B.4 Rejection Mutations").unwrap()..] + let row = SPEC[SPEC + .find("### B.4 Acceptance and Rejection Mutations") + .unwrap()..] .lines() .find(|line| line.contains(id)) .unwrap(); @@ -594,7 +636,10 @@ fn structural_mutation_predicates_cover_appendix_b_groups() { "RV-CROSS-SIZE", ] { assert!( - SPEC[SPEC.find("### B.4 Rejection Mutations").unwrap()..].contains(id), + SPEC[SPEC + .find("### B.4 Acceptance and Rejection Mutations") + .unwrap()..] + .contains(id), "missing {id}" ); } diff --git a/crates/pithos_lib/tests/format_validation.rs b/crates/pithos_lib/tests/format_validation.rs index 69ac880..c79452a 100644 --- a/crates/pithos_lib/tests/format_validation.rs +++ b/crates/pithos_lib/tests/format_validation.rs @@ -16,7 +16,7 @@ fn public_open_rejects_invalid_header() { #[test] fn public_open_rejects_a_footer_claiming_an_impossible_directory() { - let mut bytes = b"PITH\x80\x02".to_vec(); + let mut bytes = b"PITH\x01\x00".to_vec(); bytes.extend_from_slice(&u64::MAX.to_be_bytes()); bytes.extend_from_slice(&0u32.to_be_bytes()); assert!(matches!( @@ -28,6 +28,20 @@ fn public_open_rejects_a_footer_claiming_an_impossible_directory() { )); } +#[test] +fn public_open_rejects_the_legacy_varint_header() { + let (mut bytes, _) = archive_with_entries(Vec::new()); + bytes[..6].copy_from_slice(b"PITH\x80\x02"); + + assert!(matches!( + Archive::open(MemorySource::new(bytes), OpenOptions::default()), + Err(PithosError::UnsupportedFileVersion { + supported: 0x0100, + actual: 0x8002, + }) + )); +} + fn archive_with_entries(entries: Vec<(&str, EntryMetadata)>) -> (Vec, PrivateKey) { let sender = PrivateKey::generate(); let recipient = sender.duplicate(); diff --git a/crates/pithos_lib/tests/grant_readers.rs b/crates/pithos_lib/tests/grant_readers.rs index 9e71869..231d726 100644 --- a/crates/pithos_lib/tests/grant_readers.rs +++ b/crates/pithos_lib/tests/grant_readers.rs @@ -337,7 +337,7 @@ fn grant_multiple_ids_to_multiple_recipients_in_one_metadata_only_child() { assert_eq!( updated .windows(6) - .filter(|window| *window == b"PITH\x80\x02") + .filter(|window| *window == b"PITH\x01\x00") .count(), 1 ); From 1bfa63833cf33ac3cba5c2ecd131d6a52abcbcd6 Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Sat, 12 Sep 2026 22:13:44 +0200 Subject: [PATCH 040/166] feat: enforce directory record semantics --- .../pithos_lib/src/archive/path_validation.rs | 225 ++++++++++++++++-- .../src/archive/reader_private_tests.rs | 140 ++++++++++- crates/pithos_lib/src/archive/validation.rs | 91 ++++--- crates/pithos_lib/src/archive/writer.rs | 52 +++- crates/pithos_lib/src/error.rs | 6 + crates/pithos_lib/src/format/directory.rs | 34 ++- crates/pithos_lib/src/fs/append.rs | 23 +- crates/pithos_lib/src/fs/ingest.rs | 14 -- crates/pithos_lib/tests/draft_spec_vectors.rs | 29 +++ 9 files changed, 516 insertions(+), 98 deletions(-) diff --git a/crates/pithos_lib/src/archive/path_validation.rs b/crates/pithos_lib/src/archive/path_validation.rs index 3cf248e..b27b03b 100644 --- a/crates/pithos_lib/src/archive/path_validation.rs +++ b/crates/pithos_lib/src/archive/path_validation.rs @@ -90,34 +90,62 @@ pub(crate) fn validate_symlink_target(path: &str, target: &str) -> Result<(), Pi pub(crate) fn validate_entry(path: &str, entry: &FileEntry) -> Result<(), PithosError> { validate_entry_path(path)?; - match (&entry.file_type, &entry.symlink_target, &entry.block_data) { - (FileType::Symlink, Some(target), BlockDataState::Decrypted(blocks)) - if blocks.is_empty() => - { - validate_symlink_target(path, target) + if entry.permissions & !0o7777 != 0 { + return Err(PithosError::InvalidPermissions(entry.permissions)); + } + match entry.file_type { + FileType::Data | FileType::Metadata => { + if entry.symlink_target.is_some() { + return Err(PithosError::InvalidSymlinkEntry { + path: path.into(), + reason: "non-symlink has a target".into(), + }); + } } - (FileType::Symlink, None, _) => Err(PithosError::InvalidSymlinkEntry { - path: path.into(), - reason: "missing target".into(), - }), - (FileType::Symlink, Some(_), BlockDataState::Decrypted(blocks)) if !blocks.is_empty() => { - Err(PithosError::InvalidSymlinkEntry { - path: path.into(), - reason: "symlink has block references".into(), - }) + FileType::Directory => { + if entry.file_size != 0 { + return Err(PithosError::InvalidEntryCombination { + path: path.into(), + reason: "directory has nonzero file size".into(), + }); + } + if entry.symlink_target.is_some() { + return Err(PithosError::InvalidSymlinkEntry { + path: path.into(), + reason: "non-symlink has a target".into(), + }); + } + if !matches!(&entry.block_data, BlockDataState::Decrypted(blocks) if blocks.is_empty()) + { + return Err(PithosError::InvalidBlockDataState( + "directory has block material".into(), + )); + } } - (FileType::Symlink, Some(_), BlockDataState::Encrypted(_)) => { - Err(PithosError::InvalidSymlinkEntry { - path: path.into(), - reason: "encrypted symlink block data".into(), - }) + FileType::Symlink => { + if entry.file_size != 0 { + return Err(PithosError::InvalidEntryCombination { + path: path.into(), + reason: "symlink has nonzero file size".into(), + }); + } + let target = entry.symlink_target.as_deref().ok_or_else(|| { + PithosError::InvalidSymlinkEntry { + path: path.into(), + reason: "missing target".into(), + } + })?; + if !matches!(&entry.block_data, BlockDataState::Decrypted(blocks) if blocks.is_empty()) + { + return Err(PithosError::InvalidSymlinkEntry { + path: path.into(), + reason: "symlink has block material".into(), + }); + } + validate_symlink_target(path, target)?; } - (_, Some(_), _) => Err(PithosError::InvalidSymlinkEntry { - path: path.into(), - reason: "non-symlink has a target".into(), - }), - _ => Ok(()), } + Ok(()) } fn validate_candidate_hierarchy( @@ -213,13 +241,23 @@ mod tests { use super::*; fn entry(file_type: FileType, target: Option<&str>, blocks: BlockDataState) -> FileEntry { + configured_entry(file_type, target, blocks, 0, 0o644) + } + + fn configured_entry( + file_type: FileType, + target: Option<&str>, + blocks: BlockDataState, + file_size: u64, + permissions: u32, + ) -> FileEntry { FileEntry { file_type, block_data: blocks, created: 0, modified: 0, - file_size: 0, - permissions: 0o644, + file_size, + permissions, references: vec![], symlink_target: target.map(str::to_owned), } @@ -327,6 +365,141 @@ mod tests { ); } + #[test] + fn entry_semantics_cover_every_file_type_combination() { + for valid in [ + configured_entry( + FileType::Data, + None, + BlockDataState::Encrypted(vec![1]), + 12, + 0o644, + ), + configured_entry( + FileType::Metadata, + None, + BlockDataState::Decrypted(vec![].into()), + 0, + 0o600, + ), + configured_entry( + FileType::Directory, + None, + BlockDataState::Decrypted(vec![].into()), + 0, + 0o755, + ), + configured_entry( + FileType::Symlink, + Some("target"), + BlockDataState::Decrypted(vec![].into()), + 0, + 0o777, + ), + ] { + assert!(validate_entry("entry", &valid).is_ok(), "{valid:?}"); + } + + for invalid in [ + configured_entry( + FileType::Data, + Some("target"), + BlockDataState::Decrypted(vec![].into()), + 0, + 0o644, + ), + configured_entry( + FileType::Metadata, + Some("target"), + BlockDataState::Encrypted(vec![]), + 0, + 0o644, + ), + configured_entry( + FileType::Directory, + None, + BlockDataState::Encrypted(vec![]), + 0, + 0o755, + ), + configured_entry( + FileType::Directory, + None, + BlockDataState::Decrypted(vec![([1; 32], [2; 32])].into()), + 0, + 0o755, + ), + configured_entry( + FileType::Directory, + Some("target"), + BlockDataState::Decrypted(vec![].into()), + 0, + 0o755, + ), + configured_entry( + FileType::Directory, + None, + BlockDataState::Decrypted(vec![].into()), + 1, + 0o755, + ), + configured_entry( + FileType::Symlink, + None, + BlockDataState::Decrypted(vec![].into()), + 0, + 0o777, + ), + configured_entry( + FileType::Symlink, + Some("target"), + BlockDataState::Encrypted(vec![]), + 0, + 0o777, + ), + configured_entry( + FileType::Symlink, + Some("target"), + BlockDataState::Decrypted(vec![([1; 32], [2; 32])].into()), + 0, + 0o777, + ), + configured_entry( + FileType::Symlink, + Some("target"), + BlockDataState::Decrypted(vec![].into()), + 1, + 0o777, + ), + ] { + assert!(validate_entry("entry", &invalid).is_err(), "{invalid:?}"); + } + } + + #[test] + fn entry_permissions_accept_only_the_defined_twelve_bits() { + for permissions in [0, 0o7777] { + let entry = configured_entry( + FileType::Data, + None, + BlockDataState::Encrypted(vec![]), + 0, + permissions, + ); + assert!(validate_entry("entry", &entry).is_ok(), "{permissions:#o}"); + } + for permissions in [0o10000, u32::MAX] { + let entry = configured_entry( + FileType::Data, + None, + BlockDataState::Encrypted(vec![]), + 0, + permissions, + ); + assert!(validate_entry("entry", &entry).is_err(), "{permissions:#o}"); + } + } + #[test] fn archive_path_candidate_and_map_conflicts_are_order_independent() { let file = entry( diff --git a/crates/pithos_lib/src/archive/reader_private_tests.rs b/crates/pithos_lib/src/archive/reader_private_tests.rs index 9fcccef..731ac19 100644 --- a/crates/pithos_lib/src/archive/reader_private_tests.rs +++ b/crates/pithos_lib/src/archive/reader_private_tests.rs @@ -8,7 +8,8 @@ use crate::crypto::{self, PrivateKey, PublicKey}; use crate::error::PithosError; use crate::format::limits::DeserializationLimits; use crate::format::wire::{ - BlockDataState, BlockLocation, Directory, EncryptionSection, RecipientData, RecipientSection, + BlockDataState, BlockLocation, Directory, EncryptionSection, FileType, RecipientData, + RecipientSection, }; use crate::source::{ArchiveSource, MemorySource, SourceError}; use indexmap::IndexMap; @@ -133,6 +134,40 @@ fn rewrite_terminal_directory(path: &Path, mutate: impl FnOnce(&mut Directory)) std::fs::write(path, archive).unwrap(); } +fn decode_terminal_directory(path: &Path) -> Directory { + let archive = std::fs::read(path).unwrap(); + let (start, _) = directory_bounds(&archive); + crate::format::codec::decode_directory( + &mut Cursor::new(&archive[start..]), + &DeserializationLimits::default(), + ) + .unwrap() +} + +fn append_empty_directory(path: &Path, relations: Option>) { + let mut archive = std::fs::read(path).unwrap(); + let (parent_start, parent_len) = directory_bounds(&archive); + let mut directory = Directory::new( + Some((parent_start as u64, parent_len as u64)), + crate::format::entries::WireEntries::new(), + IndexMap::new(), + ); + if let Some(relations) = relations { + directory.relations = relations; + } + crate::format::codec::update_directory_len(&mut directory).unwrap(); + crate::format::codec::update_directory_crc(&mut directory).unwrap(); + crate::format::codec::encode_directory(&directory, &mut archive).unwrap(); + std::fs::write(path, archive).unwrap(); +} + +fn open_without_keys(path: &Path) -> Result, PithosError> { + Archive::open( + MemorySource::new(Arc::<[u8]>::from(std::fs::read(path).unwrap())), + OpenOptions::default(), + ) +} + fn first_block(path: &Path) -> (u64, u64) { let bytes = std::fs::read(path).unwrap(); let (start, _) = directory_bounds(&bytes); @@ -172,6 +207,109 @@ impl Write for RecordingSink { } } +#[test] +fn writers_emit_only_the_required_standard_relationship_table() { + let (_temporary, path) = fixture(); + let expected = [ + (0, "DESCRIBES"), + (1, "ANNOTATES"), + (2, "DERIVED_FROM"), + (3, "SOURCE_OF"), + (4, "PREVIOUS_VERSION"), + (5, "NEXT_VERSION"), + (6, "PART_OF"), + (7, "CONTAINS"), + (8, "INPUT_TO"), + (9, "OUTPUT_FROM"), + ]; + assert_eq!( + decode_terminal_directory(&path) + .relations + .iter() + .map(|(id, name)| (*id, name.as_str())) + .collect::>(), + expected + ); + + append_empty_directory(&path, None); + assert!(decode_terminal_directory(&path).relations.is_empty()); + open_without_keys(&path).unwrap(); +} + +#[test] +fn archive_rejects_invalid_base_standard_relationship_tables() { + let mutations: [fn(&mut Directory); 3] = [ + |directory| directory.relations.clear(), + |directory| directory.relations[0].1 = "describes".into(), + |directory| directory.relations.swap(0, 1), + ]; + for mutate in mutations { + let (_temporary, path) = fixture(); + rewrite_terminal_directory(&path, mutate); + assert!(open_without_keys(&path).is_err()); + } +} + +#[test] +fn archive_validates_custom_relationship_ids_and_names() { + for relation in [(10, "CUSTOM"), (999, "CUSTOM"), (1000, "")] { + let (_temporary, path) = fixture(); + rewrite_terminal_directory(&path, |directory| { + directory.relations.push((relation.0, relation.1.into())); + }); + assert!(open_without_keys(&path).is_err(), "{relation:?}"); + } + + let (_temporary, path) = fixture(); + rewrite_terminal_directory(&path, |directory| { + directory.relations.push((1000, "CUSTOM".into())); + }); + open_without_keys(&path).unwrap(); +} + +#[test] +fn appended_standard_relationship_repeats_must_match() { + let (_temporary, exact) = fixture(); + append_empty_directory(&exact, Some(vec![(0, "DESCRIBES".into())])); + open_without_keys(&exact).unwrap(); + + let (_temporary, conflicting) = fixture(); + append_empty_directory(&conflicting, Some(vec![(0, "describes".into())])); + assert!(open_without_keys(&conflicting).is_err()); +} + +#[test] +fn archive_rejects_no_content_sizes_and_undefined_permission_bits() { + let mutations: [fn(&mut crate::format::wire::FileEntry); 3] = [ + |entry| { + entry.file_type = FileType::Directory; + entry.block_data = BlockDataState::Decrypted(Vec::new().into()); + entry.file_size = 1; + entry.symlink_target = None; + }, + |entry| { + entry.file_type = FileType::Symlink; + entry.block_data = BlockDataState::Decrypted(Vec::new().into()); + entry.file_size = 1; + entry.symlink_target = Some("target".into()); + }, + |entry| entry.permissions = 0x1000, + ]; + for mutate in mutations { + let (_temporary, path) = fixture(); + rewrite_terminal_directory(&path, |directory| { + directory + .files + .try_for_each_mut(|_, entry| { + mutate(entry); + Ok::<_, ()>(()) + }) + .unwrap(); + }); + assert!(open_without_keys(&path).is_err()); + } +} + #[derive(Clone)] struct CountingResolver { response: Arc<[u8]>, diff --git a/crates/pithos_lib/src/archive/validation.rs b/crates/pithos_lib/src/archive/validation.rs index 9e2b6a4..9daa03a 100644 --- a/crates/pithos_lib/src/archive/validation.rs +++ b/crates/pithos_lib/src/archive/validation.rs @@ -4,6 +4,7 @@ use crate::archive::types::{ ValidatedSegment, }; use crate::error::PithosError; +use crate::format::directory::STANDARD_RELATIONSHIPS; use crate::format::wire::{ BlockDataState, BlockLocation as WireBlockLocation, Directory, FileType, }; @@ -44,12 +45,15 @@ pub(crate) fn segment_from_wire( "File id already occupied: {id}" ))); } + let path = ArchivePath::new(path)?; + let entry = entry_from_wire(path.as_str(), wire)?; entries.push(SegmentEntry { id: FileId(id), - path: ArchivePath::new(path)?, - entry: entry_from_wire(wire)?, + path, + entry, }); } + validate_relationships(directory)?; let mut descriptors = Vec::new(); for (hash, wire) in &directory.blocks { let stored_size_with_marker = @@ -99,7 +103,11 @@ pub(crate) fn segment_from_wire( }) } -fn entry_from_wire(wire: &crate::format::wire::FileEntry) -> Result { +fn entry_from_wire( + path: &str, + wire: &crate::format::wire::FileEntry, +) -> Result { + crate::archive::path_validation::validate_entry(path, wire)?; let metadata = EntryMetadata { created: wire.created, modified: wire.modified, @@ -132,25 +140,12 @@ fn entry_from_wire(wire: &crate::format::wire::FileEntry) -> Result { - require_no_content(wire, "directory")?; - if wire.symlink_target.is_some() { - return Err(PithosError::InvalidSymlinkEntry { - path: "directory".into(), - reason: "non-symlink has a target".into(), - }); - } - Ok(Entry::Directory(metadata)) - } + FileType::Directory => Ok(Entry::Directory(metadata)), FileType::Symlink => { - require_no_content(wire, "symlink")?; - let target = - wire.symlink_target - .as_deref() - .ok_or_else(|| PithosError::InvalidSymlinkEntry { - path: "symlink".into(), - reason: "missing target".into(), - })?; + let target = wire + .symlink_target + .as_deref() + .expect("validated symlink entry has a target"); Ok(Entry::Symlink { metadata, target: Arc::from(target), @@ -159,19 +154,49 @@ fn entry_from_wire(wire: &crate::format::wire::FileEntry) -> Result Result<(), PithosError> { - match &entry.block_data { - BlockDataState::Decrypted(blocks) if blocks.is_empty() => Ok(()), - BlockDataState::Decrypted(_) => Err(PithosError::InvalidBlockDataState(format!( - "{kind} has block references" - ))), - BlockDataState::Encrypted(_) => Err(PithosError::InvalidBlockDataState(format!( - "{kind} has encrypted content" - ))), +pub(crate) fn validate_relationships(directory: &Directory) -> Result<(), PithosError> { + let mut next_standard = 0usize; + for (id, name) in &directory.relations { + match *id { + 0..=9 => { + let expected = STANDARD_RELATIONSHIPS[*id as usize].1; + if name != expected { + return Err(PithosError::InvalidRelationshipDefinition { + id: *id, + reason: format!("expected standard name {expected:?}"), + }); + } + if directory.parent_directory_offset.is_none() && *id != next_standard as u64 { + return Err(PithosError::InvalidRelationshipDefinition { + id: *id, + reason: "standard definitions are out of order".into(), + }); + } + next_standard += 1; + } + 10..=999 => { + return Err(PithosError::InvalidRelationshipDefinition { + id: *id, + reason: "relationship id is reserved".into(), + }); + } + _ if name.is_empty() => { + return Err(PithosError::InvalidRelationshipDefinition { + id: *id, + reason: "custom relationship name is empty".into(), + }); + } + _ => {} + } } + if directory.parent_directory_offset.is_none() && next_standard != 10 { + let id = next_standard as u64; + return Err(PithosError::InvalidRelationshipDefinition { + id, + reason: "base directory is missing a standard definition".into(), + }); + } + Ok(()) } pub(crate) fn validate_entry(path: &ArchivePath, entry: &Entry) -> Result<(), PithosError> { diff --git a/crates/pithos_lib/src/archive/writer.rs b/crates/pithos_lib/src/archive/writer.rs index 17452a3..2c492cb 100644 --- a/crates/pithos_lib/src/archive/writer.rs +++ b/crates/pithos_lib/src/archive/writer.rs @@ -1,5 +1,6 @@ //! Streaming archive construction with a consuming publication boundary. +use crate::archive::validation::validate_relationships; use crate::archive::{AppendSnapshot, ArchivePath, FileId, Span, segment_from_wire}; use crate::archive::{validate_new_candidate, validate_symlink_target}; use crate::block; @@ -1232,6 +1233,16 @@ impl ArchiveWriter { fn validate_publishable(&self) -> Result<(), PithosError> { self.validate_entry_state()?; + validate_relationships(&self.directory)?; + if let Some(snapshot) = &self.append_snapshot { + let relationships = self + .directory + .relations + .iter() + .map(|(id, name)| (*id, name.as_str())) + .collect::>(); + snapshot.validate_child_relationships(&relationships)?; + } for section in self.directory.encryption.values() { for recipient in section.recipients.values() { if matches!(recipient.recipient_data, RecipientData::Decrypted(_)) { @@ -1273,6 +1284,9 @@ impl ArchiveWriter { _ => {} } } + for (_, path, entry) in self.directory.files.iter() { + crate::archive::path_validation::validate_entry(path, entry)?; + } self.directory .validate_references_and_accessible_blocks_with( |id| { @@ -1566,7 +1580,7 @@ mod tests { let archive = write(); assert_eq!( blake3::hash(&archive).to_hex().as_str(), - "940d5acd374e614a952fadd481e6ff6961322549bb19df446f8ff4c68f8d9fa6" + "ed87c4bc6481475e5d1a2af8906b75728231c451e1db93fca3aa4318efcdbf49" ); assert_eq!(&archive[..6], b"PITH\x01\x00"); assert_eq!( @@ -1635,6 +1649,42 @@ mod tests { )); } + #[test] + fn writer_publication_revalidates_entry_and_relationship_semantics() { + let mut writer = ArchiveWriter::create(Vec::new(), options()).unwrap(); + let before = writer.metadata_snapshot(); + assert!( + writer + .add_directory( + ArchivePath::new("invalid-permissions").unwrap(), + EntryMetadata::new(0, 0, 0o10000), + ) + .is_err() + ); + assert_eq!(writer.metadata_snapshot(), before); + + let mut writer = ArchiveWriter::create(Vec::new(), options()).unwrap(); + writer + .add_directory( + ArchivePath::new("directory").unwrap(), + EntryMetadata::new(0, 0, 0o755), + ) + .unwrap(); + writer + .directory + .files + .try_for_each_mut(|_, entry| { + entry.file_size = 1; + Ok::<_, ()>(()) + }) + .unwrap(); + assert!(writer.finish().is_err()); + + let mut writer = ArchiveWriter::create(Vec::new(), options()).unwrap(); + writer.directory.relations[0].1 = "describes".into(); + assert!(writer.finish().is_err()); + } + #[test] fn failed_content_delta_preserves_all_live_metadata_and_next_id() { let mut writer = ArchiveWriter::create(Vec::new(), options()).unwrap(); diff --git a/crates/pithos_lib/src/error.rs b/crates/pithos_lib/src/error.rs index 086cd39..472b63a 100644 --- a/crates/pithos_lib/src/error.rs +++ b/crates/pithos_lib/src/error.rs @@ -51,6 +51,8 @@ pub enum PithosError { }, #[error("Invalid block data state: {0}")] InvalidBlockDataState(String), + #[error("invalid entry combination for {path}: {reason}")] + InvalidEntryCombination { path: String, reason: String }, #[error("content is unavailable with the supplied access keys")] ContentUnavailable, #[error("append snapshot does not contain file id {0}")] @@ -81,6 +83,10 @@ pub enum PithosError { ReservedProcessingBits(u8), #[error("conflicting relationship definition for id {0}")] ConflictingRelationshipDefinition(u64), + #[error("invalid relationship definition for id {id}: {reason}")] + InvalidRelationshipDefinition { id: u64, reason: String }, + #[error("invalid permissions: {0:#o}")] + InvalidPermissions(u32), #[error("unknown relationship id {0}")] UnknownRelationshipId(u64), #[error("missing reference target file id {0}")] diff --git a/crates/pithos_lib/src/format/directory.rs b/crates/pithos_lib/src/format/directory.rs index 1b2abe1..2616961 100644 --- a/crates/pithos_lib/src/format/directory.rs +++ b/crates/pithos_lib/src/format/directory.rs @@ -2,29 +2,39 @@ use crate::error::PithosError; use crate::format::entries::WireEntries; use crate::format::wire::{BlockDataState, Directory, EncryptionSection, FileEntry, FileType}; +pub(crate) const STANDARD_RELATIONSHIPS: [(u64, &str); 10] = [ + (0, "DESCRIBES"), + (1, "ANNOTATES"), + (2, "DERIVED_FROM"), + (3, "SOURCE_OF"), + (4, "PREVIOUS_VERSION"), + (5, "NEXT_VERSION"), + (6, "PART_OF"), + (7, "CONTAINS"), + (8, "INPUT_TO"), + (9, "OUTPUT_FROM"), +]; + impl Directory { pub(crate) fn new( parent_directory_offset: Option<(u64, u64)>, files: WireEntries, encryption: indexmap::IndexMap<[u8; 32], EncryptionSection>, ) -> Self { + let relations = if parent_directory_offset.is_none() { + STANDARD_RELATIONSHIPS + .iter() + .map(|(id, name)| (*id, (*name).to_owned())) + .collect() + } else { + Vec::new() + }; Self { identifier: *b"PITHOSDR", parent_directory_offset, files, blocks: indexmap::IndexMap::new(), - relations: vec![ - (0, "Describes".into()), - (1, "Annotates".into()), - (2, "Derived_From".into()), - (3, "Source_Of".into()), - (4, "Previous_Version".into()), - (5, "Next_Version".into()), - (6, "Part_of".into()), - (7, "Contains".into()), - (8, "Input_To".into()), - (9, "Output_From".into()), - ], + relations, encryption, dir_len: 0, crc32: 0, diff --git a/crates/pithos_lib/src/fs/append.rs b/crates/pithos_lib/src/fs/append.rs index 4a1367a..573a3ea 100644 --- a/crates/pithos_lib/src/fs/append.rs +++ b/crates/pithos_lib/src/fs/append.rs @@ -603,17 +603,18 @@ mod tests { std::fs::write(&archive_path, &bytes).unwrap(); let original = bytes; - assert!(matches!( - append_files( - &archive_path, - AppendOptions::new(sender, vec![recipient]), - &[input_path], - ), - Err(FsError::Core { - source: PithosError::ConflictingRelationshipDefinition(0), - .. - }) - )); + let error = append_files( + &archive_path, + AppendOptions::new(sender, vec![recipient]), + &[input_path], + ) + .unwrap_err(); + assert!(matches!(error, FsError::Core { .. })); + assert!( + error + .to_string() + .contains("invalid relationship definition") + ); assert_eq!(std::fs::read(&archive_path).unwrap(), original); } diff --git a/crates/pithos_lib/src/fs/ingest.rs b/crates/pithos_lib/src/fs/ingest.rs index cd1832f..ffe3429 100644 --- a/crates/pithos_lib/src/fs/ingest.rs +++ b/crates/pithos_lib/src/fs/ingest.rs @@ -254,7 +254,6 @@ impl InputManifest { { return Err(PithosError::FileIdExhausted); } - snapshot.validate_child_relationships(&DEFAULT_RELATIONSHIPS)?; let mut next_id = snapshot .maximum_id() .map_or(Some(0), |id| id.0.checked_add(1)); @@ -273,19 +272,6 @@ impl InputManifest { } } -const DEFAULT_RELATIONSHIPS: [(u64, &str); 10] = [ - (0, "Describes"), - (1, "Annotates"), - (2, "Derived_From"), - (3, "Source_Of"), - (4, "Previous_Version"), - (5, "Next_Version"), - (6, "Part_of"), - (7, "Contains"), - (8, "Input_To"), - (9, "Output_From"), -]; - fn manifest_entry( source: PathBuf, path: ArchivePath, diff --git a/crates/pithos_lib/tests/draft_spec_vectors.rs b/crates/pithos_lib/tests/draft_spec_vectors.rs index dad0b27..315966b 100644 --- a/crates/pithos_lib/tests/draft_spec_vectors.rs +++ b/crates/pithos_lib/tests/draft_spec_vectors.rs @@ -137,6 +137,19 @@ fn replace_base_relationship_count(replacement: &[u8]) -> Vec { bytes } +fn replace_hello_bytes(range: std::ops::RangeInclusive, replacement: &[u8]) -> Vec { + let mut bytes = hello(); + bytes.splice(range, replacement.iter().copied()); + let directory_start = 15; + let directory_len = (bytes.len() - directory_start) as u64; + let footer = bytes.len() - 12; + bytes[footer..footer + 8].copy_from_slice(&directory_len.to_be_bytes()); + let checksum = crc32(&bytes[directory_start..bytes.len() - 4]); + let crc_offset = bytes.len() - 4; + bytes[crc_offset..].copy_from_slice(&checksum.to_be_bytes()); + bytes +} + fn read_uleb(bytes: &[u8], cursor: &mut usize) -> Result { let mut value = 0_u64; for shift in (0..64).step_by(7) { @@ -346,6 +359,22 @@ fn production_reader_rejects_overflowing_uleb128() { )); } +#[test] +fn production_reader_rejects_rv_symlink_and_permissions() { + for (id, bytes) in [ + ("RV-SYMLINK", replace_hello_bytes(0x20..=0x20, &[3])), + ( + "RV-PERMISSIONS", + replace_hello_bytes(0x66..=0x67, &[0x80, 0x20]), + ), + ] { + assert!( + Archive::open(MemorySource::new(bytes), OpenOptions::default()).is_err(), + "production reader accepted {id}" + ); + } +} + #[test] fn directory_length_calculations_are_independent() { let base_directory = &base()[6..]; From 1c3400002644a064a82fce4934fa1159ee97d5e2 Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Sat, 12 Sep 2026 22:40:20 +0200 Subject: [PATCH 041/166] feat: enforce stored hierarchy ordering --- crates/pithos_lib/benches/archive_path.rs | 24 +-- crates/pithos_lib/benches/workloads.rs | 42 ++-- crates/pithos_lib/src/archive/index.rs | 24 ++- crates/pithos_lib/src/archive/mod.rs | 64 +++++- .../pithos_lib/src/archive/path_validation.rs | 182 ++++++++++++++---- crates/pithos_lib/src/archive/snapshot.rs | 8 + crates/pithos_lib/src/archive/validation.rs | 25 +-- crates/pithos_lib/src/archive/writer.rs | 91 ++++++++- crates/pithos_lib/src/format/entries.rs | 1 + .../pithos_lib/tests/benchmark_contracts.rs | 34 +--- crates/pithos_lib/tests/common/append.rs | 8 + crates/pithos_lib/tests/format_validation.rs | 12 +- crates/pithos_lib/tests/reader.rs | 6 + crates/pithos_lib/tests/writer_failures.rs | 53 +++++ 14 files changed, 420 insertions(+), 154 deletions(-) diff --git a/crates/pithos_lib/benches/archive_path.rs b/crates/pithos_lib/benches/archive_path.rs index b791f6a..6745f31 100644 --- a/crates/pithos_lib/benches/archive_path.rs +++ b/crates/pithos_lib/benches/archive_path.rs @@ -169,8 +169,8 @@ fn bench_archive_path(c: &mut Criterion) { workloads::ConflictDirection::ExistingAncestor, ), ( - "existing-descendants", - workloads::ConflictDirection::ExistingDescendants, + "missing-ancestor", + workloads::ConflictDirection::MissingAncestor, ), ] { let workload = format!("archive_path/conflicts/{name}/1000"); @@ -197,20 +197,12 @@ fn bench_archive_path(c: &mut Criterion) { ) }); } - for (name, order) in [ - ("ancestor-first", workloads::InsertionOrder::AncestorFirst), - ( - "descendant-first", - workloads::InsertionOrder::DescendantFirst, - ), - ] { - bench_build( - c, - format!("archive_path/incremental/{name}/1000"), - &sender, - |key| workloads::build_incremental(key, recipient, 1_000, order), - ); - } + bench_build( + c, + "archive_path/incremental/ancestor-first/1000".into(), + &sender, + |key| workloads::build_incremental(key, recipient, 1_000), + ); for count in [100, 1_000] { bench_build( c, diff --git a/crates/pithos_lib/benches/workloads.rs b/crates/pithos_lib/benches/workloads.rs index c92a59b..730801a 100644 --- a/crates/pithos_lib/benches/workloads.rs +++ b/crates/pithos_lib/benches/workloads.rs @@ -20,18 +20,11 @@ const METADATA_JSON: &[u8] = br#"{"type":"benchmark-metadata"}"#; #[allow(dead_code)] // Extraction workloads use this fixed payload when they include this module. const EXTRACTION_PAYLOAD: [u8; EXTRACTION_PAYLOAD_BYTES] = [b'e'; EXTRACTION_PAYLOAD_BYTES]; -#[allow(dead_code)] // Only archive-path benchmarks incremental insertion ordering. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub enum InsertionOrder { - AncestorFirst, - DescendantFirst, -} - #[allow(dead_code)] // Only archive-path benchmarks hierarchy-conflict directions. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum ConflictDirection { ExistingAncestor, - ExistingDescendants, + MissingAncestor, } #[allow(dead_code)] // Shared fixture builders use this only in targets that create archives. @@ -58,24 +51,14 @@ fn add_file( } #[allow(dead_code)] // Only archive-path benchmarks incremental insertion. -pub fn build_incremental( - sender: PrivateKey, - recipient: PublicKey, - count: usize, - order: InsertionOrder, -) -> Vec { +pub fn build_incremental(sender: PrivateKey, recipient: PublicKey, count: usize) -> Vec { let mut writer = writer(sender, recipient); - let add_ancestor = |writer: &mut ArchiveWriter>| { - writer - .add_directory( - ArchivePath::new("root").unwrap(), - EntryMetadata::new(0, 0, 0o755), - ) - .unwrap(); - }; - if order == InsertionOrder::AncestorFirst { - add_ancestor(&mut writer); - } + writer + .add_directory( + ArchivePath::new("root").unwrap(), + EntryMetadata::new(0, 0, 0o755), + ) + .unwrap(); for index in 0..count { add_file( &mut writer, @@ -84,9 +67,6 @@ pub fn build_incremental( &EMPTY, ); } - if order == InsertionOrder::DescendantFirst { - add_ancestor(&mut writer); - } writer.finish().unwrap() } @@ -151,16 +131,16 @@ pub fn build_conflict_fixture( } "root/child".into() } - ConflictDirection::ExistingDescendants => { + ConflictDirection::MissingAncestor => { for index in 0..count { add_file( &mut writer, - format!("root/child-{index:06}"), + format!("filler-{index:06}"), EntryMetadata::new(0, 0, 0o644), &EMPTY, ); } - "root".into() + "root/child".into() } }; ConflictFixture { writer, candidate } diff --git a/crates/pithos_lib/src/archive/index.rs b/crates/pithos_lib/src/archive/index.rs index 045d1ab..2be42d8 100644 --- a/crates/pithos_lib/src/archive/index.rs +++ b/crates/pithos_lib/src/archive/index.rs @@ -3,9 +3,7 @@ use crate::archive::types::{ ArchivePath, BlockDescriptor, BlockHash, ContentState, Entry, FileId, ReadRange, RelationId, SegmentEntry, Span, ValidatedSegment, }; -use crate::archive::validation::{ - IndexLimits, validate_aggregate, validate_entry, validate_hierarchy, -}; +use crate::archive::validation::{IndexLimits, validate_aggregate, validate_entry}; use crate::error::PithosError; use indexmap::IndexMap; use std::collections::{BTreeMap, BTreeSet, HashMap}; @@ -101,9 +99,9 @@ pub(crate) fn build_effective_index( limits: IndexLimits, ) -> Result { validate_aggregate(segments, limits)?; - let mut entries = Vec::new(); + let mut entries: Vec = Vec::new(); let mut by_id = BTreeMap::new(); - let mut by_path = HashMap::new(); + let mut by_path: HashMap, usize> = HashMap::new(); let mut hierarchy = BTreeMap::new(); let mut descriptors: IndexMap = IndexMap::new(); let mut relationships: BTreeMap> = BTreeMap::new(); @@ -151,6 +149,21 @@ pub(crate) fn build_effective_index( path.as_str() ))); } + for (offset, _) in path.as_str().match_indices('/') { + let ancestor = &path.as_str()[..offset]; + let Some(ancestor_index) = by_path.get(ancestor) else { + return Err(PithosError::InvalidArchivePath { + path: path.as_str().into(), + reason: format!("missing directory ancestor {ancestor}"), + }); + }; + if !entries[*ancestor_index].entry.is_directory() { + return Err(PithosError::InvalidArchivePath { + path: path.as_str().into(), + reason: format!("file entry {ancestor} is an ancestor"), + }); + } + } let index = entries.len(); entries.push(IndexedEntry { id: *id, @@ -163,7 +176,6 @@ pub(crate) fn build_effective_index( } } - validate_hierarchy(&entries, &by_path)?; let segment_spans = segment_spans.into_iter().collect::>(); for segment in segments { for (_, descriptor) in &segment.descriptors { diff --git a/crates/pithos_lib/src/archive/mod.rs b/crates/pithos_lib/src/archive/mod.rs index 20103f8..3f2133f 100644 --- a/crates/pithos_lib/src/archive/mod.rs +++ b/crates/pithos_lib/src/archive/mod.rs @@ -65,9 +65,17 @@ mod tests { } fn segment(entries: Vec) -> ValidatedSegment { + segment_at(100, None, entries) + } + + fn segment_at( + start: u64, + parent: Option, + entries: Vec, + ) -> ValidatedSegment { ValidatedSegment { - span: Span::new(100, 10).unwrap(), - parent: None, + span: Span::new(start, 10).unwrap(), + parent, entries, descriptors: Vec::new(), relationships: Vec::new(), @@ -340,6 +348,58 @@ mod tests { assert!(build_effective_index(&[segment(adjacent)], 1_000, IndexLimits::default()).is_ok()); } + #[test] + fn hierarchy_requires_earlier_directory_ancestors_across_segments() { + let directory = |id, path: &str| SegmentEntry { + id: FileId(id), + path: ArchivePath::new(path).unwrap(), + entry: Entry::Directory(metadata()), + }; + let file = |id, path: &str| SegmentEntry { + id: FileId(id), + path: ArchivePath::new(path).unwrap(), + entry: Entry::File(ContentEntry { + metadata: metadata(), + size: 0, + content: ContentState::Available(BlockReferences::new(Vec::new())), + }), + }; + + for entries in [ + vec![file(1, "parent/child")], + vec![file(1, "top/middle/child")], + vec![file(1, "parent"), file(2, "parent/child")], + vec![directory(2, "parent/child"), directory(1, "parent")], + ] { + assert!( + build_effective_index(&[segment(entries)], 1_000, IndexLimits::default()).is_err() + ); + } + + assert!( + build_effective_index( + &[segment(vec![ + directory(1, "parent"), + file(2, "parent/child"), + ])], + 1_000, + IndexLimits::default(), + ) + .is_ok() + ); + + let base = segment_at(100, None, vec![directory(1, "parent")]); + let child = segment_at(200, Some(base.span), vec![file(2, "parent/child")]); + assert!(build_effective_index(&[base, child], 1_000, IndexLimits::default()).is_ok()); + + let early_child = segment_at(100, None, vec![file(2, "parent/child")]); + let late_parent = segment_at(200, Some(early_child.span), vec![directory(1, "parent")]); + assert!( + build_effective_index(&[early_child, late_parent], 1_000, IndexLimits::default(),) + .is_err() + ); + } + proptest! { #[test] fn component_ancestor_conflicts_ignore_adjacent_prefixes( diff --git a/crates/pithos_lib/src/archive/path_validation.rs b/crates/pithos_lib/src/archive/path_validation.rs index b27b03b..f747e3c 100644 --- a/crates/pithos_lib/src/archive/path_validation.rs +++ b/crates/pithos_lib/src/archive/path_validation.rs @@ -1,6 +1,8 @@ +use crate::archive::{AppendSnapshot, ArchivePath}; use crate::error::PithosError; use crate::format::entries::WireEntries; use crate::format::wire::{BlockDataState, FileEntry, FileType}; +use std::collections::HashMap; fn invalid_path(path: &str, reason: impl Into) -> PithosError { PithosError::InvalidArchivePath { @@ -150,19 +152,41 @@ pub(crate) fn validate_entry(path: &str, entry: &FileEntry) -> Result<(), Pithos fn validate_candidate_hierarchy( map: &WireEntries, + snapshot: Option<&AppendSnapshot>, path: &str, entry: &FileEntry, ) -> Result<(), PithosError> { for (index, _) in path.match_indices('/') { let ancestor = &path[..index]; - if map - .get_by_path(ancestor) - .is_some_and(|existing| existing.file_type != FileType::Directory) - { - return Err(PithosError::InvalidArchivePath { - path: path.into(), - reason: format!("file entry {ancestor} is an ancestor"), - }); + if let Some(existing) = map.get_by_path(ancestor) { + if existing.file_type != FileType::Directory { + return Err(PithosError::InvalidArchivePath { + path: path.into(), + reason: format!("file entry {ancestor} is an ancestor"), + }); + } + continue; + } + let ancestor_path = ArchivePath::new(ancestor)?; + match snapshot.and_then(|snapshot| snapshot.entry_at_path(&ancestor_path)) { + Some(existing) + if !matches!( + existing.kind, + crate::archive::snapshot::SnapshotEntryKind::Directory + ) => + { + return Err(PithosError::InvalidArchivePath { + path: path.into(), + reason: format!("file entry {ancestor} is an ancestor"), + }); + } + Some(_) => {} + None => { + return Err(PithosError::InvalidArchivePath { + path: path.into(), + reason: format!("missing directory ancestor {ancestor}"), + }); + } } } @@ -187,7 +211,7 @@ pub(crate) fn validate_existing_candidate( entry: &FileEntry, ) -> Result<(), PithosError> { validate_entry(path, entry)?; - validate_candidate_hierarchy(map, path, entry) + validate_candidate_hierarchy(map, None, path, entry) } pub(crate) fn validate_new_candidate( @@ -201,7 +225,22 @@ pub(crate) fn validate_new_candidate( "File path already occupied: {path}" ))); } - validate_candidate_hierarchy(map, path, entry) + validate_candidate_hierarchy(map, None, path, entry) +} + +pub(crate) fn validate_new_candidate_with_snapshot( + map: &WireEntries, + path: &str, + entry: &FileEntry, + snapshot: &AppendSnapshot, +) -> Result<(), PithosError> { + validate_entry(path, entry)?; + if map.get_by_path(path).is_some() { + return Err(PithosError::PathOccupied(format!( + "File path already occupied: {path}" + ))); + } + validate_candidate_hierarchy(map, Some(snapshot), path, entry) } pub(crate) fn validate_wire_map(map: &WireEntries) -> Result<(), PithosError> { @@ -213,26 +252,93 @@ pub(crate) fn validate_wire_map(map: &WireEntries) -> Result<(), PithosError> { } pub(crate) fn validate_wire_hierarchy(map: &WireEntries) -> Result<(), PithosError> { - let mut entries = map.iter_ordered(); - let Some((mut previous_path, mut previous_entry)) = entries.next() else { - return Ok(()); - }; - - for (path, entry) in entries { - if previous_entry.file_type != FileType::Directory - && path.starts_with(previous_path) - && path.as_bytes().get(previous_path.len()) == Some(&b'/') - { - return Err(PithosError::InvalidArchivePath { - path: previous_path.into(), - reason: format!("entry is an ancestor of {path}"), - }); + let mut earlier: HashMap<&str, &FileEntry> = HashMap::new(); + for (_, path, entry) in map.iter() { + for (offset, _) in path.match_indices('/') { + let ancestor = &path[..offset]; + if let Some(existing) = earlier.get(ancestor) { + if existing.file_type != FileType::Directory { + return Err(PithosError::InvalidArchivePath { + path: path.into(), + reason: format!("file entry {ancestor} is an ancestor"), + }); + } + } else if map.get_by_path(ancestor).is_some() { + return Err(PithosError::InvalidArchivePath { + path: path.into(), + reason: format!("ancestor {ancestor} is declared after its child"), + }); + } } + earlier.insert(path, entry); + } + Ok(()) +} - previous_path = path; - previous_entry = entry; +pub(crate) fn validate_wire_hierarchy_complete(map: &WireEntries) -> Result<(), PithosError> { + let mut earlier: HashMap<&str, &FileEntry> = HashMap::new(); + for (_, path, entry) in map.iter() { + for (offset, _) in path.match_indices('/') { + let ancestor = &path[..offset]; + let Some(existing) = earlier.get(ancestor) else { + return Err(PithosError::InvalidArchivePath { + path: path.into(), + reason: format!("missing directory ancestor {ancestor}"), + }); + }; + if existing.file_type != FileType::Directory { + return Err(PithosError::InvalidArchivePath { + path: path.into(), + reason: format!("file entry {ancestor} is an ancestor"), + }); + } + } + earlier.insert(path, entry); } + Ok(()) +} +pub(crate) fn validate_wire_hierarchy_with_snapshot( + map: &WireEntries, + snapshot: &AppendSnapshot, +) -> Result<(), PithosError> { + let mut earlier: HashMap<&str, &FileEntry> = HashMap::new(); + for (_, path, entry) in map.iter() { + for (offset, _) in path.match_indices('/') { + let ancestor = &path[..offset]; + if let Some(existing) = earlier.get(ancestor) { + if existing.file_type != FileType::Directory { + return Err(PithosError::InvalidArchivePath { + path: path.into(), + reason: format!("file entry {ancestor} is an ancestor"), + }); + } + continue; + } + let ancestor_path = ArchivePath::new(ancestor)?; + match snapshot.entry_at_path(&ancestor_path) { + Some(existing) + if !matches!( + existing.kind, + crate::archive::snapshot::SnapshotEntryKind::Directory + ) => + { + return Err(PithosError::InvalidArchivePath { + path: path.into(), + reason: format!("file entry {ancestor} is an ancestor"), + }); + } + Some(_) => {} + None => { + return Err(PithosError::InvalidArchivePath { + path: path.into(), + reason: format!("missing directory ancestor {ancestor}"), + }); + } + } + } + earlier.insert(path, entry); + } Ok(()) } @@ -544,17 +650,15 @@ mod tests { assert!(validate_wire_map(&map).is_err()); } - for order in [0, 1] { - let mut map = WireEntries::new(); - if order == 0 { - map.insert(0, "a", directory.clone()).unwrap(); - map.insert(1, "a/child", file.clone()).unwrap(); - } else { - map.insert(0, "a/child", file.clone()).unwrap(); - map.insert(1, "a", directory.clone()).unwrap(); - } - assert!(validate_wire_map(&map).is_ok()); - } + let mut parent_first = WireEntries::new(); + parent_first.insert(0, "a", directory.clone()).unwrap(); + parent_first.insert(1, "a/child", file.clone()).unwrap(); + assert!(validate_wire_map(&parent_first).is_ok()); + + let mut child_first = WireEntries::new(); + child_first.insert(0, "a/child", file.clone()).unwrap(); + child_first.insert(1, "a", directory.clone()).unwrap(); + assert!(validate_wire_map(&child_first).is_err()); } #[test] @@ -582,6 +686,10 @@ mod tests { map.insert(0, "a", directory.clone()).unwrap(); assert!(validate_new_candidate(&map, "a/child", &file).is_ok()); + let map = WireEntries::new(); + assert!(validate_new_candidate(&map, "a/child", &file).is_err()); + assert!(validate_new_candidate(&map, "a/b/child", &file).is_err()); + let mut map = WireEntries::new(); for (id, path) in [(0, "ab"), (1, "a!"), (2, "a.b"), (3, "a/child")] { map.insert(id, path, file.clone()).unwrap(); diff --git a/crates/pithos_lib/src/archive/snapshot.rs b/crates/pithos_lib/src/archive/snapshot.rs index a2e93e0..88f666e 100644 --- a/crates/pithos_lib/src/archive/snapshot.rs +++ b/crates/pithos_lib/src/archive/snapshot.rs @@ -152,6 +152,14 @@ impl AppendSnapshot { }); } } + self.ensure_candidate_successor(path, is_directory) + } + + pub(crate) fn ensure_candidate_successor( + &self, + path: &ArchivePath, + is_directory: bool, + ) -> Result<(), PithosError> { if !is_directory && let Some((successor, _)) = self.hierarchy.range(path.clone()..).next() && path.is_ancestor_of(successor) diff --git a/crates/pithos_lib/src/archive/validation.rs b/crates/pithos_lib/src/archive/validation.rs index 9daa03a..07bdd48 100644 --- a/crates/pithos_lib/src/archive/validation.rs +++ b/crates/pithos_lib/src/archive/validation.rs @@ -8,7 +8,7 @@ use crate::format::directory::STANDARD_RELATIONSHIPS; use crate::format::wire::{ BlockDataState, BlockLocation as WireBlockLocation, Directory, FileType, }; -use std::collections::{BTreeMap, HashMap, HashSet}; +use std::collections::{BTreeMap, HashSet}; use std::sync::Arc; #[derive(Clone, Copy, Debug)] @@ -240,29 +240,6 @@ fn invalid_target(path: &ArchivePath, target: &str, reason: &str) -> PithosError } } -pub(crate) fn validate_hierarchy( - entries: &[crate::archive::index::IndexedEntry], - exact_paths: &HashMap, usize>, -) -> Result<(), PithosError> { - for entry in entries { - let path = entry.path.as_str(); - // Every hierarchy conflict has a direct slash-delimited ancestor. Looking - // up only those prefixes avoids scanning lexically preceding siblings. - for (offset, _) in path.match_indices('/') { - let ancestor = &path[..offset]; - if let Some(ancestor_index) = exact_paths.get(ancestor) - && !entries[*ancestor_index].entry.is_directory() - { - return Err(PithosError::InvalidArchivePath { - path: path.into(), - reason: format!("file entry {ancestor} is an ancestor"), - }); - } - } - } - Ok(()) -} - pub(crate) fn validate_aggregate( segments: &[ValidatedSegment], limits: IndexLimits, diff --git a/crates/pithos_lib/src/archive/writer.rs b/crates/pithos_lib/src/archive/writer.rs index 2c492cb..9008301 100644 --- a/crates/pithos_lib/src/archive/writer.rs +++ b/crates/pithos_lib/src/archive/writer.rs @@ -1,5 +1,9 @@ //! Streaming archive construction with a consuming publication boundary. +use crate::archive::path_validation::{ + validate_new_candidate_with_snapshot, validate_wire_hierarchy_complete, + validate_wire_hierarchy_with_snapshot, +}; use crate::archive::validation::validate_relationships; use crate::archive::{AppendSnapshot, ArchivePath, FileId, Span, segment_from_wire}; use crate::archive::{validate_new_candidate, validate_symlink_target}; @@ -967,9 +971,16 @@ impl ArchiveWriter { fn validate_candidate(&self, path: &ArchivePath, entry: &FileEntry) -> Result<(), PithosError> { if let Some(snapshot) = &self.append_snapshot { snapshot.ensure_path_available(path)?; - snapshot.ensure_candidate_hierarchy(path, entry.file_type == FileType::Directory)?; + snapshot.ensure_candidate_successor(path, entry.file_type == FileType::Directory)?; + validate_new_candidate_with_snapshot( + &self.directory.files, + path.as_str(), + entry, + snapshot, + )?; + } else { + validate_new_candidate(&self.directory.files, path.as_str(), entry)?; } - validate_new_candidate(&self.directory.files, path.as_str(), entry)?; for reference in &entry.references { let child_has_relationship = self .directory @@ -1287,6 +1298,11 @@ impl ArchiveWriter { for (_, path, entry) in self.directory.files.iter() { crate::archive::path_validation::validate_entry(path, entry)?; } + if let Some(snapshot) = &self.append_snapshot { + validate_wire_hierarchy_with_snapshot(&self.directory.files, snapshot)?; + } else { + validate_wire_hierarchy_complete(&self.directory.files)?; + } self.directory .validate_references_and_accessible_blocks_with( |id| { @@ -1685,6 +1701,77 @@ mod tests { assert!(writer.finish().is_err()); } + #[test] + fn writer_publication_revalidates_stored_hierarchy_order() { + let mut writer = ArchiveWriter::create(Vec::new(), options()).unwrap(); + let child = writer.entry( + FileType::Directory, + EntryMetadata::new(0, 0, 0o755), + 0, + None, + ); + let parent = writer.entry( + FileType::Directory, + EntryMetadata::new(0, 0, 0o755), + 0, + None, + ); + writer + .directory + .files + .insert(0, "parent/child", child) + .unwrap(); + writer.directory.files.insert(1, "parent", parent).unwrap(); + assert!(writer.finish().is_err()); + } + + #[test] + fn append_writer_accepts_an_inherited_directory_ancestor() { + let sender = PrivateKey::generate(); + let mut parent = ArchiveWriter::create( + Vec::new(), + WriteOptions::new(sender.duplicate(), vec![sender.public_key()]), + ) + .unwrap(); + parent + .add_directory( + ArchivePath::new("parent").unwrap(), + EntryMetadata::new(0, 0, 0o755), + ) + .unwrap(); + let mut bytes = parent.finish().unwrap(); + let snapshot = Archive::open( + MemorySource::new(Arc::<[u8]>::from(bytes.clone())), + OpenOptions::default().with_access_keys(AccessKeys::new().with_key(sender.duplicate())), + ) + .unwrap() + .into_append_snapshot(); + let mut child = ArchiveWriter::append( + Vec::new(), + sender.duplicate(), + vec![sender.public_key()], + CdcConfig::default(), + snapshot, + ) + .unwrap(); + child + .add_file( + ArchivePath::new("parent/child").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(false, 0).unwrap(), + Some(0), + Cursor::new([]), + ) + .unwrap(); + bytes.extend(child.finish().unwrap()); + let archive = Archive::open( + MemorySource::new(Arc::<[u8]>::from(bytes)), + OpenOptions::default().with_access_keys(AccessKeys::new().with_key(sender)), + ) + .unwrap(); + assert!(archive.entries().any(|entry| entry.path == "parent/child")); + } + #[test] fn failed_content_delta_preserves_all_live_metadata_and_next_id() { let mut writer = ArchiveWriter::create(Vec::new(), options()).unwrap(); diff --git a/crates/pithos_lib/src/format/entries.rs b/crates/pithos_lib/src/format/entries.rs index 4ed32f2..c878c73 100644 --- a/crates/pithos_lib/src/format/entries.rs +++ b/crates/pithos_lib/src/format/entries.rs @@ -89,6 +89,7 @@ impl WireEntries { .map(|entry| (entry.id, entry.path.as_ref(), &entry.entry)) } + #[cfg(test)] pub(crate) fn iter_ordered(&self) -> impl Iterator + '_ { self.ordered_paths .iter() diff --git a/crates/pithos_lib/tests/benchmark_contracts.rs b/crates/pithos_lib/tests/benchmark_contracts.rs index 45da1c0..53426df 100644 --- a/crates/pithos_lib/tests/benchmark_contracts.rs +++ b/crates/pithos_lib/tests/benchmark_contracts.rs @@ -14,39 +14,13 @@ fn open_memory(bytes: Vec, key: PrivateKey) -> Archive { } #[test] -fn incremental_workloads_cover_both_orders_with_the_same_entries() { +fn incremental_workload_declares_the_ancestor_first() { let sender = PrivateKey::generate(); - let recipient = sender.public_key(); let ancestor = open_memory( - workloads::build_incremental( - sender.duplicate(), - recipient, - 1_000, - workloads::InsertionOrder::AncestorFirst, - ), - sender.duplicate(), - ); - let descendant = open_memory( - workloads::build_incremental( - sender.duplicate(), - recipient, - 1_000, - workloads::InsertionOrder::DescendantFirst, - ), + workloads::build_incremental(sender.duplicate(), sender.public_key(), 1_000), sender, ); - let mut ancestor_paths = ancestor - .entries() - .map(|entry| entry.path) - .collect::>(); - let mut descendant_paths = descendant - .entries() - .map(|entry| entry.path) - .collect::>(); - ancestor_paths.sort(); - descendant_paths.sort(); - assert_eq!(ancestor_paths.len(), 1_001); - assert_eq!(ancestor_paths, descendant_paths); + assert_eq!(ancestor.entries().len(), 1_001); assert_eq!( ancestor.entry("root").unwrap().unwrap().kind, EntryKind::Directory @@ -90,7 +64,7 @@ fn conflicts_are_hierarchy_conflicts_against_one_thousand_entries() { let sender = PrivateKey::generate(); for direction in [ workloads::ConflictDirection::ExistingAncestor, - workloads::ConflictDirection::ExistingDescendants, + workloads::ConflictDirection::MissingAncestor, ] { let mut fixture = workloads::build_conflict_fixture( sender.duplicate(), diff --git a/crates/pithos_lib/tests/common/append.rs b/crates/pithos_lib/tests/common/append.rs index 07e92e9..9036479 100644 --- a/crates/pithos_lib/tests/common/append.rs +++ b/crates/pithos_lib/tests/common/append.rs @@ -125,6 +125,14 @@ pub fn archive_with_entry(temporary: &tempfile::TempDir, path: &str) -> PathBuf WriteOptions::new(sender, vec![public_key("sender"), public_key("recipient1")]), ) .unwrap(); + for (index, _) in path.match_indices('/') { + writer + .add_directory( + ArchivePath::new(&path[..index]).unwrap(), + EntryMetadata::new(0, 0, 0o755), + ) + .unwrap(); + } writer .add_file( ArchivePath::new(path).unwrap(), diff --git a/crates/pithos_lib/tests/format_validation.rs b/crates/pithos_lib/tests/format_validation.rs index c79452a..1598bad 100644 --- a/crates/pithos_lib/tests/format_validation.rs +++ b/crates/pithos_lib/tests/format_validation.rs @@ -146,6 +146,12 @@ fn adversarial_wire_path_order_stays_within_open_limits() { WriteOptions::new(sender, vec![recipient.public_key()]), ) .unwrap(); + writer + .add_directory( + ArchivePath::new("root").unwrap(), + EntryMetadata::new(0, 0, 0o755), + ) + .unwrap(); for index in (0..100).rev() { writer .add_directory( @@ -154,12 +160,6 @@ fn adversarial_wire_path_order_stays_within_open_limits() { ) .unwrap(); } - writer - .add_directory( - ArchivePath::new("root").unwrap(), - EntryMetadata::new(0, 0, 0o755), - ) - .unwrap(); let archive = Archive::open( MemorySource::new(writer.finish().unwrap()), OpenOptions::default() diff --git a/crates/pithos_lib/tests/reader.rs b/crates/pithos_lib/tests/reader.rs index 103ca1a..4ed2af5 100644 --- a/crates/pithos_lib/tests/reader.rs +++ b/crates/pithos_lib/tests/reader.rs @@ -120,6 +120,12 @@ fn extraction_is_no_clobber_no_follow_and_staged() { WriteOptions::new(sender, vec![public_key("recipient1")]), ) .unwrap(); + writer + .add_directory( + ArchivePath::new("nested").unwrap(), + EntryMetadata::new(0, 0, 0o755), + ) + .unwrap(); for (path, content) in [ ("data", b"payload".as_slice()), ("nested/data", b"nested payload"), diff --git a/crates/pithos_lib/tests/writer_failures.rs b/crates/pithos_lib/tests/writer_failures.rs index 09077a0..036fb23 100644 --- a/crates/pithos_lib/tests/writer_failures.rs +++ b/crates/pithos_lib/tests/writer_failures.rs @@ -133,6 +133,59 @@ impl Read for FailingReader { Err(io::Error::other("source failure")) } } + +struct PanicOnRead; +impl Read for PanicOnRead { + fn read(&mut self, _: &mut [u8]) -> io::Result { + panic!("hierarchy validation read content") + } +} + +#[test] +fn missing_parent_fails_before_content_read_and_leaves_writer_usable() { + let sender = PrivateKey::generate(); + let reader = sender.duplicate(); + let mut writer = ArchiveWriter::create( + Vec::new(), + WriteOptions::new(sender, vec![reader.public_key()]), + ) + .unwrap(); + + assert!( + writer + .add_file( + ArchivePath::new("parent/child").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(false, 0).unwrap(), + None, + PanicOnRead, + ) + .is_err() + ); + writer + .add_directory( + ArchivePath::new("parent").unwrap(), + EntryMetadata::new(0, 0, 0o755), + ) + .unwrap(); + writer + .add_file( + ArchivePath::new("parent/child").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(false, 0).unwrap(), + Some(2), + Cursor::new(b"ok"), + ) + .unwrap(); + let archive = Archive::open( + MemorySource::new(writer.finish().unwrap()), + OpenOptions::default().with_access_keys(AccessKeys::new().with_key(reader)), + ) + .unwrap(); + let mut output = Vec::new(); + archive.copy_to("parent/child", &mut output).unwrap(); + assert_eq!(output, b"ok"); +} struct ReaderAfterBytes { bytes: Vec, served: bool, From 9eda18a60a6af927dac2374674bb9bb96e40acba Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Sun, 13 Sep 2026 10:55:56 +0200 Subject: [PATCH 042/166] feat: validate effective local block regions --- crates/pithos_lib/src/archive/index.rs | 67 ++++++--- crates/pithos_lib/src/archive/mod.rs | 128 +++++++++++++++++- crates/pithos_lib/src/archive/reader.rs | 13 +- .../src/archive/reader_private_tests.rs | 96 +++++++++++++ crates/pithos_lib/src/archive/validation.rs | 11 +- crates/pithos_lib/src/error.rs | 2 + crates/pithos_lib/tests/block_pipeline.rs | 29 ++-- crates/pithos_lib/tests/draft_spec_vectors.rs | 24 ++++ 8 files changed, 321 insertions(+), 49 deletions(-) diff --git a/crates/pithos_lib/src/archive/index.rs b/crates/pithos_lib/src/archive/index.rs index 2be42d8..c8a31df 100644 --- a/crates/pithos_lib/src/archive/index.rs +++ b/crates/pithos_lib/src/archive/index.rs @@ -5,6 +5,7 @@ use crate::archive::types::{ }; use crate::archive::validation::{IndexLimits, validate_aggregate, validate_entry}; use crate::error::PithosError; +use crate::format::wire::FileHeader; use indexmap::IndexMap; use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::sync::Arc; @@ -26,6 +27,7 @@ pub(crate) struct ArchiveIndex { descriptors: IndexMap, relationships: BTreeMap>, segment_spans: Vec, + local_spans: Vec, maximum_id: Option, } @@ -67,6 +69,10 @@ impl ArchiveIndex { self.relationships.get(&id).map(Arc::as_ref) } + pub(crate) fn local_block_spans(&self) -> impl Iterator + '_ { + self.local_spans.iter().copied() + } + #[cfg(test)] pub(crate) fn maximum_id(&self) -> Option { self.maximum_id @@ -109,6 +115,7 @@ pub(crate) fn build_effective_index( for segment in segments { validate_segment_chain(segment, &segment_spans)?; + let block_region = block_data_region(segment)?; segment_spans.insert(segment.span); for (id, name) in &segment.relationships { match relationships.get(id) { @@ -132,6 +139,7 @@ pub(crate) fn build_effective_index( }); } } else { + validate_descriptor(descriptor, archive_len, block_region)?; descriptors.insert(*hash, descriptor.clone()); } } @@ -177,10 +185,21 @@ pub(crate) fn build_effective_index( } let segment_spans = segment_spans.into_iter().collect::>(); - for segment in segments { - for (_, descriptor) in &segment.descriptors { - validate_descriptor(descriptor, archive_len, &segment_spans)?; - } + let mut local_spans = descriptors + .values() + .filter_map(|descriptor| match descriptor.location { + crate::archive::types::BlockLocation::Local(span) => Some(span), + crate::archive::types::BlockLocation::External(_) => None, + }) + .collect::>(); + local_spans.sort_unstable_by_key(|span| span.start()); + if local_spans + .windows(2) + .any(|spans| spans[0].overlaps(spans[1])) + { + return Err(PithosError::InvalidDirectoryRange { + operation: "validate block overlap", + }); } validate_references_and_content(&entries, &by_id, &relationships, &descriptors)?; let maximum_id = entries.iter().map(|entry| entry.id).max(); @@ -192,6 +211,7 @@ pub(crate) fn build_effective_index( descriptors, relationships, segment_spans, + local_spans, maximum_id, }) } @@ -227,27 +247,32 @@ fn validate_segment_chain( Ok(()) } +fn block_data_region(segment: &ValidatedSegment) -> Result { + let start = segment + .parent + .map_or(FileHeader::ENCODED_LEN as u64, Span::end); + let end = segment.span.start(); + let len = end + .checked_sub(start) + .ok_or(PithosError::InvalidDirectoryRange { + operation: "validate block data region", + })?; + Span::new(start, len) +} + fn validate_descriptor( descriptor: &BlockDescriptor, archive_len: u64, - directory_spans: &[Span], + block_region: Span, ) -> Result<(), PithosError> { - if let crate::archive::types::BlockLocation::Local(span) = descriptor.location { - if span.end() > archive_len { - return Err(PithosError::InvalidDirectoryRange { - operation: "validate block range", - }); - } - let directory_index = - directory_spans.partition_point(|directory| directory.end() <= span.start()); - if directory_spans - .get(directory_index) - .is_some_and(|directory| span.overlaps(*directory)) - { - return Err(PithosError::InvalidDirectoryRange { - operation: "validate block overlap", - }); - } + if let crate::archive::types::BlockLocation::Local(span) = descriptor.location + && (span.start() < block_region.start() + || span.end() > block_region.end() + || span.end() > archive_len) + { + return Err(PithosError::InvalidDirectoryRange { + operation: "validate block extent", + }); } Ok(()) } diff --git a/crates/pithos_lib/src/archive/mod.rs b/crates/pithos_lib/src/archive/mod.rs index 3f2133f..a0f6598 100644 --- a/crates/pithos_lib/src/archive/mod.rs +++ b/crates/pithos_lib/src/archive/mod.rs @@ -64,6 +64,15 @@ mod tests { } } + fn local_descriptor(span: Span) -> BlockDescriptor { + BlockDescriptor { + stored_size: span.len().saturating_sub(4), + original_size: 1, + processing: Processing::from_byte(0).unwrap(), + location: BlockLocation::Local(span), + } + } + fn segment(entries: Vec) -> ValidatedSegment { segment_at(100, None, entries) } @@ -174,7 +183,7 @@ mod tests { } #[test] - fn non_winning_compatible_descriptors_still_validate_every_local_range() { + fn non_winning_compatible_descriptors_do_not_impose_extent_checks() { let hash = BlockHash([9; 32]); let older = ValidatedSegment { span: Span::new(10, 5).unwrap(), @@ -194,13 +203,122 @@ mod tests { relationships: Vec::new(), }; - assert!(matches!( - build_effective_index(&[older.clone(), newer], 1_000, IndexLimits::default()), - Err(crate::error::PithosError::InvalidDirectoryRange { .. }) - )); + let index = + build_effective_index(&[older.clone(), newer], 1_000, IndexLimits::default()) + .unwrap(); + assert_eq!(index.descriptor(hash), Some(&older.descriptors[0].1)); + } + } + + #[test] + fn local_descriptors_must_stay_in_their_declaring_segment_region() { + let cases = [ + ( + "base descriptor begins before the header ends", + vec![ValidatedSegment { + span: Span::new(20, 10).unwrap(), + parent: None, + entries: Vec::new(), + descriptors: vec![( + BlockHash([1; 32]), + local_descriptor(Span::new(5, 4).unwrap()), + )], + relationships: Vec::new(), + }], + ), + ( + "base descriptor extends into its directory", + vec![ValidatedSegment { + span: Span::new(20, 10).unwrap(), + parent: None, + entries: Vec::new(), + descriptors: vec![( + BlockHash([1; 32]), + local_descriptor(Span::new(10, 11).unwrap()), + )], + relationships: Vec::new(), + }], + ), + ( + "appended descriptor points into the base block region", + vec![ + segment_at(20, None, Vec::new()), + ValidatedSegment { + span: Span::new(60, 10).unwrap(), + parent: Some(Span::new(20, 10).unwrap()), + entries: Vec::new(), + descriptors: vec![( + BlockHash([1; 32]), + local_descriptor(Span::new(10, 4).unwrap()), + )], + relationships: Vec::new(), + }, + ], + ), + ( + "appended descriptor extends into its own directory", + vec![ + segment_at(20, None, Vec::new()), + ValidatedSegment { + span: Span::new(60, 10).unwrap(), + parent: Some(Span::new(20, 10).unwrap()), + entries: Vec::new(), + descriptors: vec![( + BlockHash([1; 32]), + local_descriptor(Span::new(50, 11).unwrap()), + )], + relationships: Vec::new(), + }, + ], + ), + ]; + + for (case, segments) in cases { + assert!( + build_effective_index(&segments, 100, IndexLimits::default()).is_err(), + "accepted {case}" + ); } } + #[test] + fn overlapping_effective_local_descriptors_are_rejected() { + let mut value = segment_at(30, None, Vec::new()); + value.descriptors = vec![ + ( + BlockHash([1; 32]), + local_descriptor(Span::new(6, 10).unwrap()), + ), + ( + BlockHash([2; 32]), + local_descriptor(Span::new(12, 10).unwrap()), + ), + ]; + + assert!(build_effective_index(&[value], 40, IndexLimits::default()).is_err()); + } + + #[test] + fn adjacent_effective_local_descriptors_are_valid() { + let first = Span::new(6, 10).unwrap(); + let second = Span::new(16, 14).unwrap(); + let mut value = segment_at(30, None, Vec::new()); + value.descriptors = vec![ + (BlockHash([1; 32]), local_descriptor(first)), + (BlockHash([2; 32]), local_descriptor(second)), + ]; + + let index = build_effective_index(&[value], 40, IndexLimits::default()).unwrap(); + assert_eq!( + index.descriptor(BlockHash([1; 32])).unwrap().location, + BlockLocation::Local(first) + ); + assert_eq!( + index.descriptor(BlockHash([2; 32])).unwrap().location, + BlockLocation::Local(second) + ); + } + #[test] fn unavailable_content_is_visible_but_cannot_plan_reads() { let entry = SegmentEntry { diff --git a/crates/pithos_lib/src/archive/reader.rs b/crates/pithos_lib/src/archive/reader.rs index 76f27e6..fd466f1 100644 --- a/crates/pithos_lib/src/archive/reader.rs +++ b/crates/pithos_lib/src/archive/reader.rs @@ -313,6 +313,11 @@ where max_segments: options.limits.max_parent_directories.saturating_add(1), }; let index = build_effective_index(&segments, archive_len, index_limits)?; + for span in index.local_block_spans() { + let mut marker = [0; 4]; + source.read_exact_at(span.start(), &mut marker)?; + crate::format::codec::decode_block_marker(&mut marker.as_slice())?; + } Ok(Self { source, @@ -471,9 +476,15 @@ where let mut marker = [0; 4]; self.source.read_exact_at(span.start(), &mut marker)?; crate::format::codec::decode_block_marker(&mut marker.as_slice())?; + let payload_start = + span.start() + .checked_add(4) + .ok_or(PithosError::InvalidDirectoryRange { + operation: "read block payload", + })?; read_source( &self.source, - span.start() + 4, + payload_start, planned.descriptor.stored_size, "block", )? diff --git a/crates/pithos_lib/src/archive/reader_private_tests.rs b/crates/pithos_lib/src/archive/reader_private_tests.rs index 731ac19..db38922 100644 --- a/crates/pithos_lib/src/archive/reader_private_tests.rs +++ b/crates/pithos_lib/src/archive/reader_private_tests.rs @@ -207,6 +207,22 @@ impl Write for RecordingSink { } } +struct RecordingSource { + bytes: Arc<[u8]>, + reads: Arc>>, +} + +impl ArchiveSource for RecordingSource { + fn len(&self) -> Result { + Ok(self.bytes.len() as u64) + } + + fn read_exact_at(&self, offset: u64, output: &mut [u8]) -> Result<(), SourceError> { + self.reads.lock().unwrap().push((offset, output.len())); + MemorySource::new(Arc::clone(&self.bytes)).read_exact_at(offset, output) + } +} + #[test] fn writers_emit_only_the_required_standard_relationship_table() { let (_temporary, path) = fixture(); @@ -666,6 +682,86 @@ fn archive_rejects_stored_size_failures_before_sink_output() { assert_copy_failure_without_sink(&stored); } +#[test] +fn archive_open_validates_local_markers_without_reading_payloads_or_access_keys() { + let (_temporary, path) = fixture_with("marker validation payload", 0); + let (offset, stored_size) = first_block(&path); + let bytes = Arc::<[u8]>::from(std::fs::read(&path).unwrap()); + let reads = Arc::new(Mutex::new(Vec::new())); + Archive::open( + RecordingSource { + bytes, + reads: Arc::clone(&reads), + }, + OpenOptions::default(), + ) + .unwrap(); + let reads = reads.lock().unwrap(); + assert!( + reads.contains(&(offset, 4)), + "local marker was not read at open" + ); + let payload_start = offset + 4; + let payload_end = payload_start + stored_size; + assert!( + reads.iter().all(|(start, len)| { + let end = start.saturating_add(*len as u64); + end <= payload_start || *start >= payload_end + }), + "archive open read local payload bytes" + ); +} + +#[test] +fn archive_open_rejects_missing_or_changed_local_block_markers_without_access_keys() { + for missing in [false, true] { + let (_temporary, path) = fixture_with("invalid marker payload", 0); + if missing { + rewrite_terminal_directory(&path, |directory| { + let descriptor = directory.blocks.first_mut().unwrap().1; + descriptor.offset += 1; + descriptor.stored_size -= 1; + }); + } else { + let (offset, _) = first_block(&path); + let mut bytes = std::fs::read(&path).unwrap(); + bytes[offset as usize..offset as usize + 4].copy_from_slice(b"NOPE"); + std::fs::write(&path, bytes).unwrap(); + } + + let error = match open_without_keys(&path) { + Ok(_) => panic!("archive with an invalid local block marker opened"), + Err(error) => error, + }; + assert!(error.to_string().contains("block marker")); + } +} + +#[test] +fn encrypted_local_descriptor_stored_size_has_a_28_byte_minimum() { + for stored_size in [0, 27] { + let (_temporary, path) = fixture_with("encrypted descriptor size boundary", 0); + rewrite_terminal_directory(&path, |directory| { + let descriptor = directory.blocks.first_mut().unwrap().1; + assert!(descriptor.flags.is_encrypted()); + descriptor.stored_size = stored_size; + }); + let error = match open_without_keys(&path) { + Ok(_) => panic!("encrypted descriptor with stored size {stored_size} opened"), + Err(error) => error, + }; + assert!(error.to_string().contains("block descriptor")); + } + + let (_temporary, path) = fixture_with("encrypted descriptor size boundary", 0); + rewrite_terminal_directory(&path, |directory| { + let descriptor = directory.blocks.first_mut().unwrap().1; + assert!(descriptor.flags.is_encrypted()); + descriptor.stored_size = 28; + }); + open_without_keys(&path).unwrap(); +} + #[test] fn archive_external_blocks_validate_exact_framing_and_share_read_paths() { let (temporary, path) = fixture(); diff --git a/crates/pithos_lib/src/archive/validation.rs b/crates/pithos_lib/src/archive/validation.rs index 07bdd48..52fda95 100644 --- a/crates/pithos_lib/src/archive/validation.rs +++ b/crates/pithos_lib/src/archive/validation.rs @@ -56,6 +56,15 @@ pub(crate) fn segment_from_wire( validate_relationships(directory)?; let mut descriptors = Vec::new(); for (hash, wire) in &directory.blocks { + let processing = Processing::from_byte(wire.flags.0)?; + if matches!(wire.location, WireBlockLocation::Local) + && processing.to_byte() & 0x08 != 0 + && wire.stored_size < 28 + { + return Err(PithosError::InvalidBlockDescriptor( + "encrypted local block payload is shorter than 28 bytes", + )); + } let stored_size_with_marker = wire.stored_size .checked_add(4) @@ -75,7 +84,7 @@ pub(crate) fn segment_from_wire( BlockDescriptor { stored_size: wire.stored_size, original_size: wire.original_size, - processing: Processing::from_byte(wire.flags.0)?, + processing, location, }, )); diff --git a/crates/pithos_lib/src/error.rs b/crates/pithos_lib/src/error.rs index 472b63a..9d0ef62 100644 --- a/crates/pithos_lib/src/error.rs +++ b/crates/pithos_lib/src/error.rs @@ -51,6 +51,8 @@ pub enum PithosError { }, #[error("Invalid block data state: {0}")] InvalidBlockDataState(String), + #[error("invalid block descriptor: {0}")] + InvalidBlockDescriptor(&'static str), #[error("invalid entry combination for {path}: {reason}")] InvalidEntryCombination { path: String, reason: String }, #[error("content is unavailable with the supplied access keys")] diff --git a/crates/pithos_lib/tests/block_pipeline.rs b/crates/pithos_lib/tests/block_pipeline.rs index b621ce6..79e2bf8 100644 --- a/crates/pithos_lib/tests/block_pipeline.rs +++ b/crates/pithos_lib/tests/block_pipeline.rs @@ -1,24 +1,11 @@ mod common; -use common::util::{fixture, open, private_key}; -use pithos_lib::archive::AccessKeys; -use std::io::Write; - -struct RecordingSink(Vec); - -impl Write for RecordingSink { - fn write(&mut self, bytes: &[u8]) -> std::io::Result { - self.0.extend_from_slice(bytes); - Ok(bytes.len()) - } - - fn flush(&mut self) -> std::io::Result<()> { - Ok(()) - } -} +use common::util::fixture; +use pithos_lib::archive::{Archive, OpenOptions}; +use pithos_lib::source::FileSource; #[test] -fn public_archive_rejects_a_bad_block_marker_without_sink_output() { +fn public_archive_rejects_a_bad_block_marker_during_keyless_open() { let temporary = tempfile::tempdir().unwrap(); let path = fixture(&temporary, "recipient1"); let mut bytes = std::fs::read(&path).unwrap(); @@ -28,8 +15,8 @@ fn public_archive_rejects_a_bad_block_marker_without_sink_output() { .unwrap(); bytes[marker] ^= 1; std::fs::write(&path, bytes).unwrap(); - let archive = open(&path, AccessKeys::new().with_key(private_key("recipient1"))); - let mut sink = RecordingSink(Vec::new()); - assert!(archive.copy_to("data", &mut sink).is_err()); - assert!(sink.0.is_empty()); + assert!( + Archive::open(FileSource::open(&path).unwrap(), OpenOptions::default()).is_err(), + "bad local marker was accepted during keyless open" + ); } diff --git a/crates/pithos_lib/tests/draft_spec_vectors.rs b/crates/pithos_lib/tests/draft_spec_vectors.rs index 315966b..cac38a1 100644 --- a/crates/pithos_lib/tests/draft_spec_vectors.rs +++ b/crates/pithos_lib/tests/draft_spec_vectors.rs @@ -150,6 +150,17 @@ fn replace_hello_bytes(range: std::ops::RangeInclusive, replacement: &[u8 bytes } +fn mutate_hello_byte(offset: usize, replacement: u8) -> Vec { + let mut bytes = hello(); + bytes[offset] = replacement; + let crc_offset = bytes.len() - 4; + let directory_start = bytes.len() + - u64::from_be_bytes(bytes[crc_offset - 8..crc_offset].try_into().unwrap()) as usize; + let checksum = crc32(&bytes[directory_start..crc_offset]); + bytes[crc_offset..].copy_from_slice(&checksum.to_be_bytes()); + bytes +} + fn read_uleb(bytes: &[u8], cursor: &mut usize) -> Result { let mut value = 0_u64; for shift in (0..64).step_by(7) { @@ -375,6 +386,19 @@ fn production_reader_rejects_rv_symlink_and_permissions() { } } +#[test] +fn production_reader_rejects_rv_extent_and_short_encrypted() { + for (id, bytes) in [ + ("RV-EXTENT", mutate_hello_byte(139, 0x0f)), + ("RV-SHORT-ENCRYPTED", mutate_hello_byte(142, 0x08)), + ] { + assert!( + Archive::open(MemorySource::new(bytes), OpenOptions::default()).is_err(), + "production reader accepted {id}" + ); + } +} + #[test] fn directory_length_calculations_are_independent() { let base_directory = &base()[6..]; From f0a6b2b8e0dcb1ddb76e938401ce12e1efa3f57b Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Sun, 13 Sep 2026 14:52:16 +0200 Subject: [PATCH 043/166] feat: validate recipient grant identity --- crates/pithos_lib/src/archive/append.rs | 20 +- crates/pithos_lib/src/archive/mod.rs | 9 + crates/pithos_lib/src/archive/reader.rs | 19 +- .../src/archive/reader_private_tests.rs | 175 ++++++++++++++++++ crates/pithos_lib/src/archive/snapshot.rs | 67 ++++++- crates/pithos_lib/src/archive/types.rs | 3 + crates/pithos_lib/src/archive/validation.rs | 15 +- crates/pithos_lib/src/archive/writer.rs | 5 +- crates/pithos_lib/src/crypto.rs | 4 +- crates/pithos_lib/src/error.rs | 2 + crates/pithos_lib/src/format/codec.rs | 6 +- crates/pithos_lib/src/fs/append.rs | 125 +++++++++++-- 12 files changed, 417 insertions(+), 33 deletions(-) diff --git a/crates/pithos_lib/src/archive/append.rs b/crates/pithos_lib/src/archive/append.rs index ae19183..7bc611f 100644 --- a/crates/pithos_lib/src/archive/append.rs +++ b/crates/pithos_lib/src/archive/append.rs @@ -13,7 +13,8 @@ pub enum AppendDurability { /// Options for a direct filesystem append. pub struct AppendOptions { - pub(crate) sender: PrivateKey, + /// Private key used to open the existing archive and authorize the append. + pub(crate) access_key: PrivateKey, pub(crate) recipients: Vec, pub(crate) cdc: CdcConfig, pub(crate) durability: AppendDurability, @@ -21,9 +22,11 @@ pub struct AppendOptions { } impl AppendOptions { - pub fn new(sender: PrivateKey, recipients: Vec) -> Self { + /// Creates append options using `access_key` to open and authorize the existing archive. + /// A fresh wrapping sender is generated for each child during filesystem append operations. + pub fn new(access_key: PrivateKey, recipients: Vec) -> Self { Self { - sender, + access_key, recipients, cdc: CdcConfig::default(), durability: AppendDurability::Flush, @@ -42,7 +45,16 @@ impl AppendOptions { } pub(crate) fn validate_recipients(&self) -> Result<(), PithosError> { - WriteOptions::new(self.sender.duplicate(), self.recipients.clone()).validate() + WriteOptions::new(self.access_key.duplicate(), self.recipients.clone()).validate() + } + + pub(crate) fn recipients_with_access_key(&self) -> Vec { + let access_public = self.access_key.public_key(); + let mut recipients = self.recipients.clone(); + if !recipients.contains(&access_public) { + recipients.push(access_public); + } + recipients } } diff --git a/crates/pithos_lib/src/archive/mod.rs b/crates/pithos_lib/src/archive/mod.rs index a0f6598..6494a15 100644 --- a/crates/pithos_lib/src/archive/mod.rs +++ b/crates/pithos_lib/src/archive/mod.rs @@ -88,6 +88,7 @@ mod tests { entries, descriptors: Vec::new(), relationships: Vec::new(), + recipient_pairs: Vec::new(), } } @@ -145,6 +146,7 @@ mod tests { entries: Vec::new(), descriptors: vec![(hash, descriptor(4))], relationships: Vec::new(), + recipient_pairs: Vec::new(), }; let mut newer_descriptor = descriptor(4); newer_descriptor.stored_size = 99; @@ -154,6 +156,7 @@ mod tests { entries: Vec::new(), descriptors: vec![(hash, newer_descriptor)], relationships: Vec::new(), + recipient_pairs: Vec::new(), }; let older_before = older.clone(); let newer_before = newer.clone(); @@ -191,6 +194,7 @@ mod tests { entries: Vec::new(), descriptors: vec![(hash, descriptor(4))], relationships: Vec::new(), + recipient_pairs: Vec::new(), }; for invalid_span in [Span::new(995, 10).unwrap(), Span::new(20, 5).unwrap()] { let mut later = descriptor(4); @@ -201,6 +205,7 @@ mod tests { entries: Vec::new(), descriptors: vec![(hash, later)], relationships: Vec::new(), + recipient_pairs: Vec::new(), }; let index = @@ -224,6 +229,7 @@ mod tests { local_descriptor(Span::new(5, 4).unwrap()), )], relationships: Vec::new(), + recipient_pairs: Vec::new(), }], ), ( @@ -237,6 +243,7 @@ mod tests { local_descriptor(Span::new(10, 11).unwrap()), )], relationships: Vec::new(), + recipient_pairs: Vec::new(), }], ), ( @@ -252,6 +259,7 @@ mod tests { local_descriptor(Span::new(10, 4).unwrap()), )], relationships: Vec::new(), + recipient_pairs: Vec::new(), }, ], ), @@ -268,6 +276,7 @@ mod tests { local_descriptor(Span::new(50, 11).unwrap()), )], relationships: Vec::new(), + recipient_pairs: Vec::new(), }, ], ), diff --git a/crates/pithos_lib/src/archive/reader.rs b/crates/pithos_lib/src/archive/reader.rs index fd466f1..6369e90 100644 --- a/crates/pithos_lib/src/archive/reader.rs +++ b/crates/pithos_lib/src/archive/reader.rs @@ -14,7 +14,7 @@ use crate::error::PithosError; use crate::format::limits::DeserializationLimits; use crate::format::wire::{BlockDataState, BlockIndexEntry, Directory, FileHeader}; use crate::source::ArchiveSource; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::io::Write; /// Distinguishes archive failures from a presentation callback failure without @@ -277,6 +277,23 @@ where } raw.reverse(); + let mut first_grants = HashMap::new(); + for (directory, _) in &raw { + for (sender, section) in &directory.encryption { + for (recipient, recipient_section) in §ion.recipients { + let pair = (*sender, *recipient); + if let Some(first) = first_grants.get(&pair) + && *first != &recipient_section.recipient_data + { + return Err(PithosError::ConflictingRecipientGrant); + } + first_grants + .entry(pair) + .or_insert(&recipient_section.recipient_data); + } + } + } + let mut access = ResolvedAccess::new(); let mut recovery_order = 0usize; for (segment, (directory, _)) in raw.iter().enumerate() { diff --git a/crates/pithos_lib/src/archive/reader_private_tests.rs b/crates/pithos_lib/src/archive/reader_private_tests.rs index db38922..42b9b18 100644 --- a/crates/pithos_lib/src/archive/reader_private_tests.rs +++ b/crates/pithos_lib/src/archive/reader_private_tests.rs @@ -21,6 +21,7 @@ use std::sync::Arc; use std::sync::Mutex; use std::sync::atomic::{AtomicUsize, Ordering}; use x25519_dalek::{PublicKey as DalekPublicKey, StaticSecret}; +use zeroize::Zeroizing; fn private(name: &str) -> PrivateKey { crate::crypto::parse_private_pem( @@ -40,6 +41,18 @@ fn fixture() -> (tempfile::TempDir, std::path::PathBuf) { fixture_with("archive reader private fixture", 0) } +fn empty_fixture() -> (tempfile::TempDir, PathBuf) { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("empty.pith"); + let writer = ArchiveWriter::create( + File::create(&path).unwrap(), + WriteOptions::new(private("sender"), vec![public("recipient1")]), + ) + .unwrap(); + writer.finish().unwrap(); + (temp, path) +} + fn fixture_with(content: &str, compression_level: u8) -> (tempfile::TempDir, PathBuf) { fixture_with_encryption(content, compression_level, true) } @@ -161,6 +174,86 @@ fn append_empty_directory(path: &Path, relations: Option>) { std::fs::write(path, archive).unwrap(); } +fn append_recipient_directory(path: &Path, mutate: impl FnOnce(&mut RecipientData)) { + let mut archive = std::fs::read(path).unwrap(); + let (parent_start, parent_len) = directory_bounds(&archive); + let parent = crate::format::codec::decode_directory( + &mut Cursor::new(&archive[parent_start..]), + &DeserializationLimits::default(), + ) + .unwrap(); + let (sender, section) = parent.encryption.first().unwrap(); + let (recipient, recipient_section) = section.recipients.first().unwrap(); + let mut recipient_data = recipient_section.recipient_data.clone(); + mutate(&mut recipient_data); + let encryption = IndexMap::from_iter([( + *sender, + EncryptionSection { + recipients: IndexMap::from_iter([(*recipient, RecipientSection { recipient_data })]), + }, + )]); + let mut child = Directory::new( + Some((parent_start as u64, parent_len as u64)), + crate::format::entries::WireEntries::new(), + encryption, + ); + crate::format::codec::update_directory_len(&mut child).unwrap(); + crate::format::codec::update_directory_crc(&mut child).unwrap(); + crate::format::codec::encode_directory(&child, &mut archive).unwrap(); + std::fs::write(path, archive).unwrap(); +} + +fn set_base_recipient_records(path: &Path, records: Vec<(u64, [u8; 32])>) { + rewrite_terminal_directory(path, |directory| { + directory + .encryption + .first_mut() + .unwrap() + .1 + .recipients + .first_mut() + .unwrap() + .1 + .recipient_data = RecipientData::Decrypted(Zeroizing::new(records)); + }); +} + +fn make_terminal_recipient_count_and_id_non_minimal(path: &Path, id: u8, key: u8) { + let mut archive = std::fs::read(path).unwrap(); + let (start, _) = directory_bounds(&archive); + let pattern = [vec![1, 1, id], vec![key; 32]].concat(); + let relative = archive[start..] + .windows(pattern.len()) + .position(|window| window == pattern) + .expect("terminal decrypted recipient record"); + let record = start + relative; + archive.splice(record + 1..record + 3, [0x81, 0x00, id | 0x80, 0x00]); + let directory_len = (archive.len() - start) as u64; + let footer = archive.len() - 12; + archive[footer..footer + 8].copy_from_slice(&directory_len.to_be_bytes()); + let checksum = crc32fast::hash(&archive[start..archive.len() - 4]); + let crc_offset = archive.len() - 4; + archive[crc_offset..].copy_from_slice(&checksum.to_be_bytes()); + std::fs::write(path, archive).unwrap(); +} + +fn assert_conflicting_recipient_grant(path: &Path) { + let bytes = Arc::<[u8]>::from(std::fs::read(path).unwrap()); + for with_matching_key in [false, true] { + let options = if with_matching_key { + OpenOptions::default() + .with_access_keys(AccessKeys::new().with_key(private("recipient1"))) + } else { + OpenOptions::default() + }; + let error = match Archive::open(MemorySource::new(Arc::clone(&bytes)), options) { + Ok(_) => panic!("conflicting recipient grant was accepted"), + Err(error) => error, + }; + assert_eq!(format!("{error:?}"), "ConflictingRecipientGrant"); + } +} + fn open_without_keys(path: &Path) -> Result, PithosError> { Archive::open( MemorySource::new(Arc::<[u8]>::from(std::fs::read(path).unwrap())), @@ -553,6 +646,88 @@ fn archive_rejects_conflicting_recovered_file_keys_at_open() { )); } +#[test] +fn archive_validates_every_encoded_sender_and_recipient_key_without_access_keys() { + for sender_is_invalid in [true, false] { + let (_temporary, path) = empty_fixture(); + rewrite_terminal_directory(&path, |directory| { + if sender_is_invalid { + let section = directory.encryption.first().unwrap().1.clone(); + directory.encryption.clear(); + directory.encryption.insert([0; 32], section); + } else { + let section = directory.encryption.first_mut().unwrap().1; + let recipient = section.recipients.first().unwrap().1.clone(); + section.recipients.clear(); + section.recipients.insert([0; 32], recipient); + } + }); + + assert!(matches!( + open_without_keys(&path), + Err(PithosError::Crypt( + crypto::CryptoError::NonContributoryPublicKey + )) + )); + } +} + +#[test] +fn structurally_equal_encrypted_recipient_grants_repeat_with_or_without_access_keys() { + let (_temporary, path) = empty_fixture(); + append_recipient_directory(&path, |_| {}); + open_without_keys(&path).unwrap(); + open_path(&path, AccessKeys::new().with_key(private("recipient1"))); +} + +#[test] +fn encrypted_recipient_grant_conflicts_are_key_independent_and_compare_all_bytes() { + for mutate_last_byte in [false, true] { + let (_temporary, path) = empty_fixture(); + append_recipient_directory(&path, |data| { + let RecipientData::Encrypted(bytes) = data else { + panic!("writer recipient grant was not sealed"); + }; + let index = if mutate_last_byte { bytes.len() - 1 } else { 0 }; + bytes[index] ^= 1; + }); + assert_conflicting_recipient_grant(&path); + } +} + +#[test] +fn decrypted_recipient_grants_compare_decoded_order_and_accept_non_minimal_uleb128() { + let first = (7, [0x5a; 32]); + let second = (8, [0xa5; 32]); + + let (_temporary, equal) = empty_fixture(); + set_base_recipient_records(&equal, vec![first]); + append_recipient_directory(&equal, |_| {}); + make_terminal_recipient_count_and_id_non_minimal(&equal, 7, 0x5a); + open_without_keys(&equal).unwrap(); + open_path(&equal, AccessKeys::new().with_key(private("recipient1"))); + + let (_temporary, conflicting) = empty_fixture(); + set_base_recipient_records(&conflicting, vec![first, second]); + append_recipient_directory(&conflicting, |data| { + let RecipientData::Decrypted(records) = data else { + panic!("test grant was unexpectedly encrypted"); + }; + records.reverse(); + }); + assert_conflicting_recipient_grant(&conflicting); +} + +#[test] +fn recipient_grant_variants_must_match_across_directories() { + let (_temporary, path) = empty_fixture(); + set_base_recipient_records(&path, Vec::new()); + append_recipient_directory(&path, |data| { + *data = RecipientData::Encrypted(Vec::new()); + }); + assert_conflicting_recipient_grant(&path); +} + #[test] fn archive_ranges_are_half_open_and_invalid_ranges_do_not_acquire_payload() { let (_temporary, path) = fixture_with("0123456789abcdef", 0); diff --git a/crates/pithos_lib/src/archive/snapshot.rs b/crates/pithos_lib/src/archive/snapshot.rs index 88f666e..4145050 100644 --- a/crates/pithos_lib/src/archive/snapshot.rs +++ b/crates/pithos_lib/src/archive/snapshot.rs @@ -1,13 +1,14 @@ use crate::archive::access::ResolvedAccess; use crate::archive::index::{ArchiveIndex, build_effective_index}; use crate::archive::types::{ - ArchivePath, BlockDescriptor, BlockHash, Entry, FileId, Span, ValidatedSegment, + ArchivePath, BlockDescriptor, BlockHash, Entry, FileId, RecipientPair, Span, ValidatedSegment, }; use crate::archive::validation::IndexLimits; -use crate::crypto::FileKey; +use crate::crypto::{FileKey, PublicKey}; use crate::error::PithosError; +use crate::format::wire::EncryptionSection; use indexmap::IndexMap; -use std::collections::{BTreeMap, HashMap}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::sync::Arc; /// The secret-contained state needed to safely plan a direct append or grant. @@ -26,6 +27,7 @@ pub(crate) struct AppendSnapshot { segments: Vec, index_limits: IndexLimits, access: ResolvedAccess, + occupied_recipient_pairs: HashSet, } pub(crate) struct SnapshotEntry { @@ -58,6 +60,10 @@ impl AppendSnapshot { .map(|(id, name)| (id.0, Arc::from(name))) .collect(); let (index_entries, descriptors, maximum_id) = index.into_append_snapshot_parts(); + let occupied_recipient_pairs = segments + .iter() + .flat_map(|segment| segment.recipient_pairs.iter().copied()) + .collect(); let mut entries = BTreeMap::new(); let mut paths = HashMap::new(); let mut hierarchy = BTreeMap::new(); @@ -90,6 +96,7 @@ impl AppendSnapshot { segments, index_limits, access, + occupied_recipient_pairs, } } @@ -201,6 +208,32 @@ impl AppendSnapshot { build_effective_index(&segments, archive_len, self.index_limits).map(|_| ()) } + pub(crate) fn sender_pair_is_occupied( + &self, + sender: [u8; 32], + recipients: &[PublicKey], + ) -> bool { + recipients.iter().any(|recipient| { + self.occupied_recipient_pairs + .contains(&(sender, recipient.into_dalek_public_key().to_bytes())) + }) + } + + pub(crate) fn validate_prospective_recipient_pairs( + &self, + encryption: &IndexMap<[u8; 32], EncryptionSection>, + ) -> Result<(), PithosError> { + if encryption.iter().any(|(sender, section)| { + section.recipients.keys().any(|recipient| { + self.occupied_recipient_pairs + .contains(&(*sender, *recipient)) + }) + }) { + return Err(PithosError::ConflictingRecipientGrant); + } + Ok(()) + } + /// Borrows an opaque, zeroizing recovered file key only for content entries. pub(crate) fn with_file_key( &self, @@ -403,7 +436,7 @@ mod tests { .into_append_snapshot(); let child = ArchiveWriter::append( Vec::new(), - sender, + PrivateKey::generate(), vec![recipient.public_key()], crate::archive::CdcConfig::default(), snapshot, @@ -472,4 +505,30 @@ mod tests { Err(PithosError::SnapshotContentUnavailable(0)) )); } + + #[test] + fn prospective_append_rejects_an_ancestor_sender_recipient_pair_before_publication() { + let (bytes, sender, recipient) = archive_with_entries(); + let snapshot = + open(bytes, AccessKeys::new().with_key(recipient.duplicate())).into_append_snapshot(); + let child = ArchiveWriter::append( + Vec::new(), + sender, + vec![recipient.public_key()], + crate::archive::CdcConfig::default(), + snapshot, + ) + .unwrap(); + + let failure = match child.finish() { + Ok(_) => panic!("append repeated an ancestor sender/recipient pair"), + Err(failure) => failure, + }; + let (error, suffix) = failure.into_parts(); + assert_eq!(format!("{error:?}"), "ConflictingRecipientGrant"); + assert!( + suffix.is_empty(), + "conflicting child was partially published" + ); + } } diff --git a/crates/pithos_lib/src/archive/types.rs b/crates/pithos_lib/src/archive/types.rs index 7eb7572..bb5068d 100644 --- a/crates/pithos_lib/src/archive/types.rs +++ b/crates/pithos_lib/src/archive/types.rs @@ -304,4 +304,7 @@ pub(crate) struct ValidatedSegment { pub(crate) entries: Vec, pub(crate) descriptors: Vec<(BlockHash, BlockDescriptor)>, pub(crate) relationships: Vec<(RelationId, Arc)>, + pub(crate) recipient_pairs: Vec, } + +pub(crate) type RecipientPair = ([u8; 32], [u8; 32]); diff --git a/crates/pithos_lib/src/archive/validation.rs b/crates/pithos_lib/src/archive/validation.rs index 52fda95..56efbd3 100644 --- a/crates/pithos_lib/src/archive/validation.rs +++ b/crates/pithos_lib/src/archive/validation.rs @@ -1,7 +1,7 @@ use crate::archive::types::{ ArchivePath, BlockDescriptor, BlockHash, BlockLocation, ContentEntry, ContentState, Entry, - EntryMetadata, ExternalLocation, FileId, Processing, Reference, RelationId, SegmentEntry, Span, - ValidatedSegment, + EntryMetadata, ExternalLocation, FileId, Processing, RecipientPair, Reference, RelationId, + SegmentEntry, Span, ValidatedSegment, }; use crate::error::PithosError; use crate::format::directory::STANDARD_RELATIONSHIPS; @@ -103,12 +103,23 @@ pub(crate) fn segment_from_wire( } } } + let recipient_pairs: Vec = directory + .encryption + .iter() + .flat_map(|(sender, section)| { + section + .recipients + .keys() + .map(move |recipient| (*sender, *recipient)) + }) + .collect(); Ok(ValidatedSegment { span, parent, entries, descriptors, relationships, + recipient_pairs, }) } diff --git a/crates/pithos_lib/src/archive/writer.rs b/crates/pithos_lib/src/archive/writer.rs index 9008301..f025be0 100644 --- a/crates/pithos_lib/src/archive/writer.rs +++ b/crates/pithos_lib/src/archive/writer.rs @@ -1246,6 +1246,7 @@ impl ArchiveWriter { self.validate_entry_state()?; validate_relationships(&self.directory)?; if let Some(snapshot) = &self.append_snapshot { + snapshot.validate_prospective_recipient_pairs(&self.directory.encryption)?; let relationships = self .directory .relations @@ -1748,7 +1749,7 @@ mod tests { .into_append_snapshot(); let mut child = ArchiveWriter::append( Vec::new(), - sender.duplicate(), + PrivateKey::generate(), vec![sender.public_key()], CdcConfig::default(), snapshot, @@ -2073,7 +2074,7 @@ mod tests { .into_append_snapshot(); let mut child = ArchiveWriter::append( Vec::new(), - sender, + PrivateKey::generate(), vec![recipient], CdcConfig::default(), snapshot, diff --git a/crates/pithos_lib/src/crypto.rs b/crates/pithos_lib/src/crypto.rs index 97d1e14..081fc35 100644 --- a/crates/pithos_lib/src/crypto.rs +++ b/crates/pithos_lib/src/crypto.rs @@ -192,7 +192,7 @@ impl PublicKey { } let mut key = [0; 32]; key.copy_from_slice(&bytes[12..]); - validate_contributory_public_key(&key)?; + validate_x25519_public_key(&key)?; Ok(Self(key)) } @@ -249,7 +249,7 @@ pub(crate) fn derive_shared( Ok(SharedSecret(shared.to_bytes())) } -fn validate_contributory_public_key(public: &[u8; 32]) -> Result<(), CryptoError> { +pub(crate) fn validate_x25519_public_key(public: &[u8; 32]) -> Result<(), CryptoError> { // Low-order public keys produce the identity for every clamped X25519 scalar. let probe = StaticSecret::from([0; 32]).diffie_hellman(&DalekPublicKey::from(*public)); if probe.was_contributory() { diff --git a/crates/pithos_lib/src/error.rs b/crates/pithos_lib/src/error.rs index 9d0ef62..943b192 100644 --- a/crates/pithos_lib/src/error.rs +++ b/crates/pithos_lib/src/error.rs @@ -101,6 +101,8 @@ pub enum PithosError { AccessibleFileSizeMismatch { expected: u64, actual: u64 }, #[error("conflicting recovered file key")] ConflictingRecoveredFileKey, + #[error("conflicting recipient grant")] + ConflictingRecipientGrant, #[error("invalid half-open read range {start}..{end} for file size {file_size}")] InvalidReadRange { start: u64, diff --git a/crates/pithos_lib/src/format/codec.rs b/crates/pithos_lib/src/format/codec.rs index 8285fa7..8136444 100644 --- a/crates/pithos_lib/src/format/codec.rs +++ b/crates/pithos_lib/src/format/codec.rs @@ -400,7 +400,7 @@ fn encode_encryption_section( fn decode_encryption_section( reader: &mut R, limits: &DeserializationLimits, -) -> Result { +) -> Result { let count = bounded_len( reader.read_varint::()?, limits.max_collection_entries, @@ -410,8 +410,9 @@ fn decode_encryption_section( for _ in 0..count { let mut key = [0; 32]; reader.read_exact(&mut key)?; + crate::crypto::validate_x25519_public_key(&key)?; if recipients.contains_key(&key) { - return Err(DeserializationError::DuplicateRecipientKey); + return Err(DeserializationError::DuplicateRecipientKey.into()); } recipients.insert( key, @@ -555,6 +556,7 @@ pub(crate) fn decode_directory( for _ in 0..encryption_count { let mut key = [0; 32]; reader.read_exact(&mut key)?; + crate::crypto::validate_x25519_public_key(&key)?; if encryption.contains_key(&key) { return Err(PithosError::DuplicateSenderKey); } diff --git a/crates/pithos_lib/src/fs/append.rs b/crates/pithos_lib/src/fs/append.rs index 573a3ea..51d5600 100644 --- a/crates/pithos_lib/src/fs/append.rs +++ b/crates/pithos_lib/src/fs/append.rs @@ -2,6 +2,7 @@ use crate::archive::{ AccessKeys, AppendDurability, AppendObservation, AppendOptions, AppendSnapshot, Archive, ArchiveWriter, FileId, OpenOptions, WriterError, }; +use crate::crypto::PrivateKey; use crate::error::PithosError; use crate::fs::FsError; use crate::fs::ingest::{InputManifest, build_input_manifest}; @@ -150,7 +151,7 @@ fn append_files_impl( let bytes = Arc::new(AtomicU64::new(0)); let snapshot = open_snapshot( &locked, - spec.options.sender.duplicate(), + spec.options.access_key.duplicate(), Arc::clone(&reads), Arc::clone(&bytes), ) @@ -161,6 +162,8 @@ fn append_files_impl( archive_metadata.dev(), archive_metadata.ino(), )?; + let wrapping_sender = fresh_wrapping_sender(&snapshot, &spec.options); + let recipients = spec.options.recipients_with_access_key(); let mutated = Arc::new(AtomicBool::new(false)); let AppendSpec { options, .. } = spec; @@ -172,13 +175,8 @@ fn append_files_impl( Arc::clone(&mutated), ); sink.seek(SeekFrom::Start(original_len))?; - let mut writer = ArchiveWriter::append( - sink, - options.sender, - options.recipients, - options.cdc, - snapshot, - )?; + let mut writer = + ArchiveWriter::append(sink, wrapping_sender, recipients, options.cdc, snapshot)?; plan.manifest .ingest_planned(&plan.ids, &mut writer, options.processing) .map_err(writer_error)?; @@ -237,13 +235,15 @@ pub fn grant_readers( let bytes = Arc::new(AtomicU64::new(0)); let snapshot = open_snapshot( &locked, - options.sender.duplicate(), + options.access_key.duplicate(), Arc::clone(&reads), Arc::clone(&bytes), ) .map_err(|source| append_error("open grant snapshot", archive, source))?; validate_grant_ids(&snapshot, ids) .map_err(|source| append_error("validate grant ids", archive, source))?; + let wrapping_sender = fresh_wrapping_sender(&snapshot, &options); + let recipients = options.recipients_with_access_key(); let mutated = Arc::new(AtomicBool::new(false)); let result = (|| { @@ -254,13 +254,8 @@ pub fn grant_readers( Arc::clone(&mutated), ); sink.seek(SeekFrom::Start(original_len))?; - let mut writer = ArchiveWriter::append( - sink, - options.sender, - options.recipients, - options.cdc, - snapshot, - )?; + let mut writer = + ArchiveWriter::append(sink, wrapping_sender, recipients, options.cdc, snapshot)?; writer.grant_file_keys(&ids.iter().copied().map(FileId).collect::>())?; let mut sink = writer.finish().map_err(|error| error.into_parts().0)?; if options.durability == AppendDurability::SyncAll { @@ -335,6 +330,23 @@ fn open_snapshot( .map(Archive::into_append_snapshot) } +fn fresh_wrapping_sender(snapshot: &AppendSnapshot, options: &AppendOptions) -> PrivateKey { + let recipients = options.recipients_with_access_key(); + let access_public = options.access_key.public_key(); + loop { + let sender = PrivateKey::generate(); + let sender_public = sender.public_key(); + if sender_public != access_public + && !snapshot.sender_pair_is_occupied( + sender_public.into_dalek_public_key().to_bytes(), + &recipients, + ) + { + return sender; + } + } +} + fn validate_grant_ids(snapshot: &AppendSnapshot, ids: &[u64]) -> Result<(), PithosError> { if ids.is_empty() { return Err(PithosError::GrantRequiresFileId); @@ -511,6 +523,7 @@ mod tests { use crate::format::limits::DeserializationLimits; use crate::source::FileSource; use std::io::Cursor; + use x25519_dalek::PublicKey as DalekPublicKey; struct AppendFixture { archive: PathBuf, @@ -560,6 +573,86 @@ mod tests { AppendFixture { archive, sender } } + fn terminal_directory(path: &Path) -> crate::format::wire::Directory { + let bytes = std::fs::read(path).unwrap(); + let directory_len = + u64::from_be_bytes(bytes[bytes.len() - 12..bytes.len() - 4].try_into().unwrap()); + let directory_start = bytes.len() - usize::try_from(directory_len).unwrap(); + crate::format::codec::decode_directory( + &mut Cursor::new(&bytes[directory_start..]), + &DeserializationLimits::default(), + ) + .unwrap() + } + + #[test] + fn append_and_grant_use_fresh_wire_senders_and_retain_the_access_key_as_a_recipient() { + let temporary = tempfile::tempdir().unwrap(); + let fixture = append_fixture(&temporary); + let access_public = + DalekPublicKey::from(&fixture.sender.as_dalek_static_secret()).to_bytes(); + let base_sender = *terminal_directory(&fixture.archive) + .encryption + .first() + .unwrap() + .0; + let other = PrivateKey::generate(); + let source = fixture.append_source("appended.txt", b"appended through a fresh sender"); + + append_files( + &fixture.archive, + AppendOptions::new(fixture.sender.duplicate(), vec![other.public_key()]), + &[source], + ) + .unwrap(); + let child = terminal_directory(&fixture.archive); + let child_sender = *child.encryption.first().unwrap().0; + assert_ne!(child_sender, base_sender); + assert_ne!(child_sender, access_public); + assert!( + child + .encryption + .first() + .unwrap() + .1 + .recipients + .contains_key(&access_public) + ); + fixture + .open() + .copy_to("appended.txt", &mut Vec::new()) + .unwrap(); + + grant_readers( + &fixture.archive, + AppendOptions::new(fixture.sender.duplicate(), vec![other.public_key()]), + &[0], + ) + .unwrap(); + let grant = terminal_directory(&fixture.archive); + let grant_sender = *grant.encryption.first().unwrap().0; + assert_ne!(grant_sender, base_sender); + assert_ne!(grant_sender, child_sender); + assert_ne!(grant_sender, access_public); + assert!( + grant + .encryption + .first() + .unwrap() + .1 + .recipients + .contains_key(&access_public) + ); + + let archive = Archive::open( + FileSource::open(&fixture.archive).unwrap(), + OpenOptions::default().with_access_keys(AccessKeys::new().with_key(other)), + ) + .unwrap(); + archive.copy_to("base.txt", &mut Vec::new()).unwrap(); + archive.copy_to("appended.txt", &mut Vec::new()).unwrap(); + } + #[test] fn conflicting_ancestor_relationship_is_rejected_before_mutation() { let temporary = tempfile::tempdir().unwrap(); From 9856118c69bc9d91222595e6d376d3a0e13ef73f Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Sun, 13 Sep 2026 15:09:51 +0200 Subject: [PATCH 044/166] feat: resolve direct block lists --- crates/pithos_lib/src/archive/reader.rs | 82 ++++---- .../src/archive/reader_private_tests.rs | 186 +++++++++++++++++- crates/pithos_lib/src/format/codec.rs | 72 ++++++- crates/pithos_lib/src/format/limits.rs | 2 + crates/pithos_lib/tests/draft_spec_vectors.rs | 45 ++++- 5 files changed, 338 insertions(+), 49 deletions(-) diff --git a/crates/pithos_lib/src/archive/reader.rs b/crates/pithos_lib/src/archive/reader.rs index 6369e90..255a57e 100644 --- a/crates/pithos_lib/src/archive/reader.rs +++ b/crates/pithos_lib/src/archive/reader.rs @@ -12,7 +12,7 @@ use crate::block; use crate::crypto::{self, FileKey, PrivateKey}; use crate::error::PithosError; use crate::format::limits::DeserializationLimits; -use crate::format::wire::{BlockDataState, BlockIndexEntry, Directory, FileHeader}; +use crate::format::wire::{BlockDataEntry, BlockDataState, BlockIndexEntry, Directory, FileHeader}; use crate::source::ArchiveSource; use std::collections::{HashMap, HashSet}; use std::io::Write; @@ -32,6 +32,7 @@ struct DecodedDirectoryCounts { entries: u64, descriptors: u64, references: u64, + direct_references: u64, relationships: u64, } @@ -44,6 +45,14 @@ impl DecodedDirectoryCounts { .iter() .map(|(_, _, file)| file.references.len() as u64) .sum::(); + self.direct_references += directory + .files + .iter() + .map(|(_, _, file)| match &file.block_data { + BlockDataState::Decrypted(entries) => entries.len() as u64, + BlockDataState::Encrypted(_) => 0, + }) + .sum::(); self.relationships += directory.relations.len() as u64; } } @@ -235,6 +244,7 @@ where let mut child_start = archive_len; let mut visited = HashSet::new(); let mut decoded = DecodedDirectoryCounts::default(); + let mut remaining_block_references = options.limits.max_accessible_block_references; while let Some((start, len)) = next { validate_directory_len(len, options.limits)?; if raw.len() as u64 > options.limits.max_parent_directories { @@ -266,9 +276,10 @@ where }); } let bytes = Zeroizing::new(read_source(&source, start, len, "directory")?); - let directory = crate::format::codec::decode_complete_directory( + let directory = crate::format::codec::decode_complete_directory_with_budget( &bytes, &remaining_deserialization_limits(options.limits, &decoded), + &mut remaining_block_references, )?; decoded.record(&directory); next = directory.parent_directory_offset; @@ -308,13 +319,12 @@ where } let mut segments: Vec = Vec::new(); - let mut accessible_references = 0u64; for (segment_index, (directory, span)) in raw.into_iter().enumerate() { let mut directory = directory; resolve_block_lists( &mut directory, &mut access, - &mut accessible_references, + &mut remaining_block_references, options.limits, )?; let parent = segment_index @@ -606,6 +616,7 @@ fn deserialization_limits(limits: OpenLimits) -> DeserializationLimits { max_collection_entries: limits.max_entries, max_file_entries: limits.max_entries, max_block_descriptors: limits.max_descriptors, + max_block_references: limits.max_accessible_block_references, max_references: limits.max_references, max_relationships: limits.max_relationships, max_opaque_bytes: limits.max_opaque_metadata_bytes, @@ -622,6 +633,9 @@ fn remaining_deserialization_limits( remaining.max_block_descriptors = remaining .max_block_descriptors .saturating_sub(decoded.descriptors); + remaining.max_block_references = remaining + .max_block_references + .saturating_sub(decoded.direct_references); remaining.max_references = remaining.max_references.saturating_sub(decoded.references); remaining.max_relationships = remaining .max_relationships @@ -750,42 +764,38 @@ fn resolve_recipients( fn resolve_block_lists( directory: &mut Directory, access: &mut ResolvedAccess, - reference_count: &mut u64, + remaining_block_references: &mut u64, limits: OpenLimits, ) -> Result<(), PithosError> { directory.files.try_for_each_mut(|id, file| { - let Some(file_key) = access.key(FileId(id)) else { - return Ok(()); - }; - let BlockDataState::Encrypted(bytes) = &file.block_data else { - return Ok(()); - }; - let mut decoded_limits = deserialization_limits(limits); - decoded_limits.max_collection_entries = limits - .max_accessible_block_references - .saturating_sub(*reference_count); - let plaintext = crypto::open_file_block_list(file_key, bytes)?; - let entries = - crate::format::codec::decode_decrypted_block_list(&plaintext, &decoded_limits)?; - *reference_count = reference_count.checked_add(entries.len() as u64).ok_or( - PithosError::LimitExceeded { - field: "accessible block references", - limit: limits.max_accessible_block_references, - actual: u64::MAX, - }, - )?; - if *reference_count > limits.max_accessible_block_references { - return Err(PithosError::LimitExceeded { - field: "accessible block references", - limit: limits.max_accessible_block_references, - actual: *reference_count, - }); + let file_id = FileId(id); + match &file.block_data { + BlockDataState::Decrypted(entries) => { + record_block_keys(access, file_id, entries); + } + BlockDataState::Encrypted(bytes) => { + let Some(file_key) = access.key(file_id) else { + return Ok(()); + }; + let mut decoded_limits = deserialization_limits(limits); + decoded_limits.max_block_references = *remaining_block_references; + let plaintext = crypto::open_file_block_list(file_key, bytes)?; + let entries = crate::format::codec::decode_decrypted_block_list_with_budget( + &plaintext, + &decoded_limits, + remaining_block_references, + )?; + record_block_keys(access, file_id, &entries); + file.block_data = BlockDataState::Decrypted(entries); + } } - access.insert_block_keys( - FileId(id), - entries.iter().map(|(hash, key)| (BlockHash(*hash), key)), - ); - file.block_data = BlockDataState::Decrypted(entries); Ok(()) }) } + +fn record_block_keys(access: &mut ResolvedAccess, file_id: FileId, entries: &[BlockDataEntry]) { + access.insert_block_keys( + file_id, + entries.iter().map(|(hash, key)| (BlockHash(*hash), key)), + ); +} diff --git a/crates/pithos_lib/src/archive/reader_private_tests.rs b/crates/pithos_lib/src/archive/reader_private_tests.rs index 42b9b18..d4166ee 100644 --- a/crates/pithos_lib/src/archive/reader_private_tests.rs +++ b/crates/pithos_lib/src/archive/reader_private_tests.rs @@ -4,12 +4,12 @@ use super::reader::{ use crate::archive::{ ArchivePath, ArchiveWriter, CdcConfig, EntryMetadata, ProcessingOptions, WriteOptions, }; -use crate::crypto::{self, PrivateKey, PublicKey}; +use crate::crypto::{self, FileKey, PrivateKey, PublicKey}; use crate::error::PithosError; -use crate::format::limits::DeserializationLimits; +use crate::format::limits::{DeserializationError, DeserializationLimits}; use crate::format::wire::{ - BlockDataState, BlockLocation, Directory, EncryptionSection, FileType, RecipientData, - RecipientSection, + BlockDataState, BlockLocation, Directory, EncryptionSection, FileEntry, FileType, + RecipientData, RecipientSection, }; use crate::source::{ArchiveSource, MemorySource, SourceError}; use indexmap::IndexMap; @@ -273,6 +273,65 @@ fn first_block(path: &Path) -> (u64, u64) { (block.offset, block.stored_size) } +fn make_block_lists_direct(path: &Path, count: usize) { + rewrite_terminal_directory(path, |directory| { + let blocks = directory + .blocks + .iter() + .map(|(hash, descriptor)| (*hash, descriptor.original_size)) + .collect::>(); + let mut index = 0usize; + directory + .files + .try_for_each_mut(|_, file| { + if index < count { + let (hash, original_size) = blocks[index]; + file.block_data = BlockDataState::Decrypted(Zeroizing::new(vec![( + hash, + [index as u8 + 1; 32], + )])); + file.file_size = original_size; + index += 1; + } + Ok::<_, ()>(()) + }) + .unwrap(); + assert_eq!(index, count); + }); +} + +fn recover_first_file_key(path: &Path) -> FileKey { + let directory = decode_terminal_directory(path); + let recipient = private("recipient1").into_dalek_static_secret(); + let recipient_public = DalekPublicKey::from(&recipient).to_bytes(); + let (sender, section) = directory.encryption.first().unwrap(); + let data = §ion.recipients[&recipient_public].recipient_data; + let RecipientData::Encrypted(data) = data else { + panic!("writer recipient data was not encrypted"); + }; + let shared = crypto::derive_shared(recipient.as_bytes(), sender).unwrap(); + let plaintext = crypto::unwrap_recipient_list(&shared, data).unwrap(); + let records = crate::format::codec::decode_decrypted_recipient_list( + &plaintext, + &DeserializationLimits::default(), + ) + .unwrap(); + FileKey::from_protocol(&records[0].1) +} + +fn assert_block_reference_limit(result: Result) { + assert!(matches!( + result, + Err(PithosError::Deserialization( + DeserializationError::LimitExceeded { + field: "block references", + limit: 0, + actual: 1, + } + )) + )); +} + fn corrupt_payload(path: &Path, byte: usize) { let (offset, stored) = first_block(path); assert!(byte < stored as usize); @@ -857,6 +916,125 @@ fn archive_rejects_stored_size_failures_before_sink_output() { assert_copy_failure_without_sink(&stored); } +#[test] +fn direct_block_list_keys_are_used_without_file_or_recipient_keys() { + let (_temporary, path) = fixture_with("encrypted payload with a wrong direct key", 0); + rewrite_terminal_directory(&path, |directory| { + let hash = *directory.blocks.first().unwrap().0; + directory + .files + .try_for_each_mut(|_, file| { + file.block_data = BlockDataState::Decrypted(Zeroizing::new(vec![(hash, [0; 32])])); + Ok::<_, ()>(()) + }) + .unwrap(); + }); + + let archive = open_without_keys(&path).unwrap(); + let mut sink = RecordingSink(Vec::new()); + assert!(matches!( + archive.copy_to("data", &mut sink), + Err(PithosError::Crypt(_)) + )); + assert!(sink.0.is_empty()); +} + +#[test] +fn direct_block_lists_share_one_decode_budget_within_and_across_directories() { + let limits = OpenLimits { + max_accessible_block_references: 1, + ..OpenLimits::default() + }; + + let (_temporary, same_directory) = fixture_entries(&[("first", "one"), ("second", "two")]); + make_block_lists_direct(&same_directory, 2); + assert_block_reference_limit(Archive::open( + MemorySource::new(Arc::<[u8]>::from(std::fs::read(&same_directory).unwrap())), + OpenOptions::default().with_limits(limits), + )); + + let (_temporary, appended) = fixture_with("shared direct block", 0); + make_block_lists_direct(&appended, 1); + let mut bytes = std::fs::read(&appended).unwrap(); + let (parent_start, parent_len) = directory_bounds(&bytes); + let parent = decode_terminal_directory(&appended); + let (hash, descriptor) = parent.blocks.first().unwrap(); + let mut files = crate::format::entries::WireEntries::with_maximum_id(0); + files + .insert( + 1, + "child", + FileEntry { + file_type: FileType::Data, + block_data: BlockDataState::Decrypted(Zeroizing::new(vec![(*hash, [1; 32])])), + created: 0, + modified: 0, + file_size: descriptor.original_size, + permissions: 0o644, + references: Vec::new(), + symlink_target: None, + }, + ) + .unwrap(); + let mut child = Directory::new( + Some((parent_start as u64, parent_len as u64)), + files, + IndexMap::new(), + ); + crate::format::codec::update_directory_len(&mut child).unwrap(); + crate::format::codec::update_directory_crc(&mut child).unwrap(); + crate::format::codec::encode_directory(&child, &mut bytes).unwrap(); + assert_block_reference_limit(Archive::open( + MemorySource::new(Arc::<[u8]>::from(bytes)), + OpenOptions::default().with_limits(limits), + )); +} + +#[test] +fn direct_and_decrypted_encrypted_block_lists_share_one_open_budget() { + let (_temporary, path) = fixture_entries(&[("direct", "one"), ("encrypted", "two")]); + make_block_lists_direct(&path, 1); + let limits = OpenLimits { + max_accessible_block_references: 1, + ..OpenLimits::default() + }; + assert_block_reference_limit(Archive::open( + MemorySource::new(Arc::<[u8]>::from(std::fs::read(path).unwrap())), + OpenOptions::default() + .with_limits(limits) + .with_access_keys(AccessKeys::new().with_key(private("recipient1"))), + )); +} + +#[test] +fn decrypted_encrypted_block_list_accepts_a_non_minimal_count() { + let (_temporary, path) = fixture_with("non-minimal encrypted block list", 0); + let file_key = recover_first_file_key(&path); + rewrite_terminal_directory(&path, |directory| { + directory + .files + .try_for_each_mut(|_, file| { + let BlockDataState::Encrypted(encrypted) = &mut file.block_data else { + panic!("writer block list was not encrypted"); + }; + let plaintext = crypto::open_file_block_list(&file_key, encrypted).unwrap(); + assert_eq!(plaintext[0], 1); + let mut non_minimal = vec![0x81, 0x00]; + non_minimal.extend_from_slice(&plaintext[1..]); + *encrypted = + crypto::seal_file_block_list_with_nonce(&file_key, &non_minimal, [0x5a; 12]) + .unwrap(); + Ok::<_, ()>(()) + }) + .unwrap(); + }); + + let archive = open_path(&path, AccessKeys::new().with_key(private("recipient1"))); + let mut output = Vec::new(); + archive.copy_to("data", &mut output).unwrap(); + assert_eq!(output, b"non-minimal encrypted block list"); +} + #[test] fn archive_open_validates_local_markers_without_reading_payloads_or_access_keys() { let (_temporary, path) = fixture_with("marker validation payload", 0); diff --git a/crates/pithos_lib/src/format/codec.rs b/crates/pithos_lib/src/format/codec.rs index 8136444..6ade402 100644 --- a/crates/pithos_lib/src/format/codec.rs +++ b/crates/pithos_lib/src/format/codec.rs @@ -253,6 +253,7 @@ fn encode_block_data( fn decode_block_data( reader: &mut R, limits: &DeserializationLimits, + remaining_block_references: &mut u64, ) -> Result { let mut tag = [0]; reader.read_exact(&mut tag)?; @@ -270,7 +271,11 @@ fn decode_block_data( Ok(BlockDataState::Encrypted(bytes)) } 1 => Ok(BlockDataState::Decrypted( - decode_decrypted_block_list_reader(reader, limits)?, + decode_decrypted_block_list_reader_with_budget( + reader, + limits, + remaining_block_references, + )?, )), value => Err(DeserializationError::InvalidEnumValue(value)), } @@ -304,9 +309,10 @@ fn decode_file_entry( reader: &mut R, limits: &DeserializationLimits, remaining_references: &mut u64, + remaining_block_references: &mut u64, ) -> Result { let file_type = decode_file_type(reader)?; - let block_data = decode_block_data(reader, limits)?; + let block_data = decode_block_data(reader, limits, remaining_block_references)?; let created = reader.read_varint::()?; let modified = reader.read_varint::()?; let file_size = reader.read_varint::()?; @@ -463,9 +469,19 @@ pub(crate) fn encode_directory( Ok(()) } +#[allow(dead_code)] pub(crate) fn decode_directory( reader: &mut R, limits: &DeserializationLimits, +) -> Result { + let mut remaining_block_references = limits.max_block_references; + decode_directory_with_budget(reader, limits, &mut remaining_block_references) +} + +pub(crate) fn decode_directory_with_budget( + reader: &mut R, + limits: &DeserializationLimits, + remaining_block_references: &mut u64, ) -> Result { let mut identifier = [0; 8]; reader.read_exact(&mut identifier)?; @@ -506,7 +522,12 @@ pub(crate) fn decode_directory( files.insert( id, path, - decode_file_entry(reader, limits, &mut remaining_references)?, + decode_file_entry( + reader, + limits, + &mut remaining_references, + remaining_block_references, + )?, )?; } let block_count = bounded_len( @@ -576,9 +597,19 @@ pub(crate) fn decode_directory( Ok(directory) } +#[allow(dead_code)] pub(crate) fn decode_complete_directory( bytes: &[u8], limits: &DeserializationLimits, +) -> Result { + let mut remaining_block_references = limits.max_block_references; + decode_complete_directory_with_budget(bytes, limits, &mut remaining_block_references) +} + +pub(crate) fn decode_complete_directory_with_budget( + bytes: &[u8], + limits: &DeserializationLimits, + remaining_block_references: &mut u64, ) -> Result { if bytes.len() < MIN_DIRECTORY_LEN { return Err(PithosError::DirectoryLengthMismatch { @@ -619,7 +650,7 @@ pub(crate) fn decode_complete_directory( }); } let mut reader = Cursor::new(bytes); - let directory = decode_directory(&mut reader, limits)?; + let directory = decode_directory_with_budget(&mut reader, limits, remaining_block_references)?; if reader.position() != bytes.len() as u64 { return Err(PithosError::DirectoryConsumptionMismatch { expected: bytes.len() as u64, @@ -658,17 +689,28 @@ pub(crate) fn encode_decrypted_block_list( Ok(()) } +#[allow(dead_code)] pub(crate) fn decode_decrypted_block_list_reader( reader: &mut R, limits: &DeserializationLimits, +) -> Result>, DeserializationError> { + let mut remaining_block_references = limits.max_block_references; + decode_decrypted_block_list_reader_with_budget(reader, limits, &mut remaining_block_references) +} + +fn decode_decrypted_block_list_reader_with_budget( + reader: &mut R, + limits: &DeserializationLimits, + remaining_block_references: &mut u64, ) -> Result>, DeserializationError> { let count = bounded_len( reader.read_varint::()?, - limits.max_collection_entries, - "block index", + limits.max_block_references.min(*remaining_block_references), + "block references", )?; + *remaining_block_references -= count as u64; let mut entries = Zeroizing::new(Vec::new()); - reserve(&mut entries, count, "block index")?; + reserve(&mut entries, count, "block references")?; let mut keys = HashMap::with_capacity(count); for _ in 0..count { let mut hash = [0; 32]; @@ -686,12 +728,26 @@ pub(crate) fn decode_decrypted_block_list_reader( Ok(entries) } +#[allow(dead_code)] pub(crate) fn decode_decrypted_block_list( bytes: &[u8], limits: &DeserializationLimits, +) -> Result>, DeserializationError> { + let mut remaining_block_references = limits.max_block_references; + decode_decrypted_block_list_with_budget(bytes, limits, &mut remaining_block_references) +} + +pub(crate) fn decode_decrypted_block_list_with_budget( + bytes: &[u8], + limits: &DeserializationLimits, + remaining_block_references: &mut u64, ) -> Result>, DeserializationError> { let mut reader = Cursor::new(bytes); - let entries = decode_decrypted_block_list_reader(&mut reader, limits)?; + let entries = decode_decrypted_block_list_reader_with_budget( + &mut reader, + limits, + remaining_block_references, + )?; if reader.position() != bytes.len() as u64 { return Err(DeserializationError::InvalidLength); } diff --git a/crates/pithos_lib/src/format/limits.rs b/crates/pithos_lib/src/format/limits.rs index b8c99eb..4dfb5f9 100644 --- a/crates/pithos_lib/src/format/limits.rs +++ b/crates/pithos_lib/src/format/limits.rs @@ -41,6 +41,7 @@ pub struct DeserializationLimits { pub max_collection_entries: u64, pub max_file_entries: u64, pub max_block_descriptors: u64, + pub max_block_references: u64, pub max_references: u64, pub max_relationships: u64, } @@ -53,6 +54,7 @@ impl Default for DeserializationLimits { max_collection_entries: 1_000_000, max_file_entries: 1_000_000, max_block_descriptors: 1_000_000, + max_block_references: 1_000_000, max_references: 1_000_000, max_relationships: 1_000_000, } diff --git a/crates/pithos_lib/tests/draft_spec_vectors.rs b/crates/pithos_lib/tests/draft_spec_vectors.rs index cac38a1..e4c2b68 100644 --- a/crates/pithos_lib/tests/draft_spec_vectors.rs +++ b/crates/pithos_lib/tests/draft_spec_vectors.rs @@ -4,7 +4,7 @@ use chacha20poly1305::{ChaCha20Poly1305, KeyInit, Nonce, aead::Aead}; use crc32fast::hash as crc32; -use pithos_lib::archive::{Archive, OpenOptions}; +use pithos_lib::archive::{Archive, EntryKind, OpenLimits, OpenOptions}; use pithos_lib::error::{DeserializationError, PithosError}; use pithos_lib::source::MemorySource; use std::io::Cursor; @@ -347,6 +347,49 @@ fn production_reader_opens_the_step_1_canonical_vectors() { } } +#[test] +fn production_reader_reads_cv_local_hello_without_access_keys() { + let archive = Archive::open( + MemorySource::new(appendix_hex("CV-LOCAL-HELLO-279")), + OpenOptions::default(), + ) + .unwrap(); + assert!(matches!( + archive.entry("hello").unwrap().unwrap().kind, + EntryKind::File { + size: 5, + available: true + } + )); + let mut output = Vec::new(); + archive.copy_to("hello", &mut output).unwrap(); + assert_eq!(output, b"hello"); + output.clear(); + archive.copy_range_to("hello", 1..4, &mut output).unwrap(); + assert_eq!(output, b"ell"); +} + +#[test] +fn production_reader_rejects_a_direct_list_before_over_budget_allocation() { + let limits = OpenLimits { + max_accessible_block_references: 0, + ..OpenLimits::default() + }; + assert!(matches!( + Archive::open( + MemorySource::new(appendix_hex("CV-LOCAL-HELLO-279")), + OpenOptions::default().with_limits(limits), + ), + Err(PithosError::Deserialization( + DeserializationError::LimitExceeded { + field: "block references", + limit: 0, + actual: 1, + } + )) + )); +} + #[test] fn production_reader_accepts_bounded_non_minimal_uleb128() { Archive::open( From d56541eb540014a00a66051020f0b0f4f6464e44 Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Sun, 13 Sep 2026 15:25:15 +0200 Subject: [PATCH 045/166] feat: require external access policy --- crates/pithos_lib/src/archive/mod.rs | 4 +- crates/pithos_lib/src/archive/reader.rs | 149 +++++++++- .../src/archive/reader_private_tests.rs | 267 +++++++++++++++++- crates/pithos_lib/src/error.rs | 8 +- crates/pithos_lib/tests/public_api.rs | 31 +- 5 files changed, 435 insertions(+), 24 deletions(-) diff --git a/crates/pithos_lib/src/archive/mod.rs b/crates/pithos_lib/src/archive/mod.rs index 6494a15..bf7564f 100644 --- a/crates/pithos_lib/src/archive/mod.rs +++ b/crates/pithos_lib/src/archive/mod.rs @@ -25,8 +25,8 @@ pub(crate) use path_validation::{ }; pub(crate) use reader::ContentOperationError; pub use reader::{ - AccessKeys, Archive, ArchiveEntry, ArchiveReference, EntryKind, ExternalBlockResolver, - NoExternalBlocks, OpenLimits, OpenOptions, + AccessKeys, Archive, ArchiveEntry, ArchiveFeature, ArchiveReference, EntryKind, + ExternalBlockAccessPolicy, ExternalBlockResolver, NoExternalBlocks, OpenLimits, OpenOptions, }; pub(crate) use snapshot::AppendSnapshot; pub use types::{ArchivePath, ExternalLocation}; diff --git a/crates/pithos_lib/src/archive/reader.rs b/crates/pithos_lib/src/archive/reader.rs index 255a57e..771dd08 100644 --- a/crates/pithos_lib/src/archive/reader.rs +++ b/crates/pithos_lib/src/archive/reader.rs @@ -14,8 +14,9 @@ use crate::error::PithosError; use crate::format::limits::DeserializationLimits; use crate::format::wire::{BlockDataEntry, BlockDataState, BlockIndexEntry, Directory, FileHeader}; use crate::source::ArchiveSource; -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::io::Write; +use std::sync::Arc; /// Distinguishes archive failures from a presentation callback failure without /// making the archive core depend on the callback's error type. @@ -112,15 +113,48 @@ impl AccessKeys { } /// Resolves an opaque external block location to exactly one framed `BLCK` value. +/// +/// The resolver owns interpretation of the opaque location and selection of concrete +/// access targets. It must call the supplied policy before its initial access and before +/// every redirect. The [`ExternalLocation`] is archive-owned opaque data and is not +/// necessarily itself a concrete policy target. pub trait ExternalBlockResolver { fn resolve( &self, + policy: &dyn ExternalBlockAccessPolicy, location: &ExternalLocation, expected_len: u64, max_response_size: u64, ) -> Result, PithosError>; } +/// Describes a feature that prevents content access without changing archive structure. +#[non_exhaustive] +#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)] +pub enum ArchiveFeature { + Compression, + BlockEncryption, + EncryptedBlockList, + EncryptedRecipientList, + ExternalStorage, +} + +/// Decides whether an external resolver may access a concrete target. +/// +/// Resolvers select concrete targets from an opaque [`ExternalLocation`] and must call +/// this policy before the initial access and before every redirect. The policy is not +/// called by the archive core for the opaque location itself. +pub trait ExternalBlockAccessPolicy: Send + Sync { + fn allows(&self, target: &str) -> bool; +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum ContentAvailability { + Available, + MissingAccess, + Unsupported(ArchiveFeature), +} + /// The default resolver rejects external block reads. #[derive(Clone, Copy, Debug, Default)] pub struct NoExternalBlocks; @@ -128,11 +162,14 @@ pub struct NoExternalBlocks; impl ExternalBlockResolver for NoExternalBlocks { fn resolve( &self, + _policy: &dyn ExternalBlockAccessPolicy, _location: &ExternalLocation, _expected_len: u64, _max_response_size: u64, ) -> Result, PithosError> { - Err(PithosError::ExternalBlockSourceRequired) + Err(PithosError::UnsupportedFeature( + ArchiveFeature::ExternalStorage, + )) } } @@ -141,6 +178,8 @@ pub struct OpenOptions { limits: OpenLimits, keys: AccessKeys, external: E, + external_resolver_supplied: bool, + external_access_policy: Option>, } impl Default for OpenOptions { @@ -149,6 +188,8 @@ impl Default for OpenOptions { limits: OpenLimits::default(), keys: AccessKeys::default(), external: NoExternalBlocks, + external_resolver_supplied: false, + external_access_policy: None, } } } @@ -169,8 +210,18 @@ impl OpenOptions { limits: self.limits, keys: self.keys, external, + external_resolver_supplied: true, + external_access_policy: self.external_access_policy, } } + + pub fn with_external_access_policy( + mut self, + policy: Arc, + ) -> Self { + self.external_access_policy = Some(policy); + self + } } /// Public, secret-free entry shape returned by an opened archive. @@ -209,6 +260,7 @@ pub struct ArchiveReference { pub struct Archive { source: S, external: E, + external_access_policy: Option>, archive_len: u64, terminal_directory: Span, index: ArchiveIndex, @@ -217,6 +269,7 @@ pub struct Archive { access: ResolvedAccess, access_keys: AccessKeys, limits: OpenLimits, + content_availability: BTreeMap, } impl Archive @@ -340,6 +393,11 @@ where max_segments: options.limits.max_parent_directories.saturating_add(1), }; let index = build_effective_index(&segments, archive_len, index_limits)?; + let content_availability = classify_content_availability( + &index, + options.external_resolver_supplied, + options.external_access_policy.is_some(), + )?; for span in index.local_block_spans() { let mut marker = [0; 4]; source.read_exact_at(span.start(), &mut marker)?; @@ -349,6 +407,7 @@ where Ok(Self { source, external: options.external, + external_access_policy: options.external_access_policy, archive_len, terminal_directory: Span::new(terminal_start, terminal_len)?, index, @@ -357,13 +416,14 @@ where access, access_keys: options.keys, limits: options.limits, + content_availability, }) } pub fn entries(&self) -> impl ExactSizeIterator + '_ { self.index .entries() - .map(|entry| archive_entry(&self.index, entry)) + .map(|entry| archive_entry(&self.index, entry, &self.content_availability)) } /// Consumes the reader and transfers its validated state to append/grant planning. @@ -392,11 +452,12 @@ where Ok(self .index .entry_at_path(&path) - .map(|entry| archive_entry(&self.index, entry))) + .map(|entry| archive_entry(&self.index, entry, &self.content_availability))) } pub fn copy_to(&self, path: &str, sink: &mut W) -> Result<(), PithosError> { let (id, _) = self.content_id(path)?; + self.require_content_available(id)?; self.copy_plan(id, self.index.full_file_plan(id)?, sink) } @@ -408,6 +469,7 @@ where ) -> Result<(), PithosError> { let (id, size) = self.content_id(path)?; let range = ReadRange::new(range, size)?; + self.require_content_available(id)?; self.copy_plan(id, self.index.range_plan(id, range)?, sink) } @@ -417,6 +479,8 @@ where operation: impl FnOnce(FileId, &PrivateKey, &FileKey) -> Result, ) -> Result> { let (id, _) = self.content_id(path).map_err(ContentOperationError::Core)?; + self.require_content_available(id) + .map_err(ContentOperationError::Core)?; let key = self .access .key(id) @@ -441,6 +505,8 @@ where id: FileId, mut operation: impl FnMut(Zeroizing>) -> Result<(), CallbackError>, ) -> Result<(), ContentOperationError> { + self.require_content_available(id) + .map_err(ContentOperationError::Core)?; let plan = self .index .full_file_plan(id) @@ -466,6 +532,19 @@ where Ok((entry.id, content.size)) } + pub(crate) fn require_content_available(&self, id: FileId) -> Result<(), PithosError> { + match self.content_availability.get(&id).copied() { + Some(ContentAvailability::Available) => Ok(()), + Some(ContentAvailability::MissingAccess) => Err(PithosError::ContentUnavailable), + Some(ContentAvailability::Unsupported(feature)) => { + Err(PithosError::UnsupportedFeature(feature)) + } + None => Err(PithosError::InvalidBlockDataState( + "only data/metadata entries have content".into(), + )), + } + } + fn copy_plan( &self, id: FileId, @@ -526,6 +605,9 @@ where PithosError::ExternalBlockFraming("response size overflow".into()) })?; let response = self.external.resolve( + self.external_access_policy.as_deref().ok_or( + PithosError::UnsupportedFeature(ArchiveFeature::ExternalStorage), + )?, location, expected_len, self.limits @@ -571,15 +653,25 @@ where } } -fn entry_kind(entry: &Entry) -> EntryKind { +fn entry_kind( + entry: &Entry, + id: FileId, + content_availability: &BTreeMap, +) -> EntryKind { match entry { Entry::File(content) => EntryKind::File { size: content.size, - available: matches!(content.content, ContentState::Available(_)), + available: matches!( + content_availability.get(&id), + Some(ContentAvailability::Available) + ), }, Entry::Metadata(content) => EntryKind::Metadata { size: content.size, - available: matches!(content.content, ContentState::Available(_)), + available: matches!( + content_availability.get(&id), + Some(ContentAvailability::Available) + ), }, Entry::Directory(_) => EntryKind::Directory, Entry::Symlink { target, .. } => EntryKind::Symlink { @@ -588,12 +680,16 @@ fn entry_kind(entry: &Entry) -> EntryKind { } } -fn archive_entry(index: &ArchiveIndex, entry: &super::index::IndexedEntry) -> ArchiveEntry { +fn archive_entry( + index: &ArchiveIndex, + entry: &super::index::IndexedEntry, + content_availability: &BTreeMap, +) -> ArchiveEntry { let metadata = entry.entry.metadata(); ArchiveEntry { id: entry.id.0, path: entry.path.as_str().to_owned(), - kind: entry_kind(&entry.entry), + kind: entry_kind(&entry.entry, entry.id, content_availability), created: metadata.created, modified: metadata.modified, permissions: metadata.permissions, @@ -611,6 +707,41 @@ fn archive_entry(index: &ArchiveIndex, entry: &super::index::IndexedEntry) -> Ar } } +fn classify_content_availability( + index: &ArchiveIndex, + external_resolver_supplied: bool, + external_access_policy_supplied: bool, +) -> Result, PithosError> { + let external_supported = external_resolver_supplied && external_access_policy_supplied; + let mut availability = BTreeMap::new(); + for entry in index.entries() { + let Some(content) = entry.entry.content() else { + continue; + }; + let state = match &content.content { + ContentState::Unavailable => ContentAvailability::MissingAccess, + ContentState::Available(references) => { + let has_external_block = references.iter().any(|hash| { + matches!( + index.descriptor(hash), + Some(crate::archive::types::BlockDescriptor { + location: BlockLocation::External(_), + .. + }) + ) + }); + if has_external_block && !external_supported { + ContentAvailability::Unsupported(ArchiveFeature::ExternalStorage) + } else { + ContentAvailability::Available + } + } + }; + availability.insert(entry.id, state); + } + Ok(availability) +} + fn deserialization_limits(limits: OpenLimits) -> DeserializationLimits { DeserializationLimits { max_collection_entries: limits.max_entries, diff --git a/crates/pithos_lib/src/archive/reader_private_tests.rs b/crates/pithos_lib/src/archive/reader_private_tests.rs index d4166ee..3aa2c13 100644 --- a/crates/pithos_lib/src/archive/reader_private_tests.rs +++ b/crates/pithos_lib/src/archive/reader_private_tests.rs @@ -1,5 +1,6 @@ use super::reader::{ - AccessKeys, Archive, EntryKind, ExternalBlockResolver, OpenLimits, OpenOptions, + AccessKeys, Archive, ArchiveFeature, EntryKind, ExternalBlockAccessPolicy, + ExternalBlockResolver, OpenLimits, OpenOptions, }; use crate::archive::{ ArchivePath, ArchiveWriter, CdcConfig, EntryMetadata, ProcessingOptions, WriteOptions, @@ -485,13 +486,40 @@ struct CountingResolver { expected: Arc>>, } +const INITIAL_TARGET: &str = "https://storage.test/initial"; +const REDIRECT_TARGET: &str = "https://storage.test/redirect"; + +#[derive(Clone)] +struct RecordingPolicy { + checks: Arc>>, + denied: Option<&'static str>, +} + +impl ExternalBlockAccessPolicy for RecordingPolicy { + fn allows(&self, target: &str) -> bool { + self.checks.lock().unwrap().push(target.to_owned()); + self.denied != Some(target) + } +} + +fn allowing_policy() -> Arc { + Arc::new(RecordingPolicy { + checks: Arc::new(Mutex::new(Vec::new())), + denied: None, + }) +} + impl ExternalBlockResolver for CountingResolver { fn resolve( &self, + policy: &dyn ExternalBlockAccessPolicy, _location: &super::types::ExternalLocation, expected_len: u64, max_response_size: u64, ) -> Result, PithosError> { + if !policy.allows(INITIAL_TARGET) { + return Err(PithosError::ExternalBlockAccessDenied); + } self.calls.fetch_add(1, Ordering::Relaxed); self.expected .lock() @@ -501,6 +529,30 @@ impl ExternalBlockResolver for CountingResolver { } } +#[derive(Clone)] +struct RedirectingResolver { + response: Arc<[u8]>, + accessed: Arc>>, +} + +impl ExternalBlockResolver for RedirectingResolver { + fn resolve( + &self, + policy: &dyn ExternalBlockAccessPolicy, + _location: &super::types::ExternalLocation, + _expected_len: u64, + _max_response_size: u64, + ) -> Result, PithosError> { + for target in [INITIAL_TARGET, REDIRECT_TARGET] { + if !policy.allows(target) { + return Err(PithosError::ExternalBlockAccessDenied); + } + self.accessed.lock().unwrap().push(target.to_owned()); + } + Ok(self.response.to_vec()) + } +} + fn as_external(path: &Path) -> Vec { let bytes = std::fs::read(path).unwrap(); let (offset, stored) = first_block(path); @@ -515,6 +567,22 @@ fn as_external(path: &Path) -> Vec { response } +fn last_block_as_external(path: &Path) -> Vec { + let bytes = std::fs::read(path).unwrap(); + let directory = decode_terminal_directory(path); + let descriptor = directory.blocks.last().unwrap().1; + let start = descriptor.offset as usize; + let response = bytes[start..start + 4 + descriptor.stored_size as usize].to_vec(); + rewrite_terminal_directory(path, |directory| { + let block = directory.blocks.last_mut().unwrap().1; + block.location = BlockLocation::External { + url: "test:external".into(), + }; + block.offset = u64::MAX; + }); + response +} + struct CountingSource { bytes: Arc<[u8]>, reads: Arc, @@ -1130,7 +1198,8 @@ fn archive_external_blocks_validate_exact_framing_and_share_read_paths() { MemorySource::new(Arc::<[u8]>::from(std::fs::read(&path).unwrap())), OpenOptions::default() .with_access_keys(AccessKeys::new().with_key(private("recipient1"))) - .with_external_resolver(resolver), + .with_external_resolver(resolver) + .with_external_access_policy(allowing_policy()), ) .unwrap(); let mut full = Vec::new(); @@ -1156,7 +1225,9 @@ fn archive_external_blocks_validate_exact_framing_and_share_read_paths() { .unwrap(); assert!(matches!( missing.copy_to("data", &mut Vec::new()), - Err(PithosError::ExternalBlockSourceRequired) + Err(PithosError::UnsupportedFeature( + ArchiveFeature::ExternalStorage + )) )); let mut corrupt = response.clone(); corrupt[4] ^= 1; @@ -1175,7 +1246,8 @@ fn archive_external_blocks_validate_exact_framing_and_share_read_paths() { MemorySource::new(Arc::<[u8]>::from(std::fs::read(&path).unwrap())), OpenOptions::default() .with_access_keys(AccessKeys::new().with_key(private("recipient1"))) - .with_external_resolver(resolver), + .with_external_resolver(resolver) + .with_external_access_policy(allowing_policy()), ) .unwrap(); assert!(archive.copy_to("data", &mut Vec::new()).is_err()); @@ -1223,7 +1295,8 @@ fn archive_block_limits_reject_before_local_or_external_acquisition() { OpenOptions::default() .with_limits(limits) .with_access_keys(AccessKeys::new().with_key(private("recipient1"))) - .with_external_resolver(resolver), + .with_external_resolver(resolver) + .with_external_access_policy(allowing_policy()), ) .unwrap(); assert!(matches!( @@ -1260,5 +1333,189 @@ fn archive_block_limits_reject_before_local_or_external_acquisition() { )); assert_eq!(reads.load(Ordering::Relaxed), before); } + +#[test] +fn external_availability_requires_both_resolver_and_policy() { + let (_temporary, path) = fixture(); + let response = as_external(&path); + let bytes = Arc::<[u8]>::from(std::fs::read(path).unwrap()); + let calls = Arc::new(AtomicUsize::new(0)); + let resolver = CountingResolver { + response: Arc::from(response.clone()), + calls: Arc::clone(&calls), + expected: Arc::new(Mutex::new(Vec::new())), + }; + let keys = || AccessKeys::new().with_key(private("recipient1")); + + let default = Archive::open( + MemorySource::new(Arc::clone(&bytes)), + OpenOptions::default().with_access_keys(keys()), + ) + .unwrap(); + assert!(matches!( + default.entries().next().unwrap().kind, + EntryKind::File { + available: false, + .. + } + )); + assert!(matches!( + default.copy_range_to("data", 0..0, &mut Vec::new()), + Err(PithosError::UnsupportedFeature( + ArchiveFeature::ExternalStorage + )) + )); + + let resolver_only = Archive::open( + MemorySource::new(Arc::clone(&bytes)), + OpenOptions::default() + .with_access_keys(keys()) + .with_external_resolver(resolver.clone()), + ) + .unwrap(); + assert!(matches!( + resolver_only.copy_to("data", &mut Vec::new()), + Err(PithosError::UnsupportedFeature( + ArchiveFeature::ExternalStorage + )) + )); + assert_eq!(calls.load(Ordering::Relaxed), 0); + + let policy_only = Archive::open( + MemorySource::new(Arc::clone(&bytes)), + OpenOptions::default() + .with_access_keys(keys()) + .with_external_access_policy(allowing_policy()), + ) + .unwrap(); + assert!(matches!( + policy_only.copy_to("data", &mut Vec::new()), + Err(PithosError::UnsupportedFeature( + ArchiveFeature::ExternalStorage + )) + )); + + for options in [ + OpenOptions::default() + .with_access_keys(keys()) + .with_external_resolver(resolver.clone()) + .with_external_access_policy(allowing_policy()), + OpenOptions::default() + .with_access_keys(keys()) + .with_external_access_policy(allowing_policy()) + .with_external_resolver(resolver.clone()), + ] { + let archive = Archive::open(MemorySource::new(Arc::clone(&bytes)), options).unwrap(); + assert!(matches!( + archive.entries().next().unwrap().kind, + EntryKind::File { + available: true, + .. + } + )); + let mut output = Vec::new(); + archive.copy_to("data", &mut output).unwrap(); + assert_eq!(output, b"archive reader private fixture"); + } +} + +#[test] +fn external_policy_denial_precedes_initial_and_redirect_access() { + let (_temporary, path) = fixture(); + let response = as_external(&path); + let bytes = Arc::<[u8]>::from(std::fs::read(path).unwrap()); + let checks = Arc::new(Mutex::new(Vec::new())); + let calls = Arc::new(AtomicUsize::new(0)); + let archive = Archive::open( + MemorySource::new(Arc::clone(&bytes)), + OpenOptions::default() + .with_access_keys(AccessKeys::new().with_key(private("recipient1"))) + .with_external_resolver(CountingResolver { + response: Arc::from(response.clone()), + calls: Arc::clone(&calls), + expected: Arc::new(Mutex::new(Vec::new())), + }) + .with_external_access_policy(Arc::new(RecordingPolicy { + checks: Arc::clone(&checks), + denied: Some(INITIAL_TARGET), + })), + ) + .unwrap(); + assert!(matches!( + archive.entries().next().unwrap().kind, + EntryKind::File { + available: true, + .. + } + )); + assert!(matches!( + archive.copy_to("data", &mut Vec::new()), + Err(PithosError::ExternalBlockAccessDenied) + )); + assert_eq!(calls.load(Ordering::Relaxed), 0); + assert_eq!(&*checks.lock().unwrap(), &[INITIAL_TARGET]); + + let checks = Arc::new(Mutex::new(Vec::new())); + let accessed = Arc::new(Mutex::new(Vec::new())); + let archive = Archive::open( + MemorySource::new(bytes), + OpenOptions::default() + .with_access_keys(AccessKeys::new().with_key(private("recipient1"))) + .with_external_resolver(RedirectingResolver { + response: Arc::from(response), + accessed: Arc::clone(&accessed), + }) + .with_external_access_policy(Arc::new(RecordingPolicy { + checks: Arc::clone(&checks), + denied: Some(REDIRECT_TARGET), + })), + ) + .unwrap(); + assert!(matches!( + archive.copy_to("data", &mut Vec::new()), + Err(PithosError::ExternalBlockAccessDenied) + )); + assert_eq!(&*checks.lock().unwrap(), &[INITIAL_TARGET, REDIRECT_TARGET]); + assert_eq!(&*accessed.lock().unwrap(), &[INITIAL_TARGET]); + assert!( + !checks + .lock() + .unwrap() + .iter() + .any(|target| target == "test:external") + ); +} + +#[test] +fn unsupported_external_content_preflights_the_whole_file_and_adapters() { + let content = format!("{}{}", "x".repeat(1024), "y".repeat(1024)); + let (_temporary, path) = fixture_with_options( + &content, + 0, + true, + Some(CdcConfig::new(64, 256, 1024).unwrap()), + ); + assert!(decode_terminal_directory(&path).blocks.len() > 1); + let _response = last_block_as_external(&path); + let archive = open_path(&path, AccessKeys::new().with_key(private("recipient1"))); + let mut sink = RecordingSink(Vec::new()); + assert!(matches!( + archive.copy_to("data", &mut sink), + Err(PithosError::UnsupportedFeature( + ArchiveFeature::ExternalStorage + )) + )); + assert!(sink.0.is_empty()); + + let mut exported = Vec::new(); + assert!(matches!( + crypt4gh::export(&archive, "data", vec![public("recipient2")], &mut exported,), + Err(crate::adapters::crypt4gh::Crypt4GHError::Archive { + source: PithosError::UnsupportedFeature(ArchiveFeature::ExternalStorage), + .. + }) + )); + assert!(exported.is_empty()); +} use crate::adapters::crypt4gh; use crate::fs::extract; diff --git a/crates/pithos_lib/src/error.rs b/crates/pithos_lib/src/error.rs index 943b192..1bd8192 100644 --- a/crates/pithos_lib/src/error.rs +++ b/crates/pithos_lib/src/error.rs @@ -1,3 +1,4 @@ +use crate::archive::ArchiveFeature; use crate::crypto::CryptoError; use crate::source::SourceError; use std::io; @@ -8,6 +9,7 @@ pub use crate::format::limits::DeserializationError; /// Custom top-level error type for all of Pithos #[derive(Error, Debug)] +#[non_exhaustive] pub enum PithosError { #[error("I/O error: {0}")] Io(#[from] io::Error), @@ -57,6 +59,10 @@ pub enum PithosError { InvalidEntryCombination { path: String, reason: String }, #[error("content is unavailable with the supplied access keys")] ContentUnavailable, + #[error("archive uses unsupported feature: {0:?}")] + UnsupportedFeature(ArchiveFeature), + #[error("external block access denied by policy")] + ExternalBlockAccessDenied, #[error("append snapshot does not contain file id {0}")] SnapshotFileIdNotFound(u64), #[error("content for snapshot file id {0} is unavailable with the supplied access keys")] @@ -65,8 +71,6 @@ pub enum PithosError { SnapshotDirectoryHasNoContent(u64), #[error("snapshot file id {0} is a symlink and has no content key")] SnapshotSymlinkHasNoContent(u64), - #[error("external block source required")] - ExternalBlockSourceRequired, #[error("external block framing error: {0}")] ExternalBlockFraming(String), #[error("Block hash not found: {0:?}")] diff --git a/crates/pithos_lib/tests/public_api.rs b/crates/pithos_lib/tests/public_api.rs index b122060..3b36daa 100644 --- a/crates/pithos_lib/tests/public_api.rs +++ b/crates/pithos_lib/tests/public_api.rs @@ -1,10 +1,10 @@ use pithos_lib::adapters::{crypt4gh::Crypt4GHError, ro_crate::RoCrateError}; use pithos_lib::archive::{ - AccessKeys, AppendDurability, AppendObservation, AppendOptions, Archive, ArchivePath, - ArchiveReference, ArchiveWriter, CdcConfig, CreateError, EntryKind, EntryMetadata, - EntryReference, ExternalBlockResolver, ExternalLocation, FinishError, IncompleteWriter, - NoExternalBlocks, OpenLimits, OpenOptions, ProcessingOptions, WriteOptions, WriterError, - WrittenEntry, + AccessKeys, AppendDurability, AppendObservation, AppendOptions, Archive, ArchiveFeature, + ArchivePath, ArchiveReference, ArchiveWriter, CdcConfig, CreateError, EntryKind, EntryMetadata, + EntryReference, ExternalBlockAccessPolicy, ExternalBlockResolver, ExternalLocation, + FinishError, IncompleteWriter, NoExternalBlocks, OpenLimits, OpenOptions, ProcessingOptions, + WriteOptions, WriterError, WrittenEntry, }; use pithos_lib::crypto::{CryptoError, PrivateKey, PublicKey}; use pithos_lib::error::PithosError; @@ -19,11 +19,21 @@ struct SendSyncResolver; impl ExternalBlockResolver for SendSyncResolver { fn resolve( &self, + _policy: &dyn ExternalBlockAccessPolicy, _location: &ExternalLocation, _expected_len: u64, _max_response_size: u64, ) -> Result, PithosError> { - Err(PithosError::ExternalBlockSourceRequired) + Err(PithosError::ExternalBlockAccessDenied) + } +} + +#[derive(Clone, Copy)] +struct SendSyncPolicy; + +impl ExternalBlockAccessPolicy for SendSyncPolicy { + fn allows(&self, _target: &str) -> bool { + true } } @@ -33,7 +43,15 @@ fn standard_archive_configurations_are_send_and_sync() { assert_send_sync::(); assert_send_sync::>(); assert_send_sync::>(); + assert_send_sync::(); + assert_send_sync::>(); let _ = OpenOptions::default().with_access_keys(AccessKeys::new()); + let _ = OpenOptions::default() + .with_external_access_policy(std::sync::Arc::new(SendSyncPolicy)) + .with_external_resolver(SendSyncResolver); + let _ = OpenOptions::default() + .with_external_resolver(SendSyncResolver) + .with_external_access_policy(std::sync::Arc::new(SendSyncPolicy)); } #[test] @@ -42,6 +60,7 @@ fn selected_08_api_imports_compile() { let _ = std::any::TypeId::of::(); let _ = std::any::TypeId::of::(); let _ = std::any::TypeId::of::(); + let _ = std::any::TypeId::of::(); let _ = std::any::TypeId::of::(); let _ = std::any::TypeId::of::>>(); let _ = std::any::TypeId::of::(); From 5212a816b38cbb8dd2b02feaa701d631c11300a5 Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Sun, 13 Sep 2026 15:47:19 +0200 Subject: [PATCH 046/166] feat: add plain base archives --- AGENTS.md | 2 +- README.md | 4 +- crates/pithos/README.md | 19 +- crates/pithos/src/main.rs | 70 ++++--- crates/pithos/tests/create.rs | 120 ++++++++++++ crates/pithos_lib/README.md | 24 ++- crates/pithos_lib/src/archive/writer.rs | 175 ++++++++++++------ crates/pithos_lib/src/error.rs | 2 + crates/pithos_lib/src/lib.rs | 9 +- crates/pithos_lib/tests/draft_spec_vectors.rs | 20 +- crates/pithos_lib/tests/public_api.rs | 1 + .../pithos_lib/tests/writer_configuration.rs | 71 ++++++- 12 files changed, 424 insertions(+), 93 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 9e91ac0..c7d9725 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,7 +10,7 @@ ## Compatibility And Tests -- Current public structs and tests define 0.8 behavior; the Pithos 1.0 draft is background, not an interoperability guarantee. Do not expose old `model`, `helpers`, or wire-record paths. +- This branch implements the Pithos 1.0 draft wire rules; Cargo package versioning is separate. Do not expose old `model`, `helpers`, or wire-record paths. - Treat changes under `src/format/`, archive validation/indexing, encryption, compression, flags, indexes, or directory layout as on-disk compatibility changes. Do not casually normalize the currently inconsistent `.pto`, `.pith`, and `.pithos` extensions. - `pithos_lib` integration tests use committed fixtures and test-only PEM keys in `crates/pithos_lib/tests/data/`; reuse helpers from `tests/common/`. - Put wire-format unit coverage in `src/format/`, reader-internal coverage in `src/archive/reader_private_tests.rs`, and public extraction/range coverage in `crates/pithos_lib/tests/reader.rs`. Keep RO-Crate tests in the `ro_crate_{directory,zip,conversion}` integration targets. diff --git a/README.md b/README.md index e5178db..592b401 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@

    A secure archive format and Rust implementation for research-data packaging.

    -Pithos packages files into an encrypted, chunked archive that can be read sequentially or by byte range. The Rust implementation supports streaming creation, validated opening, filesystem ingestion and extraction, archive extension, RO-Crate conversion, and Crypt4GH export. +Pithos packages files into a chunked base or encrypted archive that can be read sequentially or by byte range. The Rust implementation supports streaming creation, validated opening, filesystem ingestion and extraction, archive extension, RO-Crate conversion, and Crypt4GH export. ## Get started @@ -56,4 +56,4 @@ The filesystem operations are Linux-only and require destination filesystem supp Version 0.8 is a source break from 0.7. Use the selected public API in `pithos_lib::archive`, `crypto`, `source`, `fs`, and `adapters`; old model, helper, and wire-record paths are not compatibility APIs. -The [Pithos 1.0 draft](spec/PITHOS_1.0.0_draft.md) is useful format background, but it is not a complete interoperability guarantee. Current library behavior and tests define the 0.8 implementation. +This branch implements the wire rules in the [Pithos 1.0 draft](spec/PITHOS_1.0.0_draft.md). Cargo package versioning is maintained separately from the wire-format version. diff --git a/crates/pithos/README.md b/crates/pithos/README.md index 8443c57..4141caf 100644 --- a/crates/pithos/README.md +++ b/crates/pithos/README.md @@ -1,6 +1,6 @@ # Pithos CLI -`pithos` creates, reads, and extends encrypted Pithos archives from the command line. It is the filesystem-oriented companion to the [`pithos_lib`](../pithos_lib/README.md) Rust API. +`pithos` creates, reads, and extends Pithos archives from the command line. It is the filesystem-oriented companion to the [`pithos_lib`](../pithos_lib/README.md) Rust API. ## Installation @@ -19,20 +19,31 @@ mkdir -p keys restored pithos --output keys keypair --prefix owner ``` -Create an archive for a file or directory. This example gives the owner access and writes `research.pith`; replace `input` with your source path. +Create a plain base archive for a file or directory without keys. Base output is local, uncompressed, and unencrypted: ```bash -pithos --secret-key keys/owner.sec.pem --public-keys keys/owner.pub.pem --output research.pith create input +pithos --output plain.pith create --plain input +pithos read list plain.pith +pithos read data plain.pith report.txt > report.txt ``` -List its entries, extract all entries to an existing destination directory, or write one entry to standard output: +Create an encrypted archive for a file or directory. This example gives the owner access and writes `research.pith`; replace `input` with your source path. ```bash +pithos --secret-key keys/owner.sec.pem --public-keys keys/owner.pub.pem --output research.pith create input pithos --secret-key keys/owner.sec.pem read list research.pith pithos --secret-key keys/owner.sec.pem --output restored read all research.pith pithos --secret-key keys/owner.sec.pem read data research.pith report.txt > report.txt ``` +Encrypted archives can be listed without a key, but reading their content requires the matching private key. Plain archives can be read without `--secret-key`: + +```bash +pithos read list research.pith +pithos read list plain.pith +pithos read data plain.pith report.txt +``` + `read data` also accepts `--ranges START:END,...` for half-open byte ranges. Use `read info` to inspect an entry and `read directory` to inspect every entry's metadata. ## More operations diff --git a/crates/pithos/src/main.rs b/crates/pithos/src/main.rs index 8b931f2..aa17ee3 100644 --- a/crates/pithos/src/main.rs +++ b/crates/pithos/src/main.rs @@ -113,6 +113,9 @@ enum PithosCommands { /// Set values for content-defined chunking #[arg(long="cdc", value_parser=parse_cdc_input, value_name = "MIN,AVG,MAX")] cdc: Option, + /// Create a local, uncompressed, unencrypted base archive + #[arg(long, conflicts_with_all = ["secret_key", "public_keys"])] + plain: bool, /// Input files #[arg(value_name = "FILES")] files: Vec, @@ -301,20 +304,20 @@ fn run() -> Result<(), PithosCliError> { let path_str = path.to_str().ok_or_else(|| { PithosCliError::InvalidArgumentError("archive path is not valid UTF-8".into()) })?; - let archive = open_archive(&file, required_private_key(&cli.secret_key)?)?; + let archive = open_archive(&file, optional_private_key(&cli.secret_key)?)?; match archive.entry(path_str)? { Some(entry) => write_stdout(format_args!("{entry:#?}"))?, None => Err(PithosError::FileNotFound(path_str.to_string()))?, } } ReadCommands::List { file } => { - let archive = open_archive(&file, required_private_key(&cli.secret_key)?)?; + let archive = open_archive(&file, optional_private_key(&cli.secret_key)?)?; for entry in archive.entries() { write_stdout(format_args!("{} {:?} {}", entry.id, entry.kind, entry.path))?; } } ReadCommands::All { file } => { - let archive = open_archive(&file, required_private_key(&cli.secret_key)?)?; + let archive = open_archive(&file, optional_private_key(&cli.secret_key)?)?; let output = cli .output .as_deref() @@ -340,7 +343,7 @@ fn run() -> Result<(), PithosCliError> { "a file output accepts exactly one archive path".into(), )); } - let archive = open_archive(&file, required_private_key(&cli.secret_key)?)?; + let archive = open_archive(&file, optional_private_key(&cli.secret_key)?)?; if let Some(output_path) = cli.output.as_ref() { if let Some(path) = paths.first() { let path = path.to_str().ok_or_else(|| { @@ -396,32 +399,46 @@ fn run() -> Result<(), PithosCliError> { } } ReadCommands::Directory { file } => { - let archive = open_archive(&file, required_private_key(&cli.secret_key)?)?; + let archive = open_archive(&file, optional_private_key(&cli.secret_key)?)?; for entry in archive.entries() { write_stdout(format_args!("{entry:#?}"))?; } } }, - PithosCommands::Create { cdc, files } => { + PithosCommands::Create { cdc, plain, files } => { + if plain && (cli.secret_key.is_some() || cli.public_keys.is_some()) { + return Err(PithosCliError::InvalidArgumentError( + "--plain conflicts with --secret-key and --public-keys".into(), + )); + } if files.is_empty() { return Err(PithosCliError::InvalidArgumentError( "No files provided".to_string(), )); } - let sender_key = required_private_key(&cli.secret_key)?; - let reader_keys: Result, PithosCliError> = cli - .public_keys - .ok_or_else(|| { - PithosCliError::InvalidArgumentError( - "at least one recipient key is required".into(), - ) - })? - .iter() - .map(|path| load_public_key_from_pem(path)) - .collect(); - let options = - WriteOptions::new(sender_key, reader_keys?).with_cdc(cdc.unwrap_or_default()); + let (options, processing) = if plain { + ( + WriteOptions::base().with_cdc(cdc.unwrap_or_default()), + ProcessingOptions::new(false, 0).expect("zero compression is valid"), + ) + } else { + let sender_key = required_private_key(&cli.secret_key)?; + let reader_keys: Result, PithosCliError> = cli + .public_keys + .ok_or_else(|| { + PithosCliError::InvalidArgumentError( + "at least one recipient key is required".into(), + ) + })? + .iter() + .map(|path| load_public_key_from_pem(path)) + .collect(); + ( + WriteOptions::new(sender_key, reader_keys?).with_cdc(cdc.unwrap_or_default()), + ProcessingOptions::default(), + ) + }; options.validate()?; let manifest = build_input_manifest(&files)?; @@ -453,7 +470,7 @@ fn run() -> Result<(), PithosCliError> { discard_incomplete_writer(writer); return Err(error.into()); } - if let Err(error) = manifest.ingest(&mut writer, ProcessingOptions::default()) { + if let Err(error) = manifest.ingest(&mut writer, processing) { discard_incomplete_writer(writer); return Err(error.into()); } @@ -552,7 +569,7 @@ fn run() -> Result<(), PithosCliError> { .map(|path| load_public_key_from_pem(path)) .collect(); - let archive = open_archive(&file, sender_key)?; + let archive = open_archive(&file, Some(sender_key))?; if let Some(destination) = cli.output { let mut output = StagedOutput::create( @@ -592,6 +609,12 @@ fn required_private_key(path: &Option) -> Result) -> Result, PithosCliError> { + path.as_ref() + .map(|path| load_private_key_from_pem(path)) + .transpose() +} + fn validate_keypair_prefix(prefix: Option) -> Result { let prefix = prefix.unwrap_or_else(|| "pithos_key".to_string()); let mut components = Path::new(&prefix).components(); @@ -944,11 +967,12 @@ fn fail_after_create_for_test() -> Result<(), PithosError> { fn open_archive( path: &std::path::Path, - key: PrivateKey, + key: Option, ) -> Result, PithosError> { + let access_keys = key.map_or_else(AccessKeys::new, |key| AccessKeys::new().with_key(key)); Archive::open( FileSource::open(path)?, - OpenOptions::default().with_access_keys(AccessKeys::new().with_key(key)), + OpenOptions::default().with_access_keys(access_keys), ) } diff --git a/crates/pithos/tests/create.rs b/crates/pithos/tests/create.rs index 7be4aba..4ea180e 100644 --- a/crates/pithos/tests/create.rs +++ b/crates/pithos/tests/create.rs @@ -88,6 +88,126 @@ fn create_reports_missing_keys_and_invalid_cdc() { let _ = fs::remove_dir_all(temporary); } +#[test] +fn plain_create_and_every_read_subcommand_work_without_keys() { + let temporary = temporary(); + let input = temporary.join("input.txt"); + let archive = temporary.join("plain.pith"); + fs::write(&input, b"plain cli content").unwrap(); + assert!( + command() + .arg("--output") + .arg(&archive) + .arg("create") + .arg("--plain") + .arg("--cdc") + .arg("64,256,1024") + .arg(&input) + .status() + .unwrap() + .success() + ); + + for arguments in [ + vec!["read", "list", archive.to_str().unwrap()], + vec!["read", "info", archive.to_str().unwrap(), "input.txt"], + vec!["read", "directory", archive.to_str().unwrap()], + ] { + assert!(command().args(arguments).status().unwrap().success()); + } + let data = command() + .args(["read", "data", archive.to_str().unwrap(), "input.txt"]) + .output() + .unwrap(); + assert!(data.status.success()); + assert_eq!(data.stdout, b"plain cli content"); + + let extracted = temporary.join("extracted"); + fs::create_dir(&extracted).unwrap(); + assert!( + command() + .arg("--output") + .arg(&extracted) + .args(["read", "all"]) + .arg(&archive) + .status() + .unwrap() + .success() + ); + assert_eq!( + fs::read(extracted.join("input.txt")).unwrap(), + b"plain cli content" + ); + let _ = fs::remove_dir_all(temporary); +} + +#[test] +fn plain_create_rejects_secret_and_public_key_options() { + let temporary = temporary(); + let input = temporary.join("input"); + fs::write(&input, b"input").unwrap(); + let private = workspace_file("keys/sender_private.pem"); + let public = workspace_file("keys/recipient1_public.pem"); + for (index, key_args) in [ + vec!["--secret-key", private.as_str()], + vec!["--public-keys", public.as_str()], + ] + .into_iter() + .enumerate() + { + let output = temporary.join(format!("conflict-{index}.pith")); + let result = command() + .args(key_args) + .arg("--output") + .arg(&output) + .arg("create") + .arg("--plain") + .arg(&input) + .output() + .unwrap(); + assert!(!result.status.success()); + assert!( + String::from_utf8(result.stderr) + .unwrap() + .contains("conflict") + ); + assert!(!output.exists()); + } + let _ = fs::remove_dir_all(temporary); +} + +#[test] +fn encrypted_archives_list_but_do_not_read_without_a_key() { + let temporary = temporary(); + let input = temporary.join("encrypted.txt"); + let archive = temporary.join("encrypted.pith"); + fs::write(&input, b"encrypted").unwrap(); + assert!(create_command(&archive, &input).status().unwrap().success()); + + let listing = command() + .args(["read", "list", archive.to_str().unwrap()]) + .output() + .unwrap(); + assert!(listing.status.success()); + assert!( + String::from_utf8(listing.stdout) + .unwrap() + .contains("available: false") + ); + + let read = command() + .args(["read", "data", archive.to_str().unwrap(), "encrypted.txt"]) + .output() + .unwrap(); + assert!(!read.status.success()); + assert!( + String::from_utf8(read.stderr) + .unwrap() + .contains("unavailable") + ); + let _ = fs::remove_dir_all(temporary); +} + #[test] fn create_reports_filesystem_errors_and_produces_a_readable_archive() { let temporary = temporary(); diff --git a/crates/pithos_lib/README.md b/crates/pithos_lib/README.md index 6f663f6..7f90569 100644 --- a/crates/pithos_lib/README.md +++ b/crates/pithos_lib/README.md @@ -1,6 +1,6 @@ # Pithos library -`pithos_lib` is the public Rust API for creating, opening, reading, extending, extracting, and adapting encrypted Pithos archives. It has no network client or transport policy. +`pithos_lib` is the public Rust API for creating, opening, reading, extending, extracting, and adapting Pithos archives. It has no network client or transport policy. This branch implements the Pithos 1.0 draft wire rules; Cargo package versioning is separate. ## Installation @@ -13,6 +13,26 @@ pithos_lib = "0.8" ## Create an archive +A base archive needs no keys and accepts only local, uncompressed, unencrypted content: + +```rust,no_run +use pithos_lib::archive::{ArchivePath, ArchiveWriter, EntryMetadata, ProcessingOptions, WriteOptions}; +use std::fs::File; + +fn main() -> Result<(), Box> { + let mut writer = ArchiveWriter::create(File::create("report.pith")?, WriteOptions::base())?; + writer.add_file( + ArchivePath::new("report.txt")?, + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(false, 0)?, + None, + File::open("report.txt")?, + )?; + writer.finish()?; + Ok(()) +} +``` + This example encrypts a local file for its owner. The recipient public key is derived from the private key only for a self-contained example; production applications should provide the intended recipients' public keys. ```rust,no_run @@ -54,7 +74,7 @@ See [`examples/create.rs`](examples/create.rs) for the maintained version and th Compiled examples are included with the package: [`create`](examples/create.rs), [`open_list`](examples/open_list.rs), [`read_ranges`](examples/read_ranges.rs), [`extract`](examples/extract.rs), [`append`](examples/append.rs), [`grant_readers`](examples/grant_readers.rs), [`ro_crate`](examples/ro_crate.rs), and [`crypt4gh`](examples/crypt4gh.rs). They are local-only and return errors to their caller. -Version 0.8 deliberately removes old model, helper, and wire-record access from the public API. Use the typed archive API instead. The [format draft](https://github.com/arunaengine/pithos/blob/main/spec/PITHOS_1.0.0_draft.md) is background information, not a complete interoperability guarantee. +The typed archive API deliberately keeps old model, helper, and wire-record access private. Use `WriteOptions::base` for plain output or `WriteOptions::new` for encrypted output. ## Read and extract semantics diff --git a/crates/pithos_lib/src/archive/writer.rs b/crates/pithos_lib/src/archive/writer.rs index f025be0..33c4d86 100644 --- a/crates/pithos_lib/src/archive/writer.rs +++ b/crates/pithos_lib/src/archive/writer.rs @@ -161,18 +161,31 @@ pub struct WrittenEntry { pub id: u64, } -/// Creation options. A writer always has one sender and at least one recipient. +enum WriteMode { + Base, + Encrypted { + sender: PrivateKey, + recipients: Vec, + }, +} + +/// Creation options for a base or encrypted archive. pub struct WriteOptions { - sender: PrivateKey, - recipients: Vec, + mode: WriteMode, cdc: CdcConfig, } impl WriteOptions { pub fn new(sender: PrivateKey, recipients: Vec) -> Self { Self { - sender, - recipients, + mode: WriteMode::Encrypted { sender, recipients }, + cdc: CdcConfig::default(), + } + } + + pub fn base() -> Self { + Self { + mode: WriteMode::Base, cdc: CdcConfig::default(), } } @@ -184,16 +197,17 @@ impl WriteOptions { /// Check all creation metadata before taking ownership of an output sink. pub fn validate(&self) -> Result<(), PithosError> { - if self.recipients.is_empty() { - return Err(PithosError::WriterRequiresRecipient); - } - let mut recipients = HashSet::with_capacity(self.recipients.len()); - if self - .recipients - .iter() - .any(|recipient| !recipients.insert(recipient)) - { - return Err(PithosError::DuplicateRecipientKey); + if let WriteMode::Encrypted { recipients, .. } = &self.mode { + if recipients.is_empty() { + return Err(PithosError::WriterRequiresRecipient); + } + let mut unique_recipients = HashSet::with_capacity(recipients.len()); + if recipients + .iter() + .any(|recipient| !unique_recipients.insert(recipient)) + { + return Err(PithosError::DuplicateRecipientKey); + } } Ok(()) } @@ -517,7 +531,7 @@ impl Write for CountingSink { /// A streaming archive writer. Dropping it leaves an intentionally incomplete sink. pub struct ArchiveWriter { - sender: StaticSecret, + mode: ArchiveWriterMode, cdc: CdcConfig, sink: CountingSink, directory: crate::format::wire::Directory, @@ -529,6 +543,17 @@ pub struct ArchiveWriter { granted_access_ids: Option>, } +enum ArchiveWriterMode { + Base, + Encrypted { sender: StaticSecret }, +} + +impl ArchiveWriterMode { + fn is_base(&self) -> bool { + matches!(self, Self::Base) + } +} + #[cfg(test)] #[derive(Debug, Eq, PartialEq)] struct MetadataSnapshot { @@ -544,16 +569,22 @@ impl ArchiveWriter { if let Err(error) = options.validate() { return Err(CreateError { error, sink }); } - let sender = options.sender.into_dalek_static_secret(); - let recipients = options - .recipients - .into_iter() - .map(PublicKey::into_dalek_public_key) - .collect::>(); - let encryption = IndexMap::from_iter([( - LegacyPublicKey::from(&sender).to_bytes(), - EncryptionSection::new(&recipients), - )]); + let WriteOptions { mode, cdc } = options; + let (mode, encryption) = match mode { + WriteMode::Base => (ArchiveWriterMode::Base, IndexMap::new()), + WriteMode::Encrypted { sender, recipients } => { + let sender = sender.into_dalek_static_secret(); + let recipients = recipients + .into_iter() + .map(PublicKey::into_dalek_public_key) + .collect::>(); + let encryption = IndexMap::from_iter([( + LegacyPublicKey::from(&sender).to_bytes(), + EncryptionSection::new(&recipients), + )]); + (ArchiveWriterMode::Encrypted { sender }, encryption) + } + }; let directory = crate::format::wire::Directory::new(None, WireEntries::new(), encryption); let mut sink = CountingSink { sink, offset: 0 }; if let Err(error) = codec::encode_header(&FileHeader::default(), &mut sink) { @@ -563,8 +594,8 @@ impl ArchiveWriter { }); } Ok(Self { - sender, - cdc: options.cdc, + mode, + cdc, sink, directory, poisoned: false, @@ -605,7 +636,7 @@ impl ArchiveWriter { encryption, ); Ok(Self { - sender, + mode: ArchiveWriterMode::Encrypted { sender }, cdc, sink: CountingSink { sink, @@ -813,6 +844,9 @@ impl ArchiveWriter { content: R, ) -> Result { self.ensure_open()?; + if self.mode.is_base() && (processing.encrypted() || processing.compression_level() != 0) { + return Err(PithosError::BaseWriterRequiresPlainProcessing.into()); + } let mut delta = self.stage_entry(file_type, path, metadata, 0, None)?; let mut stream = StreamCDC::with_level( content, @@ -919,21 +953,23 @@ impl ArchiveWriter { if let Err(error) = self.validate_unsealed_content(&delta) { return self.poison(error); } - let file_key = match self.runtime.file_key() { - Ok(file_key) => file_key, - Err(error) => return self.poison(error), - }; - let nonce = match self.runtime.block_list_nonce() { - Ok(nonce) => nonce, - Err(error) => return self.poison(error), - }; - if let Err(error) = - self.runtime - .seal_block_list(&mut delta.entry.block_data, &file_key, nonce) - { - return self.poison(error); + if !self.mode.is_base() { + let file_key = match self.runtime.file_key() { + Ok(file_key) => file_key, + Err(error) => return self.poison(error), + }; + let nonce = match self.runtime.block_list_nonce() { + Ok(nonce) => nonce, + Err(error) => return self.poison(error), + }; + if let Err(error) = + self.runtime + .seal_block_list(&mut delta.entry.block_data, &file_key, nonce) + { + return self.poison(error); + } + delta.recipient_access = Some((delta.id, file_key)); } - delta.recipient_access = Some((delta.id, file_key)); // Block bytes may already be orphaned on failure, so preparation failures // poison while every live metadata collection remains unchanged. if let Err(error) = self.prepare_delta(&delta) { @@ -1245,6 +1281,11 @@ impl ArchiveWriter { fn validate_publishable(&self) -> Result<(), PithosError> { self.validate_entry_state()?; validate_relationships(&self.directory)?; + if self.mode.is_base() && !self.directory.encryption.is_empty() { + return Err(PithosError::InvalidRecipientDataState( + "base writer has an encryption map".into(), + )); + } if let Some(snapshot) = &self.append_snapshot { snapshot.validate_prospective_recipient_pairs(&self.directory.encryption)?; let relationships = self @@ -1255,10 +1296,12 @@ impl ArchiveWriter { .collect::>(); snapshot.validate_child_relationships(&relationships)?; } - for section in self.directory.encryption.values() { - for recipient in section.recipients.values() { - if matches!(recipient.recipient_data, RecipientData::Decrypted(_)) { - return Err(PithosError::WriterUnsealedRecipientList); + if !self.mode.is_base() { + for section in self.directory.encryption.values() { + for recipient in section.recipients.values() { + if matches!(recipient.recipient_data, RecipientData::Decrypted(_)) { + return Err(PithosError::WriterUnsealedRecipientList); + } } } } @@ -1266,14 +1309,17 @@ impl ArchiveWriter { } fn seal_recipient_lists(&mut self) -> Result<(), PithosError> { - let sender = LegacyPublicKey::from(&self.sender).to_bytes(); - let Some(section) = self.directory.encryption.get_mut(&sender) else { + let ArchiveWriterMode::Encrypted { sender } = &self.mode else { + return Ok(()); + }; + let sender_public = LegacyPublicKey::from(sender).to_bytes(); + let Some(section) = self.directory.encryption.get_mut(&sender_public) else { return Ok(()); }; for (recipient_key, recipient) in &mut section.recipients { let recipient_key = LegacyPublicKey::from(*recipient_key); let shared_key = - crate::crypto::derive_shared(self.sender.as_bytes(), recipient_key.as_bytes())?; + crate::crypto::derive_shared(sender.as_bytes(), recipient_key.as_bytes())?; let nonce = self.runtime.recipient_list_nonce()?; self.runtime .seal_recipient_list(&mut recipient.recipient_data, shared_key, nonce)?; @@ -1284,16 +1330,26 @@ impl ArchiveWriter { fn validate_entry_state(&self) -> Result<(), PithosError> { for (_, _, entry) in self.directory.files.iter() { match entry.file_type { - FileType::Data | FileType::Metadata - if !matches!(entry.block_data, BlockDataState::Encrypted(_)) => - { - return Err(PithosError::WriterUnsealedBlockList); + FileType::Data | FileType::Metadata => { + let valid = if self.mode.is_base() { + matches!(entry.block_data, BlockDataState::Decrypted(_)) + } else { + matches!(entry.block_data, BlockDataState::Encrypted(_)) + }; + if !valid { + return Err(if self.mode.is_base() { + PithosError::InvalidBlockDataState( + "base content must have a decrypted block list".into(), + ) + } else { + PithosError::WriterUnsealedBlockList + }); + } } FileType::Directory | FileType::Symlink => match &entry.block_data { BlockDataState::Decrypted(entries) if entries.is_empty() => {} _ => return Err(PithosError::WriterNoContentHasBlockMaterial), }, - _ => {} } } for (_, path, entry) in self.directory.files.iter() { @@ -1337,6 +1393,15 @@ impl ArchiveWriter { } fn validate_required_access_records(&self) -> Result<(), PithosError> { + if self.mode.is_base() { + return if self.directory.encryption.is_empty() { + Ok(()) + } else { + Err(PithosError::InvalidRecipientDataState( + "base writer has recipient access records".into(), + )) + }; + } let mut content_ids = self .directory .files diff --git a/crates/pithos_lib/src/error.rs b/crates/pithos_lib/src/error.rs index 1bd8192..b1d8d70 100644 --- a/crates/pithos_lib/src/error.rs +++ b/crates/pithos_lib/src/error.rs @@ -154,6 +154,8 @@ pub enum PithosError { InvalidRecipientDataState(String), #[error("archive creation requires at least one recipient")] WriterRequiresRecipient, + #[error("base writer requires local, uncompressed, unencrypted processing")] + BaseWriterRequiresPlainProcessing, #[error("granting reader access requires at least one file or metadata id")] GrantRequiresFileId, #[error("invalid CDC configuration {min_size},{avg_size},{max_size}")] diff --git a/crates/pithos_lib/src/lib.rs b/crates/pithos_lib/src/lib.rs index b621aa6..3165b03 100644 --- a/crates/pithos_lib/src/lib.rs +++ b/crates/pithos_lib/src/lib.rs @@ -1,11 +1,12 @@ //! Pithos archive reading, writing, Linux filesystem operations, and presentation adapters. //! -//! # Public 0.8 boundary +//! # Public archive API //! //! Create archives with [`archive::ArchiveWriter`], then open them through -//! [`archive::Archive`] and an immutable [`source::ArchiveSource`]. Writer construction requires -//! a sender key and one or more recipient public keys; opening encrypted content requires the -//! matching recipient private key in [`archive::AccessKeys`]. +//! [`archive::Archive`] and an immutable [`source::ArchiveSource`]. Create either a base archive +//! with [`archive::WriteOptions::base`] for local, uncompressed, unencrypted content, or an +//! encrypted archive with a sender key and one or more recipient public keys. Opening encrypted +//! content requires the matching recipient private key in [`archive::AccessKeys`]. //! //! Payload integrity is checked when bytes are read, not when the archive is opened. In //! particular, [`archive::Archive::copy_to`] and [`archive::Archive::copy_range_to`] verify each diff --git a/crates/pithos_lib/tests/draft_spec_vectors.rs b/crates/pithos_lib/tests/draft_spec_vectors.rs index e4c2b68..926aff7 100644 --- a/crates/pithos_lib/tests/draft_spec_vectors.rs +++ b/crates/pithos_lib/tests/draft_spec_vectors.rs @@ -4,7 +4,10 @@ use chacha20poly1305::{ChaCha20Poly1305, KeyInit, Nonce, aead::Aead}; use crc32fast::hash as crc32; -use pithos_lib::archive::{Archive, EntryKind, OpenLimits, OpenOptions}; +use pithos_lib::archive::{ + Archive, ArchivePath, ArchiveWriter, EntryKind, EntryMetadata, OpenLimits, OpenOptions, + ProcessingOptions, WriteOptions, +}; use pithos_lib::error::{DeserializationError, PithosError}; use pithos_lib::source::MemorySource; use std::io::Cursor; @@ -369,6 +372,21 @@ fn production_reader_reads_cv_local_hello_without_access_keys() { assert_eq!(output, b"ell"); } +#[test] +fn production_base_writer_reproduces_cv_local_hello() { + let mut writer = ArchiveWriter::create(Vec::new(), WriteOptions::base()).unwrap(); + writer + .add_file( + ArchivePath::new("hello").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(false, 0).unwrap(), + Some(5), + Cursor::new(b"hello"), + ) + .unwrap(); + assert_eq!(writer.finish().unwrap(), appendix_hex("CV-LOCAL-HELLO-279")); +} + #[test] fn production_reader_rejects_a_direct_list_before_over_budget_allocation() { let limits = OpenLimits { diff --git a/crates/pithos_lib/tests/public_api.rs b/crates/pithos_lib/tests/public_api.rs index 3b36daa..9e253ea 100644 --- a/crates/pithos_lib/tests/public_api.rs +++ b/crates/pithos_lib/tests/public_api.rs @@ -52,6 +52,7 @@ fn standard_archive_configurations_are_send_and_sync() { let _ = OpenOptions::default() .with_external_resolver(SendSyncResolver) .with_external_access_policy(std::sync::Arc::new(SendSyncPolicy)); + let _ = WriteOptions::base(); } #[test] diff --git a/crates/pithos_lib/tests/writer_configuration.rs b/crates/pithos_lib/tests/writer_configuration.rs index 48bdb31..4bd7933 100644 --- a/crates/pithos_lib/tests/writer_configuration.rs +++ b/crates/pithos_lib/tests/writer_configuration.rs @@ -1,9 +1,23 @@ mod common; use common::writer::options; -use pithos_lib::archive::{ArchiveWriter, CdcConfig, WriteOptions}; +use pithos_lib::archive::{ + Archive, ArchivePath, ArchiveWriter, CdcConfig, EntryMetadata, OpenOptions, ProcessingOptions, + WriteOptions, WriterError, +}; use pithos_lib::crypto::PrivateKey; use pithos_lib::error::PithosError; +use pithos_lib::source::MemorySource; +use std::io::{Cursor, Read}; +use std::sync::Arc; + +struct PanicOnRead; + +impl Read for PanicOnRead { + fn read(&mut self, _buffer: &mut [u8]) -> std::io::Result { + panic!("base processing validation must precede content reads") + } +} #[test] fn creation_rejects_empty_recipients_without_losing_sink() { @@ -92,3 +106,58 @@ fn open_writer_cannot_be_recovered_as_incomplete() { let writer = ArchiveWriter::create(Vec::new(), options()).unwrap(); assert!(writer.into_incomplete().is_err()); } + +#[test] +fn empty_base_writer_reopens_without_keys() { + let bytes = ArchiveWriter::create(Vec::new(), WriteOptions::base()) + .unwrap() + .finish() + .unwrap(); + let archive = Archive::open( + MemorySource::new(Arc::<[u8]>::from(bytes)), + OpenOptions::default(), + ) + .unwrap(); + assert_eq!(archive.entries().count(), 0); +} + +#[test] +fn base_processing_rejection_precedes_reads_and_does_not_poison() { + let mut writer = ArchiveWriter::create(Vec::new(), WriteOptions::base()).unwrap(); + for (path, processing) in [ + ("encrypted", ProcessingOptions::new(true, 0).unwrap()), + ("compressed", ProcessingOptions::new(false, 1).unwrap()), + ] { + assert!(matches!( + writer.add_file( + ArchivePath::new(path).unwrap(), + EntryMetadata::new(0, 0, 0o644), + processing, + None, + PanicOnRead, + ), + Err(WriterError::Pithos( + PithosError::BaseWriterRequiresPlainProcessing + )) + )); + } + + writer + .add_file( + ArchivePath::new("plain").unwrap(), + EntryMetadata::new(1, 2, 0o640), + ProcessingOptions::new(false, 0).unwrap(), + Some(5), + Cursor::new(b"plain"), + ) + .unwrap(); + let bytes = writer.finish().unwrap(); + let archive = Archive::open( + MemorySource::new(Arc::<[u8]>::from(bytes)), + OpenOptions::default(), + ) + .unwrap(); + let mut output = Vec::new(); + archive.copy_to("plain", &mut output).unwrap(); + assert_eq!(output, b"plain"); +} From fb45d2c0ecb5fea1bc562f6ff63dcb261bf3540c Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Sun, 13 Sep 2026 16:04:32 +0200 Subject: [PATCH 047/166] feat: restore stored extraction modes --- crates/pithos/src/main.rs | 12 +- crates/pithos/tests/create.rs | 56 ++++++++ crates/pithos_lib/src/fs/extraction.rs | 177 +++++++++++++++++++++++-- crates/pithos_lib/src/fs/mod.rs | 10 +- crates/pithos_lib/tests/platform_fs.rs | 169 ++++++++++++++++++++++- crates/pithos_lib/tests/public_api.rs | 9 +- crates/pithos_lib/tests/reader.rs | 38 +++++- 7 files changed, 444 insertions(+), 27 deletions(-) diff --git a/crates/pithos/src/main.rs b/crates/pithos/src/main.rs index aa17ee3..48c9508 100644 --- a/crates/pithos/src/main.rs +++ b/crates/pithos/src/main.rs @@ -6,13 +6,13 @@ use clap::{Parser, Subcommand, ValueEnum}; use pithos_lib::adapters::crypt4gh; use pithos_lib::adapters::crypt4gh::Crypt4GHError; use pithos_lib::archive::{ - AccessKeys, AppendDurability, AppendOptions, Archive, ArchiveWriter, CdcConfig, EntryKind, - OpenOptions, ProcessingOptions, WriteOptions, WriterError, + AccessKeys, AppendDurability, AppendOptions, Archive, ArchiveWriter, CdcConfig, OpenOptions, + ProcessingOptions, WriteOptions, WriterError, }; use pithos_lib::crypto::{PrivateKey, PublicKey, generate_private_key}; use pithos_lib::error::PithosError; use pithos_lib::fs::ingest::build_input_manifest; -use pithos_lib::fs::{FsError, append_files, extract, grant_readers}; +use pithos_lib::fs::{FsError, append_files, extract_all, grant_readers}; use pithos_lib::source::FileSource; use rustix::fs::{AtFlags, Mode, OFlags, fchmod, linkat, open, openat, unlinkat}; use std::ffi::OsString; @@ -322,11 +322,7 @@ fn run() -> Result<(), PithosCliError> { .output .as_deref() .unwrap_or_else(|| std::path::Path::new(".")); - let mut entries = archive.entries().collect::>(); - entries.sort_by_key(|entry| !matches!(entry.kind, EntryKind::Directory)); - for entry in entries { - extract(&archive, &entry.path, output)?; - } + extract_all(&archive, output)?; } ReadCommands::Data { file, diff --git a/crates/pithos/tests/create.rs b/crates/pithos/tests/create.rs index 4ea180e..030330b 100644 --- a/crates/pithos/tests/create.rs +++ b/crates/pithos/tests/create.rs @@ -208,6 +208,62 @@ fn encrypted_archives_list_but_do_not_read_without_a_key() { let _ = fs::remove_dir_all(temporary); } +#[cfg(unix)] +#[test] +fn read_all_restores_archived_file_and_directory_permissions() { + use std::os::unix::fs::PermissionsExt; + + let temporary = temporary(); + let source = temporary.join("source"); + let nested = source.join("nested"); + let child = nested.join("child"); + fs::create_dir_all(&nested).unwrap(); + fs::write(&child, b"permissions").unwrap(); + fs::set_permissions(&nested, fs::Permissions::from_mode(0o555)).unwrap(); + fs::set_permissions(&child, fs::Permissions::from_mode(0o640)).unwrap(); + let archive = temporary.join("modes.pith"); + assert!( + command() + .arg("--output") + .arg(&archive) + .args(["create", "--plain"]) + .arg(&source) + .status() + .unwrap() + .success() + ); + + let output = temporary.join("output"); + fs::create_dir(&output).unwrap(); + assert!( + command() + .arg("--output") + .arg(&output) + .args(["read", "all"]) + .arg(&archive) + .status() + .unwrap() + .success() + ); + assert_eq!( + fs::metadata(output.join("nested")) + .unwrap() + .permissions() + .mode() + & 0o7777, + 0o555 + ); + assert_eq!( + fs::metadata(output.join("nested/child")) + .unwrap() + .permissions() + .mode() + & 0o7777, + 0o640 + ); + let _ = fs::remove_dir_all(temporary); +} + #[test] fn create_reports_filesystem_errors_and_produces_a_readable_archive() { let temporary = temporary(); diff --git a/crates/pithos_lib/src/fs/extraction.rs b/crates/pithos_lib/src/fs/extraction.rs index cc7bb22..c6ae3a2 100644 --- a/crates/pithos_lib/src/fs/extraction.rs +++ b/crates/pithos_lib/src/fs/extraction.rs @@ -2,10 +2,34 @@ use super::FsError; use crate::archive::{Archive, EntryKind, ExternalBlockResolver}; use crate::source::ArchiveSource; use cap_std::fs::Dir; -use rustix::fs::{AtFlags, Mode, OFlags, linkat, openat}; +use rustix::fs::{AtFlags, Mode, OFlags, fchmod, linkat, openat}; +use std::cmp::Reverse; use std::io::{self, Write}; use std::path::{Component, Path, PathBuf}; +/// Options controlling filesystem extraction behavior. +#[derive(Clone, Copy, Debug, Default)] +pub struct ExtractionOptions { + special_permissions: bool, +} + +impl ExtractionOptions { + /// Opt in to restoring setuid, setgid, and sticky permission bits. + pub fn with_special_permissions(mut self) -> Self { + self.special_permissions = true; + self + } +} + +fn applied_mode(stored_permissions: u32, options: ExtractionOptions) -> Mode { + let mask = if options.special_permissions { + 0o7777 + } else { + 0o0777 + }; + Mode::from(stored_permissions & mask) +} + fn open_dir_no_follow(parent: &Dir, component: &Path) -> io::Result { rustix::fs::openat( parent, @@ -23,6 +47,25 @@ pub fn extract( archive_path: &str, destination: &Path, ) -> Result<(), FsError> +where + S: ArchiveSource, + E: ExternalBlockResolver, +{ + extract_with_options( + archive, + archive_path, + destination, + ExtractionOptions::default(), + ) +} + +/// Extract one validated archive entry with explicit extraction options. +pub fn extract_with_options( + archive: &Archive, + archive_path: &str, + destination: &Path, + options: ExtractionOptions, +) -> Result<(), FsError> where S: ArchiveSource, E: ExternalBlockResolver, @@ -44,6 +87,7 @@ where )), })?; let root = ExtractionRoot::open(destination, true, archive_path)?; + let mode = applied_mode(entry.permissions, options); match entry.kind { EntryKind::File { .. } | EntryKind::Metadata { .. } => { let pending = root.pending_file(archive_path)?; @@ -51,14 +95,72 @@ where archive .copy_to(archive_path, &mut writer) .map_err(|source| root.archive_error("copy", archive_path, source))?; - pending.commit(archive_path)?; + pending.commit(archive_path, mode)?; } - EntryKind::Directory => root.create_dir(archive_path)?, + EntryKind::Directory => root.create_dir(archive_path, mode)?, EntryKind::Symlink { target } => root.create_symlink(archive_path, &target)?, } Ok(()) } +/// Extract every archive entry in declaration order through one destination root. +pub fn extract_all(archive: &Archive, destination: &Path) -> Result<(), FsError> +where + S: ArchiveSource, + E: ExternalBlockResolver, +{ + extract_all_with_options(archive, destination, ExtractionOptions::default()) +} + +/// Extract every archive entry in declaration order with explicit options. +pub fn extract_all_with_options( + archive: &Archive, + destination: &Path, + options: ExtractionOptions, +) -> Result<(), FsError> +where + S: ArchiveSource, + E: ExternalBlockResolver, +{ + let root = ExtractionRoot::open(destination, true, "")?; + let mut directories = Vec::new(); + + for entry in archive.entries() { + let path = entry.path; + let mode = applied_mode(entry.permissions, options); + match entry.kind { + EntryKind::File { .. } | EntryKind::Metadata { .. } => { + let pending = root.pending_file(&path)?; + let mut writer = pending.writer(&path)?; + archive + .copy_to(&path, &mut writer) + .map_err(|source| root.archive_error("copy", &path, source))?; + pending.commit(&path, mode)?; + } + EntryKind::Directory => { + let directory = root.create_dir_open(&path)?; + fchmod(&directory, Mode::RWXU).map_err(|source| { + root.operation("set temporary directory permissions", &path, source.into()) + })?; + directories.push(PendingDirectory { + path, + directory, + mode, + }); + } + EntryKind::Symlink { target } => root.create_symlink(&path, &target)?, + } + } + + directories.sort_by_key(|directory| Reverse(directory.path.split('/').count())); + for directory in directories { + fchmod(&directory.directory, directory.mode).map_err(|source| { + root.operation("set directory permissions", &directory.path, source.into()) + })?; + } + Ok(()) +} + pub(crate) struct ExtractionRoot { root: Dir, destination: PathBuf, @@ -208,17 +310,27 @@ impl ExtractionRoot { Ok((dir, final_name.to_string())) } - pub(crate) fn create_dir(&self, path: &str) -> Result<(), FsError> { + fn create_dir_open(&self, path: &str) -> Result { let (parent, name) = self.parents(path)?; match parent.symlink_metadata(&name) { Ok(_) => Err(self.collision(path, "final entry already exists")), - Err(error) if error.kind() == io::ErrorKind::NotFound => parent - .create_dir(&name) - .map_err(|source| self.operation("create directory", path, source)), + Err(error) if error.kind() == io::ErrorKind::NotFound => { + parent + .create_dir(&name) + .map_err(|source| self.operation("create directory", path, source))?; + open_dir_no_follow(&parent, Path::new(&name)) + .map_err(|source| self.operation("open directory", path, source)) + } Err(source) => Err(self.operation("inspect destination entry", path, source)), } } + fn create_dir(&self, path: &str, mode: Mode) -> Result<(), FsError> { + let directory = self.create_dir_open(path)?; + fchmod(&directory, mode) + .map_err(|source| self.operation("set directory permissions", path, source.into())) + } + pub(crate) fn create_symlink(&self, path: &str, target: &str) -> Result<(), FsError> { let (parent, name) = self.parents(path)?; match parent.symlink_metadata(&name) { @@ -274,7 +386,13 @@ impl PendingFile { }) } - pub(crate) fn commit(self, archive_path: &str) -> Result<(), FsError> { + pub(crate) fn commit(self, archive_path: &str, mode: Mode) -> Result<(), FsError> { + fchmod(&self.file, mode).map_err(|source| FsError::Operation { + operation: "set staged file permissions", + archive_path: archive_path.to_owned(), + destination: self.destination.clone(), + source: source.into(), + })?; self.file.sync_all().map_err(|source| FsError::Operation { operation: "sync staged file", archive_path: archive_path.to_owned(), @@ -306,6 +424,12 @@ impl PendingFile { } } +struct PendingDirectory { + path: String, + directory: Dir, + mode: Mode, +} + impl Write for PendingFile { fn write(&mut self, buf: &[u8]) -> io::Result { self.file.write(buf) @@ -353,7 +477,7 @@ mod tests { std::fs::write(temporary.path().join("data"), b"sentinel").unwrap(); assert!(matches!( - pending.commit("data"), + pending.commit("data", Mode::RUSR | Mode::WUSR), Err(FsError::ExtractionCollision { .. }) )); @@ -376,7 +500,7 @@ mod tests { .unwrap(); assert_eq!(std::fs::read_dir(temporary.path()).unwrap().count(), 0); - pending.commit("data").unwrap(); + pending.commit("data", Mode::RUSR | Mode::WUSR).unwrap(); assert_eq!( std::fs::read(temporary.path().join("data")).unwrap(), @@ -384,6 +508,39 @@ mod tests { ); } + #[test] + fn extraction_modes_strip_special_bits_unless_explicitly_enabled() { + assert_eq!( + applied_mode(0o7777, ExtractionOptions::default()).bits(), + 0o0777 + ); + assert_eq!( + applied_mode( + 0o7777, + ExtractionOptions::default().with_special_permissions(), + ) + .bits(), + 0o7777 + ); + } + + #[test] + fn staged_file_mode_failure_prevents_publication() { + let temporary = tempfile::tempdir().unwrap(); + let root = ExtractionRoot::open(temporary.path(), false, "data").unwrap(); + let mut pending = root.pending_file("data").unwrap(); + pending.file = cap_std::fs::File::from_std(std::fs::File::open("/dev/null").unwrap()); + + assert!(matches!( + pending.commit("data", Mode::RUSR), + Err(FsError::Operation { + operation: "set staged file permissions", + .. + }) + )); + assert!(!temporary.path().join("data").exists()); + } + #[test] fn destination_root_rejects_parent_traversal() { let Err(error) = ExtractionRoot::open(Path::new(".."), true, "data") else { diff --git a/crates/pithos_lib/src/fs/mod.rs b/crates/pithos_lib/src/fs/mod.rs index 5556e51..46c1437 100644 --- a/crates/pithos_lib/src/fs/mod.rs +++ b/crates/pithos_lib/src/fs/mod.rs @@ -3,8 +3,10 @@ //! Ingestion, extraction, append, and reader grants are Linux-only host operations. //! They keep no-follow traversal, retained source identity, metadata normalization, //! and filesystem error context out of the core archive and format layers. Extraction -//! stages regular files and publishes without clobbering an existing destination; -//! append and grants use an advisory lock that coordinates only participating writers. +//! stages regular files and publishes without clobbering an existing destination. +//! On Linux, symlink permission bits are retained as archive metadata but are not +//! applied to the symlink itself. Append and grants use an advisory lock that coordinates +//! only participating writers. mod append; mod extraction; @@ -93,4 +95,6 @@ pub enum FsError { UnsupportedEntry { path: PathBuf, kind: &'static str }, } -pub use extraction::extract; +pub use extraction::{ + ExtractionOptions, extract, extract_all, extract_all_with_options, extract_with_options, +}; diff --git a/crates/pithos_lib/tests/platform_fs.rs b/crates/pithos_lib/tests/platform_fs.rs index 23029c8..3796855 100644 --- a/crates/pithos_lib/tests/platform_fs.rs +++ b/crates/pithos_lib/tests/platform_fs.rs @@ -4,7 +4,7 @@ use pithos_lib::archive::{ }; use pithos_lib::crypto::PrivateKey; use pithos_lib::fs::ingest::{InputManifest, build_input_manifest}; -use pithos_lib::fs::{FsError, extract}; +use pithos_lib::fs::{ExtractionOptions, FsError, extract, extract_all, extract_with_options}; use pithos_lib::source::MemorySource; use std::fs::{self, File, FileTimes}; use std::os::unix::fs::PermissionsExt; @@ -136,17 +136,23 @@ fn extraction_preserves_files_directories_and_contained_dangling_symlinks() { } #[test] -fn extraction_uses_host_defaults_instead_of_archived_mode_and_timestamps() { +fn extraction_applies_exact_ordinary_modes_without_restoring_timestamps() { let sender = PrivateKey::generate(); let mut writer = ArchiveWriter::create( Vec::new(), WriteOptions::new(sender.duplicate(), vec![sender.public_key()]), ) .unwrap(); + writer + .add_directory( + ArchivePath::new("directory").unwrap(), + EntryMetadata::new(1, 1, 0o751), + ) + .unwrap(); writer .add_file( ArchivePath::new("data.bin").unwrap(), - EntryMetadata::new(1, 1, 0o111), + EntryMetadata::new(1, 1, 0o411), ProcessingOptions::default(), Some(7), std::io::Cursor::new(b"content"), @@ -160,11 +166,20 @@ fn extraction_uses_host_defaults_instead_of_archived_mode_and_timestamps() { let temporary = tempfile::tempdir().unwrap(); extract(&archive, "data.bin", temporary.path()).unwrap(); + extract(&archive, "directory", temporary.path()).unwrap(); let output = temporary.path().join("data.bin"); assert_eq!(fs::read(&output).unwrap(), b"content"); let metadata = fs::metadata(output).unwrap(); - assert_eq!(metadata.permissions().mode() & 0o111, 0); + assert_eq!(metadata.permissions().mode() & 0o7777, 0o411); + assert_eq!( + fs::metadata(temporary.path().join("directory")) + .unwrap() + .permissions() + .mode() + & 0o7777, + 0o751 + ); assert!( metadata .modified() @@ -176,6 +191,152 @@ fn extraction_uses_host_defaults_instead_of_archived_mode_and_timestamps() { ); } +#[test] +fn extraction_strips_special_bits_by_default_and_can_restore_sticky_directories() { + let mut writer = ArchiveWriter::create(Vec::new(), WriteOptions::base()).unwrap(); + writer + .add_file( + ArchivePath::new("special-file").unwrap(), + EntryMetadata::new(0, 0, 0o6754), + ProcessingOptions::new(false, 0).unwrap(), + Some(4), + std::io::Cursor::new(b"mode"), + ) + .unwrap(); + writer + .add_directory( + ArchivePath::new("sticky-default").unwrap(), + EntryMetadata::new(0, 0, 0o1751), + ) + .unwrap(); + writer + .add_directory( + ArchivePath::new("sticky-opt-in").unwrap(), + EntryMetadata::new(0, 0, 0o1755), + ) + .unwrap(); + let archive = Archive::open( + MemorySource::new(writer.finish().unwrap()), + OpenOptions::default(), + ) + .unwrap(); + let temporary = tempfile::tempdir().unwrap(); + + extract(&archive, "special-file", temporary.path()).unwrap(); + extract(&archive, "sticky-default", temporary.path()).unwrap(); + extract_with_options( + &archive, + "sticky-opt-in", + temporary.path(), + ExtractionOptions::default().with_special_permissions(), + ) + .unwrap(); + + for (path, expected) in [ + ("special-file", 0o754), + ("sticky-default", 0o751), + ("sticky-opt-in", 0o1755), + ] { + assert_eq!( + fs::symlink_metadata(temporary.path().join(path)) + .unwrap() + .permissions() + .mode() + & 0o7777, + expected + ); + } +} + +fn restrictive_directory_archive() -> Archive { + let mut writer = ArchiveWriter::create(Vec::new(), WriteOptions::base()).unwrap(); + writer + .add_directory( + ArchivePath::new("locked").unwrap(), + EntryMetadata::new(0, 0, 0o555), + ) + .unwrap(); + writer + .add_file( + ArchivePath::new("locked/child").unwrap(), + EntryMetadata::new(0, 0, 0o640), + ProcessingOptions::new(false, 0).unwrap(), + Some(5), + std::io::Cursor::new(b"child"), + ) + .unwrap(); + Archive::open( + MemorySource::new(writer.finish().unwrap()), + OpenOptions::default(), + ) + .unwrap() +} + +#[test] +fn batch_extraction_defers_restrictive_directory_modes_until_children_exist() { + let temporary = tempfile::tempdir().unwrap(); + extract_all(&restrictive_directory_archive(), temporary.path()).unwrap(); + assert_eq!( + fs::read(temporary.path().join("locked/child")).unwrap(), + b"child" + ); + assert_eq!( + fs::metadata(temporary.path().join("locked")) + .unwrap() + .permissions() + .mode() + & 0o7777, + 0o555 + ); + assert_eq!( + fs::metadata(temporary.path().join("locked/child")) + .unwrap() + .permissions() + .mode() + & 0o7777, + 0o640 + ); +} + +#[test] +fn batch_extraction_restores_modes_under_restrictive_umask() { + const CHILD: &str = "PITHOS_TEST_RESTRICTIVE_UMASK"; + if std::env::var_os(CHILD).is_some() { + // This process runs only this test, so changing its process-global umask is isolated. + unsafe { libc::umask(0o077) }; + let temporary = tempfile::tempdir().unwrap(); + extract_all(&restrictive_directory_archive(), temporary.path()).unwrap(); + assert_eq!( + fs::metadata(temporary.path().join("locked")) + .unwrap() + .permissions() + .mode() + & 0o7777, + 0o555 + ); + assert_eq!( + fs::metadata(temporary.path().join("locked/child")) + .unwrap() + .permissions() + .mode() + & 0o7777, + 0o640 + ); + return; + } + + let status = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "batch_extraction_restores_modes_under_restrictive_umask", + "--nocapture", + ]) + .env(CHILD, "1") + .status() + .unwrap(); + assert!(status.success()); +} + #[test] fn extraction_rejects_a_symlink_in_the_destination_root_path() { let temporary = tempfile::tempdir().unwrap(); diff --git a/crates/pithos_lib/tests/public_api.rs b/crates/pithos_lib/tests/public_api.rs index 9e253ea..6bb6d0c 100644 --- a/crates/pithos_lib/tests/public_api.rs +++ b/crates/pithos_lib/tests/public_api.rs @@ -8,7 +8,10 @@ use pithos_lib::archive::{ }; use pithos_lib::crypto::{CryptoError, PrivateKey, PublicKey}; use pithos_lib::error::PithosError; -use pithos_lib::fs::{FsError, ingest::InputManifest}; +use pithos_lib::fs::{ + ExtractionOptions, FsError, extract_all, extract_all_with_options, extract_with_options, + ingest::InputManifest, +}; use pithos_lib::source::{ArchiveSource, FileSource, MemorySource, SourceError}; fn assert_send_sync() {} @@ -53,6 +56,10 @@ fn standard_archive_configurations_are_send_and_sync() { .with_external_resolver(SendSyncResolver) .with_external_access_policy(std::sync::Arc::new(SendSyncPolicy)); let _ = WriteOptions::base(); + let _ = ExtractionOptions::default().with_special_permissions(); + let _ = extract_with_options::; + let _ = extract_all::; + let _ = extract_all_with_options::; } #[test] diff --git a/crates/pithos_lib/tests/reader.rs b/crates/pithos_lib/tests/reader.rs index 4ed2af5..0e87296 100644 --- a/crates/pithos_lib/tests/reader.rs +++ b/crates/pithos_lib/tests/reader.rs @@ -7,7 +7,7 @@ use pithos_lib::archive::{ WriteOptions, }; use pithos_lib::error::PithosError; -use pithos_lib::fs::extract; +use pithos_lib::fs::{ExtractionOptions, extract, extract_all, extract_all_with_options}; #[test] fn public_reader_lists_copies_ranges_extracts_and_exports() { @@ -140,6 +140,13 @@ fn extraction_is_no_clobber_no_follow_and_staged() { ) .unwrap(); } + writer + .add_symlink( + ArchivePath::new("dangling").unwrap(), + EntryMetadata::new(0, 0, 0o777), + "missing", + ) + .unwrap(); writer.finish().unwrap(); let archive = open(&path, AccessKeys::new().with_key(private_key("recipient1"))); let root = temporary.path().join("output"); @@ -153,4 +160,33 @@ fn extraction_is_no_clobber_no_follow_and_staged() { std::os::unix::fs::symlink(&outside, root.join("nested")).unwrap(); assert!(extract(&archive, "nested/data", &root).is_err()); assert!(!outside.join("data").exists()); + extract(&archive, "dangling", &root).unwrap(); + assert_eq!( + std::fs::read_link(root.join("dangling")).unwrap(), + std::path::Path::new("missing") + ); + + let batch = temporary.path().join("batch"); + std::fs::create_dir(&batch).unwrap(); + extract_all(&archive, &batch).unwrap(); + assert_eq!(std::fs::read(batch.join("data")).unwrap(), b"payload"); + assert_eq!( + std::fs::read(batch.join("nested/data")).unwrap(), + b"nested payload" + ); + assert_eq!( + std::fs::read_link(batch.join("dangling")).unwrap(), + std::path::Path::new("missing") + ); + assert!(extract_all(&archive, &batch).is_err()); + assert_eq!(std::fs::read(batch.join("data")).unwrap(), b"payload"); + + let batch_no_follow = temporary.path().join("batch-no-follow"); + std::fs::create_dir(&batch_no_follow).unwrap(); + std::os::unix::fs::symlink(&outside, batch_no_follow.join("nested")).unwrap(); + assert!( + extract_all_with_options(&archive, &batch_no_follow, ExtractionOptions::default(),) + .is_err() + ); + assert!(!outside.join("data").exists()); } From 1e45e68c2728497300783c61fcdedd3cf4e9cba4 Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Sun, 13 Sep 2026 16:45:39 +0200 Subject: [PATCH 048/166] refactor: split format components --- crates/pithos_lib/README.md | 4 +- crates/pithos_lib/src/adapters/ro_crate.rs | 5 +- crates/pithos_lib/src/archive/index.rs | 2 +- crates/pithos_lib/src/archive/mod.rs | 21 +- .../pithos_lib/src/archive/path_validation.rs | 70 +- crates/pithos_lib/src/archive/reader.rs | 63 +- .../src/archive/reader_private_tests.rs | 50 +- crates/pithos_lib/src/archive/snapshot.rs | 14 +- crates/pithos_lib/src/archive/validation.rs | 72 +- crates/pithos_lib/src/archive/writer.rs | 61 +- crates/pithos_lib/src/block.rs | 4 +- crates/pithos_lib/src/crypto.rs | 2 +- crates/pithos_lib/src/format/block.rs | 202 ++++ crates/pithos_lib/src/format/codec.rs | 965 ------------------ crates/pithos_lib/src/format/directory.rs | 602 ++++++++++- crates/pithos_lib/src/format/encryption.rs | 253 +++++ crates/pithos_lib/src/format/entries.rs | 189 ---- crates/pithos_lib/src/format/file_entry.rs | 387 +++++++ crates/pithos_lib/src/format/header.rs | 65 ++ crates/pithos_lib/src/format/mod.rs | 8 +- crates/pithos_lib/src/format/primitives.rs | 117 +++ crates/pithos_lib/src/format/wire.rs | 356 ------- crates/pithos_lib/src/fs/append.rs | 14 +- crates/pithos_lib/src/fs/extraction.rs | 3 +- crates/pithos_lib/src/fs/ingest.rs | 5 +- crates/pithos_lib/src/fuzzing.rs | 7 +- crates/pithos_lib/src/lib.rs | 4 +- crates/pithos_lib/tests/compile_fail.rs | 2 +- crates/pithos_lib/tests/draft_spec_vectors.rs | 469 ++++----- crates/pithos_lib/tests/format_validation.rs | 2 +- ...ivate.rs => crypt4gh_header_is_private.rs} | 0 ...derr => crypt4gh_header_is_private.stderr} | 2 +- 32 files changed, 2006 insertions(+), 2014 deletions(-) create mode 100644 crates/pithos_lib/src/format/block.rs delete mode 100644 crates/pithos_lib/src/format/codec.rs create mode 100644 crates/pithos_lib/src/format/encryption.rs delete mode 100644 crates/pithos_lib/src/format/entries.rs create mode 100644 crates/pithos_lib/src/format/file_entry.rs create mode 100644 crates/pithos_lib/src/format/header.rs create mode 100644 crates/pithos_lib/src/format/primitives.rs delete mode 100644 crates/pithos_lib/src/format/wire.rs rename crates/pithos_lib/tests/ui/{crypt4gh_wire_record_is_private.rs => crypt4gh_header_is_private.rs} (100%) rename crates/pithos_lib/tests/ui/{crypt4gh_wire_record_is_private.stderr => crypt4gh_header_is_private.stderr} (86%) diff --git a/crates/pithos_lib/README.md b/crates/pithos_lib/README.md index 7f90569..a58b4f5 100644 --- a/crates/pithos_lib/README.md +++ b/crates/pithos_lib/README.md @@ -1,6 +1,6 @@ # Pithos library -`pithos_lib` is the public Rust API for creating, opening, reading, extending, extracting, and adapting Pithos archives. It has no network client or transport policy. This branch implements the Pithos 1.0 draft wire rules; Cargo package versioning is separate. +`pithos_lib` is the public Rust API for creating, opening, reading, extending, extracting, and adapting Pithos archives. It has no network client or transport policy. Cargo package versioning is separate. ## Installation @@ -74,7 +74,7 @@ See [`examples/create.rs`](examples/create.rs) for the maintained version and th Compiled examples are included with the package: [`create`](examples/create.rs), [`open_list`](examples/open_list.rs), [`read_ranges`](examples/read_ranges.rs), [`extract`](examples/extract.rs), [`append`](examples/append.rs), [`grant_readers`](examples/grant_readers.rs), [`ro_crate`](examples/ro_crate.rs), and [`crypt4gh`](examples/crypt4gh.rs). They are local-only and return errors to their caller. -The typed archive API deliberately keeps old model, helper, and wire-record access private. Use `WriteOptions::base` for plain output or `WriteOptions::new` for encrypted output. +The typed archive API deliberately keeps old model, helper, and internal format access private. Use `WriteOptions::base` for plain output or `WriteOptions::new` for encrypted output. ## Read and extract semantics diff --git a/crates/pithos_lib/src/adapters/ro_crate.rs b/crates/pithos_lib/src/adapters/ro_crate.rs index 47030a2..955013b 100644 --- a/crates/pithos_lib/src/adapters/ro_crate.rs +++ b/crates/pithos_lib/src/adapters/ro_crate.rs @@ -9,6 +9,7 @@ use crate::archive::{ ArchivePath, ArchiveWriter, EntryMetadata, EntryReference, ProcessingOptions, WriterError, }; use crate::error::PithosError; +use crate::format::file_entry::VALID_PERMISSION_BITS; use cap_std::fs::{ Dir, File as CapFile, MetadataExt as CapMetadataExt, PermissionsExt as CapPermissionsExt, }; @@ -414,7 +415,7 @@ fn host_metadata(metadata: &cap_std::fs::Metadata) -> EntryMetadata { EntryMetadata::new( timestamp(metadata.created()), timestamp(metadata.modified()), - metadata.permissions().mode() & 0o7777, + metadata.permissions().mode() & VALID_PERMISSION_BITS, ) } @@ -1303,7 +1304,7 @@ fn inspect_ro_crate_zip_archive_manifest( metadata: EntryMetadata::new( zip_timestamp(&entry), zip_timestamp(&entry), - entry.unix_mode().unwrap_or(default_permissions) & 0o7777, + entry.unix_mode().unwrap_or(default_permissions) & VALID_PERMISSION_BITS, ), source: Some(ZipEntrySource { archive_index, diff --git a/crates/pithos_lib/src/archive/index.rs b/crates/pithos_lib/src/archive/index.rs index c8a31df..84c44db 100644 --- a/crates/pithos_lib/src/archive/index.rs +++ b/crates/pithos_lib/src/archive/index.rs @@ -5,7 +5,7 @@ use crate::archive::types::{ }; use crate::archive::validation::{IndexLimits, validate_aggregate, validate_entry}; use crate::error::PithosError; -use crate::format::wire::FileHeader; +use crate::format::header::FileHeader; use indexmap::IndexMap; use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::sync::Arc; diff --git a/crates/pithos_lib/src/archive/mod.rs b/crates/pithos_lib/src/archive/mod.rs index bf7564f..5b34e71 100644 --- a/crates/pithos_lib/src/archive/mod.rs +++ b/crates/pithos_lib/src/archive/mod.rs @@ -1,7 +1,7 @@ //! Validated, immutable archive state. //! //! This module deliberately has no filesystem, adapter, or writer-input -//! dependency. `Archive` is the public reader boundary; wire records and +//! dependency. `Archive` is the public reader boundary; internal format types and //! recovered access material remain crate-private. mod access; @@ -21,7 +21,7 @@ pub(crate) use access::{AccessProvenance, ResolvedAccess}; pub use append::{AppendDurability, AppendObservation, AppendOptions}; pub(crate) use index::build_effective_index; pub(crate) use path_validation::{ - validate_new_candidate, validate_symlink_target, validate_wire_map, + validate_directory_entries, validate_new_candidate, validate_symlink_target, }; pub(crate) use reader::ContentOperationError; pub use reader::{ @@ -31,7 +31,20 @@ pub use reader::{ pub(crate) use snapshot::AppendSnapshot; pub use types::{ArchivePath, ExternalLocation}; pub(crate) use types::{FileId, Span}; -pub(crate) use validation::segment_from_wire; +pub(crate) use validation::validated_segment_from_directory; + +pub(crate) fn decode_validated_directory( + bytes: &[u8], + limits: &crate::format::limits::DeserializationLimits, + remaining_block_references: &mut u64, +) -> Result { + crate::format::directory::decode_complete_directory_with_validation_and_budget( + bytes, + limits, + remaining_block_references, + |directory| validate_directory_entries(&directory.files), + ) +} pub use writer::{ ArchiveWriter, CdcConfig, CreateError, EntryMetadata, EntryReference, FinishError, IncompleteWriter, ProcessingOptions, WriteOptions, WriterError, WrittenEntry, @@ -93,7 +106,7 @@ mod tests { } #[test] - fn index_preserves_wire_order_and_component_hierarchy() { + fn index_preserves_segment_entry_order_and_component_hierarchy() { let entries = vec![ SegmentEntry { id: FileId(7), diff --git a/crates/pithos_lib/src/archive/path_validation.rs b/crates/pithos_lib/src/archive/path_validation.rs index f747e3c..ef19558 100644 --- a/crates/pithos_lib/src/archive/path_validation.rs +++ b/crates/pithos_lib/src/archive/path_validation.rs @@ -1,7 +1,7 @@ use crate::archive::{AppendSnapshot, ArchivePath}; use crate::error::PithosError; -use crate::format::entries::WireEntries; -use crate::format::wire::{BlockDataState, FileEntry, FileType}; +use crate::format::directory::DirectoryEntries; +use crate::format::file_entry::{BlockDataState, FileEntry, FileType, VALID_PERMISSION_BITS}; use std::collections::HashMap; fn invalid_path(path: &str, reason: impl Into) -> PithosError { @@ -92,7 +92,7 @@ pub(crate) fn validate_symlink_target(path: &str, target: &str) -> Result<(), Pi pub(crate) fn validate_entry(path: &str, entry: &FileEntry) -> Result<(), PithosError> { validate_entry_path(path)?; - if entry.permissions & !0o7777 != 0 { + if entry.permissions & !VALID_PERMISSION_BITS != 0 { return Err(PithosError::InvalidPermissions(entry.permissions)); } match entry.file_type { @@ -151,7 +151,7 @@ pub(crate) fn validate_entry(path: &str, entry: &FileEntry) -> Result<(), Pithos } fn validate_candidate_hierarchy( - map: &WireEntries, + map: &DirectoryEntries, snapshot: Option<&AppendSnapshot>, path: &str, entry: &FileEntry, @@ -206,7 +206,7 @@ fn validate_candidate_hierarchy( #[cfg(test)] pub(crate) fn validate_existing_candidate( - map: &WireEntries, + map: &DirectoryEntries, path: &str, entry: &FileEntry, ) -> Result<(), PithosError> { @@ -215,7 +215,7 @@ pub(crate) fn validate_existing_candidate( } pub(crate) fn validate_new_candidate( - map: &WireEntries, + map: &DirectoryEntries, path: &str, entry: &FileEntry, ) -> Result<(), PithosError> { @@ -229,7 +229,7 @@ pub(crate) fn validate_new_candidate( } pub(crate) fn validate_new_candidate_with_snapshot( - map: &WireEntries, + map: &DirectoryEntries, path: &str, entry: &FileEntry, snapshot: &AppendSnapshot, @@ -243,15 +243,17 @@ pub(crate) fn validate_new_candidate_with_snapshot( validate_candidate_hierarchy(map, Some(snapshot), path, entry) } -pub(crate) fn validate_wire_map(map: &WireEntries) -> Result<(), PithosError> { +pub(crate) fn validate_directory_entries(map: &DirectoryEntries) -> Result<(), PithosError> { for (_, path, entry) in map.iter() { validate_entry(path, entry)?; } - validate_wire_hierarchy(map) + validate_directory_entry_hierarchy(map) } -pub(crate) fn validate_wire_hierarchy(map: &WireEntries) -> Result<(), PithosError> { +pub(crate) fn validate_directory_entry_hierarchy( + map: &DirectoryEntries, +) -> Result<(), PithosError> { let mut earlier: HashMap<&str, &FileEntry> = HashMap::new(); for (_, path, entry) in map.iter() { for (offset, _) in path.match_indices('/') { @@ -275,7 +277,9 @@ pub(crate) fn validate_wire_hierarchy(map: &WireEntries) -> Result<(), PithosErr Ok(()) } -pub(crate) fn validate_wire_hierarchy_complete(map: &WireEntries) -> Result<(), PithosError> { +pub(crate) fn validate_directory_entry_hierarchy_complete( + map: &DirectoryEntries, +) -> Result<(), PithosError> { let mut earlier: HashMap<&str, &FileEntry> = HashMap::new(); for (_, path, entry) in map.iter() { for (offset, _) in path.match_indices('/') { @@ -298,8 +302,8 @@ pub(crate) fn validate_wire_hierarchy_complete(map: &WireEntries) -> Result<(), Ok(()) } -pub(crate) fn validate_wire_hierarchy_with_snapshot( - map: &WireEntries, +pub(crate) fn validate_directory_entry_hierarchy_with_snapshot( + map: &DirectoryEntries, snapshot: &AppendSnapshot, ) -> Result<(), PithosError> { let mut earlier: HashMap<&str, &FileEntry> = HashMap::new(); @@ -626,7 +630,7 @@ mod tests { for (ancestor_path, ancestor) in [("a", file.clone()), ("a", link.clone())] { for order in [0, 1] { - let mut map = WireEntries::new(); + let mut map = DirectoryEntries::new(); if order == 0 { map.insert(0, ancestor_path, ancestor.clone()).unwrap(); map.insert(1, "a/child", file.clone()).unwrap(); @@ -634,12 +638,12 @@ mod tests { map.insert(0, "a/child", file.clone()).unwrap(); map.insert(1, ancestor_path, ancestor.clone()).unwrap(); } - assert!(validate_wire_map(&map).is_err()); + assert!(validate_directory_entries(&map).is_err()); } } for order in [0, 1] { - let mut map = WireEntries::new(); + let mut map = DirectoryEntries::new(); if order == 0 { map.insert(0, "a/child", file.clone()).unwrap(); map.insert(1, "a", file.clone()).unwrap(); @@ -647,18 +651,18 @@ mod tests { map.insert(0, "a", file.clone()).unwrap(); map.insert(1, "a/child", file.clone()).unwrap(); } - assert!(validate_wire_map(&map).is_err()); + assert!(validate_directory_entries(&map).is_err()); } - let mut parent_first = WireEntries::new(); + let mut parent_first = DirectoryEntries::new(); parent_first.insert(0, "a", directory.clone()).unwrap(); parent_first.insert(1, "a/child", file.clone()).unwrap(); - assert!(validate_wire_map(&parent_first).is_ok()); + assert!(validate_directory_entries(&parent_first).is_ok()); - let mut child_first = WireEntries::new(); + let mut child_first = DirectoryEntries::new(); child_first.insert(0, "a/child", file.clone()).unwrap(); child_first.insert(1, "a", directory.clone()).unwrap(); - assert!(validate_wire_map(&child_first).is_err()); + assert!(validate_directory_entries(&child_first).is_err()); } #[test] @@ -674,23 +678,23 @@ mod tests { BlockDataState::Decrypted(vec![].into()), ); - let mut map = WireEntries::new(); + let mut map = DirectoryEntries::new(); map.insert(0, "a", file.clone()).unwrap(); assert!(validate_new_candidate(&map, "a/child", &file).is_err()); - let mut map = WireEntries::new(); + let mut map = DirectoryEntries::new(); map.insert(0, "a/child", file.clone()).unwrap(); assert!(validate_new_candidate(&map, "a", &file).is_err()); - let mut map = WireEntries::new(); + let mut map = DirectoryEntries::new(); map.insert(0, "a", directory.clone()).unwrap(); assert!(validate_new_candidate(&map, "a/child", &file).is_ok()); - let map = WireEntries::new(); + let map = DirectoryEntries::new(); assert!(validate_new_candidate(&map, "a/child", &file).is_err()); assert!(validate_new_candidate(&map, "a/b/child", &file).is_err()); - let mut map = WireEntries::new(); + let mut map = DirectoryEntries::new(); for (id, path) in [(0, "ab"), (1, "a!"), (2, "a.b"), (3, "a/child")] { map.insert(id, path, file.clone()).unwrap(); } @@ -699,7 +703,7 @@ mod tests { assert!(validate_new_candidate(&map, "a.bx", &file).is_ok()); assert!(validate_new_candidate(&map, "abx", &file).is_ok()); - let mut map = WireEntries::new(); + let mut map = DirectoryEntries::new(); map.insert(0, "ユニコード", file.clone()).unwrap(); assert!(validate_new_candidate(&map, "ユニコード/子", &file).is_err()); @@ -708,7 +712,7 @@ mod tests { .collect::>(); let ancestor = deep.join("/"); let descendant = format!("{ancestor}/leaf"); - let mut map = WireEntries::new(); + let mut map = DirectoryEntries::new(); map.insert(0, descendant, file.clone()).unwrap(); assert!(validate_new_candidate(&map, &ancestor, &file).is_err()); } @@ -720,7 +724,7 @@ mod tests { None, BlockDataState::Decrypted(vec![].into()), ); - let mut map = WireEntries::new(); + let mut map = DirectoryEntries::new(); map.insert(0, "occupied", file.clone()).unwrap(); assert!(validate_existing_candidate(&map, "occupied", &file).is_ok()); @@ -737,7 +741,7 @@ mod tests { None, BlockDataState::Decrypted(vec![].into()), ); - let mut map = WireEntries::new(); + let mut map = DirectoryEntries::new(); for (id, path) in [(0, "a"), (1, "a!"), (2, "a/child")] { map.insert(id, path, file.clone()).unwrap(); } @@ -746,17 +750,17 @@ mod tests { map.iter_ordered().map(|(path, _)| path).collect::>(), ["a", "a/child", "a!"] ); - assert!(validate_wire_map(&map).is_err()); + assert!(validate_directory_entries(&map).is_err()); } #[test] - fn descending_wire_paths_keep_component_order_without_vector_insertion() { + fn descending_insertions_produce_component_path_order() { let file = entry( FileType::Data, None, BlockDataState::Decrypted(vec![].into()), ); - let mut map = WireEntries::new(); + let mut map = DirectoryEntries::new(); for id in (0..10_000u64).rev() { map.insert(id, format!("entry-{id:05}"), file.clone()) .unwrap(); diff --git a/crates/pithos_lib/src/archive/reader.rs b/crates/pithos_lib/src/archive/reader.rs index 771dd08..ae68767 100644 --- a/crates/pithos_lib/src/archive/reader.rs +++ b/crates/pithos_lib/src/archive/reader.rs @@ -1,6 +1,6 @@ use super::{ AccessProvenance, AppendSnapshot, FileId, ResolvedAccess, Span, build_effective_index, - segment_from_wire, + decode_validated_directory, validated_segment_from_directory, }; use crate::archive::index::ArchiveIndex; use crate::archive::types::{ @@ -11,8 +11,14 @@ use crate::archive::validation::IndexLimits; use crate::block; use crate::crypto::{self, FileKey, PrivateKey}; use crate::error::PithosError; +use crate::format::block::{ + BlockIndexEntry, BlockLocation as FormatBlockLocation, ProcessingFlags, +}; +use crate::format::directory::Directory; +use crate::format::encryption::RecipientData; +use crate::format::file_entry::{BlockDataEntry, BlockDataState}; +use crate::format::header::FileHeader; use crate::format::limits::DeserializationLimits; -use crate::format::wire::{BlockDataEntry, BlockDataState, BlockIndexEntry, Directory, FileHeader}; use crate::source::ArchiveSource; use std::collections::{BTreeMap, HashMap, HashSet}; use std::io::Write; @@ -282,7 +288,7 @@ where let archive_len = source.len()?; let mut header = [0; FileHeader::ENCODED_LEN]; source.read_exact_at(0, &mut header)?; - let header = crate::format::codec::decode_header(&mut header.as_slice())?; + let header = crate::format::header::decode_header(&mut header.as_slice())?; if header.version != FileHeader::SUPPORTED_VERSION { return Err(PithosError::UnsupportedFileVersion { supported: FileHeader::SUPPORTED_VERSION, @@ -329,7 +335,7 @@ where }); } let bytes = Zeroizing::new(read_source(&source, start, len, "directory")?); - let directory = crate::format::codec::decode_complete_directory_with_budget( + let directory = decode_validated_directory( &bytes, &remaining_deserialization_limits(options.limits, &decoded), &mut remaining_block_references, @@ -383,7 +389,7 @@ where let parent = segment_index .checked_sub(1) .map(|index| segments[index].span); - segments.push(segment_from_wire(&directory, span, parent)?); + segments.push(validated_segment_from_directory(&directory, span, parent)?); } let index_limits = IndexLimits { max_entries: options.limits.max_entries, @@ -401,7 +407,7 @@ where for span in index.local_block_spans() { let mut marker = [0; 4]; source.read_exact_at(span.start(), &mut marker)?; - crate::format::codec::decode_block_marker(&mut marker.as_slice())?; + crate::format::block::decode_block_marker(&mut marker.as_slice())?; } Ok(Self { @@ -581,7 +587,7 @@ where BlockLocation::Local(span) => { let mut marker = [0; 4]; self.source.read_exact_at(span.start(), &mut marker)?; - crate::format::codec::decode_block_marker(&mut marker.as_slice())?; + crate::format::block::decode_block_marker(&mut marker.as_slice())?; let payload_start = span.start() .checked_add(4) @@ -623,7 +629,7 @@ where )); } let (mut marker, stored) = response.split_at(4); - crate::format::codec::decode_block_marker(&mut marker)?; + crate::format::block::decode_block_marker(&mut marker)?; stored.to_vec() } }; @@ -631,10 +637,8 @@ where offset: 0, stored_size: planned.descriptor.stored_size, original_size: planned.descriptor.original_size, - flags: crate::format::wire::ProcessingFlags::from_byte( - planned.descriptor.processing.to_byte(), - ), - location: crate::format::wire::BlockLocation::Local, + flags: ProcessingFlags::from_byte(planned.descriptor.processing.to_byte()), + location: FormatBlockLocation::Local, }; let key = self .access @@ -846,31 +850,30 @@ fn resolve_recipients( let secret = key.as_dalek_static_secret(); let recipient = DalekPublicKey::from(&secret).to_bytes(); for (sender_section, (sender, section)) in directory.encryption.iter().enumerate() { - let candidates: Vec<(&[u8; 32], &crate::format::wire::RecipientData)> = - if sender == &recipient { - section - .recipients - .iter() - .map(|(recipient, section)| (recipient, §ion.recipient_data)) - .collect() - } else { - section - .recipients - .get(&recipient) - .map(|section| vec![(sender, §ion.recipient_data)]) - .unwrap_or_default() - }; + let candidates: Vec<(&[u8; 32], &RecipientData)> = if sender == &recipient { + section + .recipients + .iter() + .map(|(recipient, section)| (recipient, §ion.recipient_data)) + .collect() + } else { + section + .recipients + .get(&recipient) + .map(|section| vec![(sender, §ion.recipient_data)]) + .unwrap_or_default() + }; for (recipient_section, (peer, data)) in candidates.into_iter().enumerate() { let shared = crypto::derive_shared(secret.as_bytes(), peer)?; let entries = match data { - crate::format::wire::RecipientData::Encrypted(bytes) => { + RecipientData::Encrypted(bytes) => { let plaintext = crypto::unwrap_recipient_list(&shared, bytes)?; - crate::format::codec::decode_decrypted_recipient_list( + crate::format::encryption::decode_decrypted_recipient_list( &plaintext, &decoded_limits, )? } - crate::format::wire::RecipientData::Decrypted(entries) => entries.clone(), + RecipientData::Decrypted(entries) => entries.clone(), }; for (file_id, file_key) in entries.iter() { access.insert( @@ -911,7 +914,7 @@ fn resolve_block_lists( let mut decoded_limits = deserialization_limits(limits); decoded_limits.max_block_references = *remaining_block_references; let plaintext = crypto::open_file_block_list(file_key, bytes)?; - let entries = crate::format::codec::decode_decrypted_block_list_with_budget( + let entries = crate::format::file_entry::decode_decrypted_block_list_with_budget( &plaintext, &decoded_limits, remaining_block_references, diff --git a/crates/pithos_lib/src/archive/reader_private_tests.rs b/crates/pithos_lib/src/archive/reader_private_tests.rs index 3aa2c13..a700910 100644 --- a/crates/pithos_lib/src/archive/reader_private_tests.rs +++ b/crates/pithos_lib/src/archive/reader_private_tests.rs @@ -7,11 +7,11 @@ use crate::archive::{ }; use crate::crypto::{self, FileKey, PrivateKey, PublicKey}; use crate::error::PithosError; +use crate::format::block::BlockLocation; +use crate::format::directory::{Directory, DirectoryEntries}; +use crate::format::encryption::{EncryptionSection, RecipientData, RecipientSection}; +use crate::format::file_entry::{BlockDataState, FileEntry, FileType}; use crate::format::limits::{DeserializationError, DeserializationLimits}; -use crate::format::wire::{ - BlockDataState, BlockLocation, Directory, EncryptionSection, FileEntry, FileType, - RecipientData, RecipientSection, -}; use crate::source::{ArchiveSource, MemorySource, SourceError}; use indexmap::IndexMap; use std::fs::File; @@ -133,16 +133,16 @@ fn directory_bounds(bytes: &[u8]) -> (usize, usize) { fn rewrite_terminal_directory(path: &Path, mutate: impl FnOnce(&mut Directory)) { let mut archive = std::fs::read(path).unwrap(); let (start, _) = directory_bounds(&archive); - let mut directory = crate::format::codec::decode_directory( + let mut directory = crate::format::directory::decode_directory( &mut Cursor::new(&archive[start..]), &DeserializationLimits::default(), ) .unwrap(); mutate(&mut directory); - crate::format::codec::update_directory_len(&mut directory).unwrap(); - crate::format::codec::update_directory_crc(&mut directory).unwrap(); + crate::format::directory::update_directory_len(&mut directory).unwrap(); + crate::format::directory::update_directory_crc(&mut directory).unwrap(); let mut replacement = Vec::new(); - crate::format::codec::encode_directory(&directory, &mut replacement).unwrap(); + crate::format::directory::encode_directory(&directory, &mut replacement).unwrap(); archive.truncate(start); archive.extend_from_slice(&replacement); std::fs::write(path, archive).unwrap(); @@ -151,7 +151,7 @@ fn rewrite_terminal_directory(path: &Path, mutate: impl FnOnce(&mut Directory)) fn decode_terminal_directory(path: &Path) -> Directory { let archive = std::fs::read(path).unwrap(); let (start, _) = directory_bounds(&archive); - crate::format::codec::decode_directory( + crate::format::directory::decode_directory( &mut Cursor::new(&archive[start..]), &DeserializationLimits::default(), ) @@ -163,22 +163,22 @@ fn append_empty_directory(path: &Path, relations: Option>) { let (parent_start, parent_len) = directory_bounds(&archive); let mut directory = Directory::new( Some((parent_start as u64, parent_len as u64)), - crate::format::entries::WireEntries::new(), + DirectoryEntries::new(), IndexMap::new(), ); if let Some(relations) = relations { directory.relations = relations; } - crate::format::codec::update_directory_len(&mut directory).unwrap(); - crate::format::codec::update_directory_crc(&mut directory).unwrap(); - crate::format::codec::encode_directory(&directory, &mut archive).unwrap(); + crate::format::directory::update_directory_len(&mut directory).unwrap(); + crate::format::directory::update_directory_crc(&mut directory).unwrap(); + crate::format::directory::encode_directory(&directory, &mut archive).unwrap(); std::fs::write(path, archive).unwrap(); } fn append_recipient_directory(path: &Path, mutate: impl FnOnce(&mut RecipientData)) { let mut archive = std::fs::read(path).unwrap(); let (parent_start, parent_len) = directory_bounds(&archive); - let parent = crate::format::codec::decode_directory( + let parent = crate::format::directory::decode_directory( &mut Cursor::new(&archive[parent_start..]), &DeserializationLimits::default(), ) @@ -195,12 +195,12 @@ fn append_recipient_directory(path: &Path, mutate: impl FnOnce(&mut RecipientDat )]); let mut child = Directory::new( Some((parent_start as u64, parent_len as u64)), - crate::format::entries::WireEntries::new(), + DirectoryEntries::new(), encryption, ); - crate::format::codec::update_directory_len(&mut child).unwrap(); - crate::format::codec::update_directory_crc(&mut child).unwrap(); - crate::format::codec::encode_directory(&child, &mut archive).unwrap(); + crate::format::directory::update_directory_len(&mut child).unwrap(); + crate::format::directory::update_directory_crc(&mut child).unwrap(); + crate::format::directory::encode_directory(&child, &mut archive).unwrap(); std::fs::write(path, archive).unwrap(); } @@ -265,7 +265,7 @@ fn open_without_keys(path: &Path) -> Result, PithosError> fn first_block(path: &Path) -> (u64, u64) { let bytes = std::fs::read(path).unwrap(); let (start, _) = directory_bounds(&bytes); - let directory = crate::format::codec::decode_directory( + let directory = crate::format::directory::decode_directory( &mut Cursor::new(&bytes[start..]), &DeserializationLimits::default(), ) @@ -312,7 +312,7 @@ fn recover_first_file_key(path: &Path) -> FileKey { }; let shared = crypto::derive_shared(recipient.as_bytes(), sender).unwrap(); let plaintext = crypto::unwrap_recipient_list(&shared, data).unwrap(); - let records = crate::format::codec::decode_decrypted_recipient_list( + let records = crate::format::encryption::decode_decrypted_recipient_list( &plaintext, &DeserializationLimits::default(), ) @@ -449,7 +449,7 @@ fn appended_standard_relationship_repeats_must_match() { #[test] fn archive_rejects_no_content_sizes_and_undefined_permission_bits() { - let mutations: [fn(&mut crate::format::wire::FileEntry); 3] = [ + let mutations: [fn(&mut crate::format::file_entry::FileEntry); 3] = [ |entry| { entry.file_type = FileType::Directory; entry.block_data = BlockDataState::Decrypted(Vec::new().into()); @@ -1027,7 +1027,7 @@ fn direct_block_lists_share_one_decode_budget_within_and_across_directories() { let (parent_start, parent_len) = directory_bounds(&bytes); let parent = decode_terminal_directory(&appended); let (hash, descriptor) = parent.blocks.first().unwrap(); - let mut files = crate::format::entries::WireEntries::with_maximum_id(0); + let mut files = DirectoryEntries::with_maximum_id(0); files .insert( 1, @@ -1049,9 +1049,9 @@ fn direct_block_lists_share_one_decode_budget_within_and_across_directories() { files, IndexMap::new(), ); - crate::format::codec::update_directory_len(&mut child).unwrap(); - crate::format::codec::update_directory_crc(&mut child).unwrap(); - crate::format::codec::encode_directory(&child, &mut bytes).unwrap(); + crate::format::directory::update_directory_len(&mut child).unwrap(); + crate::format::directory::update_directory_crc(&mut child).unwrap(); + crate::format::directory::encode_directory(&child, &mut bytes).unwrap(); assert_block_reference_limit(Archive::open( MemorySource::new(Arc::<[u8]>::from(bytes)), OpenOptions::default().with_limits(limits), diff --git a/crates/pithos_lib/src/archive/snapshot.rs b/crates/pithos_lib/src/archive/snapshot.rs index 4145050..91205c6 100644 --- a/crates/pithos_lib/src/archive/snapshot.rs +++ b/crates/pithos_lib/src/archive/snapshot.rs @@ -6,7 +6,7 @@ use crate::archive::types::{ use crate::archive::validation::IndexLimits; use crate::crypto::{FileKey, PublicKey}; use crate::error::PithosError; -use crate::format::wire::EncryptionSection; +use crate::format::encryption::EncryptionSection; use indexmap::IndexMap; use std::collections::{BTreeMap, HashMap, HashSet}; use std::sync::Arc; @@ -14,7 +14,7 @@ use std::sync::Arc; /// The secret-contained state needed to safely plan a direct append or grant. /// /// This is intentionally derived only from a fully opened archive. It retains validated -/// segments for pure prospective merge, but not the source, wire directories, or reader facade. +/// segments for pure prospective merge, but not the source, decoded directories, or reader facade. pub(crate) struct AppendSnapshot { archive_len: u64, terminal_directory: Span, @@ -420,7 +420,7 @@ mod tests { let first_len = u64::from_be_bytes(first[first.len() - 12..first.len() - 4].try_into().unwrap()); let first_start = first.len() - usize::try_from(first_len).unwrap(); - let first_directory = crate::format::codec::decode_directory( + let first_directory = crate::format::directory::decode_directory( &mut Cursor::new(&first[first_start..]), &DeserializationLimits::default(), ) @@ -448,7 +448,7 @@ mod tests { bytes.extend_from_slice(&child); let len = u64::from_be_bytes(bytes[bytes.len() - 12..bytes.len() - 4].try_into().unwrap()); let start = bytes.len() - usize::try_from(len).unwrap(); - let mut directory = crate::format::codec::decode_directory( + let mut directory = crate::format::directory::decode_directory( &mut Cursor::new(&bytes[start..]), &DeserializationLimits::default(), ) @@ -457,10 +457,10 @@ mod tests { let mut newer = earliest.clone(); newer.stored_size -= 1; directory.blocks.insert(hash.0, newer); - crate::format::codec::update_directory_len(&mut directory).unwrap(); - crate::format::codec::update_directory_crc(&mut directory).unwrap(); + crate::format::directory::update_directory_len(&mut directory).unwrap(); + crate::format::directory::update_directory_crc(&mut directory).unwrap(); let mut replacement = Vec::new(); - crate::format::codec::encode_directory(&directory, &mut replacement).unwrap(); + crate::format::directory::encode_directory(&directory, &mut replacement).unwrap(); bytes.truncate(start); bytes.write_all(&replacement).unwrap(); diff --git a/crates/pithos_lib/src/archive/validation.rs b/crates/pithos_lib/src/archive/validation.rs index 56efbd3..d102563 100644 --- a/crates/pithos_lib/src/archive/validation.rs +++ b/crates/pithos_lib/src/archive/validation.rs @@ -4,10 +4,9 @@ use crate::archive::types::{ SegmentEntry, Span, ValidatedSegment, }; use crate::error::PithosError; -use crate::format::directory::STANDARD_RELATIONSHIPS; -use crate::format::wire::{ - BlockDataState, BlockLocation as WireBlockLocation, Directory, FileType, -}; +use crate::format::block::BlockLocation as FormatBlockLocation; +use crate::format::directory::{Directory, STANDARD_RELATIONSHIPS}; +use crate::format::file_entry::{BlockDataState, FileEntry, FileType}; use std::collections::{BTreeMap, HashSet}; use std::sync::Arc; @@ -32,21 +31,21 @@ impl Default for IndexLimits { } } -pub(crate) fn segment_from_wire( +pub(crate) fn validated_segment_from_directory( directory: &Directory, span: Span, parent: Option, ) -> Result { let mut entries = Vec::new(); let mut entry_ids = HashSet::new(); - for (id, path, wire) in directory.files.iter() { + for (id, path, file_entry) in directory.files.iter() { if !entry_ids.insert(id) { return Err(PithosError::DuplicateFileId(format!( "File id already occupied: {id}" ))); } let path = ArchivePath::new(path)?; - let entry = entry_from_wire(path.as_str(), wire)?; + let entry = entry_from_file_entry(path.as_str(), file_entry)?; entries.push(SegmentEntry { id: FileId(id), path, @@ -55,35 +54,35 @@ pub(crate) fn segment_from_wire( } validate_relationships(directory)?; let mut descriptors = Vec::new(); - for (hash, wire) in &directory.blocks { - let processing = Processing::from_byte(wire.flags.0)?; - if matches!(wire.location, WireBlockLocation::Local) + for (hash, block_index_entry) in &directory.blocks { + let processing = Processing::from_byte(block_index_entry.flags.0)?; + if matches!(block_index_entry.location, FormatBlockLocation::Local) && processing.to_byte() & 0x08 != 0 - && wire.stored_size < 28 + && block_index_entry.stored_size < 28 { return Err(PithosError::InvalidBlockDescriptor( "encrypted local block payload is shorter than 28 bytes", )); } - let stored_size_with_marker = - wire.stored_size - .checked_add(4) - .ok_or(PithosError::InvalidDirectoryRange { - operation: "validate block range", - })?; - let location = match &wire.location { - WireBlockLocation::Local => { - BlockLocation::Local(Span::new(wire.offset, stored_size_with_marker)?) - } - WireBlockLocation::External { url } => { + let stored_size_with_marker = block_index_entry.stored_size.checked_add(4).ok_or( + PithosError::InvalidDirectoryRange { + operation: "validate block range", + }, + )?; + let location = match &block_index_entry.location { + FormatBlockLocation::Local => BlockLocation::Local(Span::new( + block_index_entry.offset, + stored_size_with_marker, + )?), + FormatBlockLocation::External { url } => { BlockLocation::External(ExternalLocation::new(url)) } }; descriptors.push(( BlockHash(*hash), BlockDescriptor { - stored_size: wire.stored_size, - original_size: wire.original_size, + stored_size: block_index_entry.stored_size, + original_size: block_index_entry.original_size, processing, location, }, @@ -123,16 +122,13 @@ pub(crate) fn segment_from_wire( }) } -fn entry_from_wire( - path: &str, - wire: &crate::format::wire::FileEntry, -) -> Result { - crate::archive::path_validation::validate_entry(path, wire)?; +fn entry_from_file_entry(path: &str, file_entry: &FileEntry) -> Result { + crate::archive::path_validation::validate_entry(path, file_entry)?; let metadata = EntryMetadata { - created: wire.created, - modified: wire.modified, - permissions: wire.permissions, - references: wire + created: file_entry.created, + modified: file_entry.modified, + permissions: file_entry.permissions, + references: file_entry .references .iter() .map(|reference| Reference { @@ -141,7 +137,7 @@ fn entry_from_wire( }) .collect(), }; - let content = || match &wire.block_data { + let content = || match &file_entry.block_data { BlockDataState::Decrypted(blocks) => { ContentState::Available(crate::archive::types::BlockReferences::new( blocks.iter().map(|(hash, _)| BlockHash(*hash)).collect(), @@ -149,20 +145,20 @@ fn entry_from_wire( } BlockDataState::Encrypted(_) => ContentState::Unavailable, }; - match wire.file_type { + match file_entry.file_type { FileType::Data => Ok(Entry::File(ContentEntry { metadata, - size: wire.file_size, + size: file_entry.file_size, content: content(), })), FileType::Metadata => Ok(Entry::Metadata(ContentEntry { metadata, - size: wire.file_size, + size: file_entry.file_size, content: content(), })), FileType::Directory => Ok(Entry::Directory(metadata)), FileType::Symlink => { - let target = wire + let target = file_entry .symlink_target .as_deref() .expect("validated symlink entry has a target"); diff --git a/crates/pithos_lib/src/archive/writer.rs b/crates/pithos_lib/src/archive/writer.rs index 33c4d86..b7fde3b 100644 --- a/crates/pithos_lib/src/archive/writer.rs +++ b/crates/pithos_lib/src/archive/writer.rs @@ -1,21 +1,21 @@ //! Streaming archive construction with a consuming publication boundary. use crate::archive::path_validation::{ - validate_new_candidate_with_snapshot, validate_wire_hierarchy_complete, - validate_wire_hierarchy_with_snapshot, + validate_directory_entry_hierarchy_complete, validate_directory_entry_hierarchy_with_snapshot, + validate_new_candidate_with_snapshot, }; use crate::archive::validation::validate_relationships; -use crate::archive::{AppendSnapshot, ArchivePath, FileId, Span, segment_from_wire}; +use crate::archive::{AppendSnapshot, ArchivePath, FileId, Span, validated_segment_from_directory}; use crate::archive::{validate_new_candidate, validate_symlink_target}; use crate::block; use crate::crypto::{FileKey, PrivateKey, PublicKey}; use crate::error::PithosError; -use crate::format::codec; -use crate::format::entries::WireEntries; -use crate::format::wire::{ - BlockDataState, BlockHeader, BlockIndexEntry, BlockLocation, EncryptionSection, FileEntry, - FileHeader, FileType, ProcessingFlags, RecipientData, Reference, -}; +use crate::format::block::{BlockHeader, BlockIndexEntry, BlockLocation, ProcessingFlags}; +use crate::format::directory::{Directory, DirectoryEntries}; +use crate::format::encryption::{EncryptionSection, RecipientData}; +use crate::format::file_entry::{BlockDataState, FileEntry, FileType, Reference}; +use crate::format::header::FileHeader; +use crate::format::{directory, header}; use fastcdc::v2020::{Normalization, StreamCDC}; use indexmap::IndexMap; use std::collections::{HashMap, HashSet}; @@ -534,7 +534,7 @@ pub struct ArchiveWriter { mode: ArchiveWriterMode, cdc: CdcConfig, sink: CountingSink, - directory: crate::format::wire::Directory, + directory: Directory, poisoned: bool, runtime: Box, append_snapshot: Option, @@ -585,9 +585,9 @@ impl ArchiveWriter { (ArchiveWriterMode::Encrypted { sender }, encryption) } }; - let directory = crate::format::wire::Directory::new(None, WireEntries::new(), encryption); + let directory = Directory::new(None, DirectoryEntries::new(), encryption); let mut sink = CountingSink { sink, offset: 0 }; - if let Err(error) = codec::encode_header(&FileHeader::default(), &mut sink) { + if let Err(error) = header::encode_header(&FileHeader::default(), &mut sink) { return Err(CreateError { error: error.into(), sink: sink.into_inner(), @@ -625,16 +625,12 @@ impl ArchiveWriter { .collect::>(); let parent = snapshot.terminal_directory(); let maximum_id = snapshot.maximum_id(); - let files = WireEntries::with_maximum_id(maximum_id.map_or(0, |id| id.0)); + let files = DirectoryEntries::with_maximum_id(maximum_id.map_or(0, |id| id.0)); let encryption = IndexMap::from_iter([( LegacyPublicKey::from(&sender).to_bytes(), EncryptionSection::new(&recipients), )]); - let directory = crate::format::wire::Directory::new( - Some((parent.start(), parent.len())), - files, - encryption, - ); + let directory = Directory::new(Some((parent.start(), parent.len())), files, encryption); Ok(Self { mode: ArchiveWriterMode::Encrypted { sender }, cdc, @@ -1200,7 +1196,7 @@ impl ArchiveWriter { } fn write_block(&mut self, bytes: &[u8]) -> Result<(), PithosError> { - codec::encode_block_marker(&BlockHeader::default(), &mut self.sink)?; + crate::format::block::encode_block_marker(&BlockHeader::default(), &mut self.sink)?; self.sink.write_all(bytes)?; Ok(()) } @@ -1257,15 +1253,18 @@ impl ArchiveWriter { self.validate_required_access_records()?; self.seal_recipient_lists()?; self.validate_publishable()?; - codec::update_directory_len(&mut self.directory)?; - codec::update_directory_crc(&mut self.directory)?; + directory::update_directory_len(&mut self.directory)?; + directory::update_directory_crc(&mut self.directory)?; if let Some(snapshot) = &self.append_snapshot { let span = Span::new(self.sink.offset, self.directory.dir_len)?; - let child = - segment_from_wire(&self.directory, span, Some(snapshot.terminal_directory()))?; + let child = validated_segment_from_directory( + &self.directory, + span, + Some(snapshot.terminal_directory()), + )?; snapshot.validate_prospective_child(child)?; } - codec::encode_directory(&self.directory, &mut self.sink)?; + directory::encode_directory(&self.directory, &mut self.sink)?; self.sink.flush()?; Ok(()) })(); @@ -1356,9 +1355,9 @@ impl ArchiveWriter { crate::archive::path_validation::validate_entry(path, entry)?; } if let Some(snapshot) = &self.append_snapshot { - validate_wire_hierarchy_with_snapshot(&self.directory.files, snapshot)?; + validate_directory_entry_hierarchy_with_snapshot(&self.directory.files, snapshot)?; } else { - validate_wire_hierarchy_complete(&self.directory.files)?; + validate_directory_entry_hierarchy_complete(&self.directory.files)?; } self.directory .validate_references_and_accessible_blocks_with( @@ -2015,7 +2014,7 @@ mod tests { #[test] fn file_id_exhaustion_does_not_mutate_or_poison_before_streaming() { let mut writer = ArchiveWriter::create(Vec::new(), options()).unwrap(); - writer.directory.files = WireEntries::with_maximum_id(u64::MAX); + writer.directory.files = DirectoryEntries::with_maximum_id(u64::MAX); assert!(matches!( writer.add_directory( ArchivePath::new("data").unwrap(), @@ -2087,9 +2086,11 @@ mod tests { )); let bytes = child.finish().unwrap(); - let directory = - codec::decode_directory(&mut Cursor::new(&bytes), &DeserializationLimits::default()) - .unwrap(); + let directory = directory::decode_directory( + &mut Cursor::new(&bytes), + &DeserializationLimits::default(), + ) + .unwrap(); assert_eq!(directory.files.len(), 0); assert_eq!(directory.blocks.len(), 0); assert_eq!(directory.encryption.len(), 1); diff --git a/crates/pithos_lib/src/block.rs b/crates/pithos_lib/src/block.rs index 5720cd0..d2ae91f 100644 --- a/crates/pithos_lib/src/block.rs +++ b/crates/pithos_lib/src/block.rs @@ -2,7 +2,7 @@ use crate::crypto::{self, BlockKey}; use crate::error::PithosError; -use crate::format::wire::{BlockIndexEntry, ProcessingFlags}; +use crate::format::block::{BlockIndexEntry, ProcessingFlags}; use std::fmt; use zeroize::Zeroizing; use zstd::bulk; @@ -153,7 +153,7 @@ fn decompress(input: &[u8], expected_size: u64) -> Result, PithosError> #[cfg(test)] mod tests { use super::*; - use crate::format::wire::BlockLocation; + use crate::format::block::BlockLocation; fn descriptor(encoded: &EncodedBlock, original_size: usize) -> BlockIndexEntry { BlockIndexEntry { diff --git a/crates/pithos_lib/src/crypto.rs b/crates/pithos_lib/src/crypto.rs index 081fc35..c9666cc 100644 --- a/crates/pithos_lib/src/crypto.rs +++ b/crates/pithos_lib/src/crypto.rs @@ -67,7 +67,7 @@ macro_rules! secret_role { } // SharedSecret is recovered only through derive_shared, while FileKey - // and BlockKey need this wire-to-owned-key conversion. + // and BlockKey need this decoded-key-to-owned-key conversion. #[allow(dead_code)] pub(crate) fn from_protocol(bytes: &[u8; 32]) -> Self { Self(*bytes) diff --git a/crates/pithos_lib/src/format/block.rs b/crates/pithos_lib/src/format/block.rs new file mode 100644 index 0000000..98e2e04 --- /dev/null +++ b/crates/pithos_lib/src/format/block.rs @@ -0,0 +1,202 @@ +use crate::format::error::SerializationError; +use crate::format::limits::{DeserializationError, DeserializationLimits}; +use crate::format::primitives::{decode_string, encode_string}; +use integer_encoding::{VarIntReader, VarIntWriter}; +use std::io::{Read, Write}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BlockHeader { + pub marker: [u8; 4], // MUST be b"BLCK" +} + +impl Default for BlockHeader { + fn default() -> Self { + BlockHeader { marker: *b"BLCK" } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ProcessingFlags(pub u8); + +impl ProcessingFlags { + const COMPRESSION_MASK: u8 = 0b0000_0111; + const ENCRYPTION_MASK: u8 = 0b0000_1000; + pub const RESERVED_MASK: u8 = 0b1111_0000; + + pub fn new(encrypted: bool, compression_level: Option) -> Self { + let mut flags = ProcessingFlags(0b0); + match compression_level { + Some(level) => { + if level > Self::COMPRESSION_MASK { + flags.set_compression_level(Self::COMPRESSION_MASK) + } else { + flags.set_compression_level(level) + } + } + None => flags.set_compression_level(3), + } + flags.set_encryption(encrypted); + flags + } + + pub fn from_byte(byte: u8) -> Self { + ProcessingFlags(byte) + } + + pub fn has_reserved_bits(&self) -> bool { + self.0 & Self::RESERVED_MASK != 0 + } + + pub fn set_encryption(&mut self, encrypted: bool) { + if encrypted { + self.0 |= Self::ENCRYPTION_MASK; + } else { + self.0 &= !Self::ENCRYPTION_MASK; + } + } + + pub fn is_encrypted(&self) -> bool { + (self.0 & Self::ENCRYPTION_MASK) != 0 + } + + pub fn set_compression_level(&mut self, mut compression_level: u8) { + compression_level = if compression_level > Self::COMPRESSION_MASK { + Self::COMPRESSION_MASK + } else { + compression_level + }; + self.0 = (self.0 & !Self::COMPRESSION_MASK) | (compression_level & Self::COMPRESSION_MASK); + } + + pub fn get_compression_level(&self) -> u8 { + self.0 & Self::COMPRESSION_MASK + } +} + +impl Default for ProcessingFlags { + fn default() -> Self { + ProcessingFlags::new(true, None) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum BlockLocation { + Local, + External { url: String }, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BlockIndexEntry { + pub offset: u64, + pub stored_size: u64, + pub original_size: u64, + pub flags: ProcessingFlags, + pub location: BlockLocation, +} + +pub(crate) fn encode_block_marker( + header: &BlockHeader, + writer: &mut W, +) -> Result<(), SerializationError> { + writer.write_all(&header.marker)?; + Ok(()) +} + +pub(crate) fn decode_block_marker( + reader: &mut R, +) -> Result { + let mut marker = [0; 4]; + reader.read_exact(&mut marker)?; + if marker != *b"BLCK" { + return Err(DeserializationError::InvalidMarker(format!( + "Read invalid block marker {marker:?}" + ))); + } + Ok(BlockHeader { marker }) +} + +pub(crate) fn encode_flags( + flags: &ProcessingFlags, + writer: &mut W, +) -> Result<(), SerializationError> { + writer.write_all(&[flags.0])?; + Ok(()) +} + +pub(crate) fn decode_flags( + reader: &mut R, +) -> Result { + let mut byte = [0]; + reader.read_exact(&mut byte)?; + let flags = ProcessingFlags::from_byte(byte[0]); + if flags.has_reserved_bits() { + return Err(DeserializationError::InvalidProcessingFlags(byte[0])); + } + Ok(flags) +} + +fn encode_location( + location: &BlockLocation, + writer: &mut W, +) -> Result<(), SerializationError> { + match location { + BlockLocation::Local => writer.write_all(&[0])?, + BlockLocation::External { url } => { + writer.write_all(&[1])?; + encode_string(writer, url)?; + } + } + Ok(()) +} + +fn decode_location( + reader: &mut R, + limits: &DeserializationLimits, +) -> Result { + let mut tag = [0]; + reader.read_exact(&mut tag)?; + match tag[0] { + 0 => Ok(BlockLocation::Local), + 1 => Ok(BlockLocation::External { + url: decode_string(reader, limits)?, + }), + value => Err(DeserializationError::InvalidEnumValue(value)), + } +} + +pub(crate) fn encode_block_index_entry( + entry: &BlockIndexEntry, + writer: &mut W, +) -> Result<(), SerializationError> { + writer.write_varint(entry.offset)?; + writer.write_varint(entry.stored_size)?; + writer.write_varint(entry.original_size)?; + encode_flags(&entry.flags, writer)?; + encode_location(&entry.location, writer) +} + +pub(crate) fn decode_block_index_entry( + reader: &mut R, + limits: &DeserializationLimits, +) -> Result { + Ok(BlockIndexEntry { + offset: reader.read_varint::()?, + stored_size: reader.read_varint::()?, + original_size: reader.read_varint::()?, + flags: decode_flags(reader)?, + location: decode_location(reader, limits)?, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn block_markers_reject_wrong_tags() { + assert!(matches!( + decode_block_marker(&mut &b"BLOK"[..]), + Err(DeserializationError::InvalidMarker(_)) + )); + } +} diff --git a/crates/pithos_lib/src/format/codec.rs b/crates/pithos_lib/src/format/codec.rs deleted file mode 100644 index 6ade402..0000000 --- a/crates/pithos_lib/src/format/codec.rs +++ /dev/null @@ -1,965 +0,0 @@ -use crate::error::PithosError; -use crate::format::entries::WireEntries; -use crate::format::error::SerializationError; -use crate::format::limits::{DeserializationError, DeserializationLimits}; -use crate::format::wire::RecipientKeyList; -use crate::format::wire::*; -use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt}; -use crc32fast::Hasher; -use indexmap::IndexMap; -use integer_encoding::{VarIntReader, VarIntWriter}; -use std::collections::{HashMap, HashSet}; -use std::io::{Cursor, Read, Write}; -use zeroize::Zeroizing; - -const DIRECTORY_MARKER: [u8; 8] = *b"PITHOSDR"; -const MIN_DIRECTORY_LEN: usize = 25; - -pub(crate) fn write_len_prefix( - writer: &mut W, - len: usize, -) -> Result<(), SerializationError> { - writer.write_varint( - u64::try_from(len) - .map_err(|_| SerializationError::Other("length does not fit in u64".to_string()))?, - )?; - Ok(()) -} - -pub(crate) fn encode_string( - writer: &mut W, - value: &str, -) -> Result<(), SerializationError> { - write_len_prefix(writer, value.len())?; - writer.write_all(value.as_bytes())?; - Ok(()) -} - -fn bounded_len(value: u64, limit: u64, field: &'static str) -> Result { - if value > limit { - return Err(DeserializationError::LimitExceeded { - field, - limit, - actual: value, - }); - } - usize::try_from(value).map_err(|_| DeserializationError::InvalidLength) -} - -fn reserve( - output: &mut Vec, - count: usize, - field: &'static str, -) -> Result<(), DeserializationError> { - output - .try_reserve(count) - .map_err(|_| DeserializationError::AllocationFailed { - field, - size: count as u64, - }) -} - -pub(crate) fn decode_string( - reader: &mut R, - limits: &DeserializationLimits, -) -> Result { - let len = bounded_len( - reader.read_varint::()?, - limits.max_string_bytes, - "string", - )?; - let mut bytes = Vec::new(); - reserve(&mut bytes, len, "string")?; - bytes.resize(len, 0); - reader.read_exact(&mut bytes)?; - Ok(String::from_utf8(bytes)?) -} - -pub(crate) fn encode_header( - header: &FileHeader, - writer: &mut W, -) -> Result<(), SerializationError> { - writer.write_all(&header.magic)?; - writer.write_u16::(header.version)?; - Ok(()) -} - -pub(crate) fn decode_header(reader: &mut R) -> Result { - let mut magic = [0; 4]; - reader.read_exact(&mut magic)?; - if magic != *b"PITH" { - return Err(DeserializationError::InvalidMarker(format!( - "Read invalid file marker {magic:?}" - ))); - } - Ok(FileHeader { - magic, - version: reader.read_u16::()?, - }) -} - -pub(crate) fn encode_block_marker( - header: &BlockHeader, - writer: &mut W, -) -> Result<(), SerializationError> { - writer.write_all(&header.marker)?; - Ok(()) -} - -pub(crate) fn decode_block_marker( - reader: &mut R, -) -> Result { - let mut marker = [0; 4]; - reader.read_exact(&mut marker)?; - if marker != *b"BLCK" { - return Err(DeserializationError::InvalidMarker(format!( - "Read invalid block marker {marker:?}" - ))); - } - Ok(BlockHeader { marker }) -} - -pub(crate) fn encode_flags( - flags: &ProcessingFlags, - writer: &mut W, -) -> Result<(), SerializationError> { - writer.write_all(&[flags.0])?; - Ok(()) -} - -pub(crate) fn decode_flags( - reader: &mut R, -) -> Result { - let mut byte = [0]; - reader.read_exact(&mut byte)?; - let flags = ProcessingFlags::from_byte(byte[0]); - if flags.has_reserved_bits() { - return Err(DeserializationError::InvalidProcessingFlags(byte[0])); - } - Ok(flags) -} - -fn encode_location( - location: &BlockLocation, - writer: &mut W, -) -> Result<(), SerializationError> { - match location { - BlockLocation::Local => writer.write_all(&[0])?, - BlockLocation::External { url } => { - writer.write_all(&[1])?; - encode_string(writer, url)?; - } - } - Ok(()) -} - -fn decode_location( - reader: &mut R, - limits: &DeserializationLimits, -) -> Result { - let mut tag = [0]; - reader.read_exact(&mut tag)?; - match tag[0] { - 0 => Ok(BlockLocation::Local), - 1 => Ok(BlockLocation::External { - url: decode_string(reader, limits)?, - }), - value => Err(DeserializationError::InvalidEnumValue(value)), - } -} - -pub(crate) fn encode_block_index_entry( - entry: &BlockIndexEntry, - writer: &mut W, -) -> Result<(), SerializationError> { - writer.write_varint(entry.offset)?; - writer.write_varint(entry.stored_size)?; - writer.write_varint(entry.original_size)?; - encode_flags(&entry.flags, writer)?; - encode_location(&entry.location, writer) -} - -pub(crate) fn decode_block_index_entry( - reader: &mut R, - limits: &DeserializationLimits, -) -> Result { - Ok(BlockIndexEntry { - offset: reader.read_varint::()?, - stored_size: reader.read_varint::()?, - original_size: reader.read_varint::()?, - flags: decode_flags(reader)?, - location: decode_location(reader, limits)?, - }) -} - -fn encode_file_type( - file_type: &FileType, - writer: &mut W, -) -> Result<(), SerializationError> { - writer.write_all(&[*file_type as u8])?; - Ok(()) -} - -pub(crate) fn decode_file_type(reader: &mut R) -> Result { - let mut tag = [0]; - reader.read_exact(&mut tag)?; - match tag[0] { - 0 => Ok(FileType::Directory), - 1 => Ok(FileType::Data), - 2 => Ok(FileType::Metadata), - 3 => Ok(FileType::Symlink), - value => Err(DeserializationError::InvalidEnumValue(value)), - } -} - -fn encode_reference( - reference: &Reference, - writer: &mut W, -) -> Result<(), SerializationError> { - writer.write_varint(reference.target_file_id)?; - writer.write_varint(reference.relationship)?; - Ok(()) -} - -fn decode_reference(reader: &mut R) -> Result { - Ok(Reference { - target_file_id: reader.read_varint::()?, - relationship: reader.read_varint::()?, - }) -} - -fn encode_block_data( - data: &BlockDataState, - writer: &mut W, -) -> Result<(), SerializationError> { - match data { - BlockDataState::Encrypted(bytes) => { - writer.write_all(&[0])?; - write_len_prefix(writer, bytes.len())?; - writer.write_all(bytes)?; - } - BlockDataState::Decrypted(entries) => { - writer.write_all(&[1])?; - write_len_prefix(writer, entries.len())?; - for (hash, key) in entries.iter() { - writer.write_all(hash)?; - writer.write_all(key)?; - } - } - } - Ok(()) -} - -fn decode_block_data( - reader: &mut R, - limits: &DeserializationLimits, - remaining_block_references: &mut u64, -) -> Result { - let mut tag = [0]; - reader.read_exact(&mut tag)?; - match tag[0] { - 0 => { - let len = bounded_len( - reader.read_varint::()?, - limits.max_opaque_bytes, - "encrypted block", - )?; - let mut bytes = Vec::new(); - reserve(&mut bytes, len, "encrypted block")?; - bytes.resize(len, 0); - reader.read_exact(&mut bytes)?; - Ok(BlockDataState::Encrypted(bytes)) - } - 1 => Ok(BlockDataState::Decrypted( - decode_decrypted_block_list_reader_with_budget( - reader, - limits, - remaining_block_references, - )?, - )), - value => Err(DeserializationError::InvalidEnumValue(value)), - } -} - -fn encode_file_entry( - entry: &FileEntry, - writer: &mut W, -) -> Result<(), SerializationError> { - encode_file_type(&entry.file_type, writer)?; - encode_block_data(&entry.block_data, writer)?; - writer.write_varint(entry.created)?; - writer.write_varint(entry.modified)?; - writer.write_varint(entry.file_size)?; - writer.write_varint(entry.permissions)?; - write_len_prefix(writer, entry.references.len())?; - for reference in &entry.references { - encode_reference(reference, writer)?; - } - match &entry.symlink_target { - Some(target) => { - writer.write_all(&[1])?; - encode_string(writer, target)?; - } - None => writer.write_all(&[0])?, - } - Ok(()) -} - -fn decode_file_entry( - reader: &mut R, - limits: &DeserializationLimits, - remaining_references: &mut u64, - remaining_block_references: &mut u64, -) -> Result { - let file_type = decode_file_type(reader)?; - let block_data = decode_block_data(reader, limits, remaining_block_references)?; - let created = reader.read_varint::()?; - let modified = reader.read_varint::()?; - let file_size = reader.read_varint::()?; - let permissions = reader.read_varint::()?; - let count = bounded_len( - reader.read_varint::()?, - *remaining_references, - "references", - )?; - *remaining_references -= count as u64; - let mut references = Vec::new(); - reserve(&mut references, count, "references")?; - for _ in 0..count { - references.push(decode_reference(reader)?); - } - let mut tag = [0]; - reader.read_exact(&mut tag)?; - let symlink_target = match tag[0] { - 0 => None, - 1 => Some(decode_string(reader, limits)?), - _ => return Err(DeserializationError::InvalidOption), - }; - Ok(FileEntry { - file_type, - block_data, - created, - modified, - file_size, - permissions, - references, - symlink_target, - }) -} - -fn encode_recipient_data( - data: &RecipientData, - writer: &mut W, -) -> Result<(), SerializationError> { - match data { - RecipientData::Encrypted(bytes) => { - writer.write_all(&[0])?; - write_len_prefix(writer, bytes.len())?; - writer.write_all(bytes)?; - } - RecipientData::Decrypted(entries) => { - writer.write_all(&[1])?; - encode_decrypted_recipient_list(entries, writer)?; - } - } - Ok(()) -} - -fn decode_recipient_data( - reader: &mut R, - limits: &DeserializationLimits, -) -> Result { - let mut tag = [0]; - reader.read_exact(&mut tag)?; - match tag[0] { - 0 => { - let len = bounded_len( - reader.read_varint::()?, - limits.max_opaque_bytes, - "encrypted recipient data", - )?; - let mut bytes = Vec::new(); - reserve(&mut bytes, len, "encrypted recipient data")?; - bytes.resize(len, 0); - reader.read_exact(&mut bytes)?; - Ok(RecipientData::Encrypted(bytes)) - } - 1 => Ok(RecipientData::Decrypted( - decode_decrypted_recipient_list_reader(reader, limits)?, - )), - value => Err(DeserializationError::InvalidEnumValue(value)), - } -} - -fn encode_encryption_section( - section: &EncryptionSection, - writer: &mut W, -) -> Result<(), SerializationError> { - write_len_prefix(writer, section.recipients.len())?; - for (key, recipient) in §ion.recipients { - writer.write_all(key)?; - encode_recipient_data(&recipient.recipient_data, writer)?; - } - Ok(()) -} - -fn decode_encryption_section( - reader: &mut R, - limits: &DeserializationLimits, -) -> Result { - let count = bounded_len( - reader.read_varint::()?, - limits.max_collection_entries, - "recipients", - )?; - let mut recipients = IndexMap::new(); - for _ in 0..count { - let mut key = [0; 32]; - reader.read_exact(&mut key)?; - crate::crypto::validate_x25519_public_key(&key)?; - if recipients.contains_key(&key) { - return Err(DeserializationError::DuplicateRecipientKey.into()); - } - recipients.insert( - key, - RecipientSection { - recipient_data: decode_recipient_data(reader, limits)?, - }, - ); - } - Ok(EncryptionSection { recipients }) -} - -pub(crate) fn encode_directory( - directory: &Directory, - writer: &mut W, -) -> Result<(), SerializationError> { - writer.write_all(&directory.identifier)?; - match directory.parent_directory_offset { - Some((start, len)) => { - writer.write_all(&[1])?; - writer.write_varint(start)?; - writer.write_varint(len)?; - } - None => writer.write_all(&[0])?, - } - write_len_prefix(writer, directory.files.len())?; - for (id, path, entry) in directory.files.iter() { - writer.write_varint::(id)?; - encode_string(writer, path)?; - encode_file_entry(entry, writer)?; - } - write_len_prefix(writer, directory.blocks.len())?; - for (hash, entry) in &directory.blocks { - writer.write_all(hash)?; - encode_block_index_entry(entry, writer)?; - } - write_len_prefix(writer, directory.relations.len())?; - for (id, name) in &directory.relations { - writer.write_varint(*id)?; - encode_string(writer, name)?; - } - write_len_prefix(writer, directory.encryption.len())?; - for (key, section) in &directory.encryption { - writer.write_all(key)?; - encode_encryption_section(section, writer)?; - } - writer.write_u64::(directory.dir_len)?; - writer.write_u32::(directory.crc32)?; - Ok(()) -} - -#[allow(dead_code)] -pub(crate) fn decode_directory( - reader: &mut R, - limits: &DeserializationLimits, -) -> Result { - let mut remaining_block_references = limits.max_block_references; - decode_directory_with_budget(reader, limits, &mut remaining_block_references) -} - -pub(crate) fn decode_directory_with_budget( - reader: &mut R, - limits: &DeserializationLimits, - remaining_block_references: &mut u64, -) -> Result { - let mut identifier = [0; 8]; - reader.read_exact(&mut identifier)?; - if identifier != DIRECTORY_MARKER { - return Err(PithosError::InvalidDirectoryMarker { - expected: DIRECTORY_MARKER, - actual: identifier, - }); - } - let mut tag = [0]; - reader.read_exact(&mut tag)?; - let parent_directory_offset = match tag[0] { - 0 => None, - 1 => Some(( - reader - .read_varint::() - .map_err(DeserializationError::from)?, - reader - .read_varint::() - .map_err(DeserializationError::from)?, - )), - _ => return Err(DeserializationError::InvalidOption.into()), - }; - let file_count = bounded_len( - reader - .read_varint::() - .map_err(DeserializationError::from)?, - limits.max_file_entries, - "files", - )?; - let mut files = WireEntries::new(); - let mut remaining_references = limits.max_references; - for _ in 0..file_count { - let id = reader - .read_varint::() - .map_err(DeserializationError::from)?; - let path = decode_string(reader, limits)?; - files.insert( - id, - path, - decode_file_entry( - reader, - limits, - &mut remaining_references, - remaining_block_references, - )?, - )?; - } - let block_count = bounded_len( - reader - .read_varint::() - .map_err(DeserializationError::from)?, - limits.max_block_descriptors, - "blocks", - )?; - let mut blocks = IndexMap::new(); - for _ in 0..block_count { - let mut hash = [0; 32]; - reader.read_exact(&mut hash)?; - if blocks.contains_key(&hash) { - return Err(PithosError::DuplicateBlockHash); - } - blocks.insert(hash, decode_block_index_entry(reader, limits)?); - } - let relation_count = bounded_len( - reader - .read_varint::() - .map_err(DeserializationError::from)?, - limits.max_relationships, - "relations", - )?; - let mut relations = Vec::new(); - reserve(&mut relations, relation_count, "relations")?; - let mut relation_ids = HashSet::new(); - for _ in 0..relation_count { - let id = reader - .read_varint::() - .map_err(DeserializationError::from)?; - let name = decode_string(reader, limits)?; - if !relation_ids.insert(id) { - return Err(PithosError::ConflictingRelationshipDefinition(id)); - } - relations.push((id, name)); - } - let encryption_count = bounded_len( - reader - .read_varint::() - .map_err(DeserializationError::from)?, - limits.max_collection_entries, - "encryption", - )?; - let mut encryption = IndexMap::new(); - for _ in 0..encryption_count { - let mut key = [0; 32]; - reader.read_exact(&mut key)?; - crate::crypto::validate_x25519_public_key(&key)?; - if encryption.contains_key(&key) { - return Err(PithosError::DuplicateSenderKey); - } - encryption.insert(key, decode_encryption_section(reader, limits)?); - } - let directory = Directory { - identifier, - parent_directory_offset, - files, - blocks, - relations, - encryption, - dir_len: reader.read_u64::()?, - crc32: reader.read_u32::()?, - }; - crate::archive::validate_wire_map(&directory.files)?; - Ok(directory) -} - -#[allow(dead_code)] -pub(crate) fn decode_complete_directory( - bytes: &[u8], - limits: &DeserializationLimits, -) -> Result { - let mut remaining_block_references = limits.max_block_references; - decode_complete_directory_with_budget(bytes, limits, &mut remaining_block_references) -} - -pub(crate) fn decode_complete_directory_with_budget( - bytes: &[u8], - limits: &DeserializationLimits, - remaining_block_references: &mut u64, -) -> Result { - if bytes.len() < MIN_DIRECTORY_LEN { - return Err(PithosError::DirectoryLengthMismatch { - expected: MIN_DIRECTORY_LEN as u64, - actual: bytes.len() as u64, - }); - } - let actual_marker: [u8; 8] = bytes[..8] - .try_into() - .expect("checked directory minimum length"); - if actual_marker != DIRECTORY_MARKER { - return Err(PithosError::InvalidDirectoryMarker { - expected: DIRECTORY_MARKER, - actual: actual_marker, - }); - } - let encoded_len = u64::from_be_bytes( - bytes[bytes.len() - 12..bytes.len() - 4] - .try_into() - .expect("checked directory footer"), - ); - if encoded_len != bytes.len() as u64 { - return Err(PithosError::DirectoryLengthMismatch { - expected: bytes.len() as u64, - actual: encoded_len, - }); - } - let encoded_crc = u32::from_be_bytes( - bytes[bytes.len() - 4..] - .try_into() - .expect("checked directory footer"), - ); - let computed_crc = crc32fast::hash(&bytes[..bytes.len() - 4]); - if encoded_crc != computed_crc { - return Err(PithosError::DirectoryChecksumMismatch { - expected: computed_crc, - actual: encoded_crc, - }); - } - let mut reader = Cursor::new(bytes); - let directory = decode_directory_with_budget(&mut reader, limits, remaining_block_references)?; - if reader.position() != bytes.len() as u64 { - return Err(PithosError::DirectoryConsumptionMismatch { - expected: bytes.len() as u64, - actual: reader.position(), - }); - } - Ok(directory) -} - -pub(crate) fn update_directory_len(directory: &mut Directory) -> Result<(), SerializationError> { - let mut bytes = Vec::new(); - encode_directory(directory, &mut bytes)?; - directory.dir_len = u64::try_from(bytes.len()) - .map_err(|_| SerializationError::Other("length does not fit in u64".to_string()))?; - Ok(()) -} - -pub(crate) fn update_directory_crc(directory: &mut Directory) -> Result<(), SerializationError> { - let mut bytes = Vec::new(); - encode_directory(directory, &mut bytes)?; - let mut hasher = Hasher::new(); - hasher.update(&bytes[..bytes.len() - 4]); - directory.crc32 = hasher.finalize(); - Ok(()) -} - -pub(crate) fn encode_decrypted_block_list( - entries: &[BlockDataEntry], - writer: &mut W, -) -> Result<(), SerializationError> { - write_len_prefix(writer, entries.len())?; - for (hash, key) in entries { - writer.write_all(hash)?; - writer.write_all(key)?; - } - Ok(()) -} - -#[allow(dead_code)] -pub(crate) fn decode_decrypted_block_list_reader( - reader: &mut R, - limits: &DeserializationLimits, -) -> Result>, DeserializationError> { - let mut remaining_block_references = limits.max_block_references; - decode_decrypted_block_list_reader_with_budget(reader, limits, &mut remaining_block_references) -} - -fn decode_decrypted_block_list_reader_with_budget( - reader: &mut R, - limits: &DeserializationLimits, - remaining_block_references: &mut u64, -) -> Result>, DeserializationError> { - let count = bounded_len( - reader.read_varint::()?, - limits.max_block_references.min(*remaining_block_references), - "block references", - )?; - *remaining_block_references -= count as u64; - let mut entries = Zeroizing::new(Vec::new()); - reserve(&mut entries, count, "block references")?; - let mut keys = HashMap::with_capacity(count); - for _ in 0..count { - let mut hash = [0; 32]; - reader.read_exact(&mut hash)?; - let mut key = [0; 32]; - reader.read_exact(&mut key)?; - if keys - .insert(hash, key) - .is_some_and(|existing| existing != key) - { - return Err(DeserializationError::DuplicateBlockReference); - } - entries.push((hash, key)); - } - Ok(entries) -} - -#[allow(dead_code)] -pub(crate) fn decode_decrypted_block_list( - bytes: &[u8], - limits: &DeserializationLimits, -) -> Result>, DeserializationError> { - let mut remaining_block_references = limits.max_block_references; - decode_decrypted_block_list_with_budget(bytes, limits, &mut remaining_block_references) -} - -pub(crate) fn decode_decrypted_block_list_with_budget( - bytes: &[u8], - limits: &DeserializationLimits, - remaining_block_references: &mut u64, -) -> Result>, DeserializationError> { - let mut reader = Cursor::new(bytes); - let entries = decode_decrypted_block_list_reader_with_budget( - &mut reader, - limits, - remaining_block_references, - )?; - if reader.position() != bytes.len() as u64 { - return Err(DeserializationError::InvalidLength); - } - Ok(entries) -} - -pub(crate) fn encode_decrypted_recipient_list( - entries: &[(u64, [u8; 32])], - writer: &mut W, -) -> Result<(), SerializationError> { - write_len_prefix(writer, entries.len())?; - for (id, key) in entries { - writer.write_varint(*id)?; - writer.write_all(key)?; - } - Ok(()) -} - -pub(crate) fn decode_decrypted_recipient_list_reader( - reader: &mut R, - limits: &DeserializationLimits, -) -> Result { - let count = bounded_len( - reader.read_varint::()?, - limits.max_collection_entries, - "recipient keys", - )?; - let mut entries = Zeroizing::new(Vec::new()); - reserve(&mut entries, count, "recipient keys")?; - let mut keys = HashMap::with_capacity(count); - for _ in 0..count { - let id = reader.read_varint::()?; - entries.push((id, [0; 32])); - reader.read_exact(&mut entries.last_mut().expect("just pushed file key").1)?; - let key = entries.last().expect("just pushed file key").1; - if keys.insert(id, key).is_some_and(|existing| existing != key) { - return Err(DeserializationError::DuplicateRecipientFileId); - } - } - Ok(entries) -} - -pub(crate) fn decode_decrypted_recipient_list( - bytes: &[u8], - limits: &DeserializationLimits, -) -> Result { - let mut reader = Cursor::new(bytes); - let entries = decode_decrypted_recipient_list_reader(&mut reader, limits)?; - if reader.position() != bytes.len() as u64 { - return Err(DeserializationError::InvalidLength); - } - Ok(entries) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn file_header_uses_the_fixed_width_1_0_encoding() { - let mut encoded = Vec::new(); - encode_header(&FileHeader::default(), &mut encoded).unwrap(); - assert_eq!(encoded, b"PITH\x01\x00"); - - let decoded = decode_header(&mut encoded.as_slice()).unwrap(); - assert_eq!(decoded.version, 0x0100); - - let error = decode_header(&mut &b"POTX\x01\x00"[..]).unwrap_err(); - assert!(error.to_string().contains("file marker")); - } - - #[test] - fn typed_uleb128_accepts_bounded_non_minimal_values() { - for (mut encoded, expected) in [ - (&[0x00][..], 0_u64), - (&[0x7f][..], 127), - (&[0x80, 0x01][..], 128), - (&[0x80, 0x00][..], 0), - (&[0x81, 0x00][..], 1), - ] { - assert_eq!(encoded.read_varint::().unwrap(), expected); - } - } - - #[test] - fn typed_uleb128_rejects_truncated_and_width_overflowing_values() { - assert!((&[0x80][..]).read_varint::().is_err()); - assert!( - (&[0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x02][..]) - .read_varint::() - .is_err() - ); - assert!( - (&[ - 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x00, - ][..]) - .read_varint::() - .is_err() - ); - assert!( - (&[0x80, 0x80, 0x80, 0x80, 0x10][..]) - .read_varint::() - .is_err() - ); - } - - #[test] - fn decrypted_recipient_lists_retain_exact_duplicates_but_reject_conflicts() { - let limits = DeserializationLimits::default(); - let mut repeated = vec![2, 7]; - repeated.extend_from_slice(&[3; 32]); - repeated.push(7); - repeated.extend_from_slice(&[3; 32]); - assert_eq!( - decode_decrypted_recipient_list(&repeated, &limits) - .unwrap() - .as_slice(), - &[(7, [3; 32]), (7, [3; 32])] - ); - - let mut conflicting = vec![2, 7]; - conflicting.extend_from_slice(&[3; 32]); - conflicting.push(7); - conflicting.extend_from_slice(&[4; 32]); - assert!(matches!( - decode_decrypted_recipient_list(&conflicting, &limits), - Err(DeserializationError::DuplicateRecipientFileId) - )); - } - - #[test] - fn complete_directory_rejects_bad_footer_and_trailing_bytes() { - let bytes = vec![ - 0x50, 0x49, 0x54, 0x48, 0x4f, 0x53, 0x44, 0x52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, - 0x19, 0xb1, 0x67, 0x40, 0x81, - ]; - assert!(decode_complete_directory(&bytes, &DeserializationLimits::default()).is_ok()); - let mut trailing = bytes.clone(); - trailing.push(0); - assert!(matches!( - decode_complete_directory(&trailing, &DeserializationLimits::default()), - Err(PithosError::DirectoryLengthMismatch { .. }) - )); - let mut corrupt = bytes; - corrupt[0] = b'X'; - assert!(matches!( - decode_complete_directory(&corrupt, &DeserializationLimits::default()), - Err(PithosError::InvalidDirectoryMarker { .. }) - )); - } - - #[test] - fn header_and_block_markers_reject_wrong_tags() { - assert!(matches!( - decode_header(&mut &b"POT\x80"[..]), - Err(DeserializationError::InvalidMarker(_)) - )); - assert!(matches!( - decode_block_marker(&mut &b"BLOK"[..]), - Err(DeserializationError::InvalidMarker(_)) - )); - } - - #[test] - fn authenticated_lists_require_exact_consumption_and_unique_keys() { - let limits = DeserializationLimits::default(); - assert!(matches!( - decode_decrypted_block_list(&[0, 0], &limits), - Err(DeserializationError::InvalidLength) - )); - let mut duplicate = vec![2]; - duplicate.extend_from_slice(&[1; 32]); - duplicate.extend_from_slice(&[2; 32]); - duplicate.extend_from_slice(&[1; 32]); - duplicate.extend_from_slice(&[3; 32]); - assert!(matches!( - decode_decrypted_block_list(&duplicate, &limits), - Err(DeserializationError::DuplicateBlockReference) - )); - let mut repeated = vec![2]; - repeated.extend_from_slice(&[1; 32]); - repeated.extend_from_slice(&[2; 32]); - repeated.extend_from_slice(&[1; 32]); - repeated.extend_from_slice(&[2; 32]); - assert_eq!( - decode_decrypted_block_list(&repeated, &limits) - .unwrap() - .len(), - 2 - ); - } - - #[test] - fn directory_rejects_duplicate_relationship_definitions() { - let mut directory = Directory { - identifier: DIRECTORY_MARKER, - parent_directory_offset: None, - blocks: IndexMap::new(), - files: WireEntries::new(), - relations: vec![(7, "same".to_owned()), (7, "same".to_owned())], - encryption: IndexMap::new(), - dir_len: 0, - crc32: 0, - }; - update_directory_len(&mut directory).unwrap(); - update_directory_crc(&mut directory).unwrap(); - let mut bytes = Vec::new(); - encode_directory(&directory, &mut bytes).unwrap(); - - assert!(matches!( - decode_complete_directory(&bytes, &DeserializationLimits::default()), - Err(PithosError::ConflictingRelationshipDefinition(7)) - )); - } -} diff --git a/crates/pithos_lib/src/format/directory.rs b/crates/pithos_lib/src/format/directory.rs index 2616961..09742bc 100644 --- a/crates/pithos_lib/src/format/directory.rs +++ b/crates/pithos_lib/src/format/directory.rs @@ -1,6 +1,224 @@ +use crate::crypto; use crate::error::PithosError; -use crate::format::entries::WireEntries; -use crate::format::wire::{BlockDataState, Directory, EncryptionSection, FileEntry, FileType}; +use crate::format::block::{BlockIndexEntry, decode_block_index_entry, encode_block_index_entry}; +use crate::format::encryption::{ + EncryptionSection, decode_encryption_section, encode_encryption_section, +}; +use crate::format::error::SerializationError; +use crate::format::file_entry::{ + BlockDataState, FileEntry, FileType, decode_file_entry, encode_file_entry, + validate_unique_block_references, +}; +use crate::format::limits::{DeserializationError, DeserializationLimits}; +use crate::format::primitives::{ + bounded_len, decode_string, encode_string, reserve, write_len_prefix, +}; +use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt}; +use crc32fast::Hasher; +use indexmap::IndexMap; +use integer_encoding::{VarIntReader, VarIntWriter}; +use std::cmp::Ordering; +use std::collections::{BTreeMap, HashSet}; +use std::io::{Cursor, Read, Write}; +use std::ops::Bound::Excluded; +use std::sync::Arc; + +const DIRECTORY_MARKER: [u8; 8] = *b"PITHOSDR"; +const MIN_DIRECTORY_LEN: usize = 25; + +#[derive(Clone, Debug, Eq, PartialEq)] +struct OrderedPath(Arc); + +impl Ord for OrderedPath { + fn cmp(&self, other: &Self) -> Ordering { + self.0.split('/').cmp(other.0.split('/')) + } +} + +impl PartialOrd for OrderedPath { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct DirectoryEntry { + id: u64, + path: Arc, + entry: FileEntry, +} + +/// Format-owned entries with stable serialization order and private indexes. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct DirectoryEntries { + by_id: IndexMap, + by_path: IndexMap, usize>, + ordered_paths: BTreeMap, + entries: Vec, + maximum_id: u64, +} + +impl DirectoryEntries { + pub(crate) fn new() -> Self { + Self::with_maximum_id(0) + } + + pub(crate) fn with_maximum_id(maximum_id: u64) -> Self { + Self { + by_id: IndexMap::new(), + by_path: IndexMap::new(), + ordered_paths: BTreeMap::new(), + entries: Vec::new(), + maximum_id, + } + } + + pub(crate) fn len(&self) -> usize { + self.entries.len() + } + + pub(crate) fn is_empty(&self) -> bool { + self.entries.is_empty() + } + + pub(crate) fn get_by_id(&self, id: u64) -> Option<&FileEntry> { + self.by_id + .get(&id) + .and_then(|index| self.entries.get(*index)) + .map(|entry| &entry.entry) + } + + pub(crate) fn get_by_path(&self, path: &str) -> Option<&FileEntry> { + self.by_path + .get(path) + .and_then(|index| self.entries.get(*index)) + .map(|entry| &entry.entry) + } + + pub(crate) fn first_path_after(&self, path: &str) -> Option<&str> { + let query = OrderedPath(Arc::from(path)); + self.ordered_paths + .range((Excluded(query), std::ops::Bound::Unbounded)) + .next() + .map(|(path, _)| path.0.as_ref()) + } + + pub(crate) fn iter(&self) -> impl ExactSizeIterator + '_ { + self.entries + .iter() + .map(|entry| (entry.id, entry.path.as_ref(), &entry.entry)) + } + + #[cfg(test)] + pub(crate) fn iter_ordered(&self) -> impl Iterator + '_ { + self.ordered_paths + .iter() + .map(|(path, index)| (path.0.as_ref(), &self.entries[*index].entry)) + } + + pub(crate) fn try_for_each_mut( + &mut self, + mut operation: impl FnMut(u64, &mut FileEntry) -> Result<(), E>, + ) -> Result<(), E> { + for entry in &mut self.entries { + operation(entry.id, &mut entry.entry)?; + } + Ok(()) + } + + pub(crate) fn next_free_id(&self, has_parent: bool) -> Result { + if self.maximum_id == 0 && self.entries.is_empty() { + Ok(u64::from(has_parent)) + } else { + self.maximum_id + .checked_add(1) + .ok_or(PithosError::FileIdExhausted) + } + } + + pub(crate) fn prevalidate_insert(&self, id: u64, path: &str) -> Result<(), PithosError> { + if self.by_id.contains_key(&id) { + return Err(PithosError::DuplicateFileId(format!( + "File id already occupied: {id}" + ))); + } + if self.by_path.contains_key(path) { + return Err(PithosError::PathOccupied(format!( + "File path already occupied: {path}" + ))); + } + Ok(()) + } + + pub(crate) fn reserve_one(&mut self) -> Result<(), PithosError> { + let size = self + .entries + .len() + .checked_add(1) + .ok_or(PithosError::AllocationFailed { + field: "file entries", + size: u64::MAX, + })?; + self.by_id + .try_reserve(1) + .map_err(|_| allocation_failed("file entry ids", size))?; + self.by_path + .try_reserve(1) + .map_err(|_| allocation_failed("file entry paths", size))?; + self.entries + .try_reserve(1) + .map_err(|_| allocation_failed("file entries", size)) + } + + /// Insert only after validation and reservation of the vector-backed indexes. + pub(crate) fn insert_prepared(&mut self, id: u64, path: impl Into>, entry: FileEntry) { + let path = path.into(); + debug_assert!(self.prevalidate_insert(id, &path).is_ok()); + let ordered = OrderedPath(Arc::clone(&path)); + let index = self.entries.len(); + self.entries.push(DirectoryEntry { + id, + path: Arc::clone(&path), + entry, + }); + self.by_id.insert(id, index); + self.by_path.insert(path, index); + self.ordered_paths.insert(ordered, index); + self.maximum_id = self.maximum_id.max(id); + } + + pub(crate) fn insert( + &mut self, + id: u64, + path: impl Into>, + entry: FileEntry, + ) -> Result<(), PithosError> { + let path = path.into(); + self.prevalidate_insert(id, &path)?; + self.reserve_one()?; + self.insert_prepared(id, path, entry); + Ok(()) + } +} + +fn allocation_failed(field: &'static str, size: usize) -> PithosError { + PithosError::AllocationFailed { + field, + size: u64::try_from(size).unwrap_or(u64::MAX), + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Directory { + pub identifier: [u8; 8], + pub parent_directory_offset: Option<(u64, u64)>, + pub blocks: IndexMap<[u8; 32], BlockIndexEntry>, + pub files: DirectoryEntries, + pub relations: Vec<(u64, String)>, + pub encryption: IndexMap<[u8; 32], EncryptionSection>, + pub dir_len: u64, + pub crc32: u32, +} pub(crate) const STANDARD_RELATIONSHIPS: [(u64, &str); 10] = [ (0, "DESCRIBES"), @@ -18,8 +236,8 @@ pub(crate) const STANDARD_RELATIONSHIPS: [(u64, &str); 10] = [ impl Directory { pub(crate) fn new( parent_directory_offset: Option<(u64, u64)>, - files: WireEntries, - encryption: indexmap::IndexMap<[u8; 32], EncryptionSection>, + files: DirectoryEntries, + encryption: IndexMap<[u8; 32], EncryptionSection>, ) -> Self { let relations = if parent_directory_offset.is_none() { STANDARD_RELATIONSHIPS @@ -30,10 +248,10 @@ impl Directory { Vec::new() }; Self { - identifier: *b"PITHOSDR", + identifier: DIRECTORY_MARKER, parent_directory_offset, files, - blocks: indexmap::IndexMap::new(), + blocks: IndexMap::new(), relations, encryption, dir_len: 0, @@ -51,7 +269,7 @@ impl Directory { .relations .iter() .map(|(id, _)| *id) - .collect::>(); + .collect::>(); for (_, _, file) in self.files.iter() { for reference in &file.references { if !relationship_ids.contains(&reference.relationship) @@ -69,16 +287,16 @@ impl Directory { continue; } if let BlockDataState::Decrypted(references) = &file.block_data { - crate::format::wire::validate_unique_block_references(references)?; + validate_unique_block_references(references)?; let actual = references.iter().try_fold(0u64, |total, (hash, _)| { let original_size = block_size(*hash).ok_or(PithosError::MissingBlockDescriptor)?; - total.checked_add(original_size).ok_or({ + total.checked_add(original_size).ok_or( PithosError::AccessibleFileSizeMismatch { expected: file.file_size, actual: u64::MAX, - } - }) + }, + ) })?; if actual != file.file_size { return Err(PithosError::AccessibleFileSizeMismatch { @@ -102,3 +320,365 @@ impl Directory { self.files.get_by_id(file_id) } } + +pub(crate) fn encode_directory( + directory: &Directory, + writer: &mut W, +) -> Result<(), SerializationError> { + writer.write_all(&directory.identifier)?; + match directory.parent_directory_offset { + Some((start, len)) => { + writer.write_all(&[1])?; + writer.write_varint(start)?; + writer.write_varint(len)?; + } + None => writer.write_all(&[0])?, + } + write_len_prefix(writer, directory.files.len())?; + for (id, path, entry) in directory.files.iter() { + writer.write_varint::(id)?; + encode_string(writer, path)?; + encode_file_entry(entry, writer)?; + } + write_len_prefix(writer, directory.blocks.len())?; + for (hash, entry) in &directory.blocks { + writer.write_all(hash)?; + encode_block_index_entry(entry, writer)?; + } + write_len_prefix(writer, directory.relations.len())?; + for (id, name) in &directory.relations { + writer.write_varint(*id)?; + encode_string(writer, name)?; + } + write_len_prefix(writer, directory.encryption.len())?; + for (key, section) in &directory.encryption { + writer.write_all(key)?; + encode_encryption_section(section, writer)?; + } + writer.write_u64::(directory.dir_len)?; + writer.write_u32::(directory.crc32)?; + Ok(()) +} + +#[allow(dead_code)] +pub(crate) fn decode_directory( + reader: &mut R, + limits: &DeserializationLimits, +) -> Result { + let mut remaining_block_references = limits.max_block_references; + decode_directory_with_budget(reader, limits, &mut remaining_block_references) +} + +pub(crate) fn decode_directory_with_budget( + reader: &mut R, + limits: &DeserializationLimits, + remaining_block_references: &mut u64, +) -> Result { + let mut identifier = [0; 8]; + reader.read_exact(&mut identifier)?; + if identifier != DIRECTORY_MARKER { + return Err(PithosError::InvalidDirectoryMarker { + expected: DIRECTORY_MARKER, + actual: identifier, + }); + } + let mut tag = [0]; + reader.read_exact(&mut tag)?; + let parent_directory_offset = match tag[0] { + 0 => None, + 1 => Some(( + reader + .read_varint::() + .map_err(DeserializationError::from)?, + reader + .read_varint::() + .map_err(DeserializationError::from)?, + )), + _ => return Err(DeserializationError::InvalidOption.into()), + }; + let file_count = bounded_len( + reader + .read_varint::() + .map_err(DeserializationError::from)?, + limits.max_file_entries, + "files", + )?; + let mut files = DirectoryEntries::new(); + let mut remaining_references = limits.max_references; + for _ in 0..file_count { + let id = reader + .read_varint::() + .map_err(DeserializationError::from)?; + let path = decode_string(reader, limits)?; + files.insert( + id, + path, + decode_file_entry( + reader, + limits, + &mut remaining_references, + remaining_block_references, + )?, + )?; + } + let block_count = bounded_len( + reader + .read_varint::() + .map_err(DeserializationError::from)?, + limits.max_block_descriptors, + "blocks", + )?; + let mut blocks = IndexMap::new(); + for _ in 0..block_count { + let mut hash = [0; 32]; + reader.read_exact(&mut hash)?; + if blocks.contains_key(&hash) { + return Err(PithosError::DuplicateBlockHash); + } + blocks.insert(hash, decode_block_index_entry(reader, limits)?); + } + let relation_count = bounded_len( + reader + .read_varint::() + .map_err(DeserializationError::from)?, + limits.max_relationships, + "relations", + )?; + let mut relations = Vec::new(); + reserve(&mut relations, relation_count, "relations")?; + let mut relation_ids = HashSet::new(); + for _ in 0..relation_count { + let id = reader + .read_varint::() + .map_err(DeserializationError::from)?; + let name = decode_string(reader, limits)?; + if !relation_ids.insert(id) { + return Err(PithosError::ConflictingRelationshipDefinition(id)); + } + relations.push((id, name)); + } + let encryption_count = bounded_len( + reader + .read_varint::() + .map_err(DeserializationError::from)?, + limits.max_collection_entries, + "encryption", + )?; + let mut encryption = IndexMap::new(); + for _ in 0..encryption_count { + let mut key = [0; 32]; + reader.read_exact(&mut key)?; + crypto::validate_x25519_public_key(&key)?; + if encryption.contains_key(&key) { + return Err(PithosError::DuplicateSenderKey); + } + encryption.insert(key, decode_encryption_section(reader, limits)?); + } + Ok(Directory { + identifier, + parent_directory_offset, + files, + blocks, + relations, + encryption, + dir_len: reader.read_u64::()?, + crc32: reader.read_u32::()?, + }) +} + +#[allow(dead_code)] +pub(crate) fn decode_complete_directory( + bytes: &[u8], + limits: &DeserializationLimits, +) -> Result { + decode_complete_directory_with_validation(bytes, limits, |_| Ok(())) +} + +pub(crate) fn decode_complete_directory_with_validation( + bytes: &[u8], + limits: &DeserializationLimits, + validate: impl Fn(&Directory) -> Result<(), PithosError>, +) -> Result { + let mut remaining_block_references = limits.max_block_references; + decode_complete_directory_with_validation_and_budget( + bytes, + limits, + &mut remaining_block_references, + validate, + ) +} + +pub(crate) fn decode_complete_directory_with_validation_and_budget( + bytes: &[u8], + limits: &DeserializationLimits, + remaining_block_references: &mut u64, + validate: impl Fn(&Directory) -> Result<(), PithosError>, +) -> Result { + if bytes.len() < MIN_DIRECTORY_LEN { + return Err(PithosError::DirectoryLengthMismatch { + expected: MIN_DIRECTORY_LEN as u64, + actual: bytes.len() as u64, + }); + } + let actual_marker: [u8; 8] = bytes[..8] + .try_into() + .expect("checked directory minimum length"); + if actual_marker != DIRECTORY_MARKER { + return Err(PithosError::InvalidDirectoryMarker { + expected: DIRECTORY_MARKER, + actual: actual_marker, + }); + } + let encoded_len = u64::from_be_bytes( + bytes[bytes.len() - 12..bytes.len() - 4] + .try_into() + .expect("checked directory footer"), + ); + if encoded_len != bytes.len() as u64 { + return Err(PithosError::DirectoryLengthMismatch { + expected: bytes.len() as u64, + actual: encoded_len, + }); + } + let encoded_crc = u32::from_be_bytes( + bytes[bytes.len() - 4..] + .try_into() + .expect("checked directory footer"), + ); + let computed_crc = crc32fast::hash(&bytes[..bytes.len() - 4]); + if encoded_crc != computed_crc { + return Err(PithosError::DirectoryChecksumMismatch { + expected: computed_crc, + actual: encoded_crc, + }); + } + let mut reader = Cursor::new(bytes); + let directory = decode_directory_with_budget(&mut reader, limits, remaining_block_references)?; + validate(&directory)?; + if reader.position() != bytes.len() as u64 { + return Err(PithosError::DirectoryConsumptionMismatch { + expected: bytes.len() as u64, + actual: reader.position(), + }); + } + Ok(directory) +} + +pub(crate) fn update_directory_len(directory: &mut Directory) -> Result<(), SerializationError> { + let mut bytes = Vec::new(); + encode_directory(directory, &mut bytes)?; + directory.dir_len = u64::try_from(bytes.len()) + .map_err(|_| SerializationError::Other("length does not fit in u64".to_string()))?; + Ok(()) +} + +pub(crate) fn update_directory_crc(directory: &mut Directory) -> Result<(), SerializationError> { + let mut bytes = Vec::new(); + encode_directory(directory, &mut bytes)?; + let mut hasher = Hasher::new(); + hasher.update(&bytes[..bytes.len() - 4]); + directory.crc32 = hasher.finalize(); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn complete_directory_rejects_bad_footer_and_trailing_bytes() { + let bytes = vec![ + 0x50, 0x49, 0x54, 0x48, 0x4f, 0x53, 0x44, 0x52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0x19, 0xb1, 0x67, 0x40, 0x81, + ]; + assert!(decode_complete_directory(&bytes, &DeserializationLimits::default()).is_ok()); + let mut trailing = bytes.clone(); + trailing.push(0); + assert!(matches!( + decode_complete_directory(&trailing, &DeserializationLimits::default()), + Err(PithosError::DirectoryLengthMismatch { .. }) + )); + let mut corrupt = bytes; + corrupt[0] = b'X'; + assert!(matches!( + decode_complete_directory(&corrupt, &DeserializationLimits::default()), + Err(PithosError::InvalidDirectoryMarker { .. }) + )); + } + + #[test] + fn complete_directory_reports_entry_semantics_before_trailing_body_data() { + let mut files = DirectoryEntries::new(); + files + .insert( + 0, + "/invalid", + FileEntry { + file_type: FileType::Directory, + block_data: BlockDataState::Decrypted(Vec::new().into()), + created: 0, + modified: 0, + file_size: 0, + permissions: 0o755, + references: Vec::new(), + symlink_target: None, + }, + ) + .unwrap(); + let mut directory = Directory::new(None, files, IndexMap::new()); + update_directory_len(&mut directory).unwrap(); + update_directory_crc(&mut directory).unwrap(); + let mut bytes = Vec::new(); + encode_directory(&directory, &mut bytes).unwrap(); + + let footer = bytes.len() - 12; + bytes.insert(footer, 0xff); + let directory_len = bytes.len() as u64; + let footer = bytes.len() - 12; + bytes[footer..footer + 8].copy_from_slice(&directory_len.to_be_bytes()); + let checksum = crc32fast::hash(&bytes[..bytes.len() - 4]); + let crc_offset = bytes.len() - 4; + bytes[crc_offset..].copy_from_slice(&checksum.to_be_bytes()); + + assert!(matches!( + decode_complete_directory_with_validation( + &bytes, + &DeserializationLimits::default(), + |directory| { + if directory.files.get_by_path("/invalid").is_some() { + Err(PithosError::InvalidArchivePath { + path: "/invalid".into(), + reason: "test".into(), + }) + } else { + Ok(()) + } + } + ), + Err(PithosError::InvalidArchivePath { path, .. }) if path == "/invalid" + )); + } + + #[test] + fn directory_rejects_duplicate_relationship_definitions() { + let mut directory = Directory { + identifier: DIRECTORY_MARKER, + parent_directory_offset: None, + blocks: IndexMap::new(), + files: DirectoryEntries::new(), + relations: vec![(7, "same".to_owned()), (7, "same".to_owned())], + encryption: IndexMap::new(), + dir_len: 0, + crc32: 0, + }; + update_directory_len(&mut directory).unwrap(); + update_directory_crc(&mut directory).unwrap(); + let mut bytes = Vec::new(); + encode_directory(&directory, &mut bytes).unwrap(); + + assert!(matches!( + decode_complete_directory(&bytes, &DeserializationLimits::default()), + Err(PithosError::ConflictingRelationshipDefinition(7)) + )); + } +} diff --git a/crates/pithos_lib/src/format/encryption.rs b/crates/pithos_lib/src/format/encryption.rs new file mode 100644 index 0000000..6c44673 --- /dev/null +++ b/crates/pithos_lib/src/format/encryption.rs @@ -0,0 +1,253 @@ +use crate::crypto::{self, SharedSecret}; +use crate::error::PithosError; +use crate::format::error::SerializationError; +use crate::format::limits::{DeserializationError, DeserializationLimits}; +use crate::format::primitives::{bounded_len, reserve, write_len_prefix}; +use indexmap::IndexMap; +use integer_encoding::{VarIntReader, VarIntWriter}; +use std::collections::HashMap; +use std::io::{Read, Write}; +use x25519_dalek::PublicKey; +use zeroize::Zeroizing; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EncryptionSection { + pub recipients: IndexMap<[u8; 32], RecipientSection>, +} + +impl EncryptionSection { + #[tracing::instrument(level = "trace", skip(recipient_pubkeys))] + pub fn new(recipient_pubkeys: &[PublicKey]) -> Self { + EncryptionSection { + recipients: IndexMap::from_iter( + recipient_pubkeys + .iter() + .map(|key| { + ( + key.to_bytes(), + RecipientSection { + recipient_data: RecipientData::Decrypted( + Zeroizing::new(Vec::new()), + ), + }, + ) + }) + .collect::>(), + ), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RecipientSection { + pub(crate) recipient_data: RecipientData, +} + +pub(crate) type RecipientKeyList = Zeroizing>; + +/// Transient recipient-list state used while encoding or opening a directory. +/// Recovered file keys are crate-private. +#[derive(Clone, PartialEq, Eq)] +pub(crate) enum RecipientData { + Encrypted(Vec), + Decrypted(RecipientKeyList), +} + +impl std::fmt::Debug for RecipientData { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Encrypted(bytes) => formatter + .debug_tuple("Encrypted") + .field(&format_args!("{} bytes", bytes.len())) + .finish(), + Self::Decrypted(entries) => formatter + .debug_tuple("Decrypted") + .field(&format_args!("{} file keys [REDACTED]", entries.len())) + .finish(), + } + } +} + +impl RecipientData { + pub(crate) fn encrypt_with_secret_and_nonce( + &mut self, + shared_key: SharedSecret, + nonce: [u8; 12], + ) -> Result<(), PithosError> { + match &self { + RecipientData::Encrypted(_) => { + return Err(PithosError::InvalidRecipientDataState( + "Recipient data already encrypted".to_string(), + )); + } + RecipientData::Decrypted(entries) => { + let mut data_bytes = Zeroizing::new(Vec::with_capacity(1 + entries.len() * 40)); + encode_decrypted_recipient_list(entries, &mut *data_bytes)?; + let encrypted_data = + crypto::wrap_recipient_list_with_nonce(&shared_key, &data_bytes, nonce)?; + *self = RecipientData::Encrypted(encrypted_data) + } + }; + Ok(()) + } +} + +fn encode_recipient_data( + data: &RecipientData, + writer: &mut W, +) -> Result<(), SerializationError> { + match data { + RecipientData::Encrypted(bytes) => { + writer.write_all(&[0])?; + write_len_prefix(writer, bytes.len())?; + writer.write_all(bytes)?; + } + RecipientData::Decrypted(entries) => { + writer.write_all(&[1])?; + encode_decrypted_recipient_list(entries, writer)?; + } + } + Ok(()) +} + +fn decode_recipient_data( + reader: &mut R, + limits: &DeserializationLimits, +) -> Result { + let mut tag = [0]; + reader.read_exact(&mut tag)?; + match tag[0] { + 0 => { + let len = bounded_len( + reader.read_varint::()?, + limits.max_opaque_bytes, + "encrypted recipient data", + )?; + let mut bytes = Vec::new(); + reserve(&mut bytes, len, "encrypted recipient data")?; + bytes.resize(len, 0); + reader.read_exact(&mut bytes)?; + Ok(RecipientData::Encrypted(bytes)) + } + 1 => Ok(RecipientData::Decrypted( + decode_decrypted_recipient_list_reader(reader, limits)?, + )), + value => Err(DeserializationError::InvalidEnumValue(value)), + } +} + +pub(crate) fn encode_encryption_section( + section: &EncryptionSection, + writer: &mut W, +) -> Result<(), SerializationError> { + write_len_prefix(writer, section.recipients.len())?; + for (key, recipient) in §ion.recipients { + writer.write_all(key)?; + encode_recipient_data(&recipient.recipient_data, writer)?; + } + Ok(()) +} + +pub(crate) fn decode_encryption_section( + reader: &mut R, + limits: &DeserializationLimits, +) -> Result { + let count = bounded_len( + reader.read_varint::()?, + limits.max_collection_entries, + "recipients", + )?; + let mut recipients = IndexMap::new(); + for _ in 0..count { + let mut key = [0; 32]; + reader.read_exact(&mut key)?; + crypto::validate_x25519_public_key(&key)?; + if recipients.contains_key(&key) { + return Err(DeserializationError::DuplicateRecipientKey.into()); + } + recipients.insert( + key, + RecipientSection { + recipient_data: decode_recipient_data(reader, limits)?, + }, + ); + } + Ok(EncryptionSection { recipients }) +} + +pub(crate) fn encode_decrypted_recipient_list( + entries: &[(u64, [u8; 32])], + writer: &mut W, +) -> Result<(), SerializationError> { + write_len_prefix(writer, entries.len())?; + for (id, key) in entries { + writer.write_varint(*id)?; + writer.write_all(key)?; + } + Ok(()) +} + +pub(crate) fn decode_decrypted_recipient_list_reader( + reader: &mut R, + limits: &DeserializationLimits, +) -> Result { + let count = bounded_len( + reader.read_varint::()?, + limits.max_collection_entries, + "recipient keys", + )?; + let mut entries = Zeroizing::new(Vec::new()); + reserve(&mut entries, count, "recipient keys")?; + let mut keys = HashMap::with_capacity(count); + for _ in 0..count { + let id = reader.read_varint::()?; + entries.push((id, [0; 32])); + reader.read_exact(&mut entries.last_mut().expect("just pushed file key").1)?; + let key = entries.last().expect("just pushed file key").1; + if keys.insert(id, key).is_some_and(|existing| existing != key) { + return Err(DeserializationError::DuplicateRecipientFileId); + } + } + Ok(entries) +} + +pub(crate) fn decode_decrypted_recipient_list( + bytes: &[u8], + limits: &DeserializationLimits, +) -> Result { + let mut reader = std::io::Cursor::new(bytes); + let entries = decode_decrypted_recipient_list_reader(&mut reader, limits)?; + if reader.position() != bytes.len() as u64 { + return Err(DeserializationError::InvalidLength); + } + Ok(entries) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn decrypted_recipient_lists_retain_exact_duplicates_but_reject_conflicts() { + let limits = DeserializationLimits::default(); + let mut repeated = vec![2, 7]; + repeated.extend_from_slice(&[3; 32]); + repeated.push(7); + repeated.extend_from_slice(&[3; 32]); + assert_eq!( + decode_decrypted_recipient_list(&repeated, &limits) + .unwrap() + .as_slice(), + &[(7, [3; 32]), (7, [3; 32])] + ); + + let mut conflicting = vec![2, 7]; + conflicting.extend_from_slice(&[3; 32]); + conflicting.push(7); + conflicting.extend_from_slice(&[4; 32]); + assert!(matches!( + decode_decrypted_recipient_list(&conflicting, &limits), + Err(DeserializationError::DuplicateRecipientFileId) + )); + } +} diff --git a/crates/pithos_lib/src/format/entries.rs b/crates/pithos_lib/src/format/entries.rs deleted file mode 100644 index c878c73..0000000 --- a/crates/pithos_lib/src/format/entries.rs +++ /dev/null @@ -1,189 +0,0 @@ -use crate::error::PithosError; -use crate::format::wire::FileEntry; -use indexmap::IndexMap; -use std::cmp::Ordering; -use std::collections::BTreeMap; -use std::ops::Bound::Excluded; -use std::sync::Arc; - -#[derive(Clone, Debug, Eq, PartialEq)] -struct OrderedPath(Arc); - -impl Ord for OrderedPath { - fn cmp(&self, other: &Self) -> Ordering { - self.0.split('/').cmp(other.0.split('/')) - } -} - -impl PartialOrd for OrderedPath { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -#[derive(Clone, Debug, Eq, PartialEq)] -struct WireEntry { - id: u64, - path: Arc, - entry: FileEntry, -} - -/// Format-owned file records with stable wire insertion order and private indexes. -#[derive(Clone, Debug, Eq, PartialEq)] -pub(crate) struct WireEntries { - by_id: IndexMap, - by_path: IndexMap, usize>, - ordered_paths: BTreeMap, - entries: Vec, - maximum_id: u64, -} - -impl WireEntries { - pub(crate) fn new() -> Self { - Self::with_maximum_id(0) - } - - pub(crate) fn with_maximum_id(maximum_id: u64) -> Self { - Self { - by_id: IndexMap::new(), - by_path: IndexMap::new(), - ordered_paths: BTreeMap::new(), - entries: Vec::new(), - maximum_id, - } - } - - pub(crate) fn len(&self) -> usize { - self.entries.len() - } - - pub(crate) fn is_empty(&self) -> bool { - self.entries.is_empty() - } - - pub(crate) fn get_by_id(&self, id: u64) -> Option<&FileEntry> { - self.by_id - .get(&id) - .and_then(|index| self.entries.get(*index)) - .map(|entry| &entry.entry) - } - - pub(crate) fn get_by_path(&self, path: &str) -> Option<&FileEntry> { - self.by_path - .get(path) - .and_then(|index| self.entries.get(*index)) - .map(|entry| &entry.entry) - } - - pub(crate) fn first_path_after(&self, path: &str) -> Option<&str> { - let query = OrderedPath(Arc::from(path)); - self.ordered_paths - .range((Excluded(query), std::ops::Bound::Unbounded)) - .next() - .map(|(path, _)| path.0.as_ref()) - } - - pub(crate) fn iter(&self) -> impl ExactSizeIterator + '_ { - self.entries - .iter() - .map(|entry| (entry.id, entry.path.as_ref(), &entry.entry)) - } - - #[cfg(test)] - pub(crate) fn iter_ordered(&self) -> impl Iterator + '_ { - self.ordered_paths - .iter() - .map(|(path, index)| (path.0.as_ref(), &self.entries[*index].entry)) - } - - pub(crate) fn try_for_each_mut( - &mut self, - mut operation: impl FnMut(u64, &mut FileEntry) -> Result<(), E>, - ) -> Result<(), E> { - for entry in &mut self.entries { - operation(entry.id, &mut entry.entry)?; - } - Ok(()) - } - - pub(crate) fn next_free_id(&self, has_parent: bool) -> Result { - if self.maximum_id == 0 && self.entries.is_empty() { - Ok(u64::from(has_parent)) - } else { - self.maximum_id - .checked_add(1) - .ok_or(PithosError::FileIdExhausted) - } - } - - pub(crate) fn prevalidate_insert(&self, id: u64, path: &str) -> Result<(), PithosError> { - if self.by_id.contains_key(&id) { - return Err(PithosError::DuplicateFileId(format!( - "File id already occupied: {id}" - ))); - } - if self.by_path.contains_key(path) { - return Err(PithosError::PathOccupied(format!( - "File path already occupied: {path}" - ))); - } - Ok(()) - } - - pub(crate) fn reserve_one(&mut self) -> Result<(), PithosError> { - let size = self - .entries - .len() - .checked_add(1) - .ok_or(PithosError::AllocationFailed { - field: "file entries", - size: u64::MAX, - })?; - self.by_id - .try_reserve(1) - .map_err(|_| allocation_failed("file entry ids", size))?; - self.by_path - .try_reserve(1) - .map_err(|_| allocation_failed("file entry paths", size))?; - self.entries - .try_reserve(1) - .map_err(|_| allocation_failed("file entries", size)) - } - - /// Insert only after validation and reservation of the vector-backed indexes. - pub(crate) fn insert_prepared(&mut self, id: u64, path: impl Into>, entry: FileEntry) { - let path = path.into(); - debug_assert!(self.prevalidate_insert(id, &path).is_ok()); - let ordered = OrderedPath(Arc::clone(&path)); - let index = self.entries.len(); - self.entries.push(WireEntry { - id, - path: Arc::clone(&path), - entry, - }); - self.by_id.insert(id, index); - self.by_path.insert(path, index); - self.ordered_paths.insert(ordered, index); - self.maximum_id = self.maximum_id.max(id); - } - - pub(crate) fn insert( - &mut self, - id: u64, - path: impl Into>, - entry: FileEntry, - ) -> Result<(), PithosError> { - let path = path.into(); - self.prevalidate_insert(id, &path)?; - self.reserve_one()?; - self.insert_prepared(id, path, entry); - Ok(()) - } -} - -fn allocation_failed(field: &'static str, size: usize) -> PithosError { - PithosError::AllocationFailed { - field, - size: u64::try_from(size).unwrap_or(u64::MAX), - } -} diff --git a/crates/pithos_lib/src/format/file_entry.rs b/crates/pithos_lib/src/format/file_entry.rs new file mode 100644 index 0000000..db4d32d --- /dev/null +++ b/crates/pithos_lib/src/format/file_entry.rs @@ -0,0 +1,387 @@ +use crate::crypto::{self, FileKey}; +use crate::error::PithosError; +use crate::format::error::SerializationError; +use crate::format::limits::{DeserializationError, DeserializationLimits}; +use crate::format::primitives::{bounded_len, decode_string, reserve, write_len_prefix}; +use integer_encoding::{VarIntReader, VarIntWriter}; +use std::collections::HashMap; +use std::fmt::{Display, Formatter}; +use std::io::{Read, Write}; +use zeroize::Zeroizing; + +pub(crate) const VALID_PERMISSION_BITS: u32 = 0o7777; + +#[repr(u8)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum FileType { + Directory = 0, + Data = 1, + Metadata = 2, + Symlink = 3, +} + +/// A block's content hash and the key used to encrypt its content. +pub type BlockDataEntry = ([u8; 32], [u8; 32]); + +/// Transient block-list state used while encoding or opening a directory. +/// Decrypted block keys are crate-private. +#[derive(Clone, PartialEq, Eq)] +pub(crate) enum BlockDataState { + Encrypted(Vec), + Decrypted(Zeroizing>), +} + +impl std::fmt::Debug for BlockDataState { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Encrypted(bytes) => formatter + .debug_tuple("Encrypted") + .field(&format_args!("{} bytes", bytes.len())) + .finish(), + Self::Decrypted(entries) => formatter + .debug_tuple("Decrypted") + .field(&format_args!("{} block keys [REDACTED]", entries.len())) + .finish(), + } + } +} + +impl BlockDataState { + pub(crate) fn encrypt_with_nonce( + &mut self, + key: &FileKey, + nonce: [u8; 12], + ) -> Result<(), PithosError> { + match &self { + BlockDataState::Encrypted(_) => { + return Err(PithosError::InvalidBlockDataState( + "Block already encrypted.".to_string(), + )); + } + BlockDataState::Decrypted(entries) => { + let mut data_bytes = Zeroizing::new(Vec::with_capacity(1 + entries.len() * 64)); + encode_decrypted_block_list(entries, &mut *data_bytes)?; + let encrypted_data = + crypto::seal_file_block_list_with_nonce(key, &data_bytes, nonce)?; + *self = BlockDataState::Encrypted(encrypted_data) + } + }; + Ok(()) + } +} + +pub(crate) fn validate_unique_block_references( + entries: &[BlockDataEntry], +) -> Result<(), PithosError> { + let mut keys = HashMap::with_capacity(entries.len()); + for (hash, key) in entries { + if keys + .insert(*hash, *key) + .is_some_and(|existing| existing != *key) + { + return Err(PithosError::DuplicateBlockReference); + } + } + Ok(()) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FileEntry { + pub file_type: FileType, + pub(crate) block_data: BlockDataState, + pub created: u64, + pub modified: u64, + pub file_size: u64, + pub permissions: u32, + pub references: Vec, + pub symlink_target: Option, +} + +impl Display for FileEntry { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_str(&format!("{:<12} {:?}\n", "Type:", self.file_type))?; + match &self.block_data { + BlockDataState::Encrypted(_) => f.write_str("Blocks: Encrypted\n")?, + BlockDataState::Decrypted(_) => f.write_str("Blocks: Decrypted\n")?, + } + f.write_str(&format!("{:<12} {}\n", "Created:", self.created))?; + f.write_str(&format!("{:<12} {}\n", "Modified:", self.modified))?; + f.write_str(&format!("{:<12} {}\n", "Size:", self.file_size))?; + f.write_str(&format!("{:<12} {:o}\n", "Permissions:", self.permissions))?; + f.write_str(&format!("{:<12} {:?}\n", "References:", self.references))?; + if let Some(target) = &self.symlink_target { + f.write_str(&format!("{:<12} {target}\n", "Target:"))?; + } + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Reference { + pub target_file_id: u64, + pub relationship: u64, +} + +fn encode_file_type( + file_type: &FileType, + writer: &mut W, +) -> Result<(), SerializationError> { + writer.write_all(&[*file_type as u8])?; + Ok(()) +} + +pub(crate) fn decode_file_type(reader: &mut R) -> Result { + let mut tag = [0]; + reader.read_exact(&mut tag)?; + match tag[0] { + 0 => Ok(FileType::Directory), + 1 => Ok(FileType::Data), + 2 => Ok(FileType::Metadata), + 3 => Ok(FileType::Symlink), + value => Err(DeserializationError::InvalidEnumValue(value)), + } +} + +fn encode_reference( + reference: &Reference, + writer: &mut W, +) -> Result<(), SerializationError> { + writer.write_varint(reference.target_file_id)?; + writer.write_varint(reference.relationship)?; + Ok(()) +} + +fn decode_reference(reader: &mut R) -> Result { + Ok(Reference { + target_file_id: reader.read_varint::()?, + relationship: reader.read_varint::()?, + }) +} + +fn encode_block_data( + data: &BlockDataState, + writer: &mut W, +) -> Result<(), SerializationError> { + match data { + BlockDataState::Encrypted(bytes) => { + writer.write_all(&[0])?; + write_len_prefix(writer, bytes.len())?; + writer.write_all(bytes)?; + } + BlockDataState::Decrypted(entries) => { + writer.write_all(&[1])?; + encode_decrypted_block_list(entries, writer)?; + } + } + Ok(()) +} + +fn decode_block_data( + reader: &mut R, + limits: &DeserializationLimits, + remaining_block_references: &mut u64, +) -> Result { + let mut tag = [0]; + reader.read_exact(&mut tag)?; + match tag[0] { + 0 => { + let len = bounded_len( + reader.read_varint::()?, + limits.max_opaque_bytes, + "encrypted block", + )?; + let mut bytes = Vec::new(); + reserve(&mut bytes, len, "encrypted block")?; + bytes.resize(len, 0); + reader.read_exact(&mut bytes)?; + Ok(BlockDataState::Encrypted(bytes)) + } + 1 => Ok(BlockDataState::Decrypted( + decode_decrypted_block_list_reader_with_budget( + reader, + limits, + remaining_block_references, + )?, + )), + value => Err(DeserializationError::InvalidEnumValue(value)), + } +} + +pub(crate) fn encode_file_entry( + entry: &FileEntry, + writer: &mut W, +) -> Result<(), SerializationError> { + encode_file_type(&entry.file_type, writer)?; + encode_block_data(&entry.block_data, writer)?; + writer.write_varint(entry.created)?; + writer.write_varint(entry.modified)?; + writer.write_varint(entry.file_size)?; + writer.write_varint(entry.permissions)?; + write_len_prefix(writer, entry.references.len())?; + for reference in &entry.references { + encode_reference(reference, writer)?; + } + match &entry.symlink_target { + Some(target) => { + writer.write_all(&[1])?; + crate::format::primitives::encode_string(writer, target)?; + } + None => writer.write_all(&[0])?, + } + Ok(()) +} + +pub(crate) fn decode_file_entry( + reader: &mut R, + limits: &DeserializationLimits, + remaining_references: &mut u64, + remaining_block_references: &mut u64, +) -> Result { + let file_type = decode_file_type(reader)?; + let block_data = decode_block_data(reader, limits, remaining_block_references)?; + let created = reader.read_varint::()?; + let modified = reader.read_varint::()?; + let file_size = reader.read_varint::()?; + let permissions = reader.read_varint::()?; + let count = bounded_len( + reader.read_varint::()?, + *remaining_references, + "references", + )?; + *remaining_references -= count as u64; + let mut references = Vec::new(); + reserve(&mut references, count, "references")?; + for _ in 0..count { + references.push(decode_reference(reader)?); + } + let mut tag = [0]; + reader.read_exact(&mut tag)?; + let symlink_target = match tag[0] { + 0 => None, + 1 => Some(decode_string(reader, limits)?), + _ => return Err(DeserializationError::InvalidOption), + }; + Ok(FileEntry { + file_type, + block_data, + created, + modified, + file_size, + permissions, + references, + symlink_target, + }) +} + +pub(crate) fn encode_decrypted_block_list( + entries: &[BlockDataEntry], + writer: &mut W, +) -> Result<(), SerializationError> { + write_len_prefix(writer, entries.len())?; + for (hash, key) in entries { + writer.write_all(hash)?; + writer.write_all(key)?; + } + Ok(()) +} + +fn decode_decrypted_block_list_reader_with_budget( + reader: &mut R, + limits: &DeserializationLimits, + remaining_block_references: &mut u64, +) -> Result>, DeserializationError> { + let count = bounded_len( + reader.read_varint::()?, + limits.max_block_references.min(*remaining_block_references), + "block references", + )?; + *remaining_block_references -= count as u64; + let mut entries = Zeroizing::new(Vec::new()); + reserve(&mut entries, count, "block references")?; + let mut keys = HashMap::with_capacity(count); + for _ in 0..count { + let mut hash = [0; 32]; + reader.read_exact(&mut hash)?; + let mut key = [0; 32]; + reader.read_exact(&mut key)?; + if keys + .insert(hash, key) + .is_some_and(|existing| existing != key) + { + return Err(DeserializationError::DuplicateBlockReference); + } + entries.push((hash, key)); + } + Ok(entries) +} + +pub(crate) fn decode_decrypted_block_list_with_budget( + bytes: &[u8], + limits: &DeserializationLimits, + remaining_block_references: &mut u64, +) -> Result>, DeserializationError> { + let mut reader = std::io::Cursor::new(bytes); + let entries = decode_decrypted_block_list_reader_with_budget( + &mut reader, + limits, + remaining_block_references, + )?; + if reader.position() != bytes.len() as u64 { + return Err(DeserializationError::InvalidLength); + } + Ok(entries) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn authenticated_lists_require_exact_consumption_and_unique_keys() { + let limits = DeserializationLimits::default(); + let mut budget = limits.max_block_references; + assert!(matches!( + decode_decrypted_block_list_with_budget(&[0, 0], &limits, &mut budget), + Err(DeserializationError::InvalidLength) + )); + let mut duplicate = vec![2]; + duplicate.extend_from_slice(&[1; 32]); + duplicate.extend_from_slice(&[2; 32]); + duplicate.extend_from_slice(&[1; 32]); + duplicate.extend_from_slice(&[3; 32]); + let mut budget = limits.max_block_references; + assert!(matches!( + decode_decrypted_block_list_with_budget(&duplicate, &limits, &mut budget), + Err(DeserializationError::DuplicateBlockReference) + )); + let mut repeated = vec![2]; + repeated.extend_from_slice(&[1; 32]); + repeated.extend_from_slice(&[2; 32]); + repeated.extend_from_slice(&[1; 32]); + repeated.extend_from_slice(&[2; 32]); + let mut budget = limits.max_block_references; + assert_eq!( + decode_decrypted_block_list_with_budget(&repeated, &limits, &mut budget) + .unwrap() + .len(), + 2 + ); + } + + #[test] + fn permission_field_uses_u32_varint_width() { + assert!( + (&[0x80, 0x80, 0x80, 0x80, 0x10][..]) + .read_varint::() + .is_err() + ); + } + + #[test] + fn transient_plaintext_key_lists_have_drop_zeroization_contracts() { + use zeroize::ZeroizeOnDrop; + fn assert_zeroize_on_drop() {} + assert_zeroize_on_drop::>>(); + assert_zeroize_on_drop::(); + } +} diff --git a/crates/pithos_lib/src/format/header.rs b/crates/pithos_lib/src/format/header.rs new file mode 100644 index 0000000..cbe0596 --- /dev/null +++ b/crates/pithos_lib/src/format/header.rs @@ -0,0 +1,65 @@ +use crate::format::error::SerializationError; +use crate::format::limits::DeserializationError; +use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt}; +use std::io::{Read, Write}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FileHeader { + pub magic: [u8; 4], // MUST be b"PITH" + pub version: u16, // Format version (e.g., 0x0100 for 1.0) +} + +impl Default for FileHeader { + fn default() -> Self { + FileHeader { + magic: *b"PITH", + version: Self::SUPPORTED_VERSION, + } + } +} + +impl FileHeader { + pub const ENCODED_LEN: usize = 6; + pub const SUPPORTED_VERSION: u16 = 0x0100; +} + +pub(crate) fn encode_header( + header: &FileHeader, + writer: &mut W, +) -> Result<(), SerializationError> { + writer.write_all(&header.magic)?; + writer.write_u16::(header.version)?; + Ok(()) +} + +pub(crate) fn decode_header(reader: &mut R) -> Result { + let mut magic = [0; 4]; + reader.read_exact(&mut magic)?; + if magic != *b"PITH" { + return Err(DeserializationError::InvalidMarker(format!( + "Read invalid file marker {magic:?}" + ))); + } + Ok(FileHeader { + magic, + version: reader.read_u16::()?, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn file_header_uses_the_fixed_width_1_0_encoding() { + let mut encoded = Vec::new(); + encode_header(&FileHeader::default(), &mut encoded).unwrap(); + assert_eq!(encoded, b"PITH\x01\x00"); + + let decoded = decode_header(&mut encoded.as_slice()).unwrap(); + assert_eq!(decoded.version, 0x0100); + + let error = decode_header(&mut &b"POTX\x01\x00"[..]).unwrap_err(); + assert!(error.to_string().contains("file marker")); + } +} diff --git a/crates/pithos_lib/src/format/mod.rs b/crates/pithos_lib/src/format/mod.rs index d309a26..cf55c6f 100644 --- a/crates/pithos_lib/src/format/mod.rs +++ b/crates/pithos_lib/src/format/mod.rs @@ -3,9 +3,11 @@ //! This module deliberately contains no archive policy, filesystem, adapter, or //! transport concerns. -pub(crate) mod codec; +pub(crate) mod block; pub(crate) mod directory; -pub(crate) mod entries; +pub(crate) mod encryption; pub(crate) mod error; +pub(crate) mod file_entry; +pub(crate) mod header; pub(crate) mod limits; -pub(crate) mod wire; +pub(crate) mod primitives; diff --git a/crates/pithos_lib/src/format/primitives.rs b/crates/pithos_lib/src/format/primitives.rs new file mode 100644 index 0000000..7a6f79b --- /dev/null +++ b/crates/pithos_lib/src/format/primitives.rs @@ -0,0 +1,117 @@ +use crate::format::error::SerializationError; +use crate::format::limits::{DeserializationError, DeserializationLimits}; +use integer_encoding::{VarIntReader, VarIntWriter}; +use std::io::{Read, Write}; + +pub(crate) fn write_len_prefix( + writer: &mut W, + len: usize, +) -> Result<(), SerializationError> { + writer.write_varint( + u64::try_from(len) + .map_err(|_| SerializationError::Other("length does not fit in u64".to_string()))?, + )?; + Ok(()) +} + +pub(crate) fn encode_string( + writer: &mut W, + value: &str, +) -> Result<(), SerializationError> { + write_len_prefix(writer, value.len())?; + writer.write_all(value.as_bytes())?; + Ok(()) +} + +pub(crate) fn bounded_len( + value: u64, + limit: u64, + field: &'static str, +) -> Result { + if value > limit { + return Err(DeserializationError::LimitExceeded { + field, + limit, + actual: value, + }); + } + usize::try_from(value).map_err(|_| DeserializationError::InvalidLength) +} + +pub(crate) fn reserve( + output: &mut Vec, + count: usize, + field: &'static str, +) -> Result<(), DeserializationError> { + output + .try_reserve(count) + .map_err(|_| DeserializationError::AllocationFailed { + field, + size: count as u64, + }) +} + +pub(crate) fn decode_string( + reader: &mut R, + limits: &DeserializationLimits, +) -> Result { + let len = bounded_len( + reader.read_varint::()?, + limits.max_string_bytes, + "string", + )?; + let mut bytes = Vec::new(); + reserve(&mut bytes, len, "string")?; + bytes.resize(len, 0); + reader.read_exact(&mut bytes)?; + Ok(String::from_utf8(bytes)?) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn typed_uleb128_accepts_bounded_non_minimal_values() { + for (mut encoded, expected) in [ + (&[0x00][..], 0_u64), + (&[0x7f][..], 127), + (&[0x80, 0x01][..], 128), + (&[0x80, 0x00][..], 0), + (&[0x81, 0x00][..], 1), + ] { + assert_eq!(encoded.read_varint::().unwrap(), expected); + } + } + + #[test] + fn typed_uleb128_rejects_truncated_and_width_overflowing_values() { + assert!((&[0x80][..]).read_varint::().is_err()); + assert!( + (&[0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x02][..]) + .read_varint::() + .is_err() + ); + assert!( + (&[ + 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x00, + ][..]) + .read_varint::() + .is_err() + ); + } + + #[test] + fn bounded_string_decode_rejects_truncation_and_limits() { + let limits = DeserializationLimits { + max_string_bytes: 2, + ..DeserializationLimits::default() + }; + assert_eq!( + decode_string(&mut &[2, b'o', b'k'][..], &limits).unwrap(), + "ok" + ); + assert!(decode_string(&mut &[3, b'o', b'k'][..], &limits).is_err()); + assert!(decode_string(&mut &[2, b'o'][..], &limits).is_err()); + } +} diff --git a/crates/pithos_lib/src/format/wire.rs b/crates/pithos_lib/src/format/wire.rs deleted file mode 100644 index 2dee765..0000000 --- a/crates/pithos_lib/src/format/wire.rs +++ /dev/null @@ -1,356 +0,0 @@ -use crate::crypto::{self, FileKey, SharedSecret}; -use std::fmt::{Display, Formatter}; -use zeroize::Zeroizing; - -use crate::error::PithosError; -use crate::format::entries::WireEntries; -use indexmap::IndexMap; -use x25519_dalek::PublicKey; - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct FileHeader { - pub magic: [u8; 4], // MUST be b"PITH" - pub version: u16, // Format version (e.g., 0x0100 for 1.0) -} - -impl Default for FileHeader { - fn default() -> Self { - FileHeader { - magic: *b"PITH", - version: Self::SUPPORTED_VERSION, - } - } -} - -impl FileHeader { - pub const ENCODED_LEN: usize = 6; - pub const SUPPORTED_VERSION: u16 = 0x0100; -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct BlockHeader { - pub marker: [u8; 4], // MUST be b"BLCK" -} - -impl Default for BlockHeader { - fn default() -> Self { - BlockHeader { marker: *b"BLCK" } - } -} - -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct ProcessingFlags(pub u8); - -impl ProcessingFlags { - // Compression level can be set from 0-7 with the first three bits - // 0 = Uncompressed - // 7 = Highest compression - const COMPRESSION_MASK: u8 = 0b0000_0111; - - // Encryption is indicated with the 4th bit - const ENCRYPTION_MASK: u8 = 0b0000_1000; - pub const RESERVED_MASK: u8 = 0b1111_0000; - - pub fn new(encrypted: bool, compression_level: Option) -> Self { - // Init - let mut flags = ProcessingFlags(0b0); - - // Set compression level - match compression_level { - Some(level) => { - if level > Self::COMPRESSION_MASK { - flags.set_compression_level(Self::COMPRESSION_MASK) // Cap at maximum - } else { - flags.set_compression_level(level) - } - } - None => flags.set_compression_level(3), // Default - } - - // Set encryption - flags.set_encryption(encrypted); - flags - } - - pub fn from_byte(byte: u8) -> Self { - ProcessingFlags(byte) - } - - pub fn has_reserved_bits(&self) -> bool { - self.0 & Self::RESERVED_MASK != 0 - } - - pub fn set_encryption(&mut self, encrypted: bool) { - if encrypted { - self.0 |= Self::ENCRYPTION_MASK; // Set bit for encryption - } else { - self.0 &= !Self::ENCRYPTION_MASK; // Clear bit for encryption - } - } - - // New function to check if encryption is enabled - pub fn is_encrypted(&self) -> bool { - (self.0 & Self::ENCRYPTION_MASK) != 0 - } - - pub fn set_compression_level(&mut self, mut compression_level: u8) { - // Sanitize - compression_level = if compression_level > Self::COMPRESSION_MASK { - Self::COMPRESSION_MASK - } else { - compression_level - }; - - // Only use the lowest 3 bits (0-7) - self.0 = (self.0 & !Self::COMPRESSION_MASK) | (compression_level & Self::COMPRESSION_MASK); - } - - pub fn get_compression_level(&self) -> u8 { - self.0 & Self::COMPRESSION_MASK - } -} - -impl Default for ProcessingFlags { - fn default() -> Self { - ProcessingFlags::new(true, None) - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum BlockLocation { - Local, // Block data at specified offset in this file - External { url: String }, // URL to external storage -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct BlockIndexEntry { - pub offset: u64, // varint - pub stored_size: u64, // varint - pub original_size: u64, // varint - pub flags: ProcessingFlags, // Compression, encryption settings - pub location: BlockLocation, // Where block data resides -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct Directory { - pub identifier: [u8; 8], // MUST be b"PITHOSDR" - pub parent_directory_offset: Option<(u64, u64)>, // (start, len) varint - pub blocks: IndexMap<[u8; 32], BlockIndexEntry>, // Blocks in this segment - pub files: WireEntries, // Files in this segment - pub relations: Vec<(u64, String)>, // Relation idx, relationname/id - pub encryption: IndexMap<[u8; 32], EncryptionSection>, // (Sender's X25519 public key, section with recipients) - pub dir_len: u64, - pub crc32: u32, // CRC32 of all preceding fields -} - -#[repr(u8)] -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -pub enum FileType { - Directory = 0, - Data = 1, - Metadata = 2, - Symlink = 3, - // 4-255 reserved -} - -/// A block's content hash and the key used to encrypt its content. -pub type BlockDataEntry = ([u8; 32], [u8; 32]); -pub(crate) type RecipientKeyList = Zeroizing>; - -/// Transient writer/reader wire state. Decrypted block keys are crate-private. -#[derive(Clone, PartialEq, Eq)] -pub(crate) enum BlockDataState { - Encrypted(Vec), // Chacha + nonce (Random key) - Decrypted(Zeroizing>), // BLAKE3 hash / Shake256 hash -} - -impl std::fmt::Debug for BlockDataState { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Encrypted(bytes) => formatter - .debug_tuple("Encrypted") - .field(&format_args!("{} bytes", bytes.len())) - .finish(), - Self::Decrypted(entries) => formatter - .debug_tuple("Decrypted") - .field(&format_args!("{} block keys [REDACTED]", entries.len())) - .finish(), - } - } -} - -impl BlockDataState { - pub(crate) fn encrypt_with_nonce( - &mut self, - key: &FileKey, - nonce: [u8; 12], - ) -> Result<(), PithosError> { - match &self { - BlockDataState::Encrypted(_) => { - return Err(PithosError::InvalidBlockDataState( - "Block already encrypted.".to_string(), - )); - } - BlockDataState::Decrypted(entries) => { - let mut data_bytes = Zeroizing::new(Vec::with_capacity(1 + entries.len() * 64)); - crate::format::codec::encode_decrypted_block_list(entries, &mut *data_bytes)?; - let encrypted_data = - crypto::seal_file_block_list_with_nonce(key, &data_bytes, nonce)?; - - *self = BlockDataState::Encrypted(encrypted_data) - } - }; - - Ok(()) - } -} - -pub(crate) fn validate_unique_block_references( - entries: &[BlockDataEntry], -) -> Result<(), PithosError> { - let mut keys = std::collections::HashMap::with_capacity(entries.len()); - for (hash, key) in entries { - if keys - .insert(*hash, *key) - .is_some_and(|existing| existing != *key) - { - return Err(PithosError::DuplicateBlockReference); - } - } - Ok(()) -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct FileEntry { - pub file_type: FileType, - pub(crate) block_data: BlockDataState, - pub created: u64, - pub modified: u64, - pub file_size: u64, - pub permissions: u32, - pub references: Vec, - pub symlink_target: Option, // Target path for symlinks -} - -impl Display for FileEntry { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.write_str(&format!("{:<12} {:?}\n", "Type:", self.file_type))?; - match &self.block_data { - BlockDataState::Encrypted(_) => f.write_str("Blocks: Encrypted\n")?, - BlockDataState::Decrypted(_) => f.write_str("Blocks: Decrypted\n")?, - } - f.write_str(&format!("{:<12} {}\n", "Created:", self.created))?; - f.write_str(&format!("{:<12} {}\n", "Modified:", self.modified))?; - f.write_str(&format!("{:<12} {}\n", "Size:", self.file_size))?; - f.write_str(&format!("{:<12} {:o}\n", "Permissions:", self.permissions))?; - f.write_str(&format!("{:<12} {:?}\n", "References:", self.references))?; - - if let Some(target) = &self.symlink_target { - f.write_str(&format!("{:<12} {target}\n", "Target:"))?; - } - Ok(()) - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct Reference { - pub target_file_id: u64, // varint - pub relationship: u64, // varint -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct EncryptionSection { - // Recipient's X25519 public key, Recipient file list - pub recipients: IndexMap<[u8; 32], RecipientSection>, -} - -impl EncryptionSection { - #[tracing::instrument(level = "trace", skip(recipient_pubkeys))] - pub fn new(recipient_pubkeys: &[PublicKey]) -> Self { - EncryptionSection { - recipients: IndexMap::from_iter( - recipient_pubkeys - .iter() - .map(|key| { - ( - key.to_bytes(), - RecipientSection { - recipient_data: RecipientData::Decrypted( - Zeroizing::new(Vec::new()), - ), - }, - ) - }) - .collect::>(), - ), - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct RecipientSection { - pub(crate) recipient_data: RecipientData, // Encrypted FileKeyEntry list -} - -/// Transient writer/reader wire state. Recovered file keys are crate-private. -#[derive(Clone, PartialEq, Eq)] -pub(crate) enum RecipientData { - Encrypted(Vec), // Chacha + nonce (Shared key PrivKey Writer <--> PubKey Reader) - Decrypted(RecipientKeyList), // Fileindex / Random key to decrypt BlockDataState -} - -impl std::fmt::Debug for RecipientData { - fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Encrypted(bytes) => formatter - .debug_tuple("Encrypted") - .field(&format_args!("{} bytes", bytes.len())) - .finish(), - Self::Decrypted(entries) => formatter - .debug_tuple("Decrypted") - .field(&format_args!("{} file keys [REDACTED]", entries.len())) - .finish(), - } - } -} - -impl RecipientData { - pub(crate) fn encrypt_with_secret_and_nonce( - &mut self, - shared_key: SharedSecret, - nonce: [u8; 12], - ) -> Result<(), PithosError> { - match &self { - RecipientData::Encrypted(_) => { - return Err(PithosError::InvalidRecipientDataState( - "Recipient data already encrypted".to_string(), - )); - } - RecipientData::Decrypted(entries) => { - let mut data_bytes = Zeroizing::new(Vec::with_capacity(1 + entries.len() * 40)); - crate::format::codec::encode_decrypted_recipient_list(entries, &mut *data_bytes)?; - - let encrypted_data = - crypto::wrap_recipient_list_with_nonce(&shared_key, &data_bytes, nonce)?; - - *self = RecipientData::Encrypted(encrypted_data) - } - }; - - Ok(()) - } -} - -#[cfg(test)] -mod zeroization_tests { - use super::*; - use zeroize::ZeroizeOnDrop; - - fn assert_zeroize_on_drop() {} - - #[test] - fn transient_plaintext_key_lists_have_drop_zeroization_contracts() { - assert_zeroize_on_drop::>>(); - assert_zeroize_on_drop::>>(); - } -} diff --git a/crates/pithos_lib/src/fs/append.rs b/crates/pithos_lib/src/fs/append.rs index 51d5600..9312615 100644 --- a/crates/pithos_lib/src/fs/append.rs +++ b/crates/pithos_lib/src/fs/append.rs @@ -573,12 +573,12 @@ mod tests { AppendFixture { archive, sender } } - fn terminal_directory(path: &Path) -> crate::format::wire::Directory { + fn terminal_directory(path: &Path) -> crate::format::directory::Directory { let bytes = std::fs::read(path).unwrap(); let directory_len = u64::from_be_bytes(bytes[bytes.len() - 12..bytes.len() - 4].try_into().unwrap()); let directory_start = bytes.len() - usize::try_from(directory_len).unwrap(); - crate::format::codec::decode_directory( + crate::format::directory::decode_directory( &mut Cursor::new(&bytes[directory_start..]), &DeserializationLimits::default(), ) @@ -586,7 +586,7 @@ mod tests { } #[test] - fn append_and_grant_use_fresh_wire_senders_and_retain_the_access_key_as_a_recipient() { + fn append_and_grant_use_fresh_senders_and_retain_the_access_key_as_a_recipient() { let temporary = tempfile::tempdir().unwrap(); let fixture = append_fixture(&temporary); let access_public = @@ -681,16 +681,16 @@ mod tests { let directory_len = u64::from_be_bytes(bytes[bytes.len() - 12..bytes.len() - 4].try_into().unwrap()); let directory_start = bytes.len() - usize::try_from(directory_len).unwrap(); - let mut directory = crate::format::codec::decode_directory( + let mut directory = crate::format::directory::decode_directory( &mut Cursor::new(&bytes[directory_start..]), &DeserializationLimits::default(), ) .unwrap(); directory.relations[0].1 = "Conflicts_With_Default".into(); - crate::format::codec::update_directory_len(&mut directory).unwrap(); - crate::format::codec::update_directory_crc(&mut directory).unwrap(); + crate::format::directory::update_directory_len(&mut directory).unwrap(); + crate::format::directory::update_directory_crc(&mut directory).unwrap(); let mut replacement = Vec::new(); - crate::format::codec::encode_directory(&directory, &mut replacement).unwrap(); + crate::format::directory::encode_directory(&directory, &mut replacement).unwrap(); bytes.truncate(directory_start); bytes.extend_from_slice(&replacement); std::fs::write(&archive_path, &bytes).unwrap(); diff --git a/crates/pithos_lib/src/fs/extraction.rs b/crates/pithos_lib/src/fs/extraction.rs index c6ae3a2..e204f6d 100644 --- a/crates/pithos_lib/src/fs/extraction.rs +++ b/crates/pithos_lib/src/fs/extraction.rs @@ -1,5 +1,6 @@ use super::FsError; use crate::archive::{Archive, EntryKind, ExternalBlockResolver}; +use crate::format::file_entry::VALID_PERMISSION_BITS; use crate::source::ArchiveSource; use cap_std::fs::Dir; use rustix::fs::{AtFlags, Mode, OFlags, fchmod, linkat, openat}; @@ -23,7 +24,7 @@ impl ExtractionOptions { fn applied_mode(stored_permissions: u32, options: ExtractionOptions) -> Mode { let mask = if options.special_permissions { - 0o7777 + VALID_PERMISSION_BITS } else { 0o0777 }; diff --git a/crates/pithos_lib/src/fs/ingest.rs b/crates/pithos_lib/src/fs/ingest.rs index ffe3429..876054d 100644 --- a/crates/pithos_lib/src/fs/ingest.rs +++ b/crates/pithos_lib/src/fs/ingest.rs @@ -3,6 +3,7 @@ use crate::archive::{ AppendSnapshot, ArchivePath, ArchiveWriter, EntryMetadata, ProcessingOptions, WriterError, }; use crate::error::PithosError; +use crate::format::file_entry::VALID_PERMISSION_BITS; use crate::fs::FsError; use cap_std::fs::{ Dir, FileTypeExt as CapFileTypeExt, MetadataExt as CapMetadataExt, @@ -59,7 +60,7 @@ fn metadata(metadata: &fs::Metadata) -> EntryMetadata { EntryMetadata::new( timestamp(metadata.created()), timestamp(metadata.modified()), - metadata.permissions().mode() & 0o7777, + metadata.permissions().mode() & VALID_PERMISSION_BITS, ) } @@ -73,7 +74,7 @@ fn cap_metadata(metadata: &cap_std::fs::Metadata) -> EntryMetadata { EntryMetadata::new( timestamp(metadata.created()), timestamp(metadata.modified()), - metadata.permissions().mode() & 0o7777, + metadata.permissions().mode() & VALID_PERMISSION_BITS, ) } diff --git a/crates/pithos_lib/src/fuzzing.rs b/crates/pithos_lib/src/fuzzing.rs index c77eb8b..9d5e875 100644 --- a/crates/pithos_lib/src/fuzzing.rs +++ b/crates/pithos_lib/src/fuzzing.rs @@ -26,14 +26,17 @@ pub enum FuzzOutcome { } pub fn decode_header(data: &[u8]) -> FuzzOutcome { - match crate::format::codec::decode_header(&mut Cursor::new(data)) { + match crate::format::header::decode_header(&mut Cursor::new(data)) { Ok(_) => FuzzOutcome::Accepted, Err(_) => FuzzOutcome::Rejected(FuzzErrorCategory::Deserialization), } } pub fn decode_directory(data: &[u8]) -> FuzzOutcome { - match crate::format::codec::decode_complete_directory(data, &DeserializationLimits::default()) { + let limits = DeserializationLimits::default(); + let mut remaining_block_references = limits.max_block_references; + match crate::archive::decode_validated_directory(data, &limits, &mut remaining_block_references) + { Ok(_) => FuzzOutcome::Accepted, Err(error) => FuzzOutcome::Rejected(classify_archive_error(&error)), } diff --git a/crates/pithos_lib/src/lib.rs b/crates/pithos_lib/src/lib.rs index 3165b03..9b2225b 100644 --- a/crates/pithos_lib/src/lib.rs +++ b/crates/pithos_lib/src/lib.rs @@ -21,7 +21,7 @@ //! The Linux filesystem adapter is intentionally host-specific. Core archive reading and writing //! remain independent of filesystem traversal and presentation formats. //! -//! Wire records and compatibility modules are deliberately not part of the public API: +//! Internal format types and compatibility modules are deliberately not part of the public API: //! //! ```compile_fail //! use pithos_lib::model::structs::Directory; @@ -32,7 +32,7 @@ //! ``` //! //! ```compile_fail -//! use pithos_lib::format::wire::FileHeader; +//! use pithos_lib::format::header::FileHeader; //! ``` pub mod adapters; diff --git a/crates/pithos_lib/tests/compile_fail.rs b/crates/pithos_lib/tests/compile_fail.rs index d3cdf3f..b7c29da 100644 --- a/crates/pithos_lib/tests/compile_fail.rs +++ b/crates/pithos_lib/tests/compile_fail.rs @@ -5,5 +5,5 @@ fn secret_ownership_and_writer_typestate_are_compile_time_enforced() { cases.compile_fail("tests/ui/writer_finish_consumes_writer.rs"); cases.compile_fail("tests/ui/writer_use_after_finish.rs"); cases.compile_fail("tests/ui/incomplete_writer_cannot_resume.rs"); - cases.compile_fail("tests/ui/crypt4gh_wire_record_is_private.rs"); + cases.compile_fail("tests/ui/crypt4gh_header_is_private.rs"); } diff --git a/crates/pithos_lib/tests/draft_spec_vectors.rs b/crates/pithos_lib/tests/draft_spec_vectors.rs index 926aff7..eb09a3f 100644 --- a/crates/pithos_lib/tests/draft_spec_vectors.rs +++ b/crates/pithos_lib/tests/draft_spec_vectors.rs @@ -5,8 +5,9 @@ use chacha20poly1305::{ChaCha20Poly1305, KeyInit, Nonce, aead::Aead}; use crc32fast::hash as crc32; use pithos_lib::archive::{ - Archive, ArchivePath, ArchiveWriter, EntryKind, EntryMetadata, OpenLimits, OpenOptions, - ProcessingOptions, WriteOptions, + Archive, ArchivePath, ArchiveWriter, EntryKind, EntryMetadata, ExternalBlockAccessPolicy, + ExternalBlockResolver, ExternalLocation, OpenLimits, OpenOptions, ProcessingOptions, + WriteOptions, }; use pithos_lib::error::{DeserializationError, PithosError}; use pithos_lib::source::MemorySource; @@ -164,172 +165,103 @@ fn mutate_hello_byte(offset: usize, replacement: u8) -> Vec { bytes } -fn read_uleb(bytes: &[u8], cursor: &mut usize) -> Result { - let mut value = 0_u64; - for shift in (0..64).step_by(7) { - let byte = *bytes.get(*cursor).ok_or("truncated ULEB128")?; - *cursor += 1; - value |= u64::from(byte & 0x7f) << shift; - if byte & 0x80 == 0 { - return Ok(value); - } - } - Err("overflowing ULEB128") +fn refresh_directory(bytes: &mut [u8], directory_start: usize) { + let footer = bytes.len() - 12; + let directory_len = (bytes.len() - directory_start) as u64; + bytes[footer..footer + 8].copy_from_slice(&directory_len.to_be_bytes()); + let checksum = crc32(&bytes[directory_start..bytes.len() - 4]); + let crc_offset = bytes.len() - 4; + bytes[crc_offset..].copy_from_slice(&checksum.to_be_bytes()); } -fn take(bytes: &[u8], cursor: &mut usize, count: usize) -> Result<(), &'static str> { - *cursor = cursor.checked_add(count).ok_or("overflow")?; - if *cursor > bytes.len() { - return Err("truncated field"); +fn encoded_empty_entry(id: u64, path: &str, file_type: u8, permissions: u64) -> Vec { + let mut bytes = Vec::new(); + uleb(id, &mut bytes); + uleb(path.len() as u64, &mut bytes); + bytes.extend_from_slice(path.as_bytes()); + bytes.extend_from_slice(&[file_type, 1, 0]); // Type and an empty decrypted block list. + for value in [0, 0, 0, permissions] { + uleb(value, &mut bytes); } - Ok(()) + bytes.extend_from_slice(&[0, 0]); // No references or symlink target. + bytes } -/// Decode the finalized draft grammar sufficiently to prove exact consumption. -fn decode_directory(bytes: &[u8]) -> Result, &'static str> { - if bytes.len() < 25 || !bytes.starts_with(b"PITHOSDR") { - return Err("directory framing"); - } - if u64::from_be_bytes(bytes[bytes.len() - 12..bytes.len() - 4].try_into().unwrap()) - != bytes.len() as u64 - { - return Err("directory length"); - } - if crc32(&bytes[..bytes.len() - 4]) - != u32::from_be_bytes(bytes[bytes.len() - 4..].try_into().unwrap()) - { - return Err("directory CRC"); - } - let mut at = 8; - let parent = match *bytes.get(at).ok_or("parent tag")? { - 0 => { - at += 1; - None - } - 1 => { - at += 1; - Some((read_uleb(bytes, &mut at)?, read_uleb(bytes, &mut at)?)) - } - _ => return Err("parent tag"), - }; - let files = read_uleb(bytes, &mut at)?; - for _ in 0..files { - read_uleb(bytes, &mut at)?; - let path_len = read_uleb(bytes, &mut at)? as usize; - take(bytes, &mut at, path_len)?; - match *bytes.get(at).ok_or("file type")? { - 0..=3 => at += 1, - _ => return Err("file type"), - } - match *bytes.get(at).ok_or("block data tag")? { - 0 => { - at += 1; - let n = read_uleb(bytes, &mut at)? as usize; - take(bytes, &mut at, n)? - } - 1 => { - at += 1; - let n = read_uleb(bytes, &mut at)? as usize; - take(bytes, &mut at, n * 64)? - } - _ => return Err("block data tag"), - } - for _ in 0..4 { - read_uleb(bytes, &mut at)?; - } - let refs = read_uleb(bytes, &mut at)?; - for _ in 0..refs { - read_uleb(bytes, &mut at)?; - read_uleb(bytes, &mut at)?; - } - match *bytes.get(at).ok_or("symlink tag")? { - 0 => at += 1, - 1 => { - at += 1; - let n = read_uleb(bytes, &mut at)? as usize; - take(bytes, &mut at, n)? - } - _ => return Err("symlink tag"), - } - } - let blocks = read_uleb(bytes, &mut at)?; - for _ in 0..blocks { - take(bytes, &mut at, 32)?; - read_uleb(bytes, &mut at)?; - read_uleb(bytes, &mut at)?; - read_uleb(bytes, &mut at)?; - let flags = *bytes.get(at).ok_or("flags")?; - at += 1; - if flags & 0xf0 != 0 { - return Err("reserved flags"); - } - match *bytes.get(at).ok_or("location tag")? { - 0 => at += 1, - 1 => { - at += 1; - let n = read_uleb(bytes, &mut at)? as usize; - take(bytes, &mut at, n)? - } - _ => return Err("location tag"), - } - } - let relations = read_uleb(bytes, &mut at)?; - for _ in 0..relations { - read_uleb(bytes, &mut at)?; - let n = read_uleb(bytes, &mut at)? as usize; - take(bytes, &mut at, n)?; +fn base_with_entries(entries: &[Vec]) -> Vec { + let mut bytes = b"PITH\x01\0PITHOSDR\0".to_vec(); + uleb(entries.len() as u64, &mut bytes); + for entry in entries { + bytes.extend_from_slice(entry); } - let sections = read_uleb(bytes, &mut at)?; - if sections != 0 { - return Err("encryption sections not used by canonical vectors"); + bytes.push(0); // Blocks. + bytes.push(10); + for (id, name) in RELATIONS { + uleb(id.into(), &mut bytes); + uleb(name.len() as u64, &mut bytes); + bytes.extend_from_slice(name.as_bytes()); } - if at + 12 != bytes.len() { - return Err("directory consumption"); - } - Ok(parent) + bytes.push(0); // Encryption. + bytes.extend_from_slice(&[0; 12]); + refresh_directory(&mut bytes, 6); + bytes +} + +fn external_hello() -> Vec { + let mut bytes = hello(); + bytes.splice(143..=143, [1, 1, b'x']); + refresh_directory(&mut bytes, 15); + bytes +} + +fn cross_size_append() -> Vec { + let mut bytes = hello(); + let child_start = bytes.len(); + let mut child = directory(Some((15, 264)), true, true); + child[15..20].copy_from_slice(b"other"); + child[129] = 4; + let child_crc = crc32(&child[..child.len() - 4]); + let child_crc_offset = child.len() - 4; + child[child_crc_offset..].copy_from_slice(&child_crc.to_be_bytes()); + bytes.extend(child); + assert_eq!(child_start, 279); + bytes } -fn decode_archive(bytes: &[u8]) -> Result<(), &'static str> { - if !bytes.starts_with(b"PITH\x01\0") { - return Err("header"); +#[derive(Clone, Copy)] +struct MalformedExternalResolver; + +impl ExternalBlockResolver for MalformedExternalResolver { + fn resolve( + &self, + policy: &dyn ExternalBlockAccessPolicy, + _location: &ExternalLocation, + expected_len: u64, + _max_response_size: u64, + ) -> Result, PithosError> { + assert_eq!(expected_len, 9); + assert!(policy.allows("test://resolved")); + Ok(b"BLCKxxxx".to_vec()) } - let length = u64::from_be_bytes( - bytes - .get(bytes.len().checked_sub(12).ok_or("footer")?..bytes.len() - 4) - .ok_or("footer")? - .try_into() - .unwrap(), - ) as usize; - let start = bytes - .len() - .checked_sub(length) - .ok_or("terminal underflow")?; - let parent = decode_directory(&bytes[start..])?; - if let Some((parent_start, parent_len)) = parent { - let parent_start = usize::try_from(parent_start).map_err(|_| "parent range")?; - let parent_len = usize::try_from(parent_len).map_err(|_| "parent range")?; - let parent_end = parent_start.checked_add(parent_len).ok_or("parent range")?; - if parent_end > start || parent_end > bytes.len() { - return Err("parent range"); - } - if decode_directory(&bytes[parent_start..parent_end])?.is_some() { - return Err("base parent"); - } - } else if start != 6 && bytes.get(6..10) != Some(b"BLCK") { - return Err("base placement"); +} + +struct AllowExternal; + +impl ExternalBlockAccessPolicy for AllowExternal { + fn allows(&self, _target: &str) -> bool { + true } - Ok(()) } #[test] -fn canonical_vectors_are_generated_decoded_and_exactly_reencoded() { +fn canonical_vectors_are_generated_opened_and_exactly_reencoded() { for (id, generated) in [ ("CV-BASE-EMPTY-146", base()), ("CV-APPEND-EMPTY-28", append()), ("CV-LOCAL-HELLO-279", hello()), ] { assert_eq!(generated, appendix_hex(id), "{id} must match Appendix B"); - decode_archive(&generated).unwrap_or_else(|error| panic!("{id}: {error}")); + Archive::open(MemorySource::new(generated.clone()), OpenOptions::default()) + .unwrap_or_else(|error| panic!("production reader rejected {id}: {error}")); // The table-derived encoder is canonical, so its output is the exact re-encoding. assert_eq!( generated, @@ -409,26 +341,31 @@ fn production_reader_rejects_a_direct_list_before_over_budget_allocation() { } #[test] -fn production_reader_accepts_bounded_non_minimal_uleb128() { +fn production_reader_accepts_av_uleb_nonminimal() { + let id = "AV-ULEB-NONMINIMAL"; Archive::open( MemorySource::new(replace_base_relationship_count(&[0x8a, 0x00])), OpenOptions::default(), ) - .unwrap(); + .unwrap_or_else(|error| panic!("production reader rejected {id}: {error}")); } #[test] -fn production_reader_rejects_overflowing_uleb128() { +fn production_reader_rejects_rv_uleb() { + let id = "RV-ULEB"; let result = Archive::open( MemorySource::new(replace_base_relationship_count(&[ 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, ])), OpenOptions::default(), ); - assert!(matches!( - result, - Err(PithosError::Deserialization(DeserializationError::Io(_))) - )); + assert!( + matches!( + result, + Err(PithosError::Deserialization(DeserializationError::Io(_))) + ), + "production reader accepted {id}" + ); } #[test] @@ -479,42 +416,14 @@ fn directory_length_calculations_are_independent() { #[test] fn fixed_mutations_have_the_appendix_crc_and_reach_their_rule() { let cases = [ - ("RV-FLAGS", hello(), 142, 0x10, 0x7bd3b103, "reserved flags"), - ( - "RV-UNKNOWN-TAG", - hello(), - 143, - 0x02, - 0x86cf1282, - "location tag", - ), - ("RV-FILETYPE", hello(), 32, 0x04, 0x50f42595, "file type"), - ( - "RV-PARENT", - append(), - 161, - 0x98, - 0xb734abf4, - "parent topology", - ), - ( - "RV-EXTENT", - hello(), - 139, - 0x0f, - 0x623c8b49, - "extent placement", - ), - ( - "RV-SHORT-ENCRYPTED", - hello(), - 142, - 0x08, - 0x92564e52, - "directory consumption", - ), + ("RV-FLAGS", hello(), 142, 0x10, 0x7bd3b103), + ("RV-UNKNOWN-TAG", hello(), 143, 0x02, 0x86cf1282), + ("RV-FILETYPE", hello(), 32, 0x04, 0x50f42595), + ("RV-PARENT", append(), 161, 0x98, 0xb734abf4), + ("RV-EXTENT", hello(), 139, 0x0f, 0x623c8b49), + ("RV-SHORT-ENCRYPTED", hello(), 142, 0x08, 0x92564e52), ]; - for (id, mut bytes, offset, replacement, expected_crc, rule) in cases { + for (id, mut bytes, offset, replacement, expected_crc) in cases { let crc_offset = bytes.len() - 4; let directory_start = bytes.len() - u64::from_be_bytes(bytes[crc_offset - 8..crc_offset].try_into().unwrap()) as usize; @@ -524,23 +433,13 @@ fn fixed_mutations_have_the_appendix_crc_and_reach_their_rule() { assert_eq!( u32::from_be_bytes(bytes[crc_offset..].try_into().unwrap()), expected_crc, - "{rule}" + "{id}" ); assert_b4_crc(id, expected_crc); - match rule { - "parent topology" => { - let terminal_start = bytes.len() - - u64::from_be_bytes(bytes[crc_offset - 8..crc_offset].try_into().unwrap()) - as usize; - assert_eq!(bytes[terminal_start + 9], terminal_start as u8); - } - "directory consumption" => { - assert_ne!(bytes[142] & 0x08, 0); - assert!(bytes[140] < 28, "encrypted payload is too short"); - } - "extent placement" => assert_eq!(bytes[139], 15, "local extent begins at Directory"), - expected => assert_eq!(decode_archive(&bytes), Err(expected)), - } + assert!( + Archive::open(MemorySource::new(bytes), OpenOptions::default()).is_err(), + "production reader accepted {id}" + ); } let mut underflow = base(); underflow[140..148].copy_from_slice(&153_u64.to_be_bytes()); @@ -551,7 +450,7 @@ fn fixed_mutations_have_the_appendix_crc_and_reach_their_rule() { 0x371dda5a ); assert_b4_crc("RV-UNDERFLOW", 0x371dda5a); - assert_eq!(decode_archive(&underflow), Err("terminal underflow")); + assert!(Archive::open(MemorySource::new(underflow), OpenOptions::default()).is_err()); let mut permissions = hello(); permissions[102..104].copy_from_slice(&[0x80, 0x20]); let crc_offset = permissions.len() - 4; @@ -564,14 +463,17 @@ fn fixed_mutations_have_the_appendix_crc_and_reach_their_rule() { 0x9f6af36e ); assert_b4_crc("RV-PERMISSIONS", 0x9f6af36e); - let mut permissions_offset = 102; - assert_eq!(read_uleb(&permissions, &mut permissions_offset), Ok(0x1000)); + assert!(Archive::open(MemorySource::new(permissions), OpenOptions::default()).is_err()); let mut bad_crc = base(); bad_crc[151] = 0xd3; - assert_eq!(decode_archive(&bad_crc), Err("directory CRC")); let mut trailing = base(); trailing.push(0); - assert!(decode_archive(&trailing).is_err()); + for (id, bytes) in [("RV-CRC", bad_crc), ("RV-TRAILING", trailing)] { + assert!( + Archive::open(MemorySource::new(bytes), OpenOptions::default()).is_err(), + "production reader accepted {id}" + ); + } } fn assert_b4_crc(id: &str, crc: u32) { @@ -741,7 +643,7 @@ fn specification_consistency_has_navigation_and_review_coverage() { } #[test] -fn structural_mutation_predicates_cover_appendix_b_groups() { +fn production_reader_rejects_generated_rv_duplicates_path_hierarchy_and_relationships() { for id in [ "RV-DUPLICATES", "RV-PATH", @@ -757,93 +659,64 @@ fn structural_mutation_predicates_cover_appendix_b_groups() { "missing {id}" ); } - let unique = |values: &[&str]| { - values - .iter() - .collect::>() - .len() - == values.len() - }; - assert!(!unique(&["0", "0"])); // duplicate IDs, paths, hashes, and relationships share this predicate - assert!(!unique(&["hello", "hello"])); - assert!(!unique(&["hash", "hash"])); - assert!(!unique(&["7", "7"])); - let valid_path = |path: &str| { - !path.is_empty() - && !path.starts_with('/') - && !path.ends_with('/') - && !path.contains(['\\', '\0']) - && !path - .split('/') - .any(|part| part.is_empty() || matches!(part, "." | "..")) - && !path.as_bytes().get(1).is_some_and(|byte| *byte == b':') - }; - assert!( - ["/hell", "hell/", "a//b", "a\\b", ".", "C:"] - .iter() - .all(|path| !valid_path(path)) - ); - let valid_symlink_target = |parent_depth: usize, target: &str| { - if target.is_empty() - || target.starts_with('/') - || target.contains(['\\', '\0']) - || target.as_bytes().get(1) == Some(&b':') - { - return false; - } - let mut depth = parent_depth; - for component in target.split('/') { - if component.is_empty() || component == "." { - return false; - } - if component == ".." { - if depth == 0 { - return false; - } - depth -= 1; - } else { - depth += 1; - } + + let duplicate = base_with_entries(&[ + encoded_empty_entry(0, "first", 0, 0o755), + encoded_empty_entry(0, "second", 0, 0o755), + ]); + let invalid_path = base_with_entries(&[encoded_empty_entry(0, "/bad", 0, 0o755)]); + let child_before_parent = base_with_entries(&[ + encoded_empty_entry(0, "parent/child", 1, 0o644), + encoded_empty_entry(1, "parent", 0, 0o755), + ]); + let mut invalid_custom_relationship = base(); + invalid_custom_relationship[18] = 10; + refresh_directory(&mut invalid_custom_relationship, 6); + + for (case, bytes) in [ + ("RV-DUPLICATES", duplicate), + ("RV-PATH", invalid_path), + ("hierarchy declaration order", child_before_parent), + ("custom relationship range", invalid_custom_relationship), + ("RV-CROSS-SIZE", cross_size_append()), + ] { + assert!( + Archive::open(MemorySource::new(bytes), OpenOptions::default()).is_err(), + "production reader accepted {case}" + ); + } +} + +#[test] +fn production_reader_enforces_rv_external_availability_and_framing() { + let bytes = external_hello(); + let unavailable = + Archive::open(MemorySource::new(bytes.clone()), OpenOptions::default()).unwrap(); + assert!(matches!( + unavailable.entry("hello").unwrap().unwrap().kind, + EntryKind::File { + available: false, + .. } - true - }; - let valid_symlink = |parent_depth: usize, target: Option<&str>, blocks: usize, size: u64| { - target.is_some_and(|target| valid_symlink_target(parent_depth, target)) - && blocks == 0 - && size == 0 - }; - assert!(valid_symlink(1, Some("../target"), 0, 0)); - assert!(!valid_symlink(0, Some("../escape"), 0, 0)); - assert!(!valid_symlink(0, None, 0, 0)); - assert!(!valid_symlink(0, Some("inside"), 1, 0)); - assert!(!valid_symlink(0, Some("inside"), 0, 1)); - let external_result = |enabled: bool, response: &[u8], stored_size: usize| { - if !enabled { - "unavailable" - } else if response.starts_with(b"BLCK") && response.len() == stored_size + 4 { - "readable" - } else { - "fails before output" + )); + let mut output = Vec::new(); + assert!(unavailable.copy_to("hello", &mut output).is_err()); + assert!(output.is_empty()); + + let available = Archive::open( + MemorySource::new(bytes), + OpenOptions::default() + .with_external_resolver(MalformedExternalResolver) + .with_external_access_policy(std::sync::Arc::new(AllowExternal)), + ) + .unwrap(); + assert!(matches!( + available.entry("hello").unwrap().unwrap().kind, + EntryKind::File { + available: true, + .. } - }; - assert_eq!(external_result(false, b"", 5), "unavailable"); - assert_eq!(external_result(true, b"BLCKxxxx", 5), "fails before output"); - assert_eq!( - external_result(true, b"BLCKxxxxxx", 5), - "fails before output" - ); - assert_eq!( - external_result(true, b"BADCxxxxx", 5), - "fails before output" - ); - let merge_descriptor = |existing: Option, candidate: u64| match existing { - None => Ok(candidate), - Some(size) if size == candidate => Ok(size), - Some(_) => Err("cross-segment original_size conflict"), - }; - assert_eq!(merge_descriptor(Some(5), 5), Ok(5)); - assert_eq!( - merge_descriptor(Some(4), 5), - Err("cross-segment original_size conflict") - ); + )); + assert!(available.copy_to("hello", &mut output).is_err()); + assert!(output.is_empty()); } diff --git a/crates/pithos_lib/tests/format_validation.rs b/crates/pithos_lib/tests/format_validation.rs index 1598bad..4fbc668 100644 --- a/crates/pithos_lib/tests/format_validation.rs +++ b/crates/pithos_lib/tests/format_validation.rs @@ -138,7 +138,7 @@ fn open_limits_reject_oversized_entry_descriptor_reference_and_relationship_coun } #[test] -fn adversarial_wire_path_order_stays_within_open_limits() { +fn reverse_sibling_order_stays_within_open_limits() { let sender = PrivateKey::generate(); let recipient = sender.duplicate(); let mut writer = ArchiveWriter::create( diff --git a/crates/pithos_lib/tests/ui/crypt4gh_wire_record_is_private.rs b/crates/pithos_lib/tests/ui/crypt4gh_header_is_private.rs similarity index 100% rename from crates/pithos_lib/tests/ui/crypt4gh_wire_record_is_private.rs rename to crates/pithos_lib/tests/ui/crypt4gh_header_is_private.rs diff --git a/crates/pithos_lib/tests/ui/crypt4gh_wire_record_is_private.stderr b/crates/pithos_lib/tests/ui/crypt4gh_header_is_private.stderr similarity index 86% rename from crates/pithos_lib/tests/ui/crypt4gh_wire_record_is_private.stderr rename to crates/pithos_lib/tests/ui/crypt4gh_header_is_private.stderr index 1328285..116c66c 100644 --- a/crates/pithos_lib/tests/ui/crypt4gh_wire_record_is_private.stderr +++ b/crates/pithos_lib/tests/ui/crypt4gh_header_is_private.stderr @@ -1,5 +1,5 @@ error[E0603]: struct `Crypt4GHHeader` is private - --> tests/ui/crypt4gh_wire_record_is_private.rs:1:37 + --> tests/ui/crypt4gh_header_is_private.rs:1:37 | 1 | use pithos_lib::adapters::crypt4gh::Crypt4GHHeader; | ^^^^^^^^^^^^^^ private struct From cf0cfd84de6aa78636744f6a3ce2a429a7663656 Mon Sep 17 00:00:00 2001 From: Jannis Schlegel Date: Wed, 30 Sep 2026 15:52:14 +0200 Subject: [PATCH 049/166] refactor: enhance specification clarity with improved segment structure visualization --- crates/pithos_lib/tests/draft_spec_vectors.rs | 2 +- spec/PITHOS_1.0.0_draft.md | 21 ++++++++++++------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/crates/pithos_lib/tests/draft_spec_vectors.rs b/crates/pithos_lib/tests/draft_spec_vectors.rs index eb09a3f..3fd19fd 100644 --- a/crates/pithos_lib/tests/draft_spec_vectors.rs +++ b/crates/pithos_lib/tests/draft_spec_vectors.rs @@ -625,7 +625,7 @@ fn specification_consistency_has_navigation_and_review_coverage() { "CV-LOCAL-HELLO-279", "### 1.1 Reader's Guide", "**Illustrative data model.**", - "+-- base segment", + "|<--------------- base segment ------------>|", "### B.5 Validation Index", "[Section 4.2.5]", "**Valid ordering example:**", diff --git a/spec/PITHOS_1.0.0_draft.md b/spec/PITHOS_1.0.0_draft.md index 1817f96..4133afa 100644 --- a/spec/PITHOS_1.0.0_draft.md +++ b/spec/PITHOS_1.0.0_draft.md @@ -72,16 +72,23 @@ this document governs. A Pithos file MUST have the following structure: ``` -+-- base segment --------------------+ +-- appended terminal segment -------------------+ -| [FileHeader][Base Blocks][Base Dir] | | [Appended Blocks][Terminal Dir ... dir_len || crc32] | EOF -+-------------------------------------+ +-----------------------------------------------------+ - ^ | - +-- parent (start, len) <-------------+ - `dir_len || crc32` = final 12 bytes +File start EOF + | | + v v ++------------+-------------+----------------+-----------------+--------------------+ +| FileHeader | Base blocks | Base Directory | Appended blocks | Terminal Directory | ++------------+-------------+----------------+-----------------+--------------------+ +|<--------------- base segment ------------>|<-------- appended segment ---------->| + +Directory chain: + +[Base Directory] <- [Directory] <- ... <- [Terminal Directory] + parent_directory_offset links each Directory to its predecessor ``` Each segment ends immediately after its Directory. Encryption sections, when -present, are items in that Directory's `encryption` vector. +present, are items in that Directory's `encryption` vector. The final 12 bytes +of every Directory are `dir_len:u64be || crc32:u32be`. ### 3.1 Common Encoding Rules From 09ba350b8608fba9b1774491f24fa487b0a6a604 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 14:02:52 +0200 Subject: [PATCH 050/166] feat: add format version 1.1 with derived recipient grant keys --- Cargo.lock | 33 +++++++ crates/pithos_lib/Cargo.toml | 2 + crates/pithos_lib/src/archive/reader.rs | 37 ++++++-- .../src/archive/reader_private_tests.rs | 19 +++- crates/pithos_lib/src/archive/snapshot.rs | 8 ++ crates/pithos_lib/src/archive/writer.rs | 68 ++++++++++++-- crates/pithos_lib/src/crypto.rs | 44 ++++++++- crates/pithos_lib/src/format/header.rs | 52 +++++++++-- crates/pithos_lib/tests/append_workflow.rs | 2 +- crates/pithos_lib/tests/draft_spec_vectors.rs | 42 ++++++++- crates/pithos_lib/tests/format_validation.rs | 2 +- crates/pithos_lib/tests/grant_readers.rs | 2 +- spec/PITHOS_1.0.0_draft.md | 91 ++++++++++++++----- 13 files changed, 344 insertions(+), 58 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a9fbcea..4d01145 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -598,7 +598,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ "block-buffer 0.12.1", + "const-oid", "crypto-common 0.2.2", + "ctutils", ] [[package]] @@ -838,6 +840,24 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hkdf" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aaa26c720c68b866f2c96ef5c1264b3e6f473fe5d4ce61cd44bbe913e553018" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", +] + [[package]] name = "http" version = "1.4.2" @@ -1354,6 +1374,7 @@ dependencies = [ "criterion", "digest 0.11.3", "fastcdc", + "hkdf", "indexmap", "integer-encoding", "libc", @@ -1362,6 +1383,7 @@ dependencies = [ "proptest", "ro-crate-rs", "rustix", + "sha2", "shake", "tempfile", "thiserror", @@ -1798,6 +1820,17 @@ dependencies = [ "serde_core", ] +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest 0.11.3", +] + [[package]] name = "shake" version = "0.1.0" diff --git a/crates/pithos_lib/Cargo.toml b/crates/pithos_lib/Cargo.toml index 85c7b4e..0081877 100644 --- a/crates/pithos_lib/Cargo.toml +++ b/crates/pithos_lib/Cargo.toml @@ -36,12 +36,14 @@ crc32fast = "1.5.0" cap-std = "4.0.2" digest = "0.11.3" fastcdc = "4.0.1" +hkdf = "0.13.0" indexmap = "2.11.0" integer-encoding = "4.1.0" libc = "0.2.178" pkcs8 = { version = "0.11.0", features = ["pem", "alloc"] } ro-crate-rs = { version = "0.5.1", default-features = false } rustix = { version = "1.1.4", features = ["fs"] } +sha2 = "0.11.0" shake = "0.1.0" thiserror = { workspace = true } tracing = { workspace = true } diff --git a/crates/pithos_lib/src/archive/reader.rs b/crates/pithos_lib/src/archive/reader.rs index ae68767..726015f 100644 --- a/crates/pithos_lib/src/archive/reader.rs +++ b/crates/pithos_lib/src/archive/reader.rs @@ -17,7 +17,7 @@ use crate::format::block::{ use crate::format::directory::Directory; use crate::format::encryption::RecipientData; use crate::format::file_entry::{BlockDataEntry, BlockDataState}; -use crate::format::header::FileHeader; +use crate::format::header::{FileHeader, FormatVersion}; use crate::format::limits::DeserializationLimits; use crate::source::ArchiveSource; use std::collections::{BTreeMap, HashMap, HashSet}; @@ -268,6 +268,7 @@ pub struct Archive { external: E, external_access_policy: Option>, archive_len: u64, + version: FormatVersion, terminal_directory: Span, index: ArchiveIndex, segments: Vec, @@ -289,12 +290,12 @@ where let mut header = [0; FileHeader::ENCODED_LEN]; source.read_exact_at(0, &mut header)?; let header = crate::format::header::decode_header(&mut header.as_slice())?; - if header.version != FileHeader::SUPPORTED_VERSION { - return Err(PithosError::UnsupportedFileVersion { - supported: FileHeader::SUPPORTED_VERSION, + let version = FormatVersion::from_wire(header.version).ok_or( + PithosError::UnsupportedFileVersion { + supported: FormatVersion::CURRENT.wire(), actual: header.version, - }); - } + }, + )?; let (terminal_start, terminal_len) = terminal_span(&source, archive_len, options.limits)?; let mut raw = Vec::new(); @@ -368,6 +369,7 @@ where let mut recovery_order = 0usize; for (segment, (directory, _)) in raw.iter().enumerate() { resolve_recipients( + version, directory, &options.keys, segment, @@ -415,6 +417,7 @@ where external: options.external, external_access_policy: options.external_access_policy, archive_len, + version, terminal_directory: Span::new(terminal_start, terminal_len)?, index, segments, @@ -436,6 +439,7 @@ where pub(crate) fn into_append_snapshot(self) -> AppendSnapshot { let Self { archive_len, + version, terminal_directory, index, segments, @@ -445,6 +449,7 @@ where } = self; AppendSnapshot::new( archive_len, + version, terminal_directory, index, segments, @@ -838,6 +843,7 @@ fn read_source( } fn resolve_recipients( + version: FormatVersion, directory: &Directory, keys: &AccessKeys, segment: usize, @@ -850,23 +856,34 @@ fn resolve_recipients( let secret = key.as_dalek_static_secret(); let recipient = DalekPublicKey::from(&secret).to_bytes(); for (sender_section, (sender, section)) in directory.encryption.iter().enumerate() { - let candidates: Vec<(&[u8; 32], &RecipientData)> = if sender == &recipient { + // Each candidate is the grant's peer key, its recipient key, and its data. + let candidates: Vec<(&[u8; 32], &[u8; 32], &RecipientData)> = if sender == &recipient { section .recipients .iter() - .map(|(recipient, section)| (recipient, §ion.recipient_data)) + .map(|(recipient, section)| (recipient, recipient, §ion.recipient_data)) .collect() } else { section .recipients .get(&recipient) - .map(|section| vec![(sender, §ion.recipient_data)]) + .map(|section| vec![(sender, &recipient, §ion.recipient_data)]) .unwrap_or_default() }; - for (recipient_section, (peer, data)) in candidates.into_iter().enumerate() { + for (recipient_section, (peer, grant_recipient, data)) in + candidates.into_iter().enumerate() + { let shared = crypto::derive_shared(secret.as_bytes(), peer)?; let entries = match data { RecipientData::Encrypted(bytes) => { + let nonce = crypto::sealed_nonce(bytes)?; + let shared = crypto::grant_wrapping_key( + version, + shared, + sender, + grant_recipient, + &nonce, + ); let plaintext = crypto::unwrap_recipient_list(&shared, bytes)?; crate::format::encryption::decode_decrypted_recipient_list( &plaintext, diff --git a/crates/pithos_lib/src/archive/reader_private_tests.rs b/crates/pithos_lib/src/archive/reader_private_tests.rs index a700910..7b9a41e 100644 --- a/crates/pithos_lib/src/archive/reader_private_tests.rs +++ b/crates/pithos_lib/src/archive/reader_private_tests.rs @@ -311,6 +311,13 @@ fn recover_first_file_key(path: &Path) -> FileKey { panic!("writer recipient data was not encrypted"); }; let shared = crypto::derive_shared(recipient.as_bytes(), sender).unwrap(); + let shared = crypto::grant_wrapping_key( + crate::format::header::FormatVersion::V1_1, + shared, + sender, + &recipient_public, + &crypto::sealed_nonce(data).unwrap(), + ); let plaintext = crypto::unwrap_recipient_list(&shared, data).unwrap(); let records = crate::format::encryption::decode_decrypted_recipient_list( &plaintext, @@ -747,10 +754,18 @@ fn archive_rejects_conflicting_recovered_file_keys_at_open() { rewrite_terminal_directory(&path, |directory| { let sender = StaticSecret::from([9; 32]); let recipient = DalekPublicKey::from(&reader); - let shared = sender.diffie_hellman(&recipient); + let shared = crypto::derive_shared(sender.as_bytes(), recipient.as_bytes()).unwrap(); + let nonce = [5; 12]; + let key = crypto::grant_wrapping_key( + crate::format::header::FormatVersion::V1_1, + shared, + &DalekPublicKey::from(&sender).to_bytes(), + recipient.as_bytes(), + &nonce, + ); let mut plaintext = vec![1, 0]; plaintext.extend_from_slice(&[0x77; 32]); - let encrypted = crypto::seal_crypt4gh_payload(shared.as_bytes(), &plaintext).unwrap(); + let encrypted = crypto::wrap_recipient_list_with_nonce(&key, &plaintext, nonce).unwrap(); directory.encryption.insert( DalekPublicKey::from(&sender).to_bytes(), EncryptionSection { diff --git a/crates/pithos_lib/src/archive/snapshot.rs b/crates/pithos_lib/src/archive/snapshot.rs index 91205c6..b325c14 100644 --- a/crates/pithos_lib/src/archive/snapshot.rs +++ b/crates/pithos_lib/src/archive/snapshot.rs @@ -7,6 +7,7 @@ use crate::archive::validation::IndexLimits; use crate::crypto::{FileKey, PublicKey}; use crate::error::PithosError; use crate::format::encryption::EncryptionSection; +use crate::format::header::FormatVersion; use indexmap::IndexMap; use std::collections::{BTreeMap, HashMap, HashSet}; use std::sync::Arc; @@ -17,6 +18,7 @@ use std::sync::Arc; /// segments for pure prospective merge, but not the source, decoded directories, or reader facade. pub(crate) struct AppendSnapshot { archive_len: u64, + version: FormatVersion, terminal_directory: Span, maximum_id: Option, descriptors: IndexMap, @@ -49,6 +51,7 @@ pub(crate) enum SnapshotEntryKind { impl AppendSnapshot { pub(crate) fn new( archive_len: u64, + version: FormatVersion, terminal_directory: Span, index: ArchiveIndex, segments: Vec, @@ -86,6 +89,7 @@ impl AppendSnapshot { } Self { archive_len, + version, terminal_directory, maximum_id, descriptors, @@ -104,6 +108,10 @@ impl AppendSnapshot { self.archive_len } + pub(crate) fn version(&self) -> FormatVersion { + self.version + } + pub(crate) fn terminal_directory(&self) -> Span { self.terminal_directory } diff --git a/crates/pithos_lib/src/archive/writer.rs b/crates/pithos_lib/src/archive/writer.rs index b7fde3b..b59bf6c 100644 --- a/crates/pithos_lib/src/archive/writer.rs +++ b/crates/pithos_lib/src/archive/writer.rs @@ -14,7 +14,7 @@ use crate::format::block::{BlockHeader, BlockIndexEntry, BlockLocation, Processi use crate::format::directory::{Directory, DirectoryEntries}; use crate::format::encryption::{EncryptionSection, RecipientData}; use crate::format::file_entry::{BlockDataState, FileEntry, FileType, Reference}; -use crate::format::header::FileHeader; +use crate::format::header::FormatVersion; use crate::format::{directory, header}; use fastcdc::v2020::{Normalization, StreamCDC}; use indexmap::IndexMap; @@ -532,6 +532,7 @@ impl Write for CountingSink { /// A streaming archive writer. Dropping it leaves an intentionally incomplete sink. pub struct ArchiveWriter { mode: ArchiveWriterMode, + version: FormatVersion, cdc: CdcConfig, sink: CountingSink, directory: Directory, @@ -587,7 +588,7 @@ impl ArchiveWriter { }; let directory = Directory::new(None, DirectoryEntries::new(), encryption); let mut sink = CountingSink { sink, offset: 0 }; - if let Err(error) = header::encode_header(&FileHeader::default(), &mut sink) { + if let Err(error) = header::encode_header(&FormatVersion::CURRENT.header(), &mut sink) { return Err(CreateError { error: error.into(), sink: sink.into_inner(), @@ -595,6 +596,7 @@ impl ArchiveWriter { } Ok(Self { mode, + version: FormatVersion::CURRENT, cdc, sink, directory, @@ -633,6 +635,7 @@ impl ArchiveWriter { let directory = Directory::new(Some((parent.start(), parent.len())), files, encryption); Ok(Self { mode: ArchiveWriterMode::Encrypted { sender }, + version: snapshot.version(), cdc, sink: CountingSink { sink, @@ -1316,10 +1319,15 @@ impl ArchiveWriter { return Ok(()); }; for (recipient_key, recipient) in &mut section.recipients { - let recipient_key = LegacyPublicKey::from(*recipient_key); - let shared_key = - crate::crypto::derive_shared(sender.as_bytes(), recipient_key.as_bytes())?; + let shared = crate::crypto::derive_shared(sender.as_bytes(), recipient_key)?; let nonce = self.runtime.recipient_list_nonce()?; + let shared_key = crate::crypto::grant_wrapping_key( + self.version, + shared, + &sender_public, + recipient_key, + &nonce, + ); self.runtime .seal_recipient_list(&mut recipient.recipient_data, shared_key, nonce)?; } @@ -1583,6 +1591,52 @@ mod tests { ) } + #[test] + fn recipient_grants_follow_the_archive_version_rules() { + for (version, other) in [ + (FormatVersion::V1_0, FormatVersion::V1_1), + (FormatVersion::V1_1, FormatVersion::V1_0), + ] { + let sender = PrivateKey::generate(); + let mut writer = ArchiveWriter::create( + Vec::new(), + WriteOptions::new(sender.duplicate(), vec![sender.public_key()]), + ) + .unwrap(); + writer.version = version; + writer + .add_file( + ArchivePath::new("data").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::default(), + None, + Cursor::new(b"content"), + ) + .unwrap(); + let mut bytes = writer.finish().unwrap(); + bytes[4..6].copy_from_slice(&version.wire().to_be_bytes()); + let open = |bytes: Vec| { + Archive::open( + MemorySource::new(bytes), + OpenOptions::default() + .with_access_keys(AccessKeys::new().with_key(sender.duplicate())), + ) + }; + let mut output = Vec::new(); + open(bytes.clone()) + .unwrap() + .copy_to("data", &mut output) + .unwrap(); + assert_eq!(output, b"content"); + + bytes[4..6].copy_from_slice(&other.wire().to_be_bytes()); + assert!( + open(bytes).is_err(), + "{other:?} rules opened a {version:?} grant" + ); + } + } + #[test] fn injected_transform_failure_poison_preserves_live_directory() { let mut writer = ArchiveWriter::with_test_runtime( @@ -1661,9 +1715,9 @@ mod tests { let archive = write(); assert_eq!( blake3::hash(&archive).to_hex().as_str(), - "ed87c4bc6481475e5d1a2af8906b75728231c451e1db93fca3aa4318efcdbf49" + "e2a3b75a71ab0e1db03f0e1b5fa9a9f183822248616397e6d727455eb7730eb2" ); - assert_eq!(&archive[..6], b"PITH\x01\x00"); + assert_eq!(&archive[..6], b"PITH\x01\x01"); assert_eq!( archive .windows(8) diff --git a/crates/pithos_lib/src/crypto.rs b/crates/pithos_lib/src/crypto.rs index c9666cc..f75bf40 100644 --- a/crates/pithos_lib/src/crypto.rs +++ b/crates/pithos_lib/src/crypto.rs @@ -1,19 +1,23 @@ //! Current Pithos cryptographic roles and protocol operations. //! //! The byte formats deliberately remain in the established protocol: every sealed value is -//! `nonce || ciphertext || tag` and always uses empty associated data. +//! `nonce || ciphertext || tag` and always uses empty associated data. Version 1.1 changes only +//! how a recipient grant's wrapping key is derived. +use crate::format::header::FormatVersion; use chacha20poly1305::{ ChaCha20Poly1305, KeyInit, Nonce, aead::{Aead, Generate, Payload}, }; use digest::{ExtendableOutput, Update, XofReader}; +use hkdf::Hkdf; use pkcs8::der::EncodePem; use pkcs8::der::pem::PemLabel; use pkcs8::{ Document, LineEnding, ObjectIdentifier, PrivateKeyInfoRef, SecretDocument, SubjectPublicKeyInfoRef, der, }; +use sha2::Sha256; use std::fmt; use std::str::FromStr; use thiserror::Error; @@ -249,6 +253,44 @@ pub(crate) fn derive_shared( Ok(SharedSecret(shared.to_bytes())) } +/// HKDF info prefix of a version 1.1 recipient grant; the sender and recipient keys follow. +const GRANT_KEY_INFO: &[u8] = b"pithos 1.1 recipient grant"; + +/// Wrapping key of one recipient grant. Version 1.0 uses the raw shared secret. Version 1.1 +/// derives a key per grant with HKDF-SHA256, salted with the grant nonce, bound to both keys. +pub(crate) fn grant_wrapping_key( + version: FormatVersion, + shared: SharedSecret, + sender: &[u8; 32], + recipient: &[u8; 32], + nonce: &[u8; 12], +) -> SharedSecret { + match version { + FormatVersion::V1_0 => shared, + FormatVersion::V1_1 => { + let mut info = Zeroizing::new([0u8; GRANT_KEY_INFO.len() + 64]); + info[..GRANT_KEY_INFO.len()].copy_from_slice(GRANT_KEY_INFO); + info[GRANT_KEY_INFO.len()..GRANT_KEY_INFO.len() + 32].copy_from_slice(sender); + info[GRANT_KEY_INFO.len() + 32..].copy_from_slice(recipient); + let mut key = SharedSecret([0; 32]); + Hkdf::::new(Some(&nonce[..]), shared.expose_for_protocol()) + .expand(&info[..], &mut key.0) + .expect("32 bytes is a valid HKDF-SHA256 output length"); + key + } + } +} + +/// The nonce stored at the start of a sealed value, needed before its key can be derived. +pub(crate) fn sealed_nonce(payload: &[u8]) -> Result<[u8; 12], CryptoError> { + if payload.len() < 15 { + return Err(CryptoError::EncryptedPayloadTooShort); + } + let mut nonce = [0; 12]; + nonce.copy_from_slice(&payload[..12]); + Ok(nonce) +} + pub(crate) fn validate_x25519_public_key(public: &[u8; 32]) -> Result<(), CryptoError> { // Low-order public keys produce the identity for every clamped X25519 scalar. let probe = StaticSecret::from([0; 32]).diffie_hellman(&DalekPublicKey::from(*public)); diff --git a/crates/pithos_lib/src/format/header.rs b/crates/pithos_lib/src/format/header.rs index cbe0596..e5c3c3d 100644 --- a/crates/pithos_lib/src/format/header.rs +++ b/crates/pithos_lib/src/format/header.rs @@ -6,21 +6,54 @@ use std::io::{Read, Write}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct FileHeader { pub magic: [u8; 4], // MUST be b"PITH" - pub version: u16, // Format version (e.g., 0x0100 for 1.0) + pub version: u16, // Format version (0x0100 for 1.0, 0x0101 for 1.1) } impl Default for FileHeader { fn default() -> Self { FileHeader { magic: *b"PITH", - version: Self::SUPPORTED_VERSION, + version: FormatVersion::CURRENT.wire(), } } } impl FileHeader { pub const ENCODED_LEN: usize = 6; - pub const SUPPORTED_VERSION: u16 = 0x0100; +} + +/// The wire rules of one archive. An append keeps the version of the archive it extends. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum FormatVersion { + V1_0, + V1_1, +} + +impl FormatVersion { + /// The version every new archive is written with. + pub(crate) const CURRENT: Self = Self::V1_1; + + pub(crate) fn from_wire(version: u16) -> Option { + match version { + 0x0100 => Some(Self::V1_0), + 0x0101 => Some(Self::V1_1), + _ => None, + } + } + + pub(crate) fn wire(self) -> u16 { + match self { + Self::V1_0 => 0x0100, + Self::V1_1 => 0x0101, + } + } + + pub(crate) fn header(self) -> FileHeader { + FileHeader { + magic: *b"PITH", + version: self.wire(), + } + } } pub(crate) fn encode_header( @@ -51,13 +84,20 @@ mod tests { use super::*; #[test] - fn file_header_uses_the_fixed_width_1_0_encoding() { + fn file_header_uses_the_fixed_width_encoding_for_both_versions() { let mut encoded = Vec::new(); encode_header(&FileHeader::default(), &mut encoded).unwrap(); - assert_eq!(encoded, b"PITH\x01\x00"); + assert_eq!(encoded, b"PITH\x01\x01"); let decoded = decode_header(&mut encoded.as_slice()).unwrap(); - assert_eq!(decoded.version, 0x0100); + assert_eq!( + FormatVersion::from_wire(decoded.version), + Some(FormatVersion::V1_1) + ); + let mut legacy = Vec::new(); + encode_header(&FormatVersion::V1_0.header(), &mut legacy).unwrap(); + assert_eq!(legacy, b"PITH\x01\x00"); + assert_eq!(FormatVersion::from_wire(0x0102), None); let error = decode_header(&mut &b"POTX\x01\x00"[..]).unwrap_err(); assert!(error.to_string().contains("file marker")); diff --git a/crates/pithos_lib/tests/append_workflow.rs b/crates/pithos_lib/tests/append_workflow.rs index b6568ab..890078e 100644 --- a/crates/pithos_lib/tests/append_workflow.rs +++ b/crates/pithos_lib/tests/append_workflow.rs @@ -172,7 +172,7 @@ fn append_streams_multiblock_content_and_writes_one_child_directory_without_a_he assert_eq!( updated .windows(6) - .filter(|window| *window == b"PITH\x01\x00") + .filter(|window| *window == b"PITH\x01\x01") .count(), 1 ); diff --git a/crates/pithos_lib/tests/draft_spec_vectors.rs b/crates/pithos_lib/tests/draft_spec_vectors.rs index 3fd19fd..319545e 100644 --- a/crates/pithos_lib/tests/draft_spec_vectors.rs +++ b/crates/pithos_lib/tests/draft_spec_vectors.rs @@ -1,4 +1,4 @@ -//! Reproducible verification for the Pithos 1.0 draft vectors in Appendix B. +//! Reproducible verification for the Pithos draft vectors in Appendix B. //! //! This repository-only test model is not an alternate production codec. @@ -316,7 +316,10 @@ fn production_base_writer_reproduces_cv_local_hello() { Cursor::new(b"hello"), ) .unwrap(); - assert_eq!(writer.finish().unwrap(), appendix_hex("CV-LOCAL-HELLO-279")); + // A version 1.1 base archive differs from the version 1.0 vector only in its header. + let mut expected = appendix_hex("CV-LOCAL-HELLO-279"); + expected[5] = 0x01; + assert_eq!(writer.finish().unwrap(), expected); } #[test] @@ -547,9 +550,38 @@ fn processing_known_answers_match_appendix_b() { stored.extend_from_slice(&ciphertext); assert_eq!( stored, - appendix_recipient_wrap(), + appendix_recipient_wrap("PV-RECIPIENT-WRAP-01"), "recipient-wrap bytes must match Appendix B" ); + + let alice_public = PublicKey::from(&alice).to_bytes(); + assert_eq!( + alice_public, + hex("8520f0098930a754748b7ddcb43ef75a0dbf3a0d26381af4eba4a98eaa9b4e6a") + ); + let mut info = b"pithos 1.1 recipient grant".to_vec(); + info.extend_from_slice(&alice_public); + info.extend_from_slice(bob.as_bytes()); + let mut key = [0; 32]; + hkdf::Hkdf::::new(Some(nonce.as_slice()), &shared) + .expand(&info, &mut key) + .unwrap(); + assert_eq!( + key, + hex("16894ee78d378733e5148de641263d7503e3131e51ee894d60a56caa36c7caaa") + ); + let mut stored = nonce.to_vec(); + stored.extend_from_slice( + &ChaCha20Poly1305::new_from_slice(&key) + .unwrap() + .encrypt(&nonce, plaintext.as_ref()) + .unwrap(), + ); + assert_eq!( + stored, + appendix_recipient_wrap("PV-RECIPIENT-WRAP-11"), + "version 1.1 recipient-wrap bytes must match Appendix B" + ); } fn appendix_zstd_bytes(id: &str) -> Vec { @@ -567,8 +599,8 @@ fn appendix_zstd_bytes(id: &str) -> Vec { .collect() } -fn appendix_recipient_wrap() -> Vec { - let start = SPEC.find("PV-RECIPIENT-WRAP-01").unwrap(); +fn appendix_recipient_wrap(id: &str) -> Vec { + let start = SPEC.find(&format!("`{id}`")).unwrap(); let block = SPEC[start..].split("```text\n").nth(1).unwrap(); block .split("```") diff --git a/crates/pithos_lib/tests/format_validation.rs b/crates/pithos_lib/tests/format_validation.rs index 4fbc668..917f689 100644 --- a/crates/pithos_lib/tests/format_validation.rs +++ b/crates/pithos_lib/tests/format_validation.rs @@ -36,7 +36,7 @@ fn public_open_rejects_the_legacy_varint_header() { assert!(matches!( Archive::open(MemorySource::new(bytes), OpenOptions::default()), Err(PithosError::UnsupportedFileVersion { - supported: 0x0100, + supported: 0x0101, actual: 0x8002, }) )); diff --git a/crates/pithos_lib/tests/grant_readers.rs b/crates/pithos_lib/tests/grant_readers.rs index 231d726..7d7c645 100644 --- a/crates/pithos_lib/tests/grant_readers.rs +++ b/crates/pithos_lib/tests/grant_readers.rs @@ -337,7 +337,7 @@ fn grant_multiple_ids_to_multiple_recipients_in_one_metadata_only_child() { assert_eq!( updated .windows(6) - .filter(|window| *window == b"PITH\x01\x00") + .filter(|window| *window == b"PITH\x01\x01") .count(), 1 ); diff --git a/spec/PITHOS_1.0.0_draft.md b/spec/PITHOS_1.0.0_draft.md index 4133afa..e3d0a59 100644 --- a/spec/PITHOS_1.0.0_draft.md +++ b/spec/PITHOS_1.0.0_draft.md @@ -1,8 +1,8 @@ # Pithos File Format Specification -**Version:** 1.0 +**Version:** 1.1 **Status:** Draft -**Date:** July 2026 +**Date:** October 2026 **Purpose:** Next-generation file format for scientific data management, optimized for object storage with built-in deduplication, encryption, and metadata support ## 1. Introduction @@ -15,8 +15,9 @@ Pithos is an append-only archive format designed for efficient storage and shari data model only; implementations may use different declarations. Normative prose and encoded-form tables govern conformance and the bytes on disk. -This document defines Pithos 1.0. Where a current 0.8 implementation differs, -this document governs. +This document defines Pithos 1.1 and the version 1.0 rules it keeps. Version 1.0 +archives remain valid; Section 1.3 lists every difference between the versions. +Where a current 0.8 implementation differs, this document governs. ### 1.1 Reader's Guide @@ -56,6 +57,20 @@ this document governs. normal terminal-Directory lookup fails; its result is incomplete and is not a normal archive view. +### 1.3 Version Differences + +Version 1.1 keeps every version 1.0 structure and encoding. The header version +selects the rules for the whole archive, including every appended segment. +Version 1.1 differs from version 1.0 only in these points: + +1. The header version is `0x0101` (Section 4.1). +2. A recipient grant's wrapping key is derived with HKDF-SHA256 instead of using + the raw X25519 shared secret (Section 5.3). + +Readers MUST support both versions. Writers MUST create new archives as version +1.1. An append MUST follow the version of the archive it extends, so appending +to a version 1.0 archive uses the version 1.0 rules. + ## 2. Core Design Principles 1. **Append-only architecture**: New data and metadata MUST be appended, never modifying existing content @@ -133,7 +148,7 @@ A FileHeader identifies a Pithos file and its format version. #[derive(Debug, Clone, PartialEq, Eq)] pub struct FileHeader { pub magic: [u8; 4], // b"PITH" - pub version: u16, // fixed-width big-endian, 0x0100 for 1.0 + pub version: u16, // fixed-width big-endian, 0x0100 for 1.0, 0x0101 for 1.1 } ``` @@ -144,8 +159,9 @@ pub struct FileHeader { | `magic` | Exactly 4 bytes: ASCII `PITH` | | `version` | Fixed-width `u16be` | -Readers MUST reject a header whose magic is not `PITH` or whose version is not -`0x0100`. The encoded form is exactly six bytes: `PITH 01 00`. +Readers MUST reject a header whose magic is not `PITH` or whose version is +neither `0x0100` nor `0x0101`. The encoded form is exactly six bytes: `PITH 01 01` +for version 1.1 and `PITH 01 00` for version 1.0. ### 4.2 Block Storage @@ -785,11 +801,20 @@ label or length prefix is included. A block payload with encryption enabled is encrypted with its block key. A file key is 32 random bytes. It encrypts the decrypted block list for a file. -For each recipient record, the recipient wrapping key is the raw 32-byte X25519 -shared secret between the sender private key and recipient public key. That -shared secret is used directly as the ChaCha20-Poly1305 key to encrypt the -decrypted recipient list; no intermediate KDF is used. Implementations MUST -reject non-contributory X25519 public keys. +For each recipient record, the shared secret is the raw 32-byte X25519 shared +secret between the sender private key and recipient public key. The wrapping key +encrypts the decrypted recipient list with ChaCha20-Poly1305: + +- **Version 1.0:** the wrapping key is the shared secret itself. +- **Version 1.1:** the wrapping key is the 32-byte HKDF-SHA256 output + (RFC 5869) with the record's 12-byte nonce as salt, the shared secret as input + key material, and info `pithos 1.1 recipient grant` (26 ASCII bytes) followed + by the sender public key and then the recipient public key. The keys are the + ones stored in the Directory: the EncryptionSection key and the recipient + record key. + +Each version 1.1 grant therefore has its own key, bound to both public keys. +Implementations MUST reject non-contributory X25519 public keys. An implementation MUST generate every nonce independently and uniformly at random with a cryptographically secure random number generator and MUST NOT @@ -797,8 +822,8 @@ deliberately reuse a nonce under the same key. It MUST authenticate and decrypt an encrypted value successfully before using its plaintext or releasing output derived from it. -Version 1.0 deliberately has no SHAKE256 label, intermediate X25519 KDF, or -AAD. A redesign of any of these inputs requires a new format version. +Neither version uses a SHAKE256 label or AAD. A redesign of any cryptographic +input requires a new format version. ### 5.4 Compression @@ -862,13 +887,15 @@ recipient counts, ciphertext lengths, and the segment timing of grants are visible. `RecipientData::Decrypted` exposes its file-key grants and provides no grant confidentiality. -Recipient wrapping uses a raw static X25519 shared secret directly as its AEAD -key, with no KDF, label, or AAD. Consequently, the construction has no domain +Version 1.0 recipient wrapping uses a raw static X25519 shared secret directly as +its AEAD key, with no KDF, label, or AAD. That construction has no domain separation, and each static sender-recipient key pair has one nonce-collision -scope across all archives that use it; uniformly random CSPRNG nonces reduce but -cannot eliminate collision risk. The plaintext-derived Directory block hash -exposes block equality, and equal plaintext also derives the same convergent -block key. +scope across all archives that use it. Version 1.1 derives every wrapping key +with HKDF-SHA256 from the shared secret, the grant's random nonce and both public +keys. This adds domain separation and gives each grant its own key, so a nonce +can only collide within one grant. Appends to version 1.0 archives keep the +version 1.0 construction. The plaintext-derived Directory block hash exposes +block equality, and equal plaintext also derives the same convergent block key. Readers must verify each block as required by Section 5.2 before releasing its output. Networked external resolution can expose a caller to unsafe targets and @@ -882,7 +909,7 @@ policy; Sections 8.3 and 4.4.3 define these requirements. ### 8.1 Base Reader and Writer -A base reader supports the version 1.0 structure, including its required +A base reader supports the version 1.0 and 1.1 structure, including its required validation, and can read local blocks whose ProcessingFlags have compression value `0` and encryption bit `0`. It supports decrypted BlockDataState lists. A base writer can create an archive containing only such local blocks and an @@ -977,6 +1004,7 @@ Extensions MUST maintain backwards compatibility for reading. ### 10.2 Version Numbers - Version 1.0: `0x0100` +- Version 1.1: `0x0101` ### 10.3 Default Values @@ -1009,8 +1037,9 @@ as test input. Hex offsets are zero-based file offsets. Each hex line contains at most 16 bytes; `0000:` is an offset, not encoded data. Integers without a fixed-width suffix use shortest ULEB128 in these deterministic vectors, although Section 3.1 also -permits bounded non-minimal forms. `u64be` and `u32be` are big-endian. The header -is always +permits bounded non-minimal forms. `u64be` and `u32be` are big-endian. The +canonical archives are version 1.0 archives, which every reader accepts, so their +header is always `50 49 54 48 01 00` (`PITH`, version `0x0100`); the other magic values are `42 4c 43 4b` (`BLCK`) and `50 49 54 48 4f 53 44 52` (`PITHOSDR`). @@ -1172,6 +1201,20 @@ Successful decryption yields exactly the stated 34-byte RecipientData plaintext. Fixed nonces are permitted here only as test inputs; writers use random nonces as required by Section 5.3. +`PV-RECIPIENT-WRAP-11` is the version 1.1 known-answer test with the same +inputs. The sender public key is Alice's public key +`8520f0098930a754748b7ddcb43ef75a0dbf3a0d26381af4eba4a98eaa9b4e6a` and the +recipient public key is Bob's. The derived wrapping key is +`16894ee78d378733e5148de641263d7503e3131e51ee894d60a56caa36c7caaa`, and the +stored `nonce || ciphertext || tag` is: + +```text +0000: 00 01 02 03 04 05 06 07 08 09 0a 0b 25 3f 85 dd +0010: 25 c7 5f b6 81 1b 4d 7c 0d 8e 14 3f b3 fc de 6d +0020: 59 94 e4 8a 32 ca 15 61 6a 2f 26 f9 9b 3c 42 da +0030: 43 a0 94 bb cc 91 e1 c5 a6 53 62 9a d4 a1 +``` + ### B.4 Acceptance and Rejection Mutations Each `AV-*` or `RV-*` vector is a mutation of the named canonical vector. @@ -1207,4 +1250,4 @@ archive. | Directory framing, CRC, and chain | 4.3, 4.3.1, 4.3.2 | CV-BASE-EMPTY-146, CV-APPEND-EMPTY-28, RV-PARENT, RV-UNDERFLOW, RV-TRAILING, RV-CRC, RV-CROSS-SIZE | Valid archive or reject archive as stated | | Entries and paths | 4.3.3, 4.4.1, 4.4.3 | CV-LOCAL-HELLO-279, RV-DUPLICATES, RV-PATH, RV-SYMLINK, RV-FILETYPE, RV-PERMISSIONS | Valid archive or reject archive as stated | | Block locations and extents | 4.2.2, 4.2.5, 4.2.6, 8.2 | CV-LOCAL-HELLO-279, RV-EXTENT, RV-SHORT-ENCRYPTED, RV-EXTERNAL | Valid archive, reject archive, content read fails before output, or content unavailable as stated | -| Content transforms and hashes | 5.2, 5.3, 5.4 | PV-ZSTD-HELLO, PV-ZSTD-TEXT, PV-RECIPIENT-WRAP-01 | Decode/decrypt to stated output | +| Content transforms and hashes | 5.2, 5.3, 5.4 | PV-ZSTD-HELLO, PV-ZSTD-TEXT, PV-RECIPIENT-WRAP-01, PV-RECIPIENT-WRAP-11 | Decode/decrypt to stated output | From 7320575e5528bdb9029125454e41ac454aabef35 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 14:17:11 +0200 Subject: [PATCH 051/166] feat: read block lists sealed in independent pieces --- crates/pithos_lib/src/archive/index.rs | 5 + crates/pithos_lib/src/archive/reader.rs | 68 +++++++- .../src/archive/reader_private_tests.rs | 163 ++++++++++++++++++ crates/pithos_lib/src/archive/validation.rs | 2 +- crates/pithos_lib/src/error.rs | 4 + crates/pithos_lib/src/format/file_entry.rs | 94 +++++++++- crates/pithos_lib/src/format/limits.rs | 2 + spec/PITHOS_1.0.0_draft.md | 48 +++++- 8 files changed, 373 insertions(+), 13 deletions(-) diff --git a/crates/pithos_lib/src/archive/index.rs b/crates/pithos_lib/src/archive/index.rs index 84c44db..2175320 100644 --- a/crates/pithos_lib/src/archive/index.rs +++ b/crates/pithos_lib/src/archive/index.rs @@ -78,6 +78,11 @@ impl ArchiveIndex { self.maximum_id } + /// Piece key ids share the file id space, so new file ids must stay above them. + pub(crate) fn cover_piece_keys(&mut self, maximum_piece_key: Option) { + self.maximum_id = self.maximum_id.max(maximum_piece_key); + } + /// Transfers only the effective state needed to form an append snapshot. pub(crate) fn into_append_snapshot_parts( self, diff --git a/crates/pithos_lib/src/archive/reader.rs b/crates/pithos_lib/src/archive/reader.rs index 726015f..22a5af6 100644 --- a/crates/pithos_lib/src/archive/reader.rs +++ b/crates/pithos_lib/src/archive/reader.rs @@ -57,7 +57,7 @@ impl DecodedDirectoryCounts { .iter() .map(|(_, _, file)| match &file.block_data { BlockDataState::Decrypted(entries) => entries.len() as u64, - BlockDataState::Encrypted(_) => 0, + BlockDataState::Encrypted(_) | BlockDataState::Pieces(_) => 0, }) .sum::(); self.relationships += directory.relations.len() as u64; @@ -347,6 +347,7 @@ where raw.push((directory, span)); } raw.reverse(); + let maximum_piece_key = validate_piece_keys(version, &raw)?; let mut first_grants = HashMap::new(); for (directory, _) in &raw { @@ -400,7 +401,8 @@ where max_relationships: options.limits.max_relationships, max_segments: options.limits.max_parent_directories.saturating_add(1), }; - let index = build_effective_index(&segments, archive_len, index_limits)?; + let mut index = build_effective_index(&segments, archive_len, index_limits)?; + index.cover_piece_keys(maximum_piece_key.map(FileId)); let content_availability = classify_content_availability( &index, options.external_resolver_supplied, @@ -924,6 +926,39 @@ fn resolve_block_lists( BlockDataState::Decrypted(entries) => { record_block_keys(access, file_id, entries); } + BlockDataState::Pieces(pieces) => { + if pieces + .iter() + .any(|piece| access.key(FileId(piece.key_id)).is_none()) + { + return Ok(()); + } + let mut entries = Zeroizing::new(Vec::new()); + for piece in pieces { + let key = access + .key(FileId(piece.key_id)) + .expect("every piece key was checked above"); + let mut decoded_limits = deserialization_limits(limits); + decoded_limits.max_block_references = *remaining_block_references; + let plaintext = crypto::open_file_block_list(key, &piece.sealed)?; + let piece_entries = + crate::format::file_entry::decode_decrypted_block_list_with_budget( + &plaintext, + &decoded_limits, + remaining_block_references, + )?; + entries.try_reserve(piece_entries.len()).map_err(|_| { + PithosError::AllocationFailed { + field: "block list pieces", + size: piece_entries.len() as u64, + } + })?; + entries.extend_from_slice(&piece_entries); + } + crate::format::file_entry::validate_unique_block_references(&entries)?; + record_block_keys(access, file_id, &entries); + file.block_data = BlockDataState::Decrypted(entries); + } BlockDataState::Encrypted(bytes) => { let Some(file_key) = access.key(file_id) else { return Ok(()); @@ -944,6 +979,35 @@ fn resolve_block_lists( }) } +/// Checks the version 1.1 piece rules across the selected chain. Returns the largest piece +/// key id, which new file ids must stay above. +fn validate_piece_keys( + version: FormatVersion, + raw: &[(Directory, Span)], +) -> Result, PithosError> { + let mut file_ids = HashSet::new(); + let mut piece_keys = HashSet::new(); + for (directory, _) in raw { + for (id, _, file) in directory.files.iter() { + file_ids.insert(id); + if let BlockDataState::Pieces(pieces) = &file.block_data { + if version == FormatVersion::V1_0 { + return Err(PithosError::UnsupportedBlockListPieces); + } + for piece in pieces { + if !piece_keys.insert(piece.key_id) { + return Err(PithosError::PieceKeyIdConflict(piece.key_id)); + } + } + } + } + } + if let Some(conflict) = piece_keys.iter().find(|key_id| file_ids.contains(*key_id)) { + return Err(PithosError::PieceKeyIdConflict(*conflict)); + } + Ok(piece_keys.into_iter().max()) +} + fn record_block_keys(access: &mut ResolvedAccess, file_id: FileId, entries: &[BlockDataEntry]) { access.insert_block_keys( file_id, diff --git a/crates/pithos_lib/src/archive/reader_private_tests.rs b/crates/pithos_lib/src/archive/reader_private_tests.rs index 7b9a41e..a729566 100644 --- a/crates/pithos_lib/src/archive/reader_private_tests.rs +++ b/crates/pithos_lib/src/archive/reader_private_tests.rs @@ -1534,3 +1534,166 @@ fn unsupported_external_content_preflights_the_whole_file_and_adapters() { } use crate::adapters::crypt4gh; use crate::fs::extract; + +/// Splits the fixture's block list into pieces with the given key ids and grants the +/// `granted` piece keys to recipient1 under the version 1.1 grant rules. +fn split_into_pieces(path: &Path, key_ids: &[u64], granted: &[u64]) { + let file_key = recover_first_file_key(path); + rewrite_terminal_directory(path, |directory| { + let entry = directory.files.iter().next().unwrap().2.clone(); + let BlockDataState::Encrypted(sealed) = &entry.block_data else { + panic!("fixture block list was not encrypted"); + }; + let plaintext = crypto::open_file_block_list(&file_key, sealed).unwrap(); + let entries = crate::format::file_entry::decode_decrypted_block_list_with_budget( + &plaintext, + &DeserializationLimits::default(), + &mut u64::MAX.clone(), + ) + .unwrap(); + let chunk = entries.len().div_ceil(key_ids.len()).max(1); + let mut pieces = Vec::new(); + let mut grants = Vec::new(); + for (index, key_id) in key_ids.iter().enumerate() { + let start = (index * chunk).min(entries.len()); + let end = ((index + 1) * chunk).min(entries.len()); + let key = [index as u8 + 1; 32]; + let mut list = Vec::new(); + crate::format::file_entry::encode_decrypted_block_list(&entries[start..end], &mut list) + .unwrap(); + let sealed = crypto::seal_file_block_list_with_nonce( + &FileKey::from_protocol(&key), + &list, + [index as u8; 12], + ) + .unwrap(); + pieces.push(crate::format::file_entry::BlockListPiece { + key_id: *key_id, + sealed, + }); + if granted.contains(key_id) { + grants.push((*key_id, key)); + } + } + directory + .files + .try_for_each_mut(|_, file| { + file.block_data = BlockDataState::Pieces(pieces.clone()); + Ok::<_, ()>(()) + }) + .unwrap(); + + let sender = StaticSecret::from([9; 32]); + let sender_public = DalekPublicKey::from(&sender).to_bytes(); + let recipient = public("recipient1").into_dalek_public_key().to_bytes(); + let nonce = [6; 12]; + let key = crypto::grant_wrapping_key( + crate::format::header::FormatVersion::V1_1, + crypto::derive_shared(sender.as_bytes(), &recipient).unwrap(), + &sender_public, + &recipient, + &nonce, + ); + let mut records = Vec::new(); + crate::format::encryption::encode_decrypted_recipient_list(&grants, &mut records).unwrap(); + directory.encryption.insert( + sender_public, + EncryptionSection { + recipients: IndexMap::from_iter([( + recipient, + RecipientSection { + recipient_data: RecipientData::Encrypted( + crypto::wrap_recipient_list_with_nonce(&key, &records, nonce).unwrap(), + ), + }, + )]), + }, + ); + }); +} + +fn pieces_fixture() -> (tempfile::TempDir, PathBuf, String) { + let content = "piece content ".repeat(300); + let (temporary, path) = fixture_with_options( + &content, + 0, + true, + Some(CdcConfig::new(64, 256, 1024).unwrap()), + ); + (temporary, path, content) +} + +fn open_pieces(path: &Path) -> Result, PithosError> { + Archive::open( + MemorySource::new(Arc::<[u8]>::from(std::fs::read(path).unwrap())), + OpenOptions::default().with_access_keys(AccessKeys::new().with_key(private("recipient1"))), + ) +} + +#[test] +fn block_list_pieces_concatenate_in_stored_order() { + let (_temporary, path, content) = pieces_fixture(); + split_into_pieces(&path, &[5, 6, 9], &[5, 6, 9]); + let mut output = Vec::new(); + let archive = open_pieces(&path).unwrap(); + archive.copy_to("data", &mut output).unwrap(); + assert_eq!(output, content.as_bytes()); + // New file ids must stay above every piece key id. + assert_eq!( + archive.into_append_snapshot().maximum_id(), + Some(crate::archive::FileId(9)) + ); +} + +#[test] +fn a_missing_piece_key_leaves_the_content_unavailable() { + let (_temporary, path, _) = pieces_fixture(); + split_into_pieces(&path, &[5, 6], &[5]); + let archive = open_pieces(&path).unwrap(); + assert!(matches!( + archive.entry("data").unwrap().unwrap().kind, + EntryKind::File { + available: false, + .. + } + )); + assert!(archive.copy_to("data", &mut Vec::new()).is_err()); +} + +#[test] +fn piece_rules_reject_version_1_0_archives_and_shared_key_ids() { + let (_temporary, path, _) = pieces_fixture(); + split_into_pieces(&path, &[5, 6], &[5, 6]); + let mut legacy = std::fs::read(&path).unwrap(); + legacy[5] = 0x00; + assert!(matches!( + Archive::open(MemorySource::new(legacy), OpenOptions::default()), + Err(PithosError::UnsupportedBlockListPieces) + )); + + let (_temporary, path, _) = pieces_fixture(); + split_into_pieces(&path, &[0, 6], &[0, 6]); + assert!(matches!( + open_pieces(&path), + Err(PithosError::PieceKeyIdConflict(0)) + )); +} + +#[test] +fn a_tampered_piece_fails_before_any_content_is_indexed() { + let (_temporary, path, _) = pieces_fixture(); + split_into_pieces(&path, &[5, 6], &[5, 6]); + rewrite_terminal_directory(&path, |directory| { + directory + .files + .try_for_each_mut(|_, file| { + if let BlockDataState::Pieces(pieces) = &mut file.block_data { + let last = pieces[1].sealed.len() - 1; + pieces[1].sealed[last] ^= 1; + } + Ok::<_, ()>(()) + }) + .unwrap(); + }); + assert!(matches!(open_pieces(&path), Err(PithosError::Crypt(_)))); +} diff --git a/crates/pithos_lib/src/archive/validation.rs b/crates/pithos_lib/src/archive/validation.rs index d102563..924680b 100644 --- a/crates/pithos_lib/src/archive/validation.rs +++ b/crates/pithos_lib/src/archive/validation.rs @@ -143,7 +143,7 @@ fn entry_from_file_entry(path: &str, file_entry: &FileEntry) -> Result ContentState::Unavailable, + BlockDataState::Encrypted(_) | BlockDataState::Pieces(_) => ContentState::Unavailable, }; match file_entry.file_type { FileType::Data => Ok(Entry::File(ContentEntry { diff --git a/crates/pithos_lib/src/error.rs b/crates/pithos_lib/src/error.rs index b1d8d70..0f3dfcb 100644 --- a/crates/pithos_lib/src/error.rs +++ b/crates/pithos_lib/src/error.rs @@ -83,6 +83,10 @@ pub enum PithosError { DuplicateRecipientKey, #[error("duplicate encoded block reference")] DuplicateBlockReference, + #[error("block list pieces require format version 1.1")] + UnsupportedBlockListPieces, + #[error("block list piece key id {0} is already used by a file or another piece")] + PieceKeyIdConflict(u64), #[error("duplicate encoded recipient file id")] DuplicateRecipientFileId, #[error("reserved processing bits are set: {0:#04x}")] diff --git a/crates/pithos_lib/src/format/file_entry.rs b/crates/pithos_lib/src/format/file_entry.rs index db4d32d..c75457a 100644 --- a/crates/pithos_lib/src/format/file_entry.rs +++ b/crates/pithos_lib/src/format/file_entry.rs @@ -29,6 +29,15 @@ pub type BlockDataEntry = ([u8; 32], [u8; 32]); pub(crate) enum BlockDataState { Encrypted(Vec), Decrypted(Zeroizing>), + /// Version 1.1: a block list sealed in independent pieces, concatenated in stored order. + Pieces(Vec), +} + +/// One sealed part of a file's block list, opened with the key granted for `key_id`. +#[derive(Clone, PartialEq, Eq)] +pub(crate) struct BlockListPiece { + pub(crate) key_id: u64, + pub(crate) sealed: Vec, } impl std::fmt::Debug for BlockDataState { @@ -42,6 +51,10 @@ impl std::fmt::Debug for BlockDataState { .debug_tuple("Decrypted") .field(&format_args!("{} block keys [REDACTED]", entries.len())) .finish(), + Self::Pieces(pieces) => formatter + .debug_tuple("Pieces") + .field(&format_args!("{} pieces", pieces.len())) + .finish(), } } } @@ -53,7 +66,7 @@ impl BlockDataState { nonce: [u8; 12], ) -> Result<(), PithosError> { match &self { - BlockDataState::Encrypted(_) => { + BlockDataState::Encrypted(_) | BlockDataState::Pieces(_) => { return Err(PithosError::InvalidBlockDataState( "Block already encrypted.".to_string(), )); @@ -103,6 +116,7 @@ impl Display for FileEntry { match &self.block_data { BlockDataState::Encrypted(_) => f.write_str("Blocks: Encrypted\n")?, BlockDataState::Decrypted(_) => f.write_str("Blocks: Decrypted\n")?, + BlockDataState::Pieces(_) => f.write_str("Blocks: Encrypted pieces\n")?, } f.write_str(&format!("{:<12} {}\n", "Created:", self.created))?; f.write_str(&format!("{:<12} {}\n", "Modified:", self.modified))?; @@ -172,6 +186,15 @@ fn encode_block_data( writer.write_all(&[1])?; encode_decrypted_block_list(entries, writer)?; } + BlockDataState::Pieces(pieces) => { + writer.write_all(&[2])?; + write_len_prefix(writer, pieces.len())?; + for piece in pieces { + writer.write_varint(piece.key_id)?; + write_len_prefix(writer, piece.sealed.len())?; + writer.write_all(&piece.sealed)?; + } + } } Ok(()) } @@ -203,10 +226,41 @@ fn decode_block_data( remaining_block_references, )?, )), + 2 => decode_pieces(reader, limits).map(BlockDataState::Pieces), value => Err(DeserializationError::InvalidEnumValue(value)), } } +fn decode_pieces( + reader: &mut R, + limits: &DeserializationLimits, +) -> Result, DeserializationError> { + let count = bounded_len( + reader.read_varint::()?, + limits.max_collection_entries, + "block list pieces", + )?; + let mut pieces: Vec = Vec::new(); + reserve(&mut pieces, count, "block list pieces")?; + for _ in 0..count { + let key_id = reader.read_varint::()?; + if pieces.last().is_some_and(|last| last.key_id >= key_id) { + return Err(DeserializationError::UnorderedPieceKeys); + } + let len = bounded_len( + reader.read_varint::()?, + limits.max_opaque_bytes, + "sealed block list piece", + )?; + let mut sealed = Vec::new(); + reserve(&mut sealed, len, "sealed block list piece")?; + sealed.resize(len, 0); + reader.read_exact(&mut sealed)?; + pieces.push(BlockListPiece { key_id, sealed }); + } + Ok(pieces) +} + pub(crate) fn encode_file_entry( entry: &FileEntry, writer: &mut W, @@ -336,6 +390,44 @@ pub(crate) fn decode_decrypted_block_list_with_budget( mod tests { use super::*; + #[test] + fn block_list_pieces_round_trip_and_require_increasing_key_ids() { + let limits = DeserializationLimits::default(); + let pieces = BlockDataState::Pieces(vec![ + BlockListPiece { + key_id: 1, + sealed: vec![7; 28], + }, + BlockListPiece { + key_id: 300, + sealed: vec![8; 30], + }, + ]); + let mut encoded = Vec::new(); + encode_block_data(&pieces, &mut encoded).unwrap(); + assert_eq!(&encoded[..3], &[2, 2, 1]); + let mut budget = limits.max_block_references; + assert_eq!( + decode_block_data(&mut encoded.as_slice(), &limits, &mut budget).unwrap(), + pieces + ); + + for key_ids in [[3, 3], [4, 3]] { + let mut unordered = vec![2, 2]; + for key_id in key_ids { + unordered.extend_from_slice(&[key_id, 1, 0]); + } + assert!(matches!( + decode_block_data(&mut unordered.as_slice(), &limits, &mut budget), + Err(DeserializationError::UnorderedPieceKeys) + )); + } + assert!(matches!( + decode_block_data(&mut [3u8, 0].as_slice(), &limits, &mut budget), + Err(DeserializationError::InvalidEnumValue(3)) + )); + } + #[test] fn authenticated_lists_require_exact_consumption_and_unique_keys() { let limits = DeserializationLimits::default(); diff --git a/crates/pithos_lib/src/format/limits.rs b/crates/pithos_lib/src/format/limits.rs index 4dfb5f9..2342941 100644 --- a/crates/pithos_lib/src/format/limits.rs +++ b/crates/pithos_lib/src/format/limits.rs @@ -24,6 +24,8 @@ pub enum DeserializationError { DuplicateBlockReference, #[error("duplicate encoded recipient file id")] DuplicateRecipientFileId, + #[error("block list piece key ids are not strictly increasing")] + UnorderedPieceKeys, #[error("{field} exceeds limit {limit}: {actual}")] LimitExceeded { field: &'static str, diff --git a/spec/PITHOS_1.0.0_draft.md b/spec/PITHOS_1.0.0_draft.md index e3d0a59..50c6a1d 100644 --- a/spec/PITHOS_1.0.0_draft.md +++ b/spec/PITHOS_1.0.0_draft.md @@ -66,6 +66,9 @@ Version 1.1 differs from version 1.0 only in these points: 1. The header version is `0x0101` (Section 4.1). 2. A recipient grant's wrapping key is derived with HKDF-SHA256 instead of using the raw X25519 shared secret (Section 5.3). +3. A file's block list may be sealed in independent pieces, each with its own + key (BlockDataState tag `02`, Section 4.4.2). Piece key IDs share the file ID + space (Section 4.3.2). Readers MUST support both versions. Writers MUST create new archives as version 1.1. An append MUST follow the version of the archive it extends, so appending @@ -399,7 +402,8 @@ no deletion, replacement, or tombstone. File IDs and paths MUST be unique across the selected chain. Writers assign file ID 0 to the first file and assign each later file ID as the current maximum ID plus -1. Readers MUST accept unused file-ID gaps. A rename adds a file record with a +1, where the maximum also covers every piece key ID (Section 4.4.2). Readers MUST +accept unused file-ID gaps. A rename adds a file record with a new file ID and path; the old record remains in the effective archive. A repeated block hash in one Directory is invalid. Across Directories in the @@ -515,12 +519,14 @@ through `ff`. #### 4.4.2 Block Data State -BlockDataState stores either encrypted file block data or a decrypted block list. +BlockDataState stores encrypted file block data, a decrypted block list, or, in +version 1.1, a block list sealed in pieces. ```rust pub enum BlockDataState { Encrypted(Vec), // nonce || ciphertext || tag Decrypted(Vec<([u8; 32], [u8; 32])>), // Block hash and block key + Pieces(Vec<(u64, Vec)>), // Piece key ID and nonce || ciphertext || tag } ``` @@ -530,11 +536,23 @@ pub enum BlockDataState { | --- | --- | --- | | `00` | `Encrypted` | Vector of bytes | | `01` | `Decrypted` | Vector of tuples, each `block_hash[32] || block_key[32]` | +| `02` | `Pieces` | Vector of items, each ULEB128 `key_id` followed by a vector of bytes | Readers MUST reject unknown tags. A decrypted block list is a ULEB128 count followed by that many `block_hash[32] || block_key[32]` pairs. Block-list identity, reuse, and validation are defined in Section 5.2. +Tag `02` is valid only in version 1.1 archives; readers MUST reject it in a +version 1.0 archive. Each piece decrypts, with the piece key granted for its +`key_id` (Section 5.3), to a decrypted block list in the encoding above. The +file's block list is the concatenation of the piece lists in stored order. The +vector MAY be empty, which is an empty block list. Within one file, `key_id` +values MUST be strictly increasing. A piece key ID MUST NOT equal any file ID in +the effective archive and MUST NOT appear in more than one piece of the +effective archive. Readers MUST reject an archive that violates these rules. +Pieces let a writer seal parts of one file independently and join them later +without opening any of them. + #### 4.4.3 File Entry Each directory file record identifies and names one file, then stores its FileEntry body. @@ -585,8 +603,8 @@ of the FileEntry body. | `file_type` | `block_data` | `file_size` | `symlink_target` | | --- | --- | --- | --- | | `Directory` | MUST be `Decrypted` with an empty list | MUST be `0` | MUST be absent (`00`) | -| `Data` | Either state is permitted | MUST equal the sum of referenced effective descriptors' `original_size` values when the block list is available | MUST be absent (`00`) | -| `Metadata` | Either state is permitted | MUST equal the sum of referenced effective descriptors' `original_size` values when the block list is available | MUST be absent (`00`) | +| `Data` | Any state permitted by the archive version | MUST equal the sum of referenced effective descriptors' `original_size` values when the block list is available | MUST be absent (`00`) | +| `Metadata` | Any state permitted by the archive version | MUST equal the sum of referenced effective descriptors' `original_size` values when the block list is available | MUST be absent (`00`) | | `Symlink` | MUST be `Decrypted` with an empty list | MUST be `0` | MUST be present (`01`) | Readers MUST reject a combination that violates this table. For encrypted block @@ -736,7 +754,9 @@ pub enum RecipientData { Readers MUST reject unknown tags. A decrypted recipient-data list is a ULEB128 count followed by that many ULEB128 `file_id` and `file_key[32]` pairs. A repeated file ID is valid only when it has the same file key; readers MUST -reject a conflicting duplicate file ID and file key. +reject a conflicting duplicate file ID and file key. In version 1.1 the +`file_id` field may also hold a piece key ID; its `file_key` is then that +piece's key. ## 5. Content Processing @@ -751,7 +771,9 @@ The block hash is the full 32-byte default unkeyed BLAKE3 digest of the exact plaintext chunk before compression or encryption. It is a block's only identity. The Directory `blocks` vector is keyed by block hash, and each file's block list is an ordered sequence of `(block_hash, block_key)` pairs. That order -reconstructs the file. +reconstructs the file. For a file sealed in pieces, the sequence is the +concatenation of its piece lists, and every rule in this section applies to that +whole sequence. Every block hash referenced by a file MUST resolve to exactly one effective descriptor after the selected chain is merged. A repeated hash in one file is @@ -801,6 +823,8 @@ label or length prefix is included. A block payload with encryption enabled is encrypted with its block key. A file key is 32 random bytes. It encrypts the decrypted block list for a file. +In version 1.1, a piece key is 32 random bytes and encrypts the decrypted block +list of one piece in the same way. For each recipient record, the shared secret is the raw 32-byte X25519 shared secret between the sender private key and recipient public key. The wrapping key encrypts the decrypted recipient list with ChaCha20-Poly1305: @@ -897,6 +921,11 @@ can only collide within one grant. Appends to version 1.0 archives keep the version 1.0 construction. The plaintext-derived Directory block hash exposes block equality, and equal plaintext also derives the same convergent block key. +Pieces are joined in the order the Directory stores them. Their strictly +increasing key IDs and the `file_size` check detect some reordering and missing +pieces, but without authenticated metadata a modified Directory can still drop or +reorder whole pieces together with `file_size`. + Readers must verify each block as required by Section 5.2 before releasing its output. Networked external resolution can expose a caller to unsafe targets and resource exhaustion; Section 4.2.6 defines the required resolver safeguards. @@ -923,7 +952,7 @@ negotiation record. | --- | --- | | Block compression | ProcessingFlags compression bits are `1` through `7` | | Block encryption | ProcessingFlags encryption bit is `1` | -| Encrypted block lists | BlockDataState tag is `00` (`Encrypted`) | +| Encrypted block lists | BlockDataState tag is `00` (`Encrypted`) or, in version 1.1, `02` (`Pieces`) | | Encrypted recipient lists | RecipientData tag is `00` (`Encrypted`) | | External storage | BlockLocation tag is `01` (`External`) | @@ -953,8 +982,9 @@ uncompressed or unencrypted bytes. | Combined: compression and encryption, at either location | Listed | Readable only when every indicated capability is supported; otherwise unavailable | For the table, an encrypted BlockDataState requires encrypted-block-list -capability, and an encrypted RecipientData required to obtain its file key -requires encrypted-recipient-list capability. A content read that needs either +capability, and an encrypted RecipientData required to obtain its file key or +piece keys requires encrypted-recipient-list capability. Content sealed in +pieces is unavailable until the key of every piece is available. A content read that needs either unsupported list form is unavailable even when its block flags themselves are otherwise supported. From ba41fc97e7dca529fb8ee1c7450d0f5eddc24ada Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 14:25:24 +0200 Subject: [PATCH 052/166] feat: encode sealed block list pieces --- crates/pithos_lib/src/archive/mod.rs | 2 + crates/pithos_lib/src/archive/pieces.rs | 324 ++++++++++++++++++++++++ crates/pithos_lib/src/archive/writer.rs | 2 +- crates/pithos_lib/src/error.rs | 2 + crates/pithos_lib/tests/pieces.rs | 59 +++++ 5 files changed, 388 insertions(+), 1 deletion(-) create mode 100644 crates/pithos_lib/src/archive/pieces.rs create mode 100644 crates/pithos_lib/tests/pieces.rs diff --git a/crates/pithos_lib/src/archive/mod.rs b/crates/pithos_lib/src/archive/mod.rs index 5b34e71..e325fdb 100644 --- a/crates/pithos_lib/src/archive/mod.rs +++ b/crates/pithos_lib/src/archive/mod.rs @@ -8,6 +8,7 @@ mod access; mod append; mod index; mod path_validation; +mod pieces; mod planning; mod reader; #[cfg(test)] @@ -23,6 +24,7 @@ pub(crate) use index::build_effective_index; pub(crate) use path_validation::{ validate_directory_entries, validate_new_candidate, validate_symlink_target, }; +pub use pieces::{Piece, PieceEncoder}; pub(crate) use reader::ContentOperationError; pub use reader::{ AccessKeys, Archive, ArchiveEntry, ArchiveFeature, ArchiveReference, EntryKind, diff --git a/crates/pithos_lib/src/archive/pieces.rs b/crates/pithos_lib/src/archive/pieces.rs new file mode 100644 index 0000000..4227896 --- /dev/null +++ b/crates/pithos_lib/src/archive/pieces.rs @@ -0,0 +1,324 @@ +//! Independently sealed parts of one file, joined later without opening any key. +//! +//! A [`PieceEncoder`] encodes the blocks of one part and seals its block list under a fresh +//! piece key granted to the recipients. The resulting [`Piece`] holds no secret, so it can be +//! stored and later joined into a version 1.1 archive with one data file. + +use crate::archive::writer::ProcessingOptions; +use crate::block; +use crate::crypto::{self, FileKey, PublicKey}; +use crate::error::PithosError; +use crate::format::block::ProcessingFlags; +use crate::format::encryption::encode_decrypted_recipient_list; +use crate::format::file_entry::{BlockDataEntry, encode_decrypted_block_list}; +use crate::format::header::FormatVersion; +use integer_encoding::{VarIntReader, VarIntWriter}; +use std::collections::HashSet; +use std::io::{Cursor, Read}; +use x25519_dalek::{PublicKey as DalekPublicKey, StaticSecret}; +use zeroize::Zeroizing; + +const PIECE_MAGIC: &[u8; 8] = b"PITHPIEC"; +const PIECE_RECORD_VERSION: u8 = 1; +const BLOCK_MARKER_LEN: u64 = 4; + +/// Location and size of one encoded block inside its piece. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct PieceBlock { + pub(crate) hash: [u8; 32], + /// Offset of the block's `BLCK` marker from the start of the piece bytes. + pub(crate) offset: u64, + pub(crate) stored_size: u64, + pub(crate) original_size: u64, + pub(crate) flags: ProcessingFlags, +} + +/// One sealed part of a file. It contains no key material: block keys are inside the sealed +/// block list, and the piece key exists only inside the recipient grants. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Piece { + pub(crate) key_id: u64, + pub(crate) original_size: u64, + pub(crate) stored_len: u64, + pub(crate) blocks: Vec, + pub(crate) sealed_list: Vec, + pub(crate) sender: [u8; 32], + pub(crate) grants: Vec<([u8; 32], Vec)>, +} + +/// Encodes the blocks of one piece. The caller chooses block boundaries and appends every +/// returned byte string, in order, to the piece's stored bytes. +pub struct PieceEncoder { + key_id: u64, + recipients: Vec<[u8; 32]>, + flags: ProcessingFlags, + entries: Zeroizing>, + blocks: Vec, + seen: HashSet<[u8; 32]>, + stored_len: u64, + original_size: u64, +} + +impl PieceEncoder { + /// Starts a piece whose key will be granted to `recipients` under `key_id`. + /// Key ids must be at least 1, because the composed file uses file id 0. + pub fn new( + key_id: u64, + recipients: Vec, + processing: ProcessingOptions, + ) -> Result { + if key_id == 0 { + return Err(PithosError::PieceKeyIdConflict(0)); + } + if recipients.is_empty() { + return Err(PithosError::WriterRequiresRecipient); + } + let recipients = recipients + .into_iter() + .map(|recipient| recipient.into_dalek_public_key().to_bytes()) + .collect::>(); + let mut unique = HashSet::with_capacity(recipients.len()); + if recipients + .iter() + .any(|recipient| !unique.insert(*recipient)) + { + return Err(PithosError::DuplicateRecipientKey); + } + Ok(Self { + key_id, + recipients, + flags: processing.flags(), + entries: Zeroizing::new(Vec::new()), + blocks: Vec::new(), + seen: HashSet::new(), + stored_len: 0, + original_size: 0, + }) + } + + /// Encodes one non-empty block and returns `BLCK || payload` to append to the piece. + /// A block that repeats an earlier block of this piece returns no bytes. + pub fn push(&mut self, plaintext: &[u8]) -> Result, PithosError> { + if plaintext.is_empty() { + return Err(PithosError::InvalidBlockDescriptor( + "piece blocks must not be empty", + )); + } + let encoded = block::encode(plaintext, self.flags, crypto::random_nonce())?; + let original_size = plaintext.len() as u64; + self.original_size = checked_add(self.original_size, original_size)?; + self.entries + .push((encoded.hash, *encoded.key.expose_for_protocol())); + if !self.seen.insert(encoded.hash) { + return Ok(Vec::new()); + } + let stored_size = encoded.stored.len() as u64; + self.blocks.push(PieceBlock { + hash: encoded.hash, + offset: self.stored_len, + stored_size, + original_size, + flags: encoded.flags, + }); + let total = checked_add(BLOCK_MARKER_LEN, stored_size)?; + self.stored_len = checked_add(self.stored_len, total)?; + let mut bytes = Vec::with_capacity(encoded.stored.len() + 4); + bytes.extend_from_slice(b"BLCK"); + bytes.extend_from_slice(&encoded.stored); + Ok(bytes) + } + + /// Seals the block list under a fresh piece key and grants that key to every recipient. + pub fn finish(self) -> Result { + let piece_key = FileKey::from_bytes(StaticSecret::random().to_bytes()); + let mut list = Zeroizing::new(Vec::new()); + encode_decrypted_block_list(&self.entries, &mut *list)?; + let sealed_list = + crypto::seal_file_block_list_with_nonce(&piece_key, &list, crypto::random_nonce())?; + + let sender = StaticSecret::random(); + let sender_public = DalekPublicKey::from(&sender).to_bytes(); + let mut records = Zeroizing::new(Vec::new()); + encode_decrypted_recipient_list( + &[(self.key_id, *piece_key.expose_for_protocol())], + &mut *records, + )?; + let mut grants = Vec::with_capacity(self.recipients.len()); + for recipient in &self.recipients { + let nonce = crypto::random_nonce(); + let key = crypto::grant_wrapping_key( + FormatVersion::V1_1, + crypto::derive_shared(sender.as_bytes(), recipient)?, + &sender_public, + recipient, + &nonce, + ); + grants.push(( + *recipient, + crypto::wrap_recipient_list_with_nonce(&key, &records, nonce)?, + )); + } + Ok(Piece { + key_id: self.key_id, + original_size: self.original_size, + stored_len: self.stored_len, + blocks: self.blocks, + sealed_list, + sender: sender_public, + grants, + }) + } +} + +impl Piece { + pub fn key_id(&self) -> u64 { + self.key_id + } + + /// Plaintext bytes covered by this piece, counting repeated blocks each time. + pub fn original_size(&self) -> u64 { + self.original_size + } + + /// Length of the stored bytes the encoder returned for this piece. + pub fn stored_len(&self) -> u64 { + self.stored_len + } + + /// Encodes this record for storage until composition. The encoding is stable for one + /// record version and is not part of the archive format. + pub fn to_bytes(&self) -> Vec { + let mut bytes = Vec::new(); + bytes.extend_from_slice(PIECE_MAGIC); + bytes.push(PIECE_RECORD_VERSION); + for value in [ + self.key_id, + self.original_size, + self.stored_len, + self.blocks.len() as u64, + ] { + write_uleb(&mut bytes, value); + } + for block in &self.blocks { + bytes.extend_from_slice(&block.hash); + write_uleb(&mut bytes, block.offset); + write_uleb(&mut bytes, block.stored_size); + write_uleb(&mut bytes, block.original_size); + bytes.push(block.flags.0); + } + write_uleb(&mut bytes, self.sealed_list.len() as u64); + bytes.extend_from_slice(&self.sealed_list); + bytes.extend_from_slice(&self.sender); + write_uleb(&mut bytes, self.grants.len() as u64); + for (recipient, wrapped) in &self.grants { + bytes.extend_from_slice(recipient); + write_uleb(&mut bytes, wrapped.len() as u64); + bytes.extend_from_slice(wrapped); + } + bytes + } + + /// Decodes a record from [`Piece::to_bytes`] and checks that its blocks are contiguous. + pub fn from_bytes(bytes: &[u8]) -> Result { + let mut reader = Cursor::new(bytes); + if read_vec(&mut reader, PIECE_MAGIC.len())? != PIECE_MAGIC + || read_vec(&mut reader, 1)? != [PIECE_RECORD_VERSION] + { + return Err(PithosError::InvalidPieceRecord); + } + let key_id = read_uleb(&mut reader)?; + let original_size = read_uleb(&mut reader)?; + let stored_len = read_uleb(&mut reader)?; + let block_count = read_count(&mut reader, 32 + 4)?; + let mut blocks = Vec::with_capacity(block_count); + let mut offset = 0u64; + for _ in 0..block_count { + let block = PieceBlock { + hash: read_array(&mut reader)?, + offset: read_uleb(&mut reader)?, + stored_size: read_uleb(&mut reader)?, + original_size: read_uleb(&mut reader)?, + flags: ProcessingFlags::from_byte(read_vec(&mut reader, 1)?[0]), + }; + if block.offset != offset || block.flags.has_reserved_bits() { + return Err(PithosError::InvalidPieceRecord); + } + offset = checked_add(offset, checked_add(BLOCK_MARKER_LEN, block.stored_size)?)?; + blocks.push(block); + } + let list_len = read_count(&mut reader, 1)?; + let sealed_list = read_vec(&mut reader, list_len)?; + let sender = read_array(&mut reader)?; + let grant_count = read_count(&mut reader, 32 + 1)?; + let mut grants = Vec::with_capacity(grant_count); + for _ in 0..grant_count { + let recipient = read_array(&mut reader)?; + let len = read_count(&mut reader, 1)?; + grants.push((recipient, read_vec(&mut reader, len)?)); + } + if offset != stored_len + || reader.position() != bytes.len() as u64 + || key_id == 0 + || grants.is_empty() + { + return Err(PithosError::InvalidPieceRecord); + } + Ok(Self { + key_id, + original_size, + stored_len, + blocks, + sealed_list, + sender, + grants, + }) + } +} + +fn write_uleb(bytes: &mut Vec, value: u64) { + bytes + .write_varint(value) + .expect("writing to a vector cannot fail"); +} + +fn read_uleb(reader: &mut Cursor<&[u8]>) -> Result { + reader + .read_varint::() + .map_err(|_| PithosError::InvalidPieceRecord) +} + +fn read_vec(reader: &mut Cursor<&[u8]>, len: usize) -> Result, PithosError> { + let mut value = vec![0; len]; + reader + .read_exact(&mut value) + .map_err(|_| PithosError::InvalidPieceRecord)?; + Ok(value) +} + +fn read_array(reader: &mut Cursor<&[u8]>) -> Result<[u8; 32], PithosError> { + let mut value = [0; 32]; + reader + .read_exact(&mut value) + .map_err(|_| PithosError::InvalidPieceRecord)?; + Ok(value) +} + +/// Reads a count whose items need at least `item_len` bytes each, so it cannot over-allocate. +fn read_count(reader: &mut Cursor<&[u8]>, item_len: u64) -> Result { + let count = read_uleb(reader)?; + let remaining = reader.get_ref().len() as u64 - reader.position(); + if count + .checked_mul(item_len) + .is_none_or(|needed| needed > remaining) + { + return Err(PithosError::InvalidPieceRecord); + } + usize::try_from(count).map_err(|_| PithosError::InvalidPieceRecord) +} + +fn checked_add(left: u64, right: u64) -> Result { + left.checked_add(right) + .ok_or(PithosError::InvalidDirectoryRange { + operation: "add piece sizes", + }) +} diff --git a/crates/pithos_lib/src/archive/writer.rs b/crates/pithos_lib/src/archive/writer.rs index b59bf6c..100801c 100644 --- a/crates/pithos_lib/src/archive/writer.rs +++ b/crates/pithos_lib/src/archive/writer.rs @@ -109,7 +109,7 @@ impl ProcessingOptions { self.compression_level } - fn flags(self) -> ProcessingFlags { + pub(crate) fn flags(self) -> ProcessingFlags { ProcessingFlags::new(self.encrypted, Some(self.compression_level)) } } diff --git a/crates/pithos_lib/src/error.rs b/crates/pithos_lib/src/error.rs index 0f3dfcb..ea63acd 100644 --- a/crates/pithos_lib/src/error.rs +++ b/crates/pithos_lib/src/error.rs @@ -87,6 +87,8 @@ pub enum PithosError { UnsupportedBlockListPieces, #[error("block list piece key id {0} is already used by a file or another piece")] PieceKeyIdConflict(u64), + #[error("invalid piece record")] + InvalidPieceRecord, #[error("duplicate encoded recipient file id")] DuplicateRecipientFileId, #[error("reserved processing bits are set: {0:#04x}")] diff --git a/crates/pithos_lib/tests/pieces.rs b/crates/pithos_lib/tests/pieces.rs new file mode 100644 index 0000000..b800fde --- /dev/null +++ b/crates/pithos_lib/tests/pieces.rs @@ -0,0 +1,59 @@ +mod common; + +use common::util::public_key; +use pithos_lib::archive::{Piece, PieceEncoder, ProcessingOptions}; +use pithos_lib::error::PithosError; + +const BLOCK: usize = 1000; + +/// Encodes `content` as one piece in fixed blocks and returns its stored bytes and record. +fn encode(key_id: u64, content: &[u8]) -> (Vec, Piece) { + let mut encoder = PieceEncoder::new( + key_id, + vec![public_key("recipient1")], + ProcessingOptions::new(true, 3).unwrap(), + ) + .unwrap(); + let mut stored = Vec::new(); + for block in content.chunks(BLOCK) { + stored.extend(encoder.push(block).unwrap()); + } + (stored, encoder.finish().unwrap()) +} + +fn content(seed: u8, len: usize) -> Vec { + (0..len) + .map(|index| (index as u8).wrapping_mul(31) ^ seed) + .collect() +} + +#[test] +fn piece_encoders_reject_the_composed_file_id() { + assert!(matches!( + PieceEncoder::new( + 0, + vec![public_key("recipient1")], + ProcessingOptions::default() + ), + Err(PithosError::PieceKeyIdConflict(0)) + )); +} + +#[test] +fn piece_records_round_trip_and_reject_damage() { + let (_, piece) = encode(3, &content(7, 2100)); + let bytes = piece.to_bytes(); + assert_eq!(Piece::from_bytes(&bytes).unwrap(), piece); + assert_eq!(piece.key_id(), 3); + assert_eq!(piece.original_size(), 2100); + for damaged in [ + bytes[..bytes.len() - 1].to_vec(), + [bytes.as_slice(), &[0]].concat(), + [b"PITHPIEX".as_slice(), &bytes[8..]].concat(), + ] { + assert!(matches!( + Piece::from_bytes(&damaged), + Err(PithosError::InvalidPieceRecord) + )); + } +} From 312cc7679b9fa956d2eae6b240bf9e3ccdd5e17e Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 14:26:53 +0200 Subject: [PATCH 053/166] feat: compose pieces into an archive without opening keys --- crates/pithos_lib/src/archive/mod.rs | 2 +- crates/pithos_lib/src/archive/pieces.rs | 166 +++++++++++++++++++++++- crates/pithos_lib/tests/pieces.rs | 128 +++++++++++++++++- spec/PITHOS_1.0.0_draft.md | 8 ++ 4 files changed, 293 insertions(+), 11 deletions(-) diff --git a/crates/pithos_lib/src/archive/mod.rs b/crates/pithos_lib/src/archive/mod.rs index e325fdb..7509f37 100644 --- a/crates/pithos_lib/src/archive/mod.rs +++ b/crates/pithos_lib/src/archive/mod.rs @@ -24,7 +24,7 @@ pub(crate) use index::build_effective_index; pub(crate) use path_validation::{ validate_directory_entries, validate_new_candidate, validate_symlink_target, }; -pub use pieces::{Piece, PieceEncoder}; +pub use pieces::{Composition, Piece, PieceEncoder, compose}; pub(crate) use reader::ContentOperationError; pub use reader::{ AccessKeys, Archive, ArchiveEntry, ArchiveFeature, ArchiveReference, EntryKind, diff --git a/crates/pithos_lib/src/archive/pieces.rs b/crates/pithos_lib/src/archive/pieces.rs index 4227896..d89b7d5 100644 --- a/crates/pithos_lib/src/archive/pieces.rs +++ b/crates/pithos_lib/src/archive/pieces.rs @@ -2,16 +2,28 @@ //! //! A [`PieceEncoder`] encodes the blocks of one part and seals its block list under a fresh //! piece key granted to the recipients. The resulting [`Piece`] holds no secret, so it can be -//! stored and later joined into a version 1.1 archive with one data file. +//! stored and later joined by [`compose`] into a version 1.1 archive with one data file. -use crate::archive::writer::ProcessingOptions; +use crate::archive::path_validation::validate_entry; +use crate::archive::types::ArchivePath; +use crate::archive::writer::{EntryMetadata, ProcessingOptions}; use crate::block; use crate::crypto::{self, FileKey, PublicKey}; use crate::error::PithosError; -use crate::format::block::ProcessingFlags; -use crate::format::encryption::encode_decrypted_recipient_list; -use crate::format::file_entry::{BlockDataEntry, encode_decrypted_block_list}; -use crate::format::header::FormatVersion; +use crate::format::block::{BlockIndexEntry, BlockLocation, ProcessingFlags}; +use crate::format::directory::{ + Directory, DirectoryEntries, encode_directory, update_directory_crc, update_directory_len, +}; +use crate::format::encryption::{ + EncryptionSection, RecipientData, RecipientSection, encode_decrypted_recipient_list, +}; +use crate::format::file_entry::{ + BlockDataEntry, BlockDataState, BlockListPiece, FileEntry, FileType, + encode_decrypted_block_list, +}; +use crate::format::header::{FileHeader, FormatVersion, encode_header}; +use crate::format::limits::DeserializationError; +use indexmap::IndexMap; use integer_encoding::{VarIntReader, VarIntWriter}; use std::collections::HashSet; use std::io::{Cursor, Read}; @@ -322,3 +334,145 @@ fn checked_add(left: u64, right: u64) -> Result { operation: "add piece sizes", }) } + +/// A composed version 1.1 archive. Write [`Composition::header`], then the stored bytes of +/// every piece in order (at [`Composition::piece_offsets`]), then the directory. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct Composition { + piece_offsets: Vec, + directory: Vec, + archive_len: u64, +} + +impl Composition { + pub fn header(&self) -> [u8; FileHeader::ENCODED_LEN] { + let mut header = [0; FileHeader::ENCODED_LEN]; + encode_header(&FormatVersion::V1_1.header(), &mut header.as_mut_slice()) + .expect("the header fits its fixed length"); + header + } + + /// Archive offset at which each piece's stored bytes start. + pub fn piece_offsets(&self) -> &[u64] { + &self.piece_offsets + } + + pub fn directory(&self) -> &[u8] { + &self.directory + } + + pub fn archive_len(&self) -> u64 { + self.archive_len + } +} + +/// Joins pieces, in content order, into an archive with one data file at `path`. +/// It opens no key, so it works without access to the content. Key ids must increase, +/// and `path` must be at the archive root because the archive declares no directories. +pub fn compose( + path: ArchivePath, + metadata: EntryMetadata, + pieces: &[Piece], +) -> Result { + if path.as_str().contains('/') { + return Err(PithosError::InvalidArchivePath { + path: path.as_str().to_owned(), + reason: "a composed file must be at the archive root".into(), + }); + } + if let Some(reference) = metadata.references.first() { + return Err(PithosError::MissingReferenceTarget( + reference.target_file_id, + )); + } + let mut piece_offsets = Vec::with_capacity(pieces.len()); + let mut offset = FileHeader::ENCODED_LEN as u64; + let mut file_size = 0u64; + let mut blocks: IndexMap<[u8; 32], BlockIndexEntry> = IndexMap::new(); + let mut encryption = IndexMap::new(); + let mut previous_key = 0u64; + for piece in pieces { + if piece.key_id <= previous_key { + return Err(DeserializationError::UnorderedPieceKeys.into()); + } + previous_key = piece.key_id; + piece_offsets.push(offset); + for block in &piece.blocks { + let entry = BlockIndexEntry { + offset: checked_add(offset, block.offset)?, + stored_size: block.stored_size, + original_size: block.original_size, + flags: block.flags, + location: BlockLocation::Local, + }; + // Equal hashes have equal convergent keys, so the first stored copy serves all. + match blocks.get(&block.hash) { + Some(existing) if existing.original_size != entry.original_size => { + return Err(PithosError::BlockIndexConflict { + hash: block.hash, + existing_original_size: existing.original_size, + new_original_size: entry.original_size, + }); + } + Some(_) => {} + None => { + blocks.insert(block.hash, entry); + } + } + } + let recipients = piece + .grants + .iter() + .map(|(recipient, wrapped)| { + let data = RecipientData::Encrypted(wrapped.clone()); + ( + *recipient, + RecipientSection { + recipient_data: data, + }, + ) + }) + .collect(); + if encryption + .insert(piece.sender, EncryptionSection { recipients }) + .is_some() + { + return Err(PithosError::DuplicateSenderKey); + } + offset = checked_add(offset, piece.stored_len)?; + file_size = checked_add(file_size, piece.original_size)?; + } + + let entry = FileEntry { + file_type: FileType::Data, + block_data: BlockDataState::Pieces( + pieces + .iter() + .map(|piece| BlockListPiece { + key_id: piece.key_id, + sealed: piece.sealed_list.clone(), + }) + .collect(), + ), + created: metadata.created, + modified: metadata.modified, + file_size, + permissions: metadata.permissions, + references: Vec::new(), + symlink_target: None, + }; + validate_entry(path.as_str(), &entry)?; + let mut files = DirectoryEntries::new(); + files.insert(0, path.as_str(), entry)?; + let mut directory = Directory::new(None, files, encryption); + directory.blocks = blocks; + update_directory_len(&mut directory)?; + update_directory_crc(&mut directory)?; + let mut bytes = Vec::new(); + encode_directory(&directory, &mut bytes)?; + Ok(Composition { + piece_offsets, + archive_len: checked_add(offset, bytes.len() as u64)?, + directory: bytes, + }) +} diff --git a/crates/pithos_lib/tests/pieces.rs b/crates/pithos_lib/tests/pieces.rs index b800fde..b633931 100644 --- a/crates/pithos_lib/tests/pieces.rs +++ b/crates/pithos_lib/tests/pieces.rs @@ -1,8 +1,12 @@ mod common; -use common::util::public_key; -use pithos_lib::archive::{Piece, PieceEncoder, ProcessingOptions}; -use pithos_lib::error::PithosError; +use common::util::{private_key, public_key}; +use pithos_lib::archive::{ + AccessKeys, Archive, ArchivePath, EntryKind, EntryMetadata, OpenOptions, Piece, PieceEncoder, + ProcessingOptions, compose, +}; +use pithos_lib::error::{DeserializationError, PithosError}; +use pithos_lib::source::MemorySource; const BLOCK: usize = 1000; @@ -21,6 +25,40 @@ fn encode(key_id: u64, content: &[u8]) -> (Vec, Piece) { (stored, encoder.finish().unwrap()) } +/// Composes the pieces from their stored records only, as a node without keys would. +fn assemble(parts: &[(Vec, Piece)]) -> Vec { + let pieces = parts + .iter() + .map(|(_, piece)| Piece::from_bytes(&piece.to_bytes()).unwrap()) + .collect::>(); + let composition = compose( + ArchivePath::new("object").unwrap(), + EntryMetadata::new(1, 2, 0o640), + &pieces, + ) + .unwrap(); + let mut archive = composition.header().to_vec(); + for ((stored, _), offset) in parts.iter().zip(composition.piece_offsets()) { + assert_eq!(archive.len() as u64, *offset); + archive.extend_from_slice(stored); + } + archive.extend_from_slice(composition.directory()); + assert_eq!(archive.len() as u64, composition.archive_len()); + archive +} + +fn open(bytes: Vec, keys: AccessKeys) -> Archive { + Archive::open( + MemorySource::new(bytes), + OpenOptions::default().with_access_keys(keys), + ) + .unwrap() +} + +fn recipient() -> AccessKeys { + AccessKeys::new().with_key(private_key("recipient1")) +} + fn content(seed: u8, len: usize) -> Vec { (0..len) .map(|index| (index as u8).wrapping_mul(31) ^ seed) @@ -28,7 +66,89 @@ fn content(seed: u8, len: usize) -> Vec { } #[test] -fn piece_encoders_reject_the_composed_file_id() { +fn composed_pieces_read_back_as_one_file_with_ranges_across_pieces() { + let first = content(1, 2500); + let second = [content(2, 1000), content(1, 1000)].concat(); + let third = content(3, 10); + let parts = vec![encode(1, &first), encode(4, &second), encode(9, &third)]; + let archive = open(assemble(&parts), recipient()); + let expected = [first, second, third].concat(); + + let mut output = Vec::new(); + archive.copy_to("object", &mut output).unwrap(); + assert_eq!(output, expected); + + let mut range = Vec::new(); + archive + .copy_range_to("object", 2400..3600, &mut range) + .unwrap(); + assert_eq!(range, expected[2400..3600]); + let entry = archive.entry("object").unwrap().unwrap(); + assert_eq!(entry.permissions, 0o640); + assert!(matches!( + entry.kind, + EntryKind::File { + size: 4510, + available: true + } + )); +} + +#[test] +fn a_reader_without_a_granted_key_sees_unavailable_content() { + let parts = vec![encode(1, &content(1, 1500))]; + let bytes = assemble(&parts); + for keys in [ + AccessKeys::new(), + AccessKeys::new().with_key(private_key("recipient2")), + ] { + let archive = open(bytes.clone(), keys); + assert!(matches!( + archive.entry("object").unwrap().unwrap().kind, + EntryKind::File { + available: false, + .. + } + )); + assert!(archive.copy_to("object", &mut Vec::new()).is_err()); + } +} + +#[test] +fn an_empty_composition_is_an_empty_file() { + let archive = open(assemble(&[]), recipient()); + let mut output = Vec::new(); + archive.copy_to("object", &mut output).unwrap(); + assert!(output.is_empty()); +} + +#[test] +fn composition_requires_increasing_key_ids_and_a_root_path() { + let (_, low) = encode(2, b"low"); + let (_, high) = encode(5, b"high"); + for pieces in [ + vec![high.clone(), low.clone()], + vec![low.clone(), low.clone()], + ] { + assert!(matches!( + compose( + ArchivePath::new("object").unwrap(), + EntryMetadata::new(0, 0, 0o644), + &pieces, + ), + Err(PithosError::Deserialization( + DeserializationError::UnorderedPieceKeys + )) + )); + } + assert!(matches!( + compose( + ArchivePath::new("nested/object").unwrap(), + EntryMetadata::new(0, 0, 0o644), + &[low], + ), + Err(PithosError::InvalidArchivePath { .. }) + )); assert!(matches!( PieceEncoder::new( 0, diff --git a/spec/PITHOS_1.0.0_draft.md b/spec/PITHOS_1.0.0_draft.md index 50c6a1d..de412fa 100644 --- a/spec/PITHOS_1.0.0_draft.md +++ b/spec/PITHOS_1.0.0_draft.md @@ -1056,6 +1056,14 @@ average, 512 KB maximum, and 48-byte window. respectively. A writer may sample 4096 bytes and require a 0.85 compression ratio before compressing. These suggestions are non-normative. +**Joining independently written parts:** a version 1.1 writer may encode each +part of a file on its own, seal that part's block list as one piece under a fresh +piece key, and grant the piece key with a fresh sender key per piece. A later +step can then join the parts into one archive without any key: it copies the +stored blocks, keeps every sealed piece and grant unchanged, and writes one +Directory. Because block keys are convergent, a block that repeats in two parts +needs only one effective descriptor. + ## Appendix B. Conformance Examples and Vectors This appendix is informative. The cited sections remain authoritative. Complete From 7e4e5c31e90f389699650b2f067d766fe54e0034 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 14:31:11 +0200 Subject: [PATCH 054/166] feat: check archive metadata against a trusted digest --- crates/pithos_lib/src/archive/mod.rs | 9 +++++ crates/pithos_lib/src/archive/pieces.rs | 5 +++ crates/pithos_lib/src/archive/reader.rs | 28 +++++++++++++++ .../src/archive/reader_private_tests.rs | 16 +++++++++ crates/pithos_lib/src/error.rs | 2 ++ crates/pithos_lib/tests/pieces.rs | 35 +++++++++++++++++++ spec/PITHOS_1.0.0_draft.md | 15 +++++++- 7 files changed, 109 insertions(+), 1 deletion(-) diff --git a/crates/pithos_lib/src/archive/mod.rs b/crates/pithos_lib/src/archive/mod.rs index 7509f37..7978637 100644 --- a/crates/pithos_lib/src/archive/mod.rs +++ b/crates/pithos_lib/src/archive/mod.rs @@ -35,6 +35,15 @@ pub use types::{ArchivePath, ExternalLocation}; pub(crate) use types::{FileId, Span}; pub(crate) use validation::validated_segment_from_directory; +/// The metadata digest: BLAKE3 over each directory's BLAKE3 hash, from base to terminal. +pub(crate) fn metadata_digest(directory_hashes: &[[u8; 32]]) -> [u8; 32] { + let mut hasher = blake3::Hasher::new(); + for hash in directory_hashes { + hasher.update(hash); + } + *hasher.finalize().as_bytes() +} + pub(crate) fn decode_validated_directory( bytes: &[u8], limits: &crate::format::limits::DeserializationLimits, diff --git a/crates/pithos_lib/src/archive/pieces.rs b/crates/pithos_lib/src/archive/pieces.rs index d89b7d5..e78f951 100644 --- a/crates/pithos_lib/src/archive/pieces.rs +++ b/crates/pithos_lib/src/archive/pieces.rs @@ -364,6 +364,11 @@ impl Composition { pub fn archive_len(&self) -> u64 { self.archive_len } + + /// The digest [`crate::archive::Archive::metadata_digest`] reports for this archive. + pub fn metadata_digest(&self) -> [u8; 32] { + crate::archive::metadata_digest(&[*blake3::hash(&self.directory).as_bytes()]) + } } /// Joins pieces, in content order, into an archive with one data file at `path`. diff --git a/crates/pithos_lib/src/archive/reader.rs b/crates/pithos_lib/src/archive/reader.rs index 22a5af6..22a26b7 100644 --- a/crates/pithos_lib/src/archive/reader.rs +++ b/crates/pithos_lib/src/archive/reader.rs @@ -186,6 +186,7 @@ pub struct OpenOptions { external: E, external_resolver_supplied: bool, external_access_policy: Option>, + expected_metadata_digest: Option<[u8; 32]>, } impl Default for OpenOptions { @@ -196,6 +197,7 @@ impl Default for OpenOptions { external: NoExternalBlocks, external_resolver_supplied: false, external_access_policy: None, + expected_metadata_digest: None, } } } @@ -218,9 +220,17 @@ impl OpenOptions { external, external_resolver_supplied: true, external_access_policy: self.external_access_policy, + expected_metadata_digest: self.expected_metadata_digest, } } + /// Requires the archive's metadata digest to equal `digest`, for example a value kept in + /// trusted storage. A mismatch fails before any metadata is decrypted or used. + pub fn with_expected_metadata_digest(mut self, digest: [u8; 32]) -> Self { + self.expected_metadata_digest = Some(digest); + self + } + pub fn with_external_access_policy( mut self, policy: Arc, @@ -269,6 +279,7 @@ pub struct Archive { external_access_policy: Option>, archive_len: u64, version: FormatVersion, + metadata_digest: [u8; 32], terminal_directory: Span, index: ArchiveIndex, segments: Vec, @@ -305,6 +316,7 @@ where let mut visited = HashSet::new(); let mut decoded = DecodedDirectoryCounts::default(); let mut remaining_block_references = options.limits.max_accessible_block_references; + let mut directory_hashes = Vec::new(); while let Some((start, len)) = next { validate_directory_len(len, options.limits)?; if raw.len() as u64 > options.limits.max_parent_directories { @@ -336,6 +348,7 @@ where }); } let bytes = Zeroizing::new(read_source(&source, start, len, "directory")?); + directory_hashes.push(*blake3::hash(&bytes).as_bytes()); let directory = decode_validated_directory( &bytes, &remaining_deserialization_limits(options.limits, &decoded), @@ -347,6 +360,14 @@ where raw.push((directory, span)); } raw.reverse(); + directory_hashes.reverse(); + let metadata_digest = crate::archive::metadata_digest(&directory_hashes); + if options + .expected_metadata_digest + .is_some_and(|expected| expected != metadata_digest) + { + return Err(PithosError::MetadataDigestMismatch); + } let maximum_piece_key = validate_piece_keys(version, &raw)?; let mut first_grants = HashMap::new(); @@ -420,6 +441,7 @@ where external_access_policy: options.external_access_policy, archive_len, version, + metadata_digest, terminal_directory: Span::new(terminal_start, terminal_len)?, index, segments, @@ -431,6 +453,12 @@ where }) } + /// BLAKE3 over the hashes of every directory, from the base to the terminal directory. + /// Keeping it in trusted storage lets a later open detect changed metadata. + pub fn metadata_digest(&self) -> [u8; 32] { + self.metadata_digest + } + pub fn entries(&self) -> impl ExactSizeIterator + '_ { self.index .entries() diff --git a/crates/pithos_lib/src/archive/reader_private_tests.rs b/crates/pithos_lib/src/archive/reader_private_tests.rs index a729566..6ba3616 100644 --- a/crates/pithos_lib/src/archive/reader_private_tests.rs +++ b/crates/pithos_lib/src/archive/reader_private_tests.rs @@ -1697,3 +1697,19 @@ fn a_tampered_piece_fails_before_any_content_is_indexed() { }); assert!(matches!(open_pieces(&path), Err(PithosError::Crypt(_)))); } + +#[test] +fn an_append_changes_the_metadata_digest() { + let (_temporary, path) = fixture(); + let before = open_path(&path, AccessKeys::new()).metadata_digest(); + append_empty_directory(&path, None); + let after = open_path(&path, AccessKeys::new()).metadata_digest(); + assert_ne!(before, after); + assert!(matches!( + Archive::open( + MemorySource::new(Arc::<[u8]>::from(std::fs::read(&path).unwrap())), + OpenOptions::default().with_expected_metadata_digest(before), + ), + Err(PithosError::MetadataDigestMismatch) + )); +} diff --git a/crates/pithos_lib/src/error.rs b/crates/pithos_lib/src/error.rs index ea63acd..4c42c31 100644 --- a/crates/pithos_lib/src/error.rs +++ b/crates/pithos_lib/src/error.rs @@ -89,6 +89,8 @@ pub enum PithosError { PieceKeyIdConflict(u64), #[error("invalid piece record")] InvalidPieceRecord, + #[error("archive metadata does not match the expected digest")] + MetadataDigestMismatch, #[error("duplicate encoded recipient file id")] DuplicateRecipientFileId, #[error("reserved processing bits are set: {0:#04x}")] diff --git a/crates/pithos_lib/tests/pieces.rs b/crates/pithos_lib/tests/pieces.rs index b633931..3a6f326 100644 --- a/crates/pithos_lib/tests/pieces.rs +++ b/crates/pithos_lib/tests/pieces.rs @@ -94,6 +94,41 @@ fn composed_pieces_read_back_as_one_file_with_ranges_across_pieces() { )); } +#[test] +fn the_composition_digest_guards_the_metadata_on_open() { + let parts = vec![encode(1, &content(1, 1500))]; + let pieces = parts + .iter() + .map(|(_, piece)| piece.clone()) + .collect::>(); + let digest = compose( + ArchivePath::new("object").unwrap(), + EntryMetadata::new(1, 2, 0o640), + &pieces, + ) + .unwrap() + .metadata_digest(); + let bytes = assemble(&parts); + let opened = Archive::open( + MemorySource::new(bytes.clone()), + OpenOptions::default() + .with_access_keys(recipient()) + .with_expected_metadata_digest(digest), + ) + .unwrap(); + assert_eq!(opened.metadata_digest(), digest); + + let mut wrong = digest; + wrong[0] ^= 1; + assert!(matches!( + Archive::open( + MemorySource::new(bytes), + OpenOptions::default().with_expected_metadata_digest(wrong), + ), + Err(PithosError::MetadataDigestMismatch) + )); +} + #[test] fn a_reader_without_a_granted_key_sees_unavailable_content() { let parts = vec![encode(1, &content(1, 1500))]; diff --git a/spec/PITHOS_1.0.0_draft.md b/spec/PITHOS_1.0.0_draft.md index de412fa..f931dfd 100644 --- a/spec/PITHOS_1.0.0_draft.md +++ b/spec/PITHOS_1.0.0_draft.md @@ -486,6 +486,17 @@ data/raw (too late; parent `data` not yet declared) data (too late; a descendant already occurred) ``` +#### 4.3.4 Metadata Digest + +The metadata digest of an archive is the BLAKE3 hash of the concatenated 32-byte +BLAKE3 hashes of each Directory's exact bytes, in selected-chain order from the +base Directory to the terminal Directory. It is not stored in the archive and is +the same for both versions. An application MAY keep the digest in storage it +trusts and supply it when it opens the archive again. A reader given an expected +digest MUST compare it after validating the Directory framing and before +decrypting, merging, or otherwise using any metadata, and MUST reject the archive +on a mismatch. Every append changes the digest. + ### 4.4 File Representation #### 4.4.1 File Types @@ -897,7 +908,9 @@ When archiving directory trees: ## 7. Security Considerations Directory CRC-32 detects accidental corruption of the serialized Directory bytes -it covers; it is not authentication. Pithos 1.0 provides no archive-wide origin +it covers; it is not authentication. An application that keeps the metadata +digest (Section 4.3.4) in trusted storage gains metadata integrity for its own +later reads; the digest does not authenticate archive origin to anyone else. Pithos 1.0 provides no archive-wide origin authentication or metadata integrity: an unauthenticated Directory can replace both a block hash and its referenced content. AEAD authenticates each encrypted value's ciphertext, but its empty AAD does not bind that value to its surrounding From 1ad78862b477ddcf37c4dc5677c7c3326d8af95c Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 14:39:22 +0200 Subject: [PATCH 055/166] feat: grant readers every piece key of a composed file --- crates/pithos_lib/src/archive/access.rs | 22 ++++++++---- crates/pithos_lib/src/archive/reader.rs | 4 +++ crates/pithos_lib/src/archive/snapshot.rs | 43 ++++++++++++----------- crates/pithos_lib/src/archive/writer.rs | 26 ++++++++------ crates/pithos_lib/src/fs/append.rs | 2 +- crates/pithos_lib/tests/pieces.rs | 28 +++++++++++++-- spec/PITHOS_1.0.0_draft.md | 3 +- 7 files changed, 87 insertions(+), 41 deletions(-) diff --git a/crates/pithos_lib/src/archive/access.rs b/crates/pithos_lib/src/archive/access.rs index ab00908..742fd74 100644 --- a/crates/pithos_lib/src/archive/access.rs +++ b/crates/pithos_lib/src/archive/access.rs @@ -8,6 +8,8 @@ pub(crate) struct ResolvedAccess { keys: BTreeMap, block_keys: BTreeMap>, provenance: BTreeMap, + /// Piece key ids of each file whose block list was sealed in pieces. + pieces: BTreeMap>, } #[derive(Clone, Copy, Eq, Ord, PartialEq, PartialOrd)] @@ -25,6 +27,7 @@ impl ResolvedAccess { keys: BTreeMap::new(), block_keys: BTreeMap::new(), provenance: BTreeMap::new(), + pieces: BTreeMap::new(), } } @@ -60,12 +63,19 @@ impl ResolvedAccess { self.provenance.get(&id).copied() } - pub(crate) fn with_file_key( - &self, - id: FileId, - operation: impl FnOnce(&FileKey) -> T, - ) -> Option { - self.keys.get(&id).map(operation) + pub(crate) fn insert_pieces(&mut self, id: FileId, key_ids: Vec) { + self.pieces.insert(id, key_ids); + } + + /// The keys a reader grant for `id` must carry: its file key, or every piece key. + pub(crate) fn grant_keys(&self, id: FileId) -> Option> { + match self.pieces.get(&id) { + Some(key_ids) => key_ids + .iter() + .map(|key_id| self.keys.get(key_id).map(|key| (*key_id, key))) + .collect(), + None => self.keys.get(&id).map(|key| vec![(id, key)]), + } } pub(crate) fn insert_block_keys<'a>( diff --git a/crates/pithos_lib/src/archive/reader.rs b/crates/pithos_lib/src/archive/reader.rs index 22a26b7..44ee6c5 100644 --- a/crates/pithos_lib/src/archive/reader.rs +++ b/crates/pithos_lib/src/archive/reader.rs @@ -984,6 +984,10 @@ fn resolve_block_lists( entries.extend_from_slice(&piece_entries); } crate::format::file_entry::validate_unique_block_references(&entries)?; + access.insert_pieces( + file_id, + pieces.iter().map(|piece| FileId(piece.key_id)).collect(), + ); record_block_keys(access, file_id, &entries); file.block_data = BlockDataState::Decrypted(entries); } diff --git a/crates/pithos_lib/src/archive/snapshot.rs b/crates/pithos_lib/src/archive/snapshot.rs index b325c14..55a80ab 100644 --- a/crates/pithos_lib/src/archive/snapshot.rs +++ b/crates/pithos_lib/src/archive/snapshot.rs @@ -242,28 +242,29 @@ impl AppendSnapshot { Ok(()) } - /// Borrows an opaque, zeroizing recovered file key only for content entries. - pub(crate) fn with_file_key( + /// Borrows the keys a reader grant for content entry `id` must carry. + pub(crate) fn with_grant_keys( &self, id: FileId, - operation: impl FnOnce(&FileKey) -> T, + operation: impl FnOnce(&[(FileId, &FileKey)]) -> T, ) -> Result { + self.ensure_content_entry(id)?; + self.access + .grant_keys(id) + .map(|keys| operation(&keys)) + .ok_or(PithosError::SnapshotContentUnavailable(id.0)) + } + + fn ensure_content_entry(&self, id: FileId) -> Result<(), PithosError> { let entry = self .entries .get(&id) .ok_or(PithosError::SnapshotFileIdNotFound(id.0))?; match entry.kind { - SnapshotEntryKind::Directory => { - return Err(PithosError::SnapshotDirectoryHasNoContent(id.0)); - } - SnapshotEntryKind::Symlink => { - return Err(PithosError::SnapshotSymlinkHasNoContent(id.0)); - } - SnapshotEntryKind::File | SnapshotEntryKind::Metadata => {} + SnapshotEntryKind::Directory => Err(PithosError::SnapshotDirectoryHasNoContent(id.0)), + SnapshotEntryKind::Symlink => Err(PithosError::SnapshotSymlinkHasNoContent(id.0)), + SnapshotEntryKind::File | SnapshotEntryKind::Metadata => Ok(()), } - self.access - .with_file_key(id, operation) - .ok_or(PithosError::SnapshotContentUnavailable(id.0)) } } @@ -364,7 +365,7 @@ mod tests { let (bytes, _sender, recipient) = archive_with_entries(); let snapshot = consume(open(bytes, AccessKeys::new().with_key(recipient))); - assert!(snapshot.with_file_key(FileId(0), |_| ()).is_ok()); + assert!(snapshot.with_grant_keys(FileId(0), |_| ()).is_ok()); } #[test] @@ -397,11 +398,11 @@ mod tests { )); assert!(snapshot.ensure_id_available(FileId(3)).is_ok()); assert!(matches!( - snapshot.with_file_key(FileId(1), |_| ()), + snapshot.with_grant_keys(FileId(1), |_| ()), Err(PithosError::SnapshotDirectoryHasNoContent(1)) )); assert!(matches!( - snapshot.with_file_key(FileId(2), |_| ()), + snapshot.with_grant_keys(FileId(2), |_| ()), Err(PithosError::SnapshotSymlinkHasNoContent(2)) )); } @@ -484,11 +485,11 @@ mod tests { let (bytes, sender, recipient) = archive_with_entries(); let sender_key = open(bytes.clone(), AccessKeys::new().with_key(sender)) .into_append_snapshot() - .with_file_key(FileId(0), |key| *key.expose_for_protocol()) + .with_grant_keys(FileId(0), |keys| *keys[0].1.expose_for_protocol()) .unwrap(); let recipient_key = open(bytes, AccessKeys::new().with_key(recipient)) .into_append_snapshot() - .with_file_key(FileId(0), |key| *key.expose_for_protocol()) + .with_grant_keys(FileId(0), |keys| *keys[0].1.expose_for_protocol()) .unwrap(); assert_eq!(sender_key, recipient_key); } @@ -501,15 +502,15 @@ mod tests { open(bytes, AccessKeys::new().with_key(private("recipient2"))).into_append_snapshot(); assert!(matches!( - unavailable.with_file_key(FileId(99), |_| ()), + unavailable.with_grant_keys(FileId(99), |_| ()), Err(PithosError::SnapshotFileIdNotFound(99)) )); assert!(matches!( - unavailable.with_file_key(FileId(0), |_| ()), + unavailable.with_grant_keys(FileId(0), |_| ()), Err(PithosError::SnapshotContentUnavailable(0)) )); assert!(matches!( - wrong.with_file_key(FileId(0), |_| ()), + wrong.with_grant_keys(FileId(0), |_| ()), Err(PithosError::SnapshotContentUnavailable(0)) )); } diff --git a/crates/pithos_lib/src/archive/writer.rs b/crates/pithos_lib/src/archive/writer.rs index 100801c..874240d 100644 --- a/crates/pithos_lib/src/archive/writer.rs +++ b/crates/pithos_lib/src/archive/writer.rs @@ -797,19 +797,22 @@ impl ArchiveWriter { .append_snapshot .as_ref() .ok_or(PithosError::GrantRequiresAppendSnapshot)?; + let mut requested_ids = HashSet::with_capacity(ids.len()); let mut granted_ids = HashSet::with_capacity(ids.len()); for id in ids { - if !granted_ids.insert(id.0) { + if !requested_ids.insert(id.0) { return Err(PithosError::DuplicateRecipientFileId); } - snapshot.with_file_key(*id, |_| ())?; + snapshot.with_grant_keys(*id, |keys| { + granted_ids.extend(keys.iter().map(|(key_id, _)| key_id.0)); + })?; } for section in self.directory.encryption.values_mut() { for recipient in section.recipients.values_mut() { let RecipientData::Decrypted(records) = &mut recipient.recipient_data else { return Err(PithosError::WriterUnsealedRecipientList); }; - reserve(records, ids.len(), "recipient access records")?; + reserve(records, granted_ids.len(), "recipient access records")?; } } let mut recipients = self @@ -819,13 +822,16 @@ impl ArchiveWriter { .flat_map(|section| section.recipients.values_mut()) .collect::>(); for id in ids { - snapshot.with_file_key(*id, |key| { - let record_key = Zeroizing::new(*key.expose_for_protocol()); - for recipient in &mut recipients { - let RecipientData::Decrypted(records) = &mut recipient.recipient_data else { - unreachable!("recipient lists were validated before grant publication"); - }; - records.push((id.0, *record_key)); + snapshot.with_grant_keys(*id, |keys| { + for (key_id, key) in keys { + let record_key = Zeroizing::new(*key.expose_for_protocol()); + for recipient in &mut recipients { + let RecipientData::Decrypted(records) = &mut recipient.recipient_data + else { + unreachable!("recipient lists were validated before grant publication"); + }; + records.push((key_id.0, *record_key)); + } } })?; } diff --git a/crates/pithos_lib/src/fs/append.rs b/crates/pithos_lib/src/fs/append.rs index 9312615..b2e43d5 100644 --- a/crates/pithos_lib/src/fs/append.rs +++ b/crates/pithos_lib/src/fs/append.rs @@ -356,7 +356,7 @@ fn validate_grant_ids(snapshot: &AppendSnapshot, ids: &[u64]) -> Result<(), Pith if !unique.insert(*id) { return Err(PithosError::DuplicateRecipientFileId); } - snapshot.with_file_key(FileId(*id), |_| ())?; + snapshot.with_grant_keys(FileId(*id), |_| ())?; } Ok(()) } diff --git a/crates/pithos_lib/tests/pieces.rs b/crates/pithos_lib/tests/pieces.rs index 3a6f326..1595f99 100644 --- a/crates/pithos_lib/tests/pieces.rs +++ b/crates/pithos_lib/tests/pieces.rs @@ -2,8 +2,8 @@ mod common; use common::util::{private_key, public_key}; use pithos_lib::archive::{ - AccessKeys, Archive, ArchivePath, EntryKind, EntryMetadata, OpenOptions, Piece, PieceEncoder, - ProcessingOptions, compose, + AccessKeys, AppendOptions, Archive, ArchivePath, EntryKind, EntryMetadata, OpenOptions, Piece, + PieceEncoder, ProcessingOptions, compose, }; use pithos_lib::error::{DeserializationError, PithosError}; use pithos_lib::source::MemorySource; @@ -212,3 +212,27 @@ fn piece_records_round_trip_and_reject_damage() { )); } } + +#[test] +fn a_reader_grant_for_a_composed_file_carries_every_piece_key() { + let first = content(4, 1200); + let second = content(5, 800); + let parts = vec![encode(2, &first), encode(3, &second)]; + let temporary = tempfile::tempdir().unwrap(); + let path = temporary.path().join("composed.pith"); + std::fs::write(&path, assemble(&parts)).unwrap(); + + pithos_lib::fs::grant_readers( + &path, + AppendOptions::new(private_key("recipient1"), vec![public_key("recipient2")]), + &[0], + ) + .unwrap(); + let archive = open( + std::fs::read(&path).unwrap(), + AccessKeys::new().with_key(private_key("recipient2")), + ); + let mut output = Vec::new(); + archive.copy_to("object", &mut output).unwrap(); + assert_eq!(output, [first, second].concat()); +} diff --git a/spec/PITHOS_1.0.0_draft.md b/spec/PITHOS_1.0.0_draft.md index f931dfd..5acd3ee 100644 --- a/spec/PITHOS_1.0.0_draft.md +++ b/spec/PITHOS_1.0.0_draft.md @@ -767,7 +767,8 @@ count followed by that many ULEB128 `file_id` and `file_key[32]` pairs. A repeated file ID is valid only when it has the same file key; readers MUST reject a conflicting duplicate file ID and file key. In version 1.1 the `file_id` field may also hold a piece key ID; its `file_key` is then that -piece's key. +piece's key. Access to a file sealed in pieces therefore needs one record +for each of its piece key IDs. ## 5. Content Processing From 78c05b5891af816ce39b95aec715135920d22e08 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 14:49:56 +0200 Subject: [PATCH 056/166] feat: make the RO-Crate and Crypt4GH adapters optional features --- crates/pithos_lib/Cargo.toml | 50 +++++++++++++++++-- crates/pithos_lib/README.md | 5 ++ crates/pithos_lib/src/adapters/mod.rs | 2 + crates/pithos_lib/src/archive/access.rs | 1 + crates/pithos_lib/src/archive/mod.rs | 1 + crates/pithos_lib/src/archive/reader.rs | 31 +++++++++--- .../src/archive/reader_private_tests.rs | 33 +++++++----- crates/pithos_lib/src/crypto.rs | 1 + crates/pithos_lib/src/lib.rs | 1 + crates/pithos_lib/tests/common/mod.rs | 1 + crates/pithos_lib/tests/pieces.rs | 19 +++++++ 11 files changed, 121 insertions(+), 24 deletions(-) diff --git a/crates/pithos_lib/Cargo.toml b/crates/pithos_lib/Cargo.toml index 0081877..5ad7e3d 100644 --- a/crates/pithos_lib/Cargo.toml +++ b/crates/pithos_lib/Cargo.toml @@ -23,12 +23,17 @@ autobenches = false bench = false [features] -fuzzing = [] +default = ["ro-crate", "crypt4gh"] +# RO-Crate directory and ZIP conversion. +ro-crate = ["dep:ro-crate-rs", "dep:zip"] +# Crypt4GH export. +crypt4gh = ["dep:blake2"] +fuzzing = ["crypt4gh"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -blake2 = "0.10.6" +blake2 = { version = "0.10.6", optional = true } blake3 = "1.8.2" byteorder = "1.5.0" chacha20poly1305 = "0.11.0" @@ -41,14 +46,14 @@ indexmap = "2.11.0" integer-encoding = "4.1.0" libc = "0.2.178" pkcs8 = { version = "0.11.0", features = ["pem", "alloc"] } -ro-crate-rs = { version = "0.5.1", default-features = false } +ro-crate-rs = { version = "0.5.1", default-features = false, optional = true } rustix = { version = "1.1.4", features = ["fs"] } sha2 = "0.11.0" shake = "0.1.0" thiserror = { workspace = true } tracing = { workspace = true } x25519-dalek = { workspace = true } -zip = { version = "8.6.0", default-features = false, features = ["deflate", "time"] } +zip = { version = "8.6.0", default-features = false, features = ["deflate", "time"], optional = true } zstd = "0.13.3" zeroize = "1.8.2" @@ -78,3 +83,40 @@ harness = false [[bench]] name = "adapters" harness = false +required-features = ["ro-crate", "crypt4gh"] + +[[test]] +name = "crypt4gh_export" +required-features = ["crypt4gh"] + +[[test]] +name = "reader" +required-features = ["crypt4gh"] + +[[test]] +name = "public_api" +required-features = ["ro-crate", "crypt4gh"] + +[[test]] +name = "ro_crate_conversion" +required-features = ["ro-crate"] + +[[test]] +name = "ro_crate_directory" +required-features = ["ro-crate"] + +[[test]] +name = "ro_crate_zip" +required-features = ["ro-crate"] + +[[example]] +name = "crypt4gh" +required-features = ["crypt4gh"] + +[[example]] +name = "ro_crate" +required-features = ["ro-crate"] + +[[test]] +name = "compile_fail" +required-features = ["crypt4gh"] diff --git a/crates/pithos_lib/README.md b/crates/pithos_lib/README.md index a58b4f5..55c529a 100644 --- a/crates/pithos_lib/README.md +++ b/crates/pithos_lib/README.md @@ -11,6 +11,10 @@ Add the crate to an application using Rust 1.88 or newer: pithos_lib = "0.8" ``` +The default features `ro-crate` and `crypt4gh` enable the RO-Crate conversion and Crypt4GH +export adapters. Use `default-features = false` to build only the archive format, reader, +writer and Linux filesystem operations. + ## Create an archive A base archive needs no keys and accepts only local, uncompressed, unencrypted content: @@ -71,6 +75,7 @@ See [`examples/create.rs`](examples/create.rs) for the maintained version and th - Read a complete entry with `copy_to` or a checked range with `copy_range_to`. - Use `fs::extract`, `fs::append_files`, `fs::grant_readers`, and `fs::ingest` for Linux host operations. - Use `adapters::ro_crate` for local directory/ZIP conversion and `adapters::crypt4gh::export` for Crypt4GH output. +- Seal parts of one file independently with `archive::PieceEncoder` and join the stored parts later with `archive::compose`, which opens no key. Keep `Composition::metadata_digest` in trusted storage and pass it to `OpenOptions::with_expected_metadata_digest`. Compiled examples are included with the package: [`create`](examples/create.rs), [`open_list`](examples/open_list.rs), [`read_ranges`](examples/read_ranges.rs), [`extract`](examples/extract.rs), [`append`](examples/append.rs), [`grant_readers`](examples/grant_readers.rs), [`ro_crate`](examples/ro_crate.rs), and [`crypt4gh`](examples/crypt4gh.rs). They are local-only and return errors to their caller. diff --git a/crates/pithos_lib/src/adapters/mod.rs b/crates/pithos_lib/src/adapters/mod.rs index 2af7891..93e83ea 100644 --- a/crates/pithos_lib/src/adapters/mod.rs +++ b/crates/pithos_lib/src/adapters/mod.rs @@ -4,5 +4,7 @@ //! context. Core archive, format, block, and crypto code do not own RO-Crate, ZIP, //! Crypt4GH, or host filesystem error types. +#[cfg(feature = "crypt4gh")] pub mod crypt4gh; +#[cfg(feature = "ro-crate")] pub mod ro_crate; diff --git a/crates/pithos_lib/src/archive/access.rs b/crates/pithos_lib/src/archive/access.rs index 742fd74..a79bcfd 100644 --- a/crates/pithos_lib/src/archive/access.rs +++ b/crates/pithos_lib/src/archive/access.rs @@ -59,6 +59,7 @@ impl ResolvedAccess { self.keys.get(&id) } + #[cfg(feature = "crypt4gh")] pub(crate) fn provenance(&self, id: FileId) -> Option { self.provenance.get(&id).copied() } diff --git a/crates/pithos_lib/src/archive/mod.rs b/crates/pithos_lib/src/archive/mod.rs index 7978637..4334598 100644 --- a/crates/pithos_lib/src/archive/mod.rs +++ b/crates/pithos_lib/src/archive/mod.rs @@ -25,6 +25,7 @@ pub(crate) use path_validation::{ validate_directory_entries, validate_new_candidate, validate_symlink_target, }; pub use pieces::{Composition, Piece, PieceEncoder, compose}; +#[cfg(feature = "crypt4gh")] pub(crate) use reader::ContentOperationError; pub use reader::{ AccessKeys, Archive, ArchiveEntry, ArchiveFeature, ArchiveReference, EntryKind, diff --git a/crates/pithos_lib/src/archive/reader.rs b/crates/pithos_lib/src/archive/reader.rs index 44ee6c5..2e32606 100644 --- a/crates/pithos_lib/src/archive/reader.rs +++ b/crates/pithos_lib/src/archive/reader.rs @@ -9,7 +9,9 @@ use crate::archive::types::{ }; use crate::archive::validation::IndexLimits; use crate::block; -use crate::crypto::{self, FileKey, PrivateKey}; +#[cfg(feature = "crypt4gh")] +use crate::crypto::FileKey; +use crate::crypto::{self, PrivateKey}; use crate::error::PithosError; use crate::format::block::{ BlockIndexEntry, BlockLocation as FormatBlockLocation, ProcessingFlags, @@ -26,6 +28,7 @@ use std::sync::Arc; /// Distinguishes archive failures from a presentation callback failure without /// making the archive core depend on the callback's error type. +#[cfg(feature = "crypt4gh")] pub(crate) enum ContentOperationError { Core(PithosError), Callback(E), @@ -285,6 +288,7 @@ pub struct Archive { segments: Vec, index_limits: IndexLimits, access: ResolvedAccess, + #[cfg(feature = "crypt4gh")] access_keys: AccessKeys, limits: OpenLimits, content_availability: BTreeMap, @@ -447,6 +451,7 @@ where segments, index_limits, access, + #[cfg(feature = "crypt4gh")] access_keys: options.keys, limits: options.limits, content_availability, @@ -514,6 +519,7 @@ where self.copy_plan(id, self.index.range_plan(id, range)?, sink) } + #[cfg(feature = "crypt4gh")] pub(crate) fn with_crypt4gh_content( &self, path: &str, @@ -522,14 +528,24 @@ where let (id, _) = self.content_id(path).map_err(ContentOperationError::Core)?; self.require_content_available(id) .map_err(ContentOperationError::Core)?; - let key = self - .access - .key(id) - .ok_or(PithosError::ContentUnavailable) - .map_err(ContentOperationError::Core)?; + let fresh_key; + let (key, key_owner) = match self.access.key(id) { + Some(key) => (key, id), + None => { + // A file sealed in pieces has no file key, so the export gets a fresh one. + let first_piece = self + .access + .grant_keys(id) + .and_then(|keys| keys.first().map(|(key_id, _)| *key_id)) + .ok_or(PithosError::ContentUnavailable) + .map_err(ContentOperationError::Core)?; + fresh_key = FileKey::from_bytes(x25519_dalek::StaticSecret::random().to_bytes()); + (&fresh_key, first_piece) + } + }; let provenance = self .access - .provenance(id) + .provenance(key_owner) .ok_or(PithosError::ContentUnavailable) .map_err(ContentOperationError::Core)?; let reader = self @@ -541,6 +557,7 @@ where operation(id, reader, key).map_err(ContentOperationError::Callback) } + #[cfg(feature = "crypt4gh")] pub(crate) fn for_each_verified_file_block( &self, id: FileId, diff --git a/crates/pithos_lib/src/archive/reader_private_tests.rs b/crates/pithos_lib/src/archive/reader_private_tests.rs index 6ba3616..38b05d3 100644 --- a/crates/pithos_lib/src/archive/reader_private_tests.rs +++ b/crates/pithos_lib/src/archive/reader_private_tests.rs @@ -1223,12 +1223,15 @@ fn archive_external_blocks_validate_exact_framing_and_share_read_paths() { archive.copy_range_to("data", 1..4, &mut range).unwrap(); let output = temporary.path().join("external-output"); extract(&archive, "data", &output).unwrap(); - let mut crypt4gh = Vec::new(); - crypt4gh::export(&archive, "data", vec![public("recipient2")], &mut crypt4gh).unwrap(); assert_eq!(std::fs::read(output.join("data")).unwrap(), full); assert_eq!(range, &full[1..4]); - assert!(!crypt4gh.is_empty()); - assert_eq!(calls.load(Ordering::Relaxed), 4); + #[cfg(feature = "crypt4gh")] + { + let mut crypt4gh = Vec::new(); + crypt4gh::export(&archive, "data", vec![public("recipient2")], &mut crypt4gh).unwrap(); + assert!(!crypt4gh.is_empty()); + assert_eq!(calls.load(Ordering::Relaxed), 4); + } let (expected_len, policy) = expected.lock().unwrap()[0]; assert_eq!(expected_len, response.len() as u64); assert_eq!(policy, OpenLimits::default().max_stored_block_bytes + 4); @@ -1522,16 +1525,20 @@ fn unsupported_external_content_preflights_the_whole_file_and_adapters() { )); assert!(sink.0.is_empty()); - let mut exported = Vec::new(); - assert!(matches!( - crypt4gh::export(&archive, "data", vec![public("recipient2")], &mut exported,), - Err(crate::adapters::crypt4gh::Crypt4GHError::Archive { - source: PithosError::UnsupportedFeature(ArchiveFeature::ExternalStorage), - .. - }) - )); - assert!(exported.is_empty()); + #[cfg(feature = "crypt4gh")] + { + let mut exported = Vec::new(); + assert!(matches!( + crypt4gh::export(&archive, "data", vec![public("recipient2")], &mut exported,), + Err(crate::adapters::crypt4gh::Crypt4GHError::Archive { + source: PithosError::UnsupportedFeature(ArchiveFeature::ExternalStorage), + .. + }) + )); + assert!(exported.is_empty()); + } } +#[cfg(feature = "crypt4gh")] use crate::adapters::crypt4gh; use crate::fs::extract; diff --git a/crates/pithos_lib/src/crypto.rs b/crates/pithos_lib/src/crypto.rs index f75bf40..65a15d8 100644 --- a/crates/pithos_lib/src/crypto.rs +++ b/crates/pithos_lib/src/crypto.rs @@ -362,6 +362,7 @@ pub(crate) fn random_nonce() -> [u8; 12] { Nonce::generate().into() } +#[cfg(feature = "crypt4gh")] pub(crate) fn seal_crypt4gh_payload( key: &[u8; 32], plaintext: &[u8], diff --git a/crates/pithos_lib/src/lib.rs b/crates/pithos_lib/src/lib.rs index 9b2225b..15673a2 100644 --- a/crates/pithos_lib/src/lib.rs +++ b/crates/pithos_lib/src/lib.rs @@ -35,6 +35,7 @@ //! use pithos_lib::format::header::FileHeader; //! ``` +#[cfg(any(feature = "ro-crate", feature = "crypt4gh"))] pub mod adapters; pub mod archive; mod block; diff --git a/crates/pithos_lib/tests/common/mod.rs b/crates/pithos_lib/tests/common/mod.rs index ada0e4a..b8e0fa8 100644 --- a/crates/pithos_lib/tests/common/mod.rs +++ b/crates/pithos_lib/tests/common/mod.rs @@ -2,6 +2,7 @@ pub mod append; pub mod archive; pub mod fixtures; pub mod keys; +#[cfg(feature = "ro-crate")] pub mod ro_crate; pub mod writer; diff --git a/crates/pithos_lib/tests/pieces.rs b/crates/pithos_lib/tests/pieces.rs index 1595f99..cc9c82b 100644 --- a/crates/pithos_lib/tests/pieces.rs +++ b/crates/pithos_lib/tests/pieces.rs @@ -236,3 +236,22 @@ fn a_reader_grant_for_a_composed_file_carries_every_piece_key() { archive.copy_to("object", &mut output).unwrap(); assert_eq!(output, [first, second].concat()); } + +#[cfg(feature = "crypt4gh")] +#[test] +fn a_composed_file_exports_to_crypt4gh_with_a_fresh_data_key() { + let parts = vec![encode(1, &content(6, 900)), encode(2, &content(7, 600))]; + let archive = open(assemble(&parts), recipient()); + let mut exported = Vec::new(); + pithos_lib::adapters::crypt4gh::export( + &archive, + "object", + vec![public_key("recipient2")], + &mut exported, + ) + .unwrap(); + assert!(exported.starts_with(b"crypt4gh")); + // One recipient packet, then one segment: nonce, 1500 ciphertext bytes and a tag. + let header_len = 16 + u32::from_le_bytes(exported[16..20].try_into().unwrap()) as usize; + assert_eq!(exported.len(), header_len + 12 + 1500 + 16); +} From 2f4b7af02e0ad3d22103eb990ce3794e85895d0e Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 14:57:12 +0200 Subject: [PATCH 057/166] test: add a version 1.1 conformance vector for sealed pieces --- crates/pithos_lib/tests/draft_spec_vectors.rs | 122 +++++++++++++++++- spec/PITHOS_1.0.0_draft.md | 84 ++++++++++++ 2 files changed, 203 insertions(+), 3 deletions(-) diff --git a/crates/pithos_lib/tests/draft_spec_vectors.rs b/crates/pithos_lib/tests/draft_spec_vectors.rs index 319545e..8f48762 100644 --- a/crates/pithos_lib/tests/draft_spec_vectors.rs +++ b/crates/pithos_lib/tests/draft_spec_vectors.rs @@ -5,9 +5,9 @@ use chacha20poly1305::{ChaCha20Poly1305, KeyInit, Nonce, aead::Aead}; use crc32fast::hash as crc32; use pithos_lib::archive::{ - Archive, ArchivePath, ArchiveWriter, EntryKind, EntryMetadata, ExternalBlockAccessPolicy, - ExternalBlockResolver, ExternalLocation, OpenLimits, OpenOptions, ProcessingOptions, - WriteOptions, + AccessKeys, Archive, ArchivePath, ArchiveWriter, EntryKind, EntryMetadata, + ExternalBlockAccessPolicy, ExternalBlockResolver, ExternalLocation, OpenLimits, OpenOptions, + ProcessingOptions, WriteOptions, }; use pithos_lib::error::{DeserializationError, PithosError}; use pithos_lib::source::MemorySource; @@ -117,6 +117,122 @@ fn hello() -> Vec { bytes } +/// CV-PIECES-HELLO-766: "hello" and " world" sealed as pieces 1 and 2, granted to Bob. +fn pieces_hello() -> Vec { + let seal = |key: &[u8; 32], nonce: [u8; 12], plaintext: &[u8]| { + let mut sealed = nonce.to_vec(); + sealed.extend( + ChaCha20Poly1305::new_from_slice(key) + .unwrap() + .encrypt(&Nonce::from(nonce), plaintext) + .unwrap(), + ); + sealed + }; + let bob = PublicKey::from(hex( + "de9edb7d7b7dc1b4d35b61c2ece435373f8343c85b78674dadfc7e146f882b4f", + )); + let mut archive = b"PITH\x01\x01".to_vec(); + let (mut pieces, mut blocks, mut encryption) = (Vec::new(), Vec::new(), Vec::new()); + for (key_id, plain, seed) in [(1u8, b"hello" as &[u8], 1u8), (2, b" world", 2)] { + let hash = *blake3::hash(plain).as_bytes(); + let block_key = shake256(plain); + let stored = seal(&block_key, [0x10 * seed; 12], plain); + blocks.extend_from_slice(&hash); + uleb(archive.len() as u64, &mut blocks); + uleb(stored.len() as u64, &mut blocks); + uleb(plain.len() as u64, &mut blocks); + blocks.extend_from_slice(&[0x08, 0]); // encrypted, uncompressed, local + archive.extend_from_slice(b"BLCK"); + archive.extend_from_slice(&stored); + + let piece_key = [seed; 32]; + let list = [&[1u8][..], &hash, &block_key].concat(); + let sealed = seal(&piece_key, [0x10 + seed; 12], &list); + pieces.push(key_id); + uleb(sealed.len() as u64, &mut pieces); + pieces.extend(sealed); + + let sender = StaticSecret::from([0x30 + seed; 32]); + let sender_public = PublicKey::from(&sender); + let nonce = [0x40 + seed; 12]; + let mut info = b"pithos 1.1 recipient grant".to_vec(); + info.extend_from_slice(sender_public.as_bytes()); + info.extend_from_slice(bob.as_bytes()); + let mut wrapping_key = [0; 32]; + hkdf::Hkdf::::new( + Some(nonce.as_slice()), + sender.diffie_hellman(&bob).as_bytes(), + ) + .expand(&info, &mut wrapping_key) + .unwrap(); + let wrapped = seal( + &wrapping_key, + nonce, + &[&[1, key_id][..], &piece_key].concat(), + ); + encryption.extend_from_slice(sender_public.as_bytes()); + encryption.push(1); + encryption.extend_from_slice(bob.as_bytes()); + encryption.push(0); + uleb(wrapped.len() as u64, &mut encryption); + encryption.extend(wrapped); + } + let mut directory = b"PITHOSDR\0\x01\0\x05hello\x01\x02\x02".to_vec(); + directory.extend(pieces); + directory.extend_from_slice(&[0, 0, 11, 0xa4, 0x03, 0, 0, 2]); + directory.extend(blocks); + directory.push(10); + for (id, name) in RELATIONS { + directory.push(id); + directory.push(name.len() as u8); + directory.extend_from_slice(name.as_bytes()); + } + directory.push(2); + directory.extend(encryption); + let length = directory.len() + 12; + directory.extend_from_slice(&(length as u64).to_be_bytes()); + directory.extend_from_slice(&crc32(&directory).to_be_bytes()); + archive.extend(directory); + archive +} + +#[test] +fn version_1_1_pieces_vector_is_rebuilt_and_read_through_its_grant() { + let bytes = pieces_hello(); + assert_eq!(bytes, appendix_hex("CV-PIECES-HELLO-766")); + + let mut der = vec![ + 0x30, 0x2e, 0x02, 0x01, 0x00, 0x30, 0x05, 0x06, 0x03, 0x2b, 0x65, 0x6e, 0x04, 0x22, 0x04, + 0x20, + ]; + der.extend_from_slice(&hex( + "5dab087e624a8a4b79e17f8b83800ee66f3bb1292618b6fd1c2f8b27ff88e0eb", + )); + let pem = pkcs8::Document::try_from(der.as_slice()) + .unwrap() + .to_pem("PRIVATE KEY", pkcs8::LineEnding::LF) + .unwrap(); + let bob = pithos_lib::crypto::parse_private_pem(pem.as_bytes()).unwrap(); + let archive = Archive::open( + MemorySource::new(bytes.clone()), + OpenOptions::default().with_access_keys(AccessKeys::new().with_key(bob)), + ) + .unwrap(); + let mut output = Vec::new(); + archive.copy_to("hello", &mut output).unwrap(); + assert_eq!(output, b"hello world"); + + let locked = Archive::open(MemorySource::new(bytes), OpenOptions::default()).unwrap(); + assert!(matches!( + locked.entry("hello").unwrap().unwrap().kind, + EntryKind::File { + size: 11, + available: false + } + )); +} + fn appendix_hex(id: &str) -> Vec { let start = SPEC.find(&format!("#### {id}")).expect("vector heading"); let text = &SPEC[start..]; diff --git a/spec/PITHOS_1.0.0_draft.md b/spec/PITHOS_1.0.0_draft.md index 5acd3ee..663b424 100644 --- a/spec/PITHOS_1.0.0_draft.md +++ b/spec/PITHOS_1.0.0_draft.md @@ -1034,6 +1034,12 @@ The format reserves space for future extensions: - FileType values 4-255 - ProcessingFlags bits 4-7 - Custom relationship types starting at 1000 +- BlockDataState tags `03` through `ff` + +A paged block index for very large files is a planned extension. It would let a +reader fetch only the descriptors and block keys a byte range needs. Version 1.1 +keeps every block list complete in its Directory, so writers of very large files +should use larger blocks to keep Directories small. Extensions MUST maintain backwards compatibility for reading. @@ -1225,6 +1231,83 @@ Its sole block plaintext is `68 65 6c 6c 6f`; its BLAKE3 hash is block key in the decrypted pair is SHAKE256(`hello`)[0..32], `1234075ae4a1e77316cf2d8000974581a343b9ebbca7e3d1db83394c30f22162`. +#### CV-PIECES-HELLO-766 + +| Item | Value | +| --- | --- | +| Purpose | Version 1.1 archive whose `Data` entry `hello` is sealed in two pieces, each granted to Bob | +| Required capability | Block encryption, encrypted block lists and encrypted recipient lists | +| Total length | 766 bytes | +| Expected result | Valid archive; with Bob's private key, read `hello` as ASCII `hello world`; without it, `hello` is listed with size 11 and is unavailable | + +Bob's keys are the RFC 7748 test keys of the recipient-wrap vectors below; his private +key is `5dab087e624a8a4b79e17f8b83800ee66f3bb1292618b6fd1c2f8b27ff88e0eb`. For +piece `n` (1 for `hello`, 2 for ` world`), the vector uses these fixed inputs: +block nonce `n0` repeated 12 times, piece key byte `0n` repeated 32 times, piece +list nonce `1n` repeated 12 times, sender private key byte `3n` repeated 32 +times, and grant nonce `4n` repeated 12 times. Block keys are convergent +(Section 5.3), and each grant record is `01 0n` followed by the piece key. + +```text +0000: 50 49 54 48 01 01 42 4c 43 4b 10 10 10 10 10 10 +0010: 10 10 10 10 10 10 8c 95 2e 75 aa 57 95 de 40 5c +0020: a4 29 e9 97 b9 ad 24 35 31 a8 cd 42 4c 43 4b 20 +0030: 20 20 20 20 20 20 20 20 20 20 20 39 5a 8e 4a 74 +0040: 68 8b bf 2a f6 2e 6d 11 92 a4 58 be e3 f4 5d 22 +0050: cb 50 49 54 48 4f 53 44 52 00 01 00 05 68 65 6c +0060: 6c 6f 01 02 02 01 5d 11 11 11 11 11 11 11 11 11 +0070: 11 11 11 b8 99 ae be f7 3e 3a 3b 8a 7e 15 a0 59 +0080: 49 88 7f a7 0a c7 21 ef 07 ac f9 30 80 6e 69 a6 +0090: 80 af 84 99 06 c0 11 e8 b7 d7 b7 99 d9 8a 6c 11 +00a0: a5 67 59 d9 1f 7c 8f 31 22 ce 10 31 e6 5a 22 c6 +00b0: 6f aa 66 d1 ae 1e a8 42 aa 3d cc 80 1f f4 59 83 +00c0: 6e 2e ec 6e 02 5d 12 12 12 12 12 12 12 12 12 12 +00d0: 12 12 a3 9a c0 1b 2b 0f 69 f5 be b3 61 ad af d5 +00e0: 7e dc 34 f6 82 9a ff f4 d5 7f 2e db fb 4d bf 85 +00f0: 7a ae 35 b6 7a 49 79 17 6a fa 8d 32 fa 15 27 a9 +0100: 12 a2 80 7f fc d9 56 c6 6a c4 3d a7 26 94 8b cc +0110: 3e 35 cf b0 81 0d b8 fe fc cf 8d 7d 92 68 83 25 +0120: fb 8c 59 00 00 0b a4 03 00 00 02 ea 8f 16 3d b3 +0130: 86 82 92 5e 44 91 c5 e5 8d 4b b3 50 6e f8 c1 4e +0140: b7 8a 86 e9 08 c5 62 4a 67 20 0f 06 21 05 08 00 +0150: cb 7a 20 2c 78 d1 fc 0b b6 98 a1 2b 4b 0c 95 13 +0160: d2 72 08 36 76 e9 55 3b 38 40 e8 93 66 71 e8 5a +0170: 2b 22 06 08 00 0a 00 09 44 45 53 43 52 49 42 45 +0180: 53 01 09 41 4e 4e 4f 54 41 54 45 53 02 0c 44 45 +0190: 52 49 56 45 44 5f 46 52 4f 4d 03 09 53 4f 55 52 +01a0: 43 45 5f 4f 46 04 10 50 52 45 56 49 4f 55 53 5f +01b0: 56 45 52 53 49 4f 4e 05 0c 4e 45 58 54 5f 56 45 +01c0: 52 53 49 4f 4e 06 07 50 41 52 54 5f 4f 46 07 08 +01d0: 43 4f 4e 54 41 49 4e 53 08 08 49 4e 50 55 54 5f +01e0: 54 4f 09 0b 4f 55 54 50 55 54 5f 46 52 4f 4d 02 +01f0: 04 f5 f2 91 62 c3 1a 8d ef a1 8e 6e 74 22 24 ee +0200: 80 6f c1 71 8a 27 8b e8 59 ba 56 20 40 2b 8f 3a +0210: 01 de 9e db 7d 7b 7d c1 b4 d3 5b 61 c2 ec e4 35 +0220: 37 3f 83 43 c8 5b 78 67 4d ad fc 7e 14 6f 88 2b +0230: 4f 00 3e 41 41 41 41 41 41 41 41 41 41 41 41 d5 +0240: 91 81 e5 13 df d5 0d b5 8d 13 41 09 e1 a3 4b ba +0250: 61 34 a5 d0 ec 4d 12 7b 5b a8 a1 04 24 87 1f 0c +0260: 18 9f 61 0a 13 c4 0b 0f 44 4b ee 26 66 e0 9a 81 +0270: b6 59 d9 22 54 73 45 1e ff fe 6b 36 db ca ef db +0280: f7 b1 89 5d e6 20 84 50 9a 7f 5b 58 bf 01 d0 64 +0290: 18 01 de 9e db 7d 7b 7d c1 b4 d3 5b 61 c2 ec e4 +02a0: 35 37 3f 83 43 c8 5b 78 67 4d ad fc 7e 14 6f 88 +02b0: 2b 4f 00 3e 42 42 42 42 42 42 42 42 42 42 42 42 +02c0: 33 72 35 6a b6 7a 39 7b 36 d7 c8 6e cf 5c 57 37 +02d0: b2 7d 9d 62 f2 ff c3 31 bc 55 32 b9 66 1f fa 09 +02e0: 9c 3d a9 dd 19 ee f5 a5 23 66 45 93 a2 10 aa 0c +02f0: ed c0 00 00 00 00 00 00 02 ad 3c f3 d9 39 +``` + +| Offset range | Field | Meaning | +| --- | --- | --- | +| `00..05` | header | Pithos 1.1, `PITH 01 01` | +| `06..2a` | first block | `BLCK` and the 33-byte sealed `hello` | +| `2b..50` | second block | `BLCK` and the 34-byte sealed ` world` | +| `51..2fd` | Directory | One file record with tag `02` and pieces 1 and 2, two encrypted block descriptors, the ten standard relationships, and two encryption sections | + +The Directory starts at `0x51` and `dir_len = 685`; `0x51 + 685 = 766`. + ### B.3 Processing Vectors | Vector ID | Stored bytes | `original_size` | Expected plaintext | BLAKE3 | @@ -1303,3 +1386,4 @@ archive. | Entries and paths | 4.3.3, 4.4.1, 4.4.3 | CV-LOCAL-HELLO-279, RV-DUPLICATES, RV-PATH, RV-SYMLINK, RV-FILETYPE, RV-PERMISSIONS | Valid archive or reject archive as stated | | Block locations and extents | 4.2.2, 4.2.5, 4.2.6, 8.2 | CV-LOCAL-HELLO-279, RV-EXTENT, RV-SHORT-ENCRYPTED, RV-EXTERNAL | Valid archive, reject archive, content read fails before output, or content unavailable as stated | | Content transforms and hashes | 5.2, 5.3, 5.4 | PV-ZSTD-HELLO, PV-ZSTD-TEXT, PV-RECIPIENT-WRAP-01, PV-RECIPIENT-WRAP-11 | Decode/decrypt to stated output | +| Version 1.1 pieces and grants | 1.3, 4.4.2, 5.3 | CV-PIECES-HELLO-766, PV-RECIPIENT-WRAP-11 | Valid archive; content readable only with the granted key | From 7e4e77f24181fa043a96378fc870a0501cad2cae Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 15:02:15 +0200 Subject: [PATCH 058/166] chore: add the grant key crates to the fuzz lockfile --- fuzz/Cargo.lock | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index 2fde604..c132b71 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -370,7 +370,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ "block-buffer 0.12.1", + "const-oid", "crypto-common 0.2.2", + "ctutils", ] [[package]] @@ -557,6 +559,24 @@ version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +[[package]] +name = "hkdf" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aaa26c720c68b866f2c96ef5c1264b3e6f473fe5d4ce61cd44bbe913e553018" +dependencies = [ + "hmac", +] + +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", +] + [[package]] name = "http" version = "1.4.2" @@ -1009,12 +1029,14 @@ dependencies = [ "crc32fast", "digest 0.11.3", "fastcdc", + "hkdf", "indexmap", "integer-encoding", "libc", "pkcs8", "ro-crate-rs", "rustix", + "sha2", "shake", "thiserror", "tracing", @@ -1266,6 +1288,17 @@ dependencies = [ "zmij", ] +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest 0.11.3", +] + [[package]] name = "shake" version = "0.1.0" From 2e8c278ad133ee01d9585eb374327eb9fb033870 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 15:45:23 +0200 Subject: [PATCH 059/166] fix: correct the grant nonce collision scope in the spec --- spec/PITHOS_1.0.0_draft.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/spec/PITHOS_1.0.0_draft.md b/spec/PITHOS_1.0.0_draft.md index 663b424..ab34eb5 100644 --- a/spec/PITHOS_1.0.0_draft.md +++ b/spec/PITHOS_1.0.0_draft.md @@ -849,7 +849,9 @@ encrypts the decrypted recipient list with ChaCha20-Poly1305: ones stored in the Directory: the EncryptionSection key and the recipient record key. -Each version 1.1 grant therefore has its own key, bound to both public keys. +Each version 1.1 grant key is therefore bound to both public keys and to the +grant nonce. Grants with different nonces have different keys; a repeated nonce +for the same key pair repeats the key (Section 7). Implementations MUST reject non-contributory X25519 public keys. An implementation MUST generate every nonce independently and uniformly at @@ -930,9 +932,13 @@ its AEAD key, with no KDF, label, or AAD. That construction has no domain separation, and each static sender-recipient key pair has one nonce-collision scope across all archives that use it. Version 1.1 derives every wrapping key with HKDF-SHA256 from the shared secret, the grant's random nonce and both public -keys. This adds domain separation and gives each grant its own key, so a nonce -can only collide within one grant. Appends to version 1.0 archives keep the -version 1.0 construction. The plaintext-derived Directory block hash exposes +keys. This adds domain separation and binds each wrapping key to its sender and +recipient. It does not shrink the nonce-collision scope: the nonce is the HKDF +salt, so a repeated grant nonce for the same sender and recipient key pair +repeats both the derived key and the AEAD nonce. Each key pair therefore still +has one nonce-collision scope across all archives that use it. A fresh sender +key for each archive or piece keeps that scope small. Appends to version 1.0 +archives keep the version 1.0 construction. The plaintext-derived Directory block hash exposes block equality, and equal plaintext also derives the same convergent block key. Pieces are joined in the order the Directory stores them. Their strictly From 61a0ef9359df3e925248728ae14f16aa8c2aeb3f Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 15:54:19 +0200 Subject: [PATCH 060/166] feat: add unique-key blocks to format version 1.1 --- crates/pithos_lib/src/archive/append.rs | 7 + crates/pithos_lib/src/archive/mod.rs | 38 +++++- crates/pithos_lib/src/archive/pieces.rs | 38 +++++- crates/pithos_lib/src/archive/reader.rs | 4 +- crates/pithos_lib/src/archive/types.rs | 24 +++- crates/pithos_lib/src/archive/validation.rs | 4 +- crates/pithos_lib/src/archive/writer.rs | 78 ++++++++++- crates/pithos_lib/src/block.rs | 121 +++++++++++++++++- crates/pithos_lib/src/crypto.rs | 16 ++- crates/pithos_lib/src/error.rs | 4 + crates/pithos_lib/src/format/block.rs | 18 ++- crates/pithos_lib/src/fs/append.rs | 4 + crates/pithos_lib/tests/draft_spec_vectors.rs | 29 ++++- spec/PITHOS_1.0.0_draft.md | 113 ++++++++++++---- 14 files changed, 452 insertions(+), 46 deletions(-) diff --git a/crates/pithos_lib/src/archive/append.rs b/crates/pithos_lib/src/archive/append.rs index 7bc611f..fd8519a 100644 --- a/crates/pithos_lib/src/archive/append.rs +++ b/crates/pithos_lib/src/archive/append.rs @@ -44,6 +44,13 @@ impl AppendOptions { self } + /// Processing for appended content. Options that need format version 1.1 are rejected + /// before any byte is written when the archive is version 1.0. + pub fn with_processing(mut self, processing: ProcessingOptions) -> Self { + self.processing = processing; + self + } + pub(crate) fn validate_recipients(&self) -> Result<(), PithosError> { WriteOptions::new(self.access_key.duplicate(), self.recipients.clone()).validate() } diff --git a/crates/pithos_lib/src/archive/mod.rs b/crates/pithos_lib/src/archive/mod.rs index 4334598..e952d8f 100644 --- a/crates/pithos_lib/src/archive/mod.rs +++ b/crates/pithos_lib/src/archive/mod.rs @@ -58,8 +58,8 @@ pub(crate) fn decode_validated_directory( ) } pub use writer::{ - ArchiveWriter, CdcConfig, CreateError, EntryMetadata, EntryReference, FinishError, - IncompleteWriter, ProcessingOptions, WriteOptions, WriterError, WrittenEntry, + ArchiveWriter, BlockKeyMode, CdcConfig, CreateError, EntryMetadata, EntryReference, + FinishError, IncompleteWriter, ProcessingOptions, WriteOptions, WriterError, WrittenEntry, }; #[cfg(test)] @@ -68,6 +68,8 @@ mod tests { use super::index::build_effective_index; use super::types::*; use super::validation::IndexLimits; + use crate::error::PithosError; + use crate::format::header::FormatVersion; use proptest::prelude::*; use std::sync::Arc; @@ -84,7 +86,7 @@ mod tests { BlockDescriptor { stored_size: size, original_size: size, - processing: Processing::from_byte(0).unwrap(), + processing: Processing::from_byte(0, FormatVersion::V1_1).unwrap(), location: BlockLocation::External(ExternalLocation::new("opaque")), } } @@ -93,7 +95,7 @@ mod tests { BlockDescriptor { stored_size: span.len().saturating_sub(4), original_size: 1, - processing: Processing::from_byte(0).unwrap(), + processing: Processing::from_byte(0, FormatVersion::V1_1).unwrap(), location: BlockLocation::Local(span), } } @@ -117,6 +119,34 @@ mod tests { } } + #[test] + fn processing_flags_follow_the_version_and_encryption_rules() { + for version in [FormatVersion::V1_0, FormatVersion::V1_1] { + assert_eq!( + Processing::from_byte(0x0b, version).unwrap().to_byte(), + 0x0b + ); + assert!(matches!( + Processing::from_byte(0x40, version), + Err(PithosError::ReservedProcessingBits(0x40)) + )); + } + assert_eq!( + Processing::from_byte(0x1b, FormatVersion::V1_1) + .unwrap() + .to_byte(), + 0x1b + ); + assert!(matches!( + Processing::from_byte(0x1b, FormatVersion::V1_0), + Err(PithosError::UnsupportedProcessingFlags(0x1b)) + )); + assert!(matches!( + Processing::from_byte(0x13, FormatVersion::V1_1), + Err(PithosError::ProcessingRequiresEncryption(0x13)) + )); + } + #[test] fn index_preserves_segment_entry_order_and_component_hierarchy() { let entries = vec![ diff --git a/crates/pithos_lib/src/archive/pieces.rs b/crates/pithos_lib/src/archive/pieces.rs index e78f951..75e0ccb 100644 --- a/crates/pithos_lib/src/archive/pieces.rs +++ b/crates/pithos_lib/src/archive/pieces.rs @@ -5,7 +5,7 @@ //! stored and later joined by [`compose`] into a version 1.1 archive with one data file. use crate::archive::path_validation::validate_entry; -use crate::archive::types::ArchivePath; +use crate::archive::types::{ArchivePath, Processing}; use crate::archive::writer::{EntryMetadata, ProcessingOptions}; use crate::block; use crate::crypto::{self, FileKey, PublicKey}; @@ -85,6 +85,7 @@ impl PieceEncoder { if recipients.is_empty() { return Err(PithosError::WriterRequiresRecipient); } + processing.validate_for(FormatVersion::V1_1)?; let recipients = recipients .into_iter() .map(|recipient| recipient.into_dalek_public_key().to_bytes()) @@ -109,7 +110,8 @@ impl PieceEncoder { } /// Encodes one non-empty block and returns `BLCK || payload` to append to the piece. - /// A block that repeats an earlier block of this piece returns no bytes. + /// With content-derived keys, a block that repeats an earlier block of this piece returns + /// no bytes. With unique keys, every block is stored. pub fn push(&mut self, plaintext: &[u8]) -> Result, PithosError> { if plaintext.is_empty() { return Err(PithosError::InvalidBlockDescriptor( @@ -252,7 +254,9 @@ impl Piece { original_size: read_uleb(&mut reader)?, flags: ProcessingFlags::from_byte(read_vec(&mut reader, 1)?[0]), }; - if block.offset != offset || block.flags.has_reserved_bits() { + if block.offset != offset + || Processing::from_byte(block.flags.0, FormatVersion::V1_1).is_err() + { return Err(PithosError::InvalidPieceRecord); } offset = checked_add(offset, checked_add(BLOCK_MARKER_LEN, block.stored_size)?)?; @@ -411,6 +415,7 @@ pub fn compose( location: BlockLocation::Local, }; // Equal hashes have equal convergent keys, so the first stored copy serves all. + // Unique-key hashes are keyed with random keys and do not repeat. match blocks.get(&block.hash) { Some(existing) if existing.original_size != entry.original_size => { return Err(PithosError::BlockIndexConflict { @@ -481,3 +486,30 @@ pub fn compose( directory: bytes, }) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::archive::BlockKeyMode; + + #[test] + fn piece_records_carry_unique_key_flags_and_reject_invalid_ones() { + let recipient = crate::crypto::PrivateKey::generate().public_key(); + let processing = ProcessingOptions::new(true, 0) + .unwrap() + .with_key_mode(BlockKeyMode::Unique) + .unwrap(); + let mut encoder = PieceEncoder::new(1, vec![recipient], processing).unwrap(); + encoder.push(b"unique block").unwrap(); + let mut piece = encoder.finish().unwrap(); + assert_eq!(piece.blocks[0].flags.0, 0x18); + assert_eq!(Piece::from_bytes(&piece.to_bytes()).unwrap(), piece); + for flags in [0x10, 0x40] { + piece.blocks[0].flags = ProcessingFlags(flags); + assert!(matches!( + Piece::from_bytes(&piece.to_bytes()), + Err(PithosError::InvalidPieceRecord) + )); + } + } +} diff --git a/crates/pithos_lib/src/archive/reader.rs b/crates/pithos_lib/src/archive/reader.rs index 2e32606..5ca6ef0 100644 --- a/crates/pithos_lib/src/archive/reader.rs +++ b/crates/pithos_lib/src/archive/reader.rs @@ -417,7 +417,9 @@ where let parent = segment_index .checked_sub(1) .map(|index| segments[index].span); - segments.push(validated_segment_from_directory(&directory, span, parent)?); + segments.push(validated_segment_from_directory( + version, &directory, span, parent, + )?); } let index_limits = IndexLimits { max_entries: options.limits.max_entries, diff --git a/crates/pithos_lib/src/archive/types.rs b/crates/pithos_lib/src/archive/types.rs index bb5068d..2b34d53 100644 --- a/crates/pithos_lib/src/archive/types.rs +++ b/crates/pithos_lib/src/archive/types.rs @@ -1,4 +1,6 @@ use crate::error::PithosError; +use crate::format::block::ProcessingFlags; +use crate::format::header::FormatVersion; use std::cmp::Ordering; use std::fmt::{Debug, Formatter}; use std::ops::Range; @@ -167,21 +169,35 @@ impl Compression { pub(crate) struct Processing { compression: Compression, encrypted: bool, + unique_key: bool, } impl Processing { - pub(crate) fn from_byte(value: u8) -> Result { - if value & 0xf0 != 0 { + /// Validates one flags byte against the rules of the archive's format version. + pub(crate) fn from_byte(value: u8, version: FormatVersion) -> Result { + if value & ProcessingFlags::RESERVED_MASK != 0 { return Err(PithosError::ReservedProcessingBits(value)); } + let encrypted = value & 0x08 != 0; + if value & ProcessingFlags::VERSION_1_1_MASK != 0 { + if version == FormatVersion::V1_0 { + return Err(PithosError::UnsupportedProcessingFlags(value)); + } + if !encrypted { + return Err(PithosError::ProcessingRequiresEncryption(value)); + } + } Ok(Self { compression: Compression::new(value & 0x07)?, - encrypted: value & 0x08 != 0, + encrypted, + unique_key: value & ProcessingFlags::UNIQUE_KEY_MASK != 0, }) } pub(crate) fn to_byte(self) -> u8 { - self.compression.level() | (u8::from(self.encrypted) << 3) + self.compression.level() + | (u8::from(self.encrypted) << 3) + | (u8::from(self.unique_key) << 4) } } diff --git a/crates/pithos_lib/src/archive/validation.rs b/crates/pithos_lib/src/archive/validation.rs index 924680b..1587669 100644 --- a/crates/pithos_lib/src/archive/validation.rs +++ b/crates/pithos_lib/src/archive/validation.rs @@ -7,6 +7,7 @@ use crate::error::PithosError; use crate::format::block::BlockLocation as FormatBlockLocation; use crate::format::directory::{Directory, STANDARD_RELATIONSHIPS}; use crate::format::file_entry::{BlockDataState, FileEntry, FileType}; +use crate::format::header::FormatVersion; use std::collections::{BTreeMap, HashSet}; use std::sync::Arc; @@ -32,6 +33,7 @@ impl Default for IndexLimits { } pub(crate) fn validated_segment_from_directory( + version: FormatVersion, directory: &Directory, span: Span, parent: Option, @@ -55,7 +57,7 @@ pub(crate) fn validated_segment_from_directory( validate_relationships(directory)?; let mut descriptors = Vec::new(); for (hash, block_index_entry) in &directory.blocks { - let processing = Processing::from_byte(block_index_entry.flags.0)?; + let processing = Processing::from_byte(block_index_entry.flags.0, version)?; if matches!(block_index_entry.location, FormatBlockLocation::Local) && processing.to_byte() & 0x08 != 0 && block_index_entry.stored_size < 28 diff --git a/crates/pithos_lib/src/archive/writer.rs b/crates/pithos_lib/src/archive/writer.rs index 874240d..3492d62 100644 --- a/crates/pithos_lib/src/archive/writer.rs +++ b/crates/pithos_lib/src/archive/writer.rs @@ -4,6 +4,7 @@ use crate::archive::path_validation::{ validate_directory_entry_hierarchy_complete, validate_directory_entry_hierarchy_with_snapshot, validate_new_candidate_with_snapshot, }; +use crate::archive::types::Processing; use crate::archive::validation::validate_relationships; use crate::archive::{AppendSnapshot, ArchivePath, FileId, Span, validated_segment_from_directory}; use crate::archive::{validate_new_candidate, validate_symlink_target}; @@ -77,11 +78,23 @@ impl Default for CdcConfig { } } +/// How the key of each encrypted block is chosen. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub enum BlockKeyMode { + /// The key is derived from the block plaintext, so equal blocks are stored once. + #[default] + ContentDerived, + /// Every stored block gets a fresh random key and a keyed block hash (version 1.1). + /// Equal blocks are stored again, and their equality stays hidden. + Unique, +} + /// Per-block processing requested for a content entry. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct ProcessingOptions { encrypted: bool, compression_level: u8, + key_mode: BlockKeyMode, } impl ProcessingOptions { @@ -89,6 +102,7 @@ impl ProcessingOptions { Self { encrypted: true, compression_level: 2, + key_mode: BlockKeyMode::ContentDerived, } } @@ -99,18 +113,37 @@ impl ProcessingOptions { Ok(Self { encrypted, compression_level, + key_mode: BlockKeyMode::ContentDerived, }) } + /// Selects the key mode. [`BlockKeyMode::Unique`] requires encryption, and writing it + /// requires a version 1.1 archive. + pub fn with_key_mode(self, key_mode: BlockKeyMode) -> Result { + let options = Self { key_mode, ..self }; + options.validate_for(FormatVersion::V1_1)?; + Ok(options) + } + pub fn encrypted(self) -> bool { self.encrypted } pub fn compression_level(self) -> u8 { self.compression_level } + pub fn key_mode(self) -> BlockKeyMode { + self.key_mode + } pub(crate) fn flags(self) -> ProcessingFlags { - ProcessingFlags::new(self.encrypted, Some(self.compression_level)) + let mut flags = ProcessingFlags::new(self.encrypted, Some(self.compression_level)); + flags.set_unique_key(self.key_mode == BlockKeyMode::Unique); + flags + } + + /// Checks that an archive of `version` may store blocks with these options. + pub(crate) fn validate_for(self, version: FormatVersion) -> Result<(), PithosError> { + Processing::from_byte(self.flags().0, version).map(|_| ()) } } @@ -119,6 +152,7 @@ impl Default for ProcessingOptions { Self { encrypted: true, compression_level: 3, + key_mode: BlockKeyMode::ContentDerived, } } } @@ -852,6 +886,7 @@ impl ArchiveWriter { if self.mode.is_base() && (processing.encrypted() || processing.compression_level() != 0) { return Err(PithosError::BaseWriterRequiresPlainProcessing.into()); } + processing.validate_for(self.version)?; let mut delta = self.stage_entry(file_type, path, metadata, 0, None)?; let mut stream = StreamCDC::with_level( content, @@ -1267,6 +1302,7 @@ impl ArchiveWriter { if let Some(snapshot) = &self.append_snapshot { let span = Span::new(self.sink.offset, self.directory.dir_len)?; let child = validated_segment_from_directory( + self.version, &self.directory, span, Some(snapshot.terminal_directory()), @@ -1643,6 +1679,46 @@ mod tests { } } + #[test] + fn version_1_0_writers_reject_unique_keys_before_writing_blocks() { + let mut writer = ArchiveWriter::create(Vec::new(), options()).unwrap(); + writer.version = FormatVersion::V1_0; + let unique = ProcessingOptions::default() + .with_key_mode(BlockKeyMode::Unique) + .unwrap(); + assert!(matches!( + writer.add_file( + ArchivePath::new("data").unwrap(), + EntryMetadata::new(0, 0, 0o644), + unique, + None, + Cursor::new(b"content"), + ), + Err(WriterError::Pithos( + PithosError::UnsupportedProcessingFlags(_) + )) + )); + assert!(!writer.poisoned); + assert_eq!(writer.sink.offset, header::FileHeader::ENCODED_LEN as u64); + assert_eq!(writer.metadata_snapshot().descriptors, 0); + } + + #[test] + fn unique_key_options_require_encryption() { + assert!(matches!( + ProcessingOptions::new(false, 0) + .unwrap() + .with_key_mode(BlockKeyMode::Unique), + Err(PithosError::ProcessingRequiresEncryption(0x10)) + )); + let unique = ProcessingOptions::new(true, 5) + .unwrap() + .with_key_mode(BlockKeyMode::Unique) + .unwrap(); + assert_eq!(unique.key_mode(), BlockKeyMode::Unique); + assert_eq!(unique.flags().0, 0x1d); + } + #[test] fn injected_transform_failure_poison_preserves_live_directory() { let mut writer = ArchiveWriter::with_test_runtime( diff --git a/crates/pithos_lib/src/block.rs b/crates/pithos_lib/src/block.rs index d2ae91f..beb4105 100644 --- a/crates/pithos_lib/src/block.rs +++ b/crates/pithos_lib/src/block.rs @@ -31,13 +31,30 @@ impl fmt::Debug for EncodedBlock { } } +/// Encodes one block. A unique-key block gets a fresh random key on every call. pub fn encode( plaintext: &[u8], requested_flags: ProcessingFlags, nonce: [u8; 12], ) -> Result { - let hash = crypto::block_hash(plaintext); - let key = crypto::derive_block_key(plaintext); + let key = if requested_flags.is_unique_key() { + crypto::random_block_key() + } else { + crypto::derive_block_key(plaintext) + }; + encode_with_key(plaintext, requested_flags, key, nonce) +} + +fn encode_with_key( + plaintext: &[u8], + requested_flags: ProcessingFlags, + key: BlockKey, + nonce: [u8; 12], +) -> Result { + if requested_flags.is_unique_key() && !requested_flags.is_encrypted() { + return Err(PithosError::ProcessingRequiresEncryption(requested_flags.0)); + } + let hash = block_hash(requested_flags, &key, plaintext); let mut flags = requested_flags; let compression_level = zstd_level(flags); let mut stored = Zeroizing::new( @@ -101,7 +118,7 @@ pub(crate) fn verify( actual: plaintext.len() as u64, }); } - let actual_hash = crypto::block_hash(&plaintext); + let actual_hash = block_hash(meta.flags, key, &plaintext); if actual_hash != expected_hash { return Err(PithosError::BlockHashMismatch { expected: expected_hash, @@ -111,6 +128,15 @@ pub(crate) fn verify( Ok(plaintext) } +/// The block hash: plain BLAKE3, or the keyed identity of a unique-key block. +fn block_hash(flags: ProcessingFlags, key: &BlockKey, plaintext: &[u8]) -> [u8; 32] { + if flags.is_unique_key() { + crypto::keyed_block_hash(key, plaintext) + } else { + crypto::block_hash(plaintext) + } +} + pub(crate) fn zstd_level(flags: ProcessingFlags) -> i32 { match flags.get_compression_level() { 0 => 0, @@ -190,6 +216,95 @@ mod tests { ); } + fn limits() -> Limits { + Limits { + max_stored_bytes: 1024, + max_decoded_bytes: 1024, + } + } + + fn unique_flags(compression_level: u8) -> ProcessingFlags { + let mut flags = ProcessingFlags::new(true, Some(compression_level)); + flags.set_unique_key(true); + flags + } + + #[test] + fn unique_key_vector_matches_the_specification() { + let key = BlockKey::from_bytes(std::array::from_fn(|index| index as u8)); + let nonce = std::array::from_fn(|index| 0xa0 + index as u8); + let encoded = encode_with_key(b"hello", unique_flags(0), key, nonce).unwrap(); + assert_eq!(encoded.flags.0, 0x18); + // PV-UNIQUE-KEY-HELLO in Appendix B. + assert_eq!( + blake3::Hash::from(encoded.hash).to_hex().as_str(), + "ca2bb927d0c0ac7196480fdf9c5101615bd8bc349663ffd9c3fef37228675a62" + ); + assert_eq!( + &encoded.stored[12..], + [ + 0x64, 0xce, 0x14, 0x33, 0x22, 0xbc, 0x47, 0xc9, 0xe5, 0xcc, 0xec, 0xe2, 0xdd, 0x29, + 0xf5, 0xe4, 0x22, 0xc2, 0xe7, 0xbc, 0xd2 + ] + ); + let meta = descriptor(&encoded, 5); + let plain = verify( + encoded.stored.clone(), + &encoded.key, + encoded.hash, + &meta, + limits(), + ) + .unwrap(); + assert_eq!(&*plain, b"hello"); + + // Without bit 4 the reader expects the plain BLAKE3 digest instead. + let mut convergent = meta.clone(); + convergent.flags.set_unique_key(false); + assert!(matches!( + verify( + encoded.stored, + &encoded.key, + encoded.hash, + &convergent, + limits() + ), + Err(PithosError::BlockHashMismatch { .. }) + )); + } + + #[test] + fn every_unique_key_encoding_has_a_fresh_key_and_identity() { + let plain = b"the same plaintext block"; + let first = encode(plain, unique_flags(3), [1; 12]).unwrap(); + let second = encode(plain, unique_flags(3), [1; 12]).unwrap(); + assert_ne!( + first.key.expose_for_protocol(), + second.key.expose_for_protocol() + ); + assert_ne!(first.hash, second.hash); + assert_ne!(first.hash, crypto::block_hash(plain)); + let meta = descriptor(&second, plain.len()); + assert!(matches!( + verify( + second.stored.clone(), + &first.key, + second.hash, + &meta, + limits() + ), + Err(PithosError::Crypt(_)) + )); + assert!(matches!( + verify(second.stored, &second.key, first.hash, &meta, limits()), + Err(PithosError::BlockHashMismatch { .. }) + )); + assert!(matches!( + encode(plain, ProcessingFlags(0x10), [0; 12]), + Err(PithosError::ProcessingRequiresEncryption(0x10)) + )); + } + #[test] fn corruption_never_returns_unverified_plaintext() { let plain = b"compressible ".repeat(1024); diff --git a/crates/pithos_lib/src/crypto.rs b/crates/pithos_lib/src/crypto.rs index 65a15d8..f5f3b88 100644 --- a/crates/pithos_lib/src/crypto.rs +++ b/crates/pithos_lib/src/crypto.rs @@ -6,7 +6,7 @@ use crate::format::header::FormatVersion; use chacha20poly1305::{ - ChaCha20Poly1305, KeyInit, Nonce, + ChaCha20Poly1305, Key, KeyInit, Nonce, aead::{Aead, Generate, Payload}, }; use digest::{ExtendableOutput, Update, XofReader}; @@ -313,6 +313,20 @@ pub(crate) fn block_hash(plaintext: &[u8]) -> [u8; 32] { *blake3::hash(plaintext).as_bytes() } +/// BLAKE3 key derivation context of the identity subkey of a unique block key. +const BLOCK_IDENTITY_CONTEXT: &str = "pithos 1.1 block identity"; + +/// A fresh random key for one unique-key block. +pub(crate) fn random_block_key() -> BlockKey { + BlockKey(Key::generate().into()) +} + +/// The keyed block hash of a unique-key block, keyed with a subkey of its block key. +pub(crate) fn keyed_block_hash(key: &BlockKey, plaintext: &[u8]) -> [u8; 32] { + let subkey = Zeroizing::new(blake3::derive_key(BLOCK_IDENTITY_CONTEXT, &key.0)); + *blake3::keyed_hash(&subkey, plaintext).as_bytes() +} + pub(crate) fn seal_block_with_nonce( key: &BlockKey, plaintext: &[u8], diff --git a/crates/pithos_lib/src/error.rs b/crates/pithos_lib/src/error.rs index 4c42c31..cd035a7 100644 --- a/crates/pithos_lib/src/error.rs +++ b/crates/pithos_lib/src/error.rs @@ -95,6 +95,10 @@ pub enum PithosError { DuplicateRecipientFileId, #[error("reserved processing bits are set: {0:#04x}")] ReservedProcessingBits(u8), + #[error("processing flags {0:#04x} require format version 1.1")] + UnsupportedProcessingFlags(u8), + #[error("processing flags {0:#04x} require encryption")] + ProcessingRequiresEncryption(u8), #[error("conflicting relationship definition for id {0}")] ConflictingRelationshipDefinition(u64), #[error("invalid relationship definition for id {id}: {reason}")] diff --git a/crates/pithos_lib/src/format/block.rs b/crates/pithos_lib/src/format/block.rs index 98e2e04..2b0e1de 100644 --- a/crates/pithos_lib/src/format/block.rs +++ b/crates/pithos_lib/src/format/block.rs @@ -21,7 +21,11 @@ pub struct ProcessingFlags(pub u8); impl ProcessingFlags { const COMPRESSION_MASK: u8 = 0b0000_0111; const ENCRYPTION_MASK: u8 = 0b0000_1000; - pub const RESERVED_MASK: u8 = 0b1111_0000; + /// Version 1.1: the block key is random and the block hash is a keyed identity. + pub const UNIQUE_KEY_MASK: u8 = 0b0001_0000; + /// Bits that are valid only in version 1.1 archives and only with encryption. + pub const VERSION_1_1_MASK: u8 = Self::UNIQUE_KEY_MASK; + pub const RESERVED_MASK: u8 = 0b1110_0000; pub fn new(encrypted: bool, compression_level: Option) -> Self { let mut flags = ProcessingFlags(0b0); @@ -59,6 +63,18 @@ impl ProcessingFlags { (self.0 & Self::ENCRYPTION_MASK) != 0 } + pub fn set_unique_key(&mut self, unique: bool) { + if unique { + self.0 |= Self::UNIQUE_KEY_MASK; + } else { + self.0 &= !Self::UNIQUE_KEY_MASK; + } + } + + pub fn is_unique_key(&self) -> bool { + (self.0 & Self::UNIQUE_KEY_MASK) != 0 + } + pub fn set_compression_level(&mut self, mut compression_level: u8) { compression_level = if compression_level > Self::COMPRESSION_MASK { Self::COMPRESSION_MASK diff --git a/crates/pithos_lib/src/fs/append.rs b/crates/pithos_lib/src/fs/append.rs index b2e43d5..aadfb4c 100644 --- a/crates/pithos_lib/src/fs/append.rs +++ b/crates/pithos_lib/src/fs/append.rs @@ -156,6 +156,10 @@ fn append_files_impl( Arc::clone(&bytes), ) .map_err(|source| append_error("open append snapshot", archive, source))?; + spec.options + .processing + .validate_for(snapshot.version()) + .map_err(|source| append_error("validate append processing", archive, source))?; let plan = AppendPlan::build( &spec, &snapshot, diff --git a/crates/pithos_lib/tests/draft_spec_vectors.rs b/crates/pithos_lib/tests/draft_spec_vectors.rs index 8f48762..5142948 100644 --- a/crates/pithos_lib/tests/draft_spec_vectors.rs +++ b/crates/pithos_lib/tests/draft_spec_vectors.rs @@ -666,7 +666,7 @@ fn processing_known_answers_match_appendix_b() { stored.extend_from_slice(&ciphertext); assert_eq!( stored, - appendix_recipient_wrap("PV-RECIPIENT-WRAP-01"), + appendix_text_block("PV-RECIPIENT-WRAP-01"), "recipient-wrap bytes must match Appendix B" ); @@ -695,11 +695,34 @@ fn processing_known_answers_match_appendix_b() { ); assert_eq!( stored, - appendix_recipient_wrap("PV-RECIPIENT-WRAP-11"), + appendix_text_block("PV-RECIPIENT-WRAP-11"), "version 1.1 recipient-wrap bytes must match Appendix B" ); } +#[test] +fn unique_key_block_known_answer_matches_appendix_b() { + let key: [u8; 32] = std::array::from_fn(|index| index as u8); + let nonce: [u8; 12] = std::array::from_fn(|index| 0xa0 + index as u8); + let subkey = blake3::derive_key("pithos 1.1 block identity", &key); + let identity = blake3::keyed_hash(&subkey, b"hello"); + for value in [blake3::Hash::from(subkey).to_hex(), identity.to_hex()] { + assert!( + SPEC.contains(value.as_str()), + "{value} missing from Appendix B" + ); + } + assert_ne!(identity, blake3::hash(b"hello")); + let mut stored = nonce.to_vec(); + stored.extend_from_slice( + &ChaCha20Poly1305::new_from_slice(&key) + .unwrap() + .encrypt(&Nonce::from(nonce), b"hello".as_ref()) + .unwrap(), + ); + assert_eq!(stored, appendix_text_block("PV-UNIQUE-KEY-HELLO")); +} + fn appendix_zstd_bytes(id: &str) -> Vec { let row = SPEC .lines() @@ -715,7 +738,7 @@ fn appendix_zstd_bytes(id: &str) -> Vec { .collect() } -fn appendix_recipient_wrap(id: &str) -> Vec { +fn appendix_text_block(id: &str) -> Vec { let start = SPEC.find(&format!("`{id}`")).unwrap(); let block = SPEC[start..].split("```text\n").nth(1).unwrap(); block diff --git a/spec/PITHOS_1.0.0_draft.md b/spec/PITHOS_1.0.0_draft.md index ab34eb5..6c934aa 100644 --- a/spec/PITHOS_1.0.0_draft.md +++ b/spec/PITHOS_1.0.0_draft.md @@ -69,6 +69,9 @@ Version 1.1 differs from version 1.0 only in these points: 3. A file's block list may be sealed in independent pieces, each with its own key (BlockDataState tag `02`, Section 4.4.2). Piece key IDs share the file ID space (Section 4.3.2). +4. An encrypted block may use a unique random key instead of its convergent key + (ProcessingFlags bit 4, Section 4.2.3). Its block hash is then a keyed BLAKE3 + identity (Section 5.2), and the block is never deduplicated (Section 5.3). Readers MUST support both versions. Writers MUST create new archives as version 1.1. An append MUST follow the version of the archive it extends, so appending @@ -214,7 +217,7 @@ The directory block sequence is a vector. Each item is encoded as follows: | `offset` | ULEB128 `u64`: local block offset, or zero for `External` | | `stored_size` | ULEB128 `u64` | | `original_size` | ULEB128 `u64` | -| `flags` | One ProcessingFlags byte: bits 0-2 compression level, bit 3 encryption enabled, bits 4-7 zero | +| `flags` | One ProcessingFlags byte as defined in Section 4.2.3 | | `location` | One tag byte: `00` local, or `01` followed by an external location identifier string | Readers MUST reject duplicate block hashes in one directory and MUST use the @@ -222,7 +225,8 @@ Readers MUST reject duplicate block hashes in one directory and MUST use the #### 4.2.3 Processing Flags -ProcessingFlags records the compression level and whether a block is encrypted. +ProcessingFlags records the compression level, whether a block is encrypted and, +in version 1.1, how an encrypted block is keyed. ```rust /// Processing flags packed into one byte @@ -242,7 +246,10 @@ bitflags::bitflags! { // Bit 3: Encryption enabled const ENCRYPTION_ENABLED = 0b0000_1000; - // Bits 4-7: Reserved for future use (MUST be zero) + // Bit 4: Unique random block key (version 1.1, requires bit 3) + const UNIQUE_KEY = 0b0001_0000; + + // Bits 5-7: Reserved for future use (MUST be zero) } } ``` @@ -253,10 +260,15 @@ bitflags::bitflags! { | --- | --- | | 0-2 | Compression: `0` means the stored payload is not compressed; `1` through `7` each mean the stored payload is one standard Zstandard frame | | 3 | Encryption enabled: `0` is disabled; `1` is enabled | -| 4-7 | Reserved; all bits MUST be zero | +| 4 | Unique key (version 1.1): `0` means the block key is convergent; `1` means it is a unique random key (Section 5.3) | +| 5-7 | Reserved; all bits MUST be zero | ProcessingFlags is stored as exactly one byte. Readers MUST reject a value with -any reserved bit set. +any reserved bit set. Bit 4 is valid only in a version 1.1 archive and only +when bit 3 is set. Readers MUST reject a descriptor that sets bit 4 in a version +1.0 archive or without bit 3. The same rules apply wherever a writer records +these flags. For example, an encrypted unique-key block with no compression has +the flags byte `18`. #### 4.2.4 Block Location @@ -779,10 +791,24 @@ not a compatibility requirement; see Appendix A. ### 5.2 Block Hashing -The block hash is the full 32-byte default unkeyed BLAKE3 digest of the exact -plaintext chunk before compression or encryption. It is a block's only -identity. The Directory `blocks` vector is keyed by block hash, and each file's -block list is an ordered sequence of `(block_hash, block_key)` pairs. That order +The block hash is a block's only identity. It is 32 bytes and covers the exact +plaintext chunk before compression or encryption. Its form depends on +ProcessingFlags bit 4 of the block's effective descriptor: + +- **Convergent key (bit 4 is `0`):** the full default unkeyed BLAKE3 digest of + the plaintext. +- **Unique key (bit 4 is `1`):** a keyed BLAKE3 identity. The 32-byte identity + subkey is BLAKE3 `derive_key` with the context string + `pithos 1.1 block identity` (25 ASCII bytes) and the block key as key + material. The block hash is BLAKE3 `keyed_hash` of the plaintext under the + identity subkey. Equal plaintext under different random keys has different + block hashes. + +A keyed block hash identifies one stored block. It is not a content hash of the +block or of its file and cannot be compared with a plain BLAKE3 digest. + +The Directory `blocks` vector is keyed by block hash, and each file's block list +is an ordered sequence of `(block_hash, block_key)` pairs. That order reconstructs the file. For a file sealed in pieces, the sequence is the concatenation of its piece lists, and every rule in this section applies to that whole sequence. @@ -802,8 +828,9 @@ empty file has `file_size` zero and an empty block list. Readers MUST retrieve each stored block, authenticate and decrypt it when encrypted, decompress it when compressed using the recorded original size as the output bound, require the resulting plaintext length to equal the recorded -original size, compute the complete plaintext digest, and compare it with the -block hash before releasing any output derived from that block. +original size, compute the block hash of the complete plaintext in the form +selected above, and compare it with the stored block hash before releasing any +output derived from that block. Readers MAY use bounded memory or temporary spill-backed storage while performing this verification, but MUST NOT release output derived from a block @@ -829,10 +856,19 @@ produces a 16-byte authentication tag. Every encrypted value is stored as `nonce || ciphertext || tag`; the nonce is part of the stored byte vector. The additional authenticated data (AAD) is empty. -The block key is the first 32 output bytes of `SHAKE256(plaintext)`, where -`plaintext` is the exact block plaintext before compression or encryption. No -label or length prefix is included. A block payload with encryption enabled is -encrypted with its block key. +By default the block key is convergent: the first 32 output bytes of +`SHAKE256(plaintext)`, where `plaintext` is the exact block plaintext before +compression or encryption. No label or length prefix is included. A block +payload with encryption enabled is encrypted with its block key. + +In version 1.1, a block whose ProcessingFlags bit 4 is set has a unique key +instead: 32 bytes generated uniformly at random with a cryptographically secure +random number generator. A writer MUST generate a fresh unique key for every +block it encodes in this mode, including a block whose plaintext it has stored +before, and MUST NOT use that key for any other block. Unique-key blocks are +therefore never deduplicated: each occurrence is stored with its own key and +block hash (Section 5.2). The block list carries the unique key like any other +block key. A file key is 32 random bytes. It encrypts the decrypted block list for a file. In version 1.1, a piece key is 32 random bytes and encrypts the decrypted block @@ -896,7 +932,7 @@ Zstandard versions. 1. Write file header 2. Process files in correct directory order 3. Chunk content using content-defined chunking -4. Deduplicate blocks by hash +4. Deduplicate blocks by hash (unique-key blocks never repeat a hash) 5. Write a directory, including its encryption sections when present 6. Validate complete structure @@ -941,6 +977,15 @@ key for each archive or piece keeps that scope small. Appends to version 1.0 archives keep the version 1.0 construction. The plaintext-derived Directory block hash exposes block equality, and equal plaintext also derives the same convergent block key. +Unique-key blocks (Section 5.3) hide one thing: whether two stored blocks have +equal plaintext, from anyone who does not hold the block key. Their block hashes +are keyed with random keys, so equal plaintext gives unrelated hashes in the +same or in different archives. Unique keys do not hide stored and original block +sizes, the number of blocks, the ProcessingFlags, or file sizes. A reader that +holds a block key can test a guessed plaintext against that block's hash. A whole-file content hash +kept outside the archive is separate from the keyed block identity and still +reveals equal files. + Pieces are joined in the order the Directory stores them. Their strictly increasing key IDs and the `file_size` check detect some reordering and missing pieces, but without authenticated metadata a modified Directory can still drop or @@ -972,14 +1017,15 @@ negotiation record. | --- | --- | | Block compression | ProcessingFlags compression bits are `1` through `7` | | Block encryption | ProcessingFlags encryption bit is `1` | +| Unique block keys | ProcessingFlags bit 4 is `1` (version 1.1) | | Encrypted block lists | BlockDataState tag is `00` (`Encrypted`) or, in version 1.1, `02` (`Pieces`) | | Encrypted recipient lists | RecipientData tag is `00` (`Encrypted`) | | External storage | BlockLocation tag is `01` (`External`) | -Compression, block encryption, encrypted block lists, encrypted recipient -lists, and external storage are optional capabilities. An implementation that -supports an optional capability MUST process it according to its definition in -this specification. +Compression, block encryption, unique block keys, encrypted block lists, +encrypted recipient lists, and external storage are optional capabilities. An +implementation that supports an optional capability MUST process it according +to its definition in this specification. ### 8.2 Unavailable Content @@ -998,6 +1044,7 @@ uncompressed or unencrypted bytes. | Plain local: local, compression `0`, encryption `0` | Listed | Readable by a base reader | | Compressed local: local, compression `1` through `7`, encryption `0` | Listed | Readable only with block-compression capability; otherwise unavailable | | Encrypted local: local, compression `0`, encryption `1` | Listed | Readable only with block-encryption capability and any encrypted-list capability needed to obtain its block key; otherwise unavailable | +| Unique-key local: local, encryption `1`, bit 4 `1` | Listed | Readable only with block-encryption and unique-block-key capability and any encrypted-list capability needed to obtain its block key; otherwise unavailable | | External: BlockLocation `External` | Listed | Readable only with external-storage capability and every capability required by its flags and data states; otherwise unavailable | | Combined: compression and encryption, at either location | Listed | Readable only when every indicated capability is supported; otherwise unavailable | @@ -1038,7 +1085,7 @@ Non-POSIX platforms MAY retain them as metadata without an ACL mapping. The format reserves space for future extensions: - FileType values 4-255 -- ProcessingFlags bits 4-7 +- ProcessingFlags bits 5-7 - Custom relationship types starting at 1000 - BlockDataState tags `03` through `ff` @@ -1087,8 +1134,9 @@ part of a file on its own, seal that part's block list as one piece under a fres piece key, and grant the piece key with a fresh sender key per piece. A later step can then join the parts into one archive without any key: it copies the stored blocks, keeps every sealed piece and grant unchanged, and writes one -Directory. Because block keys are convergent, a block that repeats in two parts -needs only one effective descriptor. +Directory. Because convergent block keys are equal for equal plaintext, a block +that repeats in two parts needs only one effective descriptor. Unique-key blocks +never repeat. ## Appendix B. Conformance Examples and Vectors @@ -1356,6 +1404,23 @@ stored `nonce || ciphertext || tag` is: 0030: 43 a0 94 bb cc 91 e1 c5 a6 53 62 9a d4 a1 ``` +`PV-UNIQUE-KEY-HELLO` is a version 1.1 known-answer test for one unique-key block +(Sections 5.2 and 5.3). The plaintext is ASCII `hello` and the flags byte is `18` +(no compression, encryption, unique key). The test inputs are block key bytes +`00` through `1f` and nonce bytes `a0` through `ab`; writers use random values. +The identity subkey is +`7523b18f1fdbfe99d23177669ef03ce0ca6925cfe372b8a26cc99804db196d6b`, and the +block hash is +`ca2bb927d0c0ac7196480fdf9c5101615bd8bc349663ffd9c3fef37228675a62`, not the +plain BLAKE3 digest of `hello`. The stored ChaCha20-Poly1305 +`nonce || ciphertext || tag` is: + +```text +0000: a0 a1 a2 a3 a4 a5 a6 a7 a8 a9 aa ab 64 ce 14 33 +0010: 22 bc 47 c9 e5 cc ec e2 dd 29 f5 e4 22 c2 e7 bc +0020: d2 +``` + ### B.4 Acceptance and Rejection Mutations Each `AV-*` or `RV-*` vector is a mutation of the named canonical vector. @@ -1391,5 +1456,5 @@ archive. | Directory framing, CRC, and chain | 4.3, 4.3.1, 4.3.2 | CV-BASE-EMPTY-146, CV-APPEND-EMPTY-28, RV-PARENT, RV-UNDERFLOW, RV-TRAILING, RV-CRC, RV-CROSS-SIZE | Valid archive or reject archive as stated | | Entries and paths | 4.3.3, 4.4.1, 4.4.3 | CV-LOCAL-HELLO-279, RV-DUPLICATES, RV-PATH, RV-SYMLINK, RV-FILETYPE, RV-PERMISSIONS | Valid archive or reject archive as stated | | Block locations and extents | 4.2.2, 4.2.5, 4.2.6, 8.2 | CV-LOCAL-HELLO-279, RV-EXTENT, RV-SHORT-ENCRYPTED, RV-EXTERNAL | Valid archive, reject archive, content read fails before output, or content unavailable as stated | -| Content transforms and hashes | 5.2, 5.3, 5.4 | PV-ZSTD-HELLO, PV-ZSTD-TEXT, PV-RECIPIENT-WRAP-01, PV-RECIPIENT-WRAP-11 | Decode/decrypt to stated output | +| Content transforms and hashes | 5.2, 5.3, 5.4 | PV-ZSTD-HELLO, PV-ZSTD-TEXT, PV-RECIPIENT-WRAP-01, PV-RECIPIENT-WRAP-11, PV-UNIQUE-KEY-HELLO | Decode/decrypt to stated output | | Version 1.1 pieces and grants | 1.3, 4.4.2, 5.3 | CV-PIECES-HELLO-766, PV-RECIPIENT-WRAP-11 | Valid archive; content readable only with the granted key | From 9ef1dedf8fd83549da97eaa0fd770dd30f4d6691 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 15:55:21 +0200 Subject: [PATCH 061/166] test: cover unique-key blocks in archives, pieces and appends --- .../src/archive/reader_private_tests.rs | 120 +++++++++++++++++- crates/pithos_lib/tests/append_workflow.rs | 64 +++++++++- crates/pithos_lib/tests/pieces.rs | 38 ++++-- 3 files changed, 211 insertions(+), 11 deletions(-) diff --git a/crates/pithos_lib/src/archive/reader_private_tests.rs b/crates/pithos_lib/src/archive/reader_private_tests.rs index 38b05d3..24775bb 100644 --- a/crates/pithos_lib/src/archive/reader_private_tests.rs +++ b/crates/pithos_lib/src/archive/reader_private_tests.rs @@ -3,7 +3,8 @@ use super::reader::{ ExternalBlockResolver, OpenLimits, OpenOptions, }; use crate::archive::{ - ArchivePath, ArchiveWriter, CdcConfig, EntryMetadata, ProcessingOptions, WriteOptions, + ArchivePath, ArchiveWriter, BlockKeyMode, CdcConfig, EntryMetadata, ProcessingOptions, + WriteOptions, }; use crate::crypto::{self, FileKey, PrivateKey, PublicKey}; use crate::error::PithosError; @@ -1720,3 +1721,120 @@ fn an_append_changes_the_metadata_digest() { Err(PithosError::MetadataDigestMismatch) )); } + +fn fixture_with_processing( + content: &[u8], + processing: ProcessingOptions, +) -> (tempfile::TempDir, PathBuf) { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("processing.pith"); + let options = WriteOptions::new(private("sender"), vec![public("recipient1")]) + .with_cdc(CdcConfig::new(64, 256, 1024).unwrap()); + let mut writer = ArchiveWriter::create(File::create(&path).unwrap(), options).unwrap(); + writer + .add_file( + ArchivePath::new("data").unwrap(), + EntryMetadata::new(0, 0, 0o644), + processing, + Some(content.len() as u64), + Cursor::new(content), + ) + .unwrap(); + writer.finish().unwrap(); + (temp, path) +} + +fn unique_processing() -> ProcessingOptions { + ProcessingOptions::new(true, 0) + .unwrap() + .with_key_mode(BlockKeyMode::Unique) + .unwrap() +} + +#[test] +fn unique_key_blocks_round_trip_and_store_every_repeat() { + // Zero bytes split into two equal 1024-byte chunks. + let content = vec![0u8; 2048]; + let (_convergent_dir, convergent) = + fixture_with_processing(&content, ProcessingOptions::new(true, 0).unwrap()); + assert_eq!(decode_terminal_directory(&convergent).blocks.len(), 1); + + let (_temporary, path) = fixture_with_processing(&content, unique_processing()); + let blocks = decode_terminal_directory(&path).blocks; + assert_eq!(blocks.len(), 2); + assert!(blocks.values().all(|block| block.flags.0 == 0x18)); + assert!( + !blocks + .keys() + .any(|hash| *hash == *blake3::hash(&content[..1024]).as_bytes()) + ); + let archive = open_path(&path, AccessKeys::new().with_key(private("recipient1"))); + let mut output = Vec::new(); + archive.copy_to("data", &mut output).unwrap(); + assert_eq!(output, content); +} + +#[test] +fn unique_key_flags_are_rejected_in_version_1_0_archives() { + for (processing, expected) in [ + (ProcessingOptions::new(true, 0).unwrap(), None), + (unique_processing(), Some(0x18)), + ] { + let (_temporary, path) = fixture_with_processing(b"version gated block", processing); + let mut legacy = std::fs::read(&path).unwrap(); + legacy[5] = 0x00; + let result = Archive::open(MemorySource::new(legacy), OpenOptions::default()); + match expected { + None => assert!(result.is_ok()), + Some(flags) => assert!(matches!( + result, + Err(PithosError::UnsupportedProcessingFlags(actual)) if actual == flags + )), + } + } +} + +#[test] +fn unique_key_flags_require_encryption() { + let (_temporary, path) = fixture_with_processing(b"plain unique block", unique_processing()); + rewrite_terminal_directory(&path, |directory| { + directory + .blocks + .first_mut() + .unwrap() + .1 + .flags + .set_encryption(false); + }); + assert!(matches!( + open_without_keys(&path), + Err(PithosError::ProcessingRequiresEncryption(0x10)) + )); +} + +#[test] +fn unique_key_blocks_verify_payload_and_identity_before_output() { + let content = b"unique key payload long enough for every corruption position"; + for byte in [0, 12, 40] { + let (_temporary, path) = fixture_with_processing(content, unique_processing()); + corrupt_payload(&path, byte); + assert_copy_failure_without_sink(&path); + } + let (_temporary, path) = fixture_with_processing(content, unique_processing()); + rewrite_terminal_directory(&path, |directory| { + directory + .blocks + .first_mut() + .unwrap() + .1 + .flags + .set_unique_key(false); + }); + let archive = open_path(&path, AccessKeys::new().with_key(private("recipient1"))); + let mut sink = RecordingSink(Vec::new()); + assert!(matches!( + archive.copy_to("data", &mut sink), + Err(PithosError::BlockHashMismatch { .. }) + )); + assert!(sink.0.is_empty()); +} diff --git a/crates/pithos_lib/tests/append_workflow.rs b/crates/pithos_lib/tests/append_workflow.rs index 890078e..5148b6d 100644 --- a/crates/pithos_lib/tests/append_workflow.rs +++ b/crates/pithos_lib/tests/append_workflow.rs @@ -1,9 +1,69 @@ mod common; use common::append::{WITHHELD_ID, append, append_fixture, append_with_cdc, archive_with_entry}; -use pithos_lib::archive::CdcConfig; +use common::util::{open, private_key, public_key}; +use pithos_lib::archive::{ + AccessKeys, AppendOptions, ArchivePath, ArchiveWriter, BlockKeyMode, CdcConfig, EntryMetadata, + ProcessingOptions, WriteOptions, +}; use pithos_lib::error::PithosError; -use pithos_lib::fs::FsError; +use pithos_lib::fs::{FsError, append_files}; +use std::io::Cursor; +use std::path::PathBuf; + +/// A version 1.0 base archive. It has no grants, so only its header differs from 1.1. +fn version_1_0_base_archive(temporary: &tempfile::TempDir) -> PathBuf { + let mut writer = ArchiveWriter::create(Vec::new(), WriteOptions::base()).unwrap(); + writer + .add_file( + ArchivePath::new("base.txt").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(false, 0).unwrap(), + None, + Cursor::new(b"base payload"), + ) + .unwrap(); + let mut bytes = writer.finish().unwrap(); + bytes[4..6].copy_from_slice(&[0x01, 0x00]); + let path = temporary.path().join("legacy.pith"); + std::fs::write(&path, bytes).unwrap(); + path +} + +#[test] +fn appends_to_version_1_0_archives_reject_version_1_1_processing_before_writing() { + let temporary = tempfile::tempdir().unwrap(); + let archive = version_1_0_base_archive(&temporary); + let original = std::fs::read(&archive).unwrap(); + let source = temporary.path().join("appended.txt"); + std::fs::write(&source, b"appended payload").unwrap(); + let options = || AppendOptions::new(private_key("sender"), vec![public_key("recipient1")]); + let unique = ProcessingOptions::default() + .with_key_mode(BlockKeyMode::Unique) + .unwrap(); + assert!(matches!( + append_files( + &archive, + options().with_processing(unique), + std::slice::from_ref(&source) + ), + Err(FsError::Core { + source: PithosError::UnsupportedProcessingFlags(_), + .. + }) + )); + assert_eq!(std::fs::read(&archive).unwrap(), original); + + append_files(&archive, options(), &[source]).unwrap(); + let mut contents = Vec::new(); + open( + &archive, + AccessKeys::new().with_key(private_key("recipient1")), + ) + .copy_to("appended.txt", &mut contents) + .unwrap(); + assert_eq!(contents, b"appended payload"); +} #[test] fn direct_append_file_workflow_adds_a_file() { diff --git a/crates/pithos_lib/tests/pieces.rs b/crates/pithos_lib/tests/pieces.rs index cc9c82b..7312c8a 100644 --- a/crates/pithos_lib/tests/pieces.rs +++ b/crates/pithos_lib/tests/pieces.rs @@ -2,8 +2,8 @@ mod common; use common::util::{private_key, public_key}; use pithos_lib::archive::{ - AccessKeys, AppendOptions, Archive, ArchivePath, EntryKind, EntryMetadata, OpenOptions, Piece, - PieceEncoder, ProcessingOptions, compose, + AccessKeys, AppendOptions, Archive, ArchivePath, BlockKeyMode, EntryKind, EntryMetadata, + OpenOptions, Piece, PieceEncoder, ProcessingOptions, compose, }; use pithos_lib::error::{DeserializationError, PithosError}; use pithos_lib::source::MemorySource; @@ -12,12 +12,12 @@ const BLOCK: usize = 1000; /// Encodes `content` as one piece in fixed blocks and returns its stored bytes and record. fn encode(key_id: u64, content: &[u8]) -> (Vec, Piece) { - let mut encoder = PieceEncoder::new( - key_id, - vec![public_key("recipient1")], - ProcessingOptions::new(true, 3).unwrap(), - ) - .unwrap(); + encode_with(key_id, content, ProcessingOptions::new(true, 3).unwrap()) +} + +fn encode_with(key_id: u64, content: &[u8], processing: ProcessingOptions) -> (Vec, Piece) { + let mut encoder = + PieceEncoder::new(key_id, vec![public_key("recipient1")], processing).unwrap(); let mut stored = Vec::new(); for block in content.chunks(BLOCK) { stored.extend(encoder.push(block).unwrap()); @@ -94,6 +94,28 @@ fn composed_pieces_read_back_as_one_file_with_ranges_across_pieces() { )); } +#[test] +fn unique_key_pieces_store_every_block_and_compose_with_convergent_pieces() { + let repeated = [content(8, BLOCK), content(8, BLOCK)].concat(); + let convergent = ProcessingOptions::new(true, 0).unwrap(); + let unique = convergent.with_key_mode(BlockKeyMode::Unique).unwrap(); + let parts = vec![ + encode_with(1, &repeated, unique), + encode_with(2, &repeated, convergent), + encode_with(3, &repeated, unique), + ]; + // Each stored block is `BLCK`, a nonce, the plaintext and a tag. Unique keys store both + // copies; convergent keys store the repeat once. + let stored_block = (4 + 12 + BLOCK + 16) as u64; + assert_eq!(parts[0].1.stored_len(), 2 * stored_block); + assert_eq!(parts[1].1.stored_len(), stored_block); + assert_eq!(parts[2].1.stored_len(), 2 * stored_block); + let archive = open(assemble(&parts), recipient()); + let mut output = Vec::new(); + archive.copy_to("object", &mut output).unwrap(); + assert_eq!(output, repeated.repeat(3)); +} + #[test] fn the_composition_digest_guards_the_metadata_on_open() { let parts = vec![encode(1, &content(1, 1500))]; From 9c755015114eabe0e6302ffaaae1f6187cac07de Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 16:08:56 +0200 Subject: [PATCH 062/166] feat: add AES-256-GCM block payloads to format version 1.1 --- Cargo.lock | 61 ++++++++++++ crates/pithos_lib/Cargo.toml | 1 + crates/pithos_lib/src/archive/mod.rs | 34 ++++--- crates/pithos_lib/src/archive/pieces.rs | 10 +- crates/pithos_lib/src/archive/types.rs | 3 + crates/pithos_lib/src/archive/writer.rs | 89 +++++++++++++----- crates/pithos_lib/src/block.rs | 81 +++++++++++++++- crates/pithos_lib/src/crypto.rs | 56 +++++++++-- crates/pithos_lib/src/format/block.rs | 18 +++- crates/pithos_lib/tests/draft_spec_vectors.rs | 25 +++++ spec/PITHOS_1.0.0_draft.md | 93 ++++++++++++++++--- 11 files changed, 397 insertions(+), 74 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4d01145..358b68e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18,6 +18,31 @@ dependencies = [ "inout", ] +[[package]] +name = "aes" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8eb277bec05f56a0e0591f155a484cbd0f4f07ff2905051a48c72f004f7ed58" +dependencies = [ + "cipher", + "cpubits", + "cpufeatures", +] + +[[package]] +name = "aes-gcm" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f2b8006a0c83f52b62ba44a97b58bf76fe2f70a329e588f67f89691d93d498f" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ctutils", + "ghash", +] + [[package]] name = "aho-corasick" version = "1.1.4" @@ -423,6 +448,12 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpubits" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae" + [[package]] name = "cpufeatures" version = "0.3.0" @@ -528,6 +559,15 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "ctr" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baaca1c4b237092596f64d571e9db6ce4109c4ef9742e27590f1709594461f21" +dependencies = [ + "cipher", +] + [[package]] name = "ctutils" version = "0.4.2" @@ -811,6 +851,15 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "ghash" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eecf2d5dc9b66b732b97707a0210906b1d30523eb773193ab777c0c84b3e8d5" +dependencies = [ + "polyval", +] + [[package]] name = "glob" version = "0.3.4" @@ -1365,6 +1414,7 @@ dependencies = [ name = "pithos_lib" version = "0.8.0" dependencies = [ + "aes-gcm", "blake2", "blake3", "byteorder", @@ -1456,6 +1506,17 @@ dependencies = [ "universal-hash", ] +[[package]] +name = "polyval" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0fa31d631f2b2cb2a544d0aa321ce847a94764d701ca2becc411138b93d49cd" +dependencies = [ + "cpubits", + "cpufeatures", + "universal-hash", +] + [[package]] name = "potential_utf" version = "0.1.5" diff --git a/crates/pithos_lib/Cargo.toml b/crates/pithos_lib/Cargo.toml index 5ad7e3d..3cd02ee 100644 --- a/crates/pithos_lib/Cargo.toml +++ b/crates/pithos_lib/Cargo.toml @@ -33,6 +33,7 @@ fuzzing = ["crypt4gh"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +aes-gcm = "0.11.1" blake2 = { version = "0.10.6", optional = true } blake3 = "1.8.2" byteorder = "1.5.0" diff --git a/crates/pithos_lib/src/archive/mod.rs b/crates/pithos_lib/src/archive/mod.rs index e952d8f..ba229a5 100644 --- a/crates/pithos_lib/src/archive/mod.rs +++ b/crates/pithos_lib/src/archive/mod.rs @@ -59,7 +59,8 @@ pub(crate) fn decode_validated_directory( } pub use writer::{ ArchiveWriter, BlockKeyMode, CdcConfig, CreateError, EntryMetadata, EntryReference, - FinishError, IncompleteWriter, ProcessingOptions, WriteOptions, WriterError, WrittenEntry, + FinishError, IncompleteWriter, PayloadCipher, ProcessingOptions, WriteOptions, WriterError, + WrittenEntry, }; #[cfg(test)] @@ -131,20 +132,23 @@ mod tests { Err(PithosError::ReservedProcessingBits(0x40)) )); } - assert_eq!( - Processing::from_byte(0x1b, FormatVersion::V1_1) - .unwrap() - .to_byte(), - 0x1b - ); - assert!(matches!( - Processing::from_byte(0x1b, FormatVersion::V1_0), - Err(PithosError::UnsupportedProcessingFlags(0x1b)) - )); - assert!(matches!( - Processing::from_byte(0x13, FormatVersion::V1_1), - Err(PithosError::ProcessingRequiresEncryption(0x13)) - )); + for flags in [0x1b, 0x2b, 0x3b] { + assert_eq!( + Processing::from_byte(flags, FormatVersion::V1_1) + .unwrap() + .to_byte(), + flags + ); + assert!(matches!( + Processing::from_byte(flags, FormatVersion::V1_0), + Err(PithosError::UnsupportedProcessingFlags(actual)) if actual == flags + )); + let plain = flags & !0x08; + assert!(matches!( + Processing::from_byte(plain, FormatVersion::V1_1), + Err(PithosError::ProcessingRequiresEncryption(actual)) if actual == plain + )); + } } #[test] diff --git a/crates/pithos_lib/src/archive/pieces.rs b/crates/pithos_lib/src/archive/pieces.rs index 75e0ccb..e76ee46 100644 --- a/crates/pithos_lib/src/archive/pieces.rs +++ b/crates/pithos_lib/src/archive/pieces.rs @@ -490,21 +490,23 @@ pub fn compose( #[cfg(test)] mod tests { use super::*; - use crate::archive::BlockKeyMode; + use crate::archive::{BlockKeyMode, PayloadCipher}; #[test] - fn piece_records_carry_unique_key_flags_and_reject_invalid_ones() { + fn piece_records_carry_version_1_1_flags_and_reject_invalid_ones() { let recipient = crate::crypto::PrivateKey::generate().public_key(); let processing = ProcessingOptions::new(true, 0) .unwrap() .with_key_mode(BlockKeyMode::Unique) + .unwrap() + .with_cipher(PayloadCipher::Aes256Gcm) .unwrap(); let mut encoder = PieceEncoder::new(1, vec![recipient], processing).unwrap(); encoder.push(b"unique block").unwrap(); let mut piece = encoder.finish().unwrap(); - assert_eq!(piece.blocks[0].flags.0, 0x18); + assert_eq!(piece.blocks[0].flags.0, 0x38); assert_eq!(Piece::from_bytes(&piece.to_bytes()).unwrap(), piece); - for flags in [0x10, 0x40] { + for flags in [0x10, 0x20, 0x40] { piece.blocks[0].flags = ProcessingFlags(flags); assert!(matches!( Piece::from_bytes(&piece.to_bytes()), diff --git a/crates/pithos_lib/src/archive/types.rs b/crates/pithos_lib/src/archive/types.rs index 2b34d53..2a1fb00 100644 --- a/crates/pithos_lib/src/archive/types.rs +++ b/crates/pithos_lib/src/archive/types.rs @@ -170,6 +170,7 @@ pub(crate) struct Processing { compression: Compression, encrypted: bool, unique_key: bool, + aes_256_gcm: bool, } impl Processing { @@ -191,6 +192,7 @@ impl Processing { compression: Compression::new(value & 0x07)?, encrypted, unique_key: value & ProcessingFlags::UNIQUE_KEY_MASK != 0, + aes_256_gcm: value & ProcessingFlags::AES_256_GCM_MASK != 0, }) } @@ -198,6 +200,7 @@ impl Processing { self.compression.level() | (u8::from(self.encrypted) << 3) | (u8::from(self.unique_key) << 4) + | (u8::from(self.aes_256_gcm) << 5) } } diff --git a/crates/pithos_lib/src/archive/writer.rs b/crates/pithos_lib/src/archive/writer.rs index 3492d62..ce6dab7 100644 --- a/crates/pithos_lib/src/archive/writer.rs +++ b/crates/pithos_lib/src/archive/writer.rs @@ -89,12 +89,25 @@ pub enum BlockKeyMode { Unique, } +/// The AEAD cipher that seals each new encrypted block payload. +/// +/// The cipher belongs to each stored block, not to the archive. A content-derived block that +/// reuses an existing descriptor keeps that descriptor's cipher, so an archive may mix ciphers. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub enum PayloadCipher { + #[default] + ChaCha20Poly1305, + /// AES-256-GCM under a key derived from the block key (version 1.1). + Aes256Gcm, +} + /// Per-block processing requested for a content entry. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct ProcessingOptions { encrypted: bool, compression_level: u8, key_mode: BlockKeyMode, + cipher: PayloadCipher, } impl ProcessingOptions { @@ -103,6 +116,7 @@ impl ProcessingOptions { encrypted: true, compression_level: 2, key_mode: BlockKeyMode::ContentDerived, + cipher: PayloadCipher::ChaCha20Poly1305, } } @@ -114,6 +128,7 @@ impl ProcessingOptions { encrypted, compression_level, key_mode: BlockKeyMode::ContentDerived, + cipher: PayloadCipher::ChaCha20Poly1305, }) } @@ -125,6 +140,14 @@ impl ProcessingOptions { Ok(options) } + /// Selects the payload cipher. [`PayloadCipher::Aes256Gcm`] requires encryption, and + /// writing it requires a version 1.1 archive. + pub fn with_cipher(self, cipher: PayloadCipher) -> Result { + let options = Self { cipher, ..self }; + options.validate_for(FormatVersion::V1_1)?; + Ok(options) + } + pub fn encrypted(self) -> bool { self.encrypted } @@ -134,10 +157,14 @@ impl ProcessingOptions { pub fn key_mode(self) -> BlockKeyMode { self.key_mode } + pub fn cipher(self) -> PayloadCipher { + self.cipher + } pub(crate) fn flags(self) -> ProcessingFlags { let mut flags = ProcessingFlags::new(self.encrypted, Some(self.compression_level)); flags.set_unique_key(self.key_mode == BlockKeyMode::Unique); + flags.set_aes_256_gcm(self.cipher == PayloadCipher::Aes256Gcm); flags } @@ -153,6 +180,7 @@ impl Default for ProcessingOptions { encrypted: true, compression_level: 3, key_mode: BlockKeyMode::ContentDerived, + cipher: PayloadCipher::ChaCha20Poly1305, } } } @@ -1680,43 +1708,54 @@ mod tests { } #[test] - fn version_1_0_writers_reject_unique_keys_before_writing_blocks() { - let mut writer = ArchiveWriter::create(Vec::new(), options()).unwrap(); - writer.version = FormatVersion::V1_0; + fn version_1_0_writers_reject_version_1_1_processing_before_writing_blocks() { let unique = ProcessingOptions::default() .with_key_mode(BlockKeyMode::Unique) .unwrap(); - assert!(matches!( - writer.add_file( - ArchivePath::new("data").unwrap(), - EntryMetadata::new(0, 0, 0o644), - unique, - None, - Cursor::new(b"content"), - ), - Err(WriterError::Pithos( - PithosError::UnsupportedProcessingFlags(_) - )) - )); - assert!(!writer.poisoned); - assert_eq!(writer.sink.offset, header::FileHeader::ENCODED_LEN as u64); - assert_eq!(writer.metadata_snapshot().descriptors, 0); + let aes = ProcessingOptions::default() + .with_cipher(PayloadCipher::Aes256Gcm) + .unwrap(); + for processing in [unique, aes] { + let mut writer = ArchiveWriter::create(Vec::new(), options()).unwrap(); + writer.version = FormatVersion::V1_0; + assert!(matches!( + writer.add_file( + ArchivePath::new("data").unwrap(), + EntryMetadata::new(0, 0, 0o644), + processing, + None, + Cursor::new(b"content"), + ), + Err(WriterError::Pithos( + PithosError::UnsupportedProcessingFlags(_) + )) + )); + assert!(!writer.poisoned); + assert_eq!(writer.sink.offset, header::FileHeader::ENCODED_LEN as u64); + assert_eq!(writer.metadata_snapshot().descriptors, 0); + } } #[test] - fn unique_key_options_require_encryption() { + fn version_1_1_processing_options_require_encryption() { + let plain = ProcessingOptions::new(false, 0).unwrap(); assert!(matches!( - ProcessingOptions::new(false, 0) - .unwrap() - .with_key_mode(BlockKeyMode::Unique), + plain.with_key_mode(BlockKeyMode::Unique), Err(PithosError::ProcessingRequiresEncryption(0x10)) )); - let unique = ProcessingOptions::new(true, 5) + assert!(matches!( + plain.with_cipher(PayloadCipher::Aes256Gcm), + Err(PithosError::ProcessingRequiresEncryption(0x20)) + )); + let both = ProcessingOptions::new(true, 5) .unwrap() .with_key_mode(BlockKeyMode::Unique) + .unwrap() + .with_cipher(PayloadCipher::Aes256Gcm) .unwrap(); - assert_eq!(unique.key_mode(), BlockKeyMode::Unique); - assert_eq!(unique.flags().0, 0x1d); + assert_eq!(both.key_mode(), BlockKeyMode::Unique); + assert_eq!(both.cipher(), PayloadCipher::Aes256Gcm); + assert_eq!(both.flags().0, 0x3d); } #[test] diff --git a/crates/pithos_lib/src/block.rs b/crates/pithos_lib/src/block.rs index beb4105..995d17f 100644 --- a/crates/pithos_lib/src/block.rs +++ b/crates/pithos_lib/src/block.rs @@ -51,7 +51,8 @@ fn encode_with_key( key: BlockKey, nonce: [u8; 12], ) -> Result { - if requested_flags.is_unique_key() && !requested_flags.is_encrypted() { + if requested_flags.0 & ProcessingFlags::VERSION_1_1_MASK != 0 && !requested_flags.is_encrypted() + { return Err(PithosError::ProcessingRequiresEncryption(requested_flags.0)); } let hash = block_hash(requested_flags, &key, plaintext); @@ -65,7 +66,9 @@ fn encode_with_key( plaintext.to_vec() }, ); - if flags.is_encrypted() { + if flags.is_aes_256_gcm() { + stored = Zeroizing::new(crypto::seal_block_aes_with_nonce(&key, &stored, nonce)?); + } else if flags.is_encrypted() { stored = Zeroizing::new(crypto::seal_block_with_nonce(&key, &stored, nonce)?); } Ok(EncodedBlock { @@ -104,7 +107,9 @@ pub(crate) fn verify( actual: stored.len() as u64, }); } - let mut plaintext = if meta.flags.is_encrypted() { + let mut plaintext = if meta.flags.is_aes_256_gcm() { + crypto::open_block_aes(key, &stored)? + } else if meta.flags.is_encrypted() { crypto::open_block(key, &stored)? } else { stored @@ -273,6 +278,76 @@ mod tests { )); } + #[test] + fn aes_256_gcm_vector_matches_the_specification() { + let nonce = std::array::from_fn(|index| 0xb0 + index as u8); + let key = crypto::derive_block_key(b"hello"); + let encoded = encode_with_key(b"hello", ProcessingFlags(0x28), key, nonce).unwrap(); + // PV-AES-GCM-HELLO in Appendix B. + assert_eq!(encoded.hash, crypto::block_hash(b"hello")); + assert_eq!( + &encoded.stored[12..], + [ + 0x37, 0x8d, 0x6e, 0x4e, 0x73, 0xd7, 0xd5, 0xba, 0x27, 0x60, 0x8f, 0xef, 0x88, 0xed, + 0x83, 0x96, 0x88, 0x2e, 0x54, 0xd5, 0xcd + ] + ); + let meta = descriptor(&encoded, 5); + let plain = verify( + encoded.stored.clone(), + &encoded.key, + encoded.hash, + &meta, + limits(), + ) + .unwrap(); + assert_eq!(&*plain, b"hello"); + + // The same key opens neither cipher's payload as the other. + let mut chacha = meta.clone(); + chacha.flags.set_aes_256_gcm(false); + assert!(matches!( + verify( + encoded.stored, + &encoded.key, + encoded.hash, + &chacha, + limits() + ), + Err(PithosError::Crypt(_)) + )); + assert!(matches!( + encode(b"hello", ProcessingFlags(0x20), [0; 12]), + Err(PithosError::ProcessingRequiresEncryption(0x20)) + )); + } + + #[test] + fn unique_keys_and_aes_256_gcm_combine() { + let plain = b"unique key sealed with AES-256-GCM"; + let mut flags = unique_flags(0); + flags.set_aes_256_gcm(true); + let encoded = encode(plain, flags, [3; 12]).unwrap(); + assert_eq!(encoded.flags.0, 0x38); + assert_eq!(encoded.hash, crypto::keyed_block_hash(&encoded.key, plain)); + let meta = descriptor(&encoded, plain.len()); + let output = verify( + encoded.stored.clone(), + &encoded.key, + encoded.hash, + &meta, + limits(), + ) + .unwrap(); + assert_eq!(&*output, plain); + let mut corrupt = encoded.stored; + corrupt[20] ^= 1; + assert!(matches!( + verify(corrupt, &encoded.key, encoded.hash, &meta, limits()), + Err(PithosError::Crypt(_)) + )); + } + #[test] fn every_unique_key_encoding_has_a_fresh_key_and_identity() { let plain = b"the same plaintext block"; diff --git a/crates/pithos_lib/src/crypto.rs b/crates/pithos_lib/src/crypto.rs index f5f3b88..b7bde72 100644 --- a/crates/pithos_lib/src/crypto.rs +++ b/crates/pithos_lib/src/crypto.rs @@ -1,13 +1,14 @@ //! Current Pithos cryptographic roles and protocol operations. //! //! The byte formats deliberately remain in the established protocol: every sealed value is -//! `nonce || ciphertext || tag` and always uses empty associated data. Version 1.1 changes only -//! how a recipient grant's wrapping key is derived. +//! `nonce || ciphertext || tag` and always uses empty associated data. Version 1.1 derives +//! recipient grant keys with HKDF and adds unique block keys and AES-256-GCM block payloads. use crate::format::header::FormatVersion; +use aes_gcm::Aes256Gcm; use chacha20poly1305::{ ChaCha20Poly1305, Key, KeyInit, Nonce, - aead::{Aead, Generate, Payload}, + aead::{self, Aead, AeadCore, Generate, Payload, consts::U12}, }; use digest::{ExtendableOutput, Update, XofReader}; use hkdf::Hkdf; @@ -342,6 +343,28 @@ pub(crate) fn open_block( open_empty_aad(key.expose_for_protocol(), payload) } +/// BLAKE3 key derivation context of the AES-256-GCM payload key of a block key. +const AES_PAYLOAD_CONTEXT: &str = "pithos 1.1 aes-256-gcm payload"; + +fn aes_payload_key(key: &BlockKey) -> Zeroizing<[u8; 32]> { + Zeroizing::new(blake3::derive_key(AES_PAYLOAD_CONTEXT, &key.0)) +} + +pub(crate) fn seal_block_aes_with_nonce( + key: &BlockKey, + plaintext: &[u8], + nonce: [u8; 12], +) -> Result, CryptoError> { + seal_with::(&aes_payload_key(key), plaintext, nonce) +} + +pub(crate) fn open_block_aes( + key: &BlockKey, + payload: &[u8], +) -> Result>, CryptoError> { + open_with::(&aes_payload_key(key), payload) +} + pub(crate) fn seal_file_block_list_with_nonce( key: &FileKey, plaintext: &[u8], @@ -389,11 +412,22 @@ fn seal_empty_aad( plaintext: &[u8], nonce: [u8; 12], ) -> Result, CryptoError> { - let cipher = - ChaCha20Poly1305::new_from_slice(key).map_err(|_| CryptoError::CipherInitialization)?; + seal_with::(key, plaintext, nonce) +} + +fn open_empty_aad(key: &[u8; 32], payload: &[u8]) -> Result>, CryptoError> { + open_with::(key, payload) +} + +fn seal_with>( + key: &[u8; 32], + plaintext: &[u8], + nonce: [u8; 12], +) -> Result, CryptoError> { + let cipher = C::new_from_slice(key).map_err(|_| CryptoError::CipherInitialization)?; let ciphertext = cipher .encrypt( - &Nonce::from(nonce), + &aead::Nonce::::from(nonce), Payload { msg: plaintext, aad: b"", @@ -406,19 +440,21 @@ fn seal_empty_aad( Ok(payload) } -fn open_empty_aad(key: &[u8; 32], payload: &[u8]) -> Result>, CryptoError> { +fn open_with>( + key: &[u8; 32], + payload: &[u8], +) -> Result>, CryptoError> { if payload.len() < 15 { return Err(CryptoError::EncryptedPayloadTooShort); } let (nonce, ciphertext) = payload.split_at(12); - let cipher = - ChaCha20Poly1305::new_from_slice(key).map_err(|_| CryptoError::CipherInitialization)?; + let cipher = C::new_from_slice(key).map_err(|_| CryptoError::CipherInitialization)?; let nonce: [u8; 12] = nonce .try_into() .map_err(|_| CryptoError::EncryptedPayloadTooShort)?; cipher .decrypt( - &Nonce::from(nonce), + &aead::Nonce::::from(nonce), Payload { msg: ciphertext, aad: b"", diff --git a/crates/pithos_lib/src/format/block.rs b/crates/pithos_lib/src/format/block.rs index 2b0e1de..bd9f060 100644 --- a/crates/pithos_lib/src/format/block.rs +++ b/crates/pithos_lib/src/format/block.rs @@ -23,9 +23,11 @@ impl ProcessingFlags { const ENCRYPTION_MASK: u8 = 0b0000_1000; /// Version 1.1: the block key is random and the block hash is a keyed identity. pub const UNIQUE_KEY_MASK: u8 = 0b0001_0000; + /// Version 1.1: the payload is sealed with AES-256-GCM instead of ChaCha20-Poly1305. + pub const AES_256_GCM_MASK: u8 = 0b0010_0000; /// Bits that are valid only in version 1.1 archives and only with encryption. - pub const VERSION_1_1_MASK: u8 = Self::UNIQUE_KEY_MASK; - pub const RESERVED_MASK: u8 = 0b1110_0000; + pub const VERSION_1_1_MASK: u8 = Self::UNIQUE_KEY_MASK | Self::AES_256_GCM_MASK; + pub const RESERVED_MASK: u8 = 0b1100_0000; pub fn new(encrypted: bool, compression_level: Option) -> Self { let mut flags = ProcessingFlags(0b0); @@ -75,6 +77,18 @@ impl ProcessingFlags { (self.0 & Self::UNIQUE_KEY_MASK) != 0 } + pub fn set_aes_256_gcm(&mut self, aes: bool) { + if aes { + self.0 |= Self::AES_256_GCM_MASK; + } else { + self.0 &= !Self::AES_256_GCM_MASK; + } + } + + pub fn is_aes_256_gcm(&self) -> bool { + (self.0 & Self::AES_256_GCM_MASK) != 0 + } + pub fn set_compression_level(&mut self, mut compression_level: u8) { compression_level = if compression_level > Self::COMPRESSION_MASK { Self::COMPRESSION_MASK diff --git a/crates/pithos_lib/tests/draft_spec_vectors.rs b/crates/pithos_lib/tests/draft_spec_vectors.rs index 5142948..bc0e29f 100644 --- a/crates/pithos_lib/tests/draft_spec_vectors.rs +++ b/crates/pithos_lib/tests/draft_spec_vectors.rs @@ -723,6 +723,31 @@ fn unique_key_block_known_answer_matches_appendix_b() { assert_eq!(stored, appendix_text_block("PV-UNIQUE-KEY-HELLO")); } +#[test] +fn aes_256_gcm_block_known_answer_matches_appendix_b() { + use aes_gcm::Aes256Gcm; + let nonce: [u8; 12] = std::array::from_fn(|index| 0xb0 + index as u8); + let block_key = shake256(b"hello"); + let payload_key = blake3::derive_key("pithos 1.1 aes-256-gcm payload", &block_key); + for value in [ + blake3::Hash::from(block_key).to_hex(), + blake3::Hash::from(payload_key).to_hex(), + ] { + assert!( + SPEC.contains(value.as_str()), + "{value} missing from Appendix B" + ); + } + let mut stored = nonce.to_vec(); + stored.extend_from_slice( + &Aes256Gcm::new_from_slice(&payload_key) + .unwrap() + .encrypt(&aes_gcm::Nonce::from(nonce), b"hello".as_ref()) + .unwrap(), + ); + assert_eq!(stored, appendix_text_block("PV-AES-GCM-HELLO")); +} + fn appendix_zstd_bytes(id: &str) -> Vec { let row = SPEC .lines() diff --git a/spec/PITHOS_1.0.0_draft.md b/spec/PITHOS_1.0.0_draft.md index 6c934aa..38f99e1 100644 --- a/spec/PITHOS_1.0.0_draft.md +++ b/spec/PITHOS_1.0.0_draft.md @@ -72,6 +72,9 @@ Version 1.1 differs from version 1.0 only in these points: 4. An encrypted block may use a unique random key instead of its convergent key (ProcessingFlags bit 4, Section 4.2.3). Its block hash is then a keyed BLAKE3 identity (Section 5.2), and the block is never deduplicated (Section 5.3). +5. An encrypted block payload may use AES-256-GCM instead of ChaCha20-Poly1305 + (ProcessingFlags bit 5, Sections 4.2.3 and 5.3). Block lists, pieces and + recipient grants keep ChaCha20-Poly1305. Readers MUST support both versions. Writers MUST create new archives as version 1.1. An append MUST follow the version of the archive it extends, so appending @@ -226,7 +229,7 @@ Readers MUST reject duplicate block hashes in one directory and MUST use the #### 4.2.3 Processing Flags ProcessingFlags records the compression level, whether a block is encrypted and, -in version 1.1, how an encrypted block is keyed. +in version 1.1, how an encrypted block is keyed and sealed. ```rust /// Processing flags packed into one byte @@ -249,7 +252,10 @@ bitflags::bitflags! { // Bit 4: Unique random block key (version 1.1, requires bit 3) const UNIQUE_KEY = 0b0001_0000; - // Bits 5-7: Reserved for future use (MUST be zero) + // Bit 5: AES-256-GCM payload (version 1.1, requires bit 3) + const AES_256_GCM = 0b0010_0000; + + // Bits 6-7: Reserved for future use (MUST be zero) } } ``` @@ -261,14 +267,15 @@ bitflags::bitflags! { | 0-2 | Compression: `0` means the stored payload is not compressed; `1` through `7` each mean the stored payload is one standard Zstandard frame | | 3 | Encryption enabled: `0` is disabled; `1` is enabled | | 4 | Unique key (version 1.1): `0` means the block key is convergent; `1` means it is a unique random key (Section 5.3) | -| 5-7 | Reserved; all bits MUST be zero | +| 5 | AES-256-GCM (version 1.1): `0` means the payload is sealed with ChaCha20-Poly1305; `1` means it is sealed with AES-256-GCM (Section 5.3) | +| 6-7 | Reserved; all bits MUST be zero | ProcessingFlags is stored as exactly one byte. Readers MUST reject a value with -any reserved bit set. Bit 4 is valid only in a version 1.1 archive and only -when bit 3 is set. Readers MUST reject a descriptor that sets bit 4 in a version -1.0 archive or without bit 3. The same rules apply wherever a writer records -these flags. For example, an encrypted unique-key block with no compression has -the flags byte `18`. +any reserved bit set. Bits 4 and 5 are valid only in a version 1.1 archive and +only when bit 3 is set. Readers MUST reject a descriptor that sets either bit in +a version 1.0 archive or without bit 3. The same rules apply wherever a writer +records these flags. For example, encrypted blocks with no compression have the +flags byte `18` with a unique key, `28` with AES-256-GCM, and `38` with both. #### 4.2.4 Block Location @@ -849,10 +856,12 @@ until its required transforms, size, and hash have all been verified. ### 5.3 Convergent Encryption Encryption is optional. An implementation that supports encryption MUST use -X25519, SHAKE256, and ChaCha20-Poly1305 as specified here. +X25519, SHAKE256, and ChaCha20-Poly1305 as specified here. Version 1.1 adds +BLAKE3 key derivation and, for blocks that set ProcessingFlags bit 5, +AES-256-GCM. -All encryption keys are 32 bytes. ChaCha20-Poly1305 uses a 12-byte nonce and -produces a 16-byte authentication tag. Every encrypted value is stored as +All encryption keys are 32 bytes. ChaCha20-Poly1305 and AES-256-GCM each use a +12-byte nonce and produce a 16-byte authentication tag. Every encrypted value is stored as `nonce || ciphertext || tag`; the nonce is part of the stored byte vector. The additional authenticated data (AAD) is empty. @@ -870,6 +879,22 @@ therefore never deduplicated: each occurrence is stored with its own key and block hash (Section 5.2). The block list carries the unique key like any other block key. +In version 1.1, a block whose ProcessingFlags bit 5 is set has an AES-256-GCM +payload instead of a ChaCha20-Poly1305 payload. Its payload key is the 32-byte +BLAKE3 `derive_key` output with the context string +`pithos 1.1 aes-256-gcm payload` (30 ASCII bytes) and the block key as key +material. The block key itself is convergent or unique as described above, and +it is the key stored in the block list. The payload is stored as +`nonce || ciphertext || tag` with empty AAD. Block lists, pieces and recipient +grants always use ChaCha20-Poly1305. + +The cipher is a property of each effective descriptor, not of the archive. One +archive, and one file, MAY contain blocks sealed with both ciphers. A writer +that reuses an existing descriptor for a convergent block hash, by +deduplication or composition, keeps that descriptor's cipher. Requesting +AES-256-GCM therefore does not guarantee that every block of a file uses it. +Readers MUST select the cipher from each block's effective descriptor. + A file key is 32 random bytes. It encrypts the decrypted block list for a file. In version 1.1, a piece key is 32 random bytes and encrypts the decrypted block list of one piece in the same way. @@ -896,6 +921,21 @@ deliberately reuse a nonce under the same key. It MUST authenticate and decrypt an encrypted value successfully before using its plaintext or releasing output derived from it. +**Nonce scope of block payloads.** A ChaCha20-Poly1305 payload uses the block key +as its AEAD key, and an AES-256-GCM payload uses the derived payload key, so the +two ciphers never share an AEAD key. A unique key seals exactly one payload. A +convergent key is the same for every encryption of the same block plaintext, by +any writer and in any archive. All those encryptions share one key and one +random-nonce scope. NIST SP 800-38D Section 8.3 limits AES-GCM with random +96-bit nonces to at most 2^32 invocations of the encryption function per key. +Writers SHOULD apply the same limit to ChaCha20-Poly1305 block payloads. A +repeated nonce under one key exposes the XOR of the two stored payloads, which +can differ in compression, and AES-256-GCM also loses authenticity under that +key; the block hash check still rejects altered plaintext. Writers SHOULD reuse +an existing stored block instead of encrypting a convergent block again. +Deployments that may encrypt one plaintext more often than this bound across +all archives SHOULD use unique keys. + Neither version uses a SHAKE256 label or AAD. A redesign of any cryptographic input requires a new format version. @@ -986,6 +1026,10 @@ holds a block key can test a guessed plaintext against that block's hash. A whol kept outside the archive is separate from the keyed block identity and still reveals equal files. +The payload cipher (ChaCha20-Poly1305 or AES-256-GCM) does not change what is +hidden. Every encryption of one convergent block shares one key and one nonce +scope across all archives; Section 5.3 states the resulting invocation bound. + Pieces are joined in the order the Directory stores them. Their strictly increasing key IDs and the `file_size` check detect some reordering and missing pieces, but without authenticated metadata a modified Directory can still drop or @@ -1018,12 +1062,14 @@ negotiation record. | Block compression | ProcessingFlags compression bits are `1` through `7` | | Block encryption | ProcessingFlags encryption bit is `1` | | Unique block keys | ProcessingFlags bit 4 is `1` (version 1.1) | +| AES-256-GCM payloads | ProcessingFlags bit 5 is `1` (version 1.1) | | Encrypted block lists | BlockDataState tag is `00` (`Encrypted`) or, in version 1.1, `02` (`Pieces`) | | Encrypted recipient lists | RecipientData tag is `00` (`Encrypted`) | | External storage | BlockLocation tag is `01` (`External`) | -Compression, block encryption, unique block keys, encrypted block lists, -encrypted recipient lists, and external storage are optional capabilities. An +Compression, block encryption, unique block keys, AES-256-GCM payloads, +encrypted block lists, encrypted recipient lists, and external storage are +optional capabilities. An implementation that supports an optional capability MUST process it according to its definition in this specification. @@ -1045,6 +1091,7 @@ uncompressed or unencrypted bytes. | Compressed local: local, compression `1` through `7`, encryption `0` | Listed | Readable only with block-compression capability; otherwise unavailable | | Encrypted local: local, compression `0`, encryption `1` | Listed | Readable only with block-encryption capability and any encrypted-list capability needed to obtain its block key; otherwise unavailable | | Unique-key local: local, encryption `1`, bit 4 `1` | Listed | Readable only with block-encryption and unique-block-key capability and any encrypted-list capability needed to obtain its block key; otherwise unavailable | +| AES-256-GCM local: local, encryption `1`, bit 5 `1` | Listed | Readable only with block-encryption and AES-256-GCM capability and any encrypted-list capability needed to obtain its block key; otherwise unavailable | | External: BlockLocation `External` | Listed | Readable only with external-storage capability and every capability required by its flags and data states; otherwise unavailable | | Combined: compression and encryption, at either location | Listed | Readable only when every indicated capability is supported; otherwise unavailable | @@ -1085,7 +1132,7 @@ Non-POSIX platforms MAY retain them as metadata without an ACL mapping. The format reserves space for future extensions: - FileType values 4-255 -- ProcessingFlags bits 5-7 +- ProcessingFlags bits 6-7 - Custom relationship types starting at 1000 - BlockDataState tags `03` through `ff` @@ -1421,6 +1468,22 @@ plain BLAKE3 digest of `hello`. The stored ChaCha20-Poly1305 0020: d2 ``` +`PV-AES-GCM-HELLO` is a version 1.1 known-answer test for one AES-256-GCM block +with a convergent key (Section 5.3). The plaintext is ASCII `hello` and the flags +byte is `28` (no compression, encryption, AES-256-GCM). The block key is +`SHAKE256("hello")`, +`1234075ae4a1e77316cf2d8000974581a343b9ebbca7e3d1db83394c30f22162`, and the +derived payload key is +`7207849ab2398b3765b8c0ec4c43646d7e0ccdae28ed3d393f037a5a0febf245`. The block +hash is the plain BLAKE3 digest of `hello`. With the test nonce bytes `b0` +through `bb`, the stored `nonce || ciphertext || tag` is: + +```text +0000: b0 b1 b2 b3 b4 b5 b6 b7 b8 b9 ba bb 37 8d 6e 4e +0010: 73 d7 d5 ba 27 60 8f ef 88 ed 83 96 88 2e 54 d5 +0020: cd +``` + ### B.4 Acceptance and Rejection Mutations Each `AV-*` or `RV-*` vector is a mutation of the named canonical vector. @@ -1456,5 +1519,5 @@ archive. | Directory framing, CRC, and chain | 4.3, 4.3.1, 4.3.2 | CV-BASE-EMPTY-146, CV-APPEND-EMPTY-28, RV-PARENT, RV-UNDERFLOW, RV-TRAILING, RV-CRC, RV-CROSS-SIZE | Valid archive or reject archive as stated | | Entries and paths | 4.3.3, 4.4.1, 4.4.3 | CV-LOCAL-HELLO-279, RV-DUPLICATES, RV-PATH, RV-SYMLINK, RV-FILETYPE, RV-PERMISSIONS | Valid archive or reject archive as stated | | Block locations and extents | 4.2.2, 4.2.5, 4.2.6, 8.2 | CV-LOCAL-HELLO-279, RV-EXTENT, RV-SHORT-ENCRYPTED, RV-EXTERNAL | Valid archive, reject archive, content read fails before output, or content unavailable as stated | -| Content transforms and hashes | 5.2, 5.3, 5.4 | PV-ZSTD-HELLO, PV-ZSTD-TEXT, PV-RECIPIENT-WRAP-01, PV-RECIPIENT-WRAP-11, PV-UNIQUE-KEY-HELLO | Decode/decrypt to stated output | +| Content transforms and hashes | 5.2, 5.3, 5.4 | PV-ZSTD-HELLO, PV-ZSTD-TEXT, PV-RECIPIENT-WRAP-01, PV-RECIPIENT-WRAP-11, PV-UNIQUE-KEY-HELLO, PV-AES-GCM-HELLO | Decode/decrypt to stated output | | Version 1.1 pieces and grants | 1.3, 4.4.2, 5.3 | CV-PIECES-HELLO-766, PV-RECIPIENT-WRAP-11 | Valid archive; content readable only with the granted key | From bd87e2d2dea3a8d59bca6348362e870ea3e793ba Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 16:10:28 +0200 Subject: [PATCH 063/166] test: cover AES-256-GCM payloads and mixed ciphers --- .../src/archive/reader_private_tests.rs | 120 ++++++++++++++++-- crates/pithos_lib/tests/append_workflow.rs | 31 +++-- crates/pithos_lib/tests/pieces.rs | 29 ++++- 3 files changed, 152 insertions(+), 28 deletions(-) diff --git a/crates/pithos_lib/src/archive/reader_private_tests.rs b/crates/pithos_lib/src/archive/reader_private_tests.rs index 24775bb..d745ea8 100644 --- a/crates/pithos_lib/src/archive/reader_private_tests.rs +++ b/crates/pithos_lib/src/archive/reader_private_tests.rs @@ -3,8 +3,8 @@ use super::reader::{ ExternalBlockResolver, OpenLimits, OpenOptions, }; use crate::archive::{ - ArchivePath, ArchiveWriter, BlockKeyMode, CdcConfig, EntryMetadata, ProcessingOptions, - WriteOptions, + ArchivePath, ArchiveWriter, BlockKeyMode, CdcConfig, EntryMetadata, PayloadCipher, + ProcessingOptions, WriteOptions, }; use crate::crypto::{self, FileKey, PrivateKey, PublicKey}; use crate::error::PithosError; @@ -1751,6 +1751,13 @@ fn unique_processing() -> ProcessingOptions { .unwrap() } +fn aes_processing() -> ProcessingOptions { + ProcessingOptions::new(true, 0) + .unwrap() + .with_cipher(PayloadCipher::Aes256Gcm) + .unwrap() +} + #[test] fn unique_key_blocks_round_trip_and_store_every_repeat() { // Zero bytes split into two equal 1024-byte chunks. @@ -1775,10 +1782,17 @@ fn unique_key_blocks_round_trip_and_store_every_repeat() { } #[test] -fn unique_key_flags_are_rejected_in_version_1_0_archives() { +fn version_1_1_flags_are_rejected_in_version_1_0_archives() { for (processing, expected) in [ (ProcessingOptions::new(true, 0).unwrap(), None), (unique_processing(), Some(0x18)), + (aes_processing(), Some(0x28)), + ( + unique_processing() + .with_cipher(PayloadCipher::Aes256Gcm) + .unwrap(), + Some(0x38), + ), ] { let (_temporary, path) = fixture_with_processing(b"version gated block", processing); let mut legacy = std::fs::read(&path).unwrap(); @@ -1795,8 +1809,34 @@ fn unique_key_flags_are_rejected_in_version_1_0_archives() { } #[test] -fn unique_key_flags_require_encryption() { - let (_temporary, path) = fixture_with_processing(b"plain unique block", unique_processing()); +fn version_1_1_flags_require_encryption() { + for (processing, expected) in [(unique_processing(), 0x10), (aes_processing(), 0x20)] { + let (_temporary, path) = fixture_with_processing(b"plain block", processing); + rewrite_terminal_directory(&path, |directory| { + directory + .blocks + .first_mut() + .unwrap() + .1 + .flags + .set_encryption(false); + }); + assert!(matches!( + open_without_keys(&path), + Err(PithosError::ProcessingRequiresEncryption(actual)) if actual == expected + )); + } +} + +#[test] +fn unique_key_blocks_verify_payload_and_identity_before_output() { + let content = b"unique key payload long enough for every corruption position"; + for byte in [0, 12, 40] { + let (_temporary, path) = fixture_with_processing(content, unique_processing()); + corrupt_payload(&path, byte); + assert_copy_failure_without_sink(&path); + } + let (_temporary, path) = fixture_with_processing(content, unique_processing()); rewrite_terminal_directory(&path, |directory| { directory .blocks @@ -1804,23 +1844,26 @@ fn unique_key_flags_require_encryption() { .unwrap() .1 .flags - .set_encryption(false); + .set_unique_key(false); }); + let archive = open_path(&path, AccessKeys::new().with_key(private("recipient1"))); + let mut sink = RecordingSink(Vec::new()); assert!(matches!( - open_without_keys(&path), - Err(PithosError::ProcessingRequiresEncryption(0x10)) + archive.copy_to("data", &mut sink), + Err(PithosError::BlockHashMismatch { .. }) )); + assert!(sink.0.is_empty()); } #[test] -fn unique_key_blocks_verify_payload_and_identity_before_output() { - let content = b"unique key payload long enough for every corruption position"; +fn aes_256_gcm_blocks_verify_payload_and_cipher_before_output() { + let content = b"AES-256-GCM payload long enough for every corruption position"; for byte in [0, 12, 40] { - let (_temporary, path) = fixture_with_processing(content, unique_processing()); + let (_temporary, path) = fixture_with_processing(content, aes_processing()); corrupt_payload(&path, byte); assert_copy_failure_without_sink(&path); } - let (_temporary, path) = fixture_with_processing(content, unique_processing()); + let (_temporary, path) = fixture_with_processing(content, aes_processing()); rewrite_terminal_directory(&path, |directory| { directory .blocks @@ -1828,13 +1871,62 @@ fn unique_key_blocks_verify_payload_and_identity_before_output() { .unwrap() .1 .flags - .set_unique_key(false); + .set_aes_256_gcm(false); }); let archive = open_path(&path, AccessKeys::new().with_key(private("recipient1"))); let mut sink = RecordingSink(Vec::new()); assert!(matches!( archive.copy_to("data", &mut sink), - Err(PithosError::BlockHashMismatch { .. }) + Err(PithosError::Crypt(_)) )); assert!(sink.0.is_empty()); } + +#[test] +fn ciphers_mix_per_descriptor_and_reused_blocks_keep_their_cipher() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("mixed.pith"); + let options = WriteOptions::new(private("sender"), vec![public("recipient1")]); + let mut writer = ArchiveWriter::create(File::create(&path).unwrap(), options).unwrap(); + // Each content is below the minimum chunk size, so every file is one block. + let shared = b"block first written with ChaCha20-Poly1305".as_slice(); + let other = b"block written with AES-256-GCM".as_slice(); + let files = [ + ("chacha", shared, ProcessingOptions::new(true, 0).unwrap()), + ("aes-reused", shared, aes_processing()), + ("aes", other, aes_processing()), + ( + "unique-aes", + shared, + aes_processing() + .with_key_mode(BlockKeyMode::Unique) + .unwrap(), + ), + ]; + for (name, content, processing) in files { + writer + .add_file( + ArchivePath::new(name).unwrap(), + EntryMetadata::new(0, 0, 0o644), + processing, + None, + Cursor::new(content), + ) + .unwrap(); + } + writer.finish().unwrap(); + + // The AES request for the shared block reuses its ChaCha20-Poly1305 descriptor. + let flags = decode_terminal_directory(&path) + .blocks + .values() + .map(|block| block.flags.0) + .collect::>(); + assert_eq!(flags, [0x08, 0x28, 0x38]); + let archive = open_path(&path, AccessKeys::new().with_key(private("recipient1"))); + for (name, expected, _) in files { + let mut output = Vec::new(); + archive.copy_to(name, &mut output).unwrap(); + assert_eq!(output, expected, "{name}"); + } +} diff --git a/crates/pithos_lib/tests/append_workflow.rs b/crates/pithos_lib/tests/append_workflow.rs index 5148b6d..b48746b 100644 --- a/crates/pithos_lib/tests/append_workflow.rs +++ b/crates/pithos_lib/tests/append_workflow.rs @@ -4,7 +4,7 @@ use common::append::{WITHHELD_ID, append, append_fixture, append_with_cdc, archi use common::util::{open, private_key, public_key}; use pithos_lib::archive::{ AccessKeys, AppendOptions, ArchivePath, ArchiveWriter, BlockKeyMode, CdcConfig, EntryMetadata, - ProcessingOptions, WriteOptions, + PayloadCipher, ProcessingOptions, WriteOptions, }; use pithos_lib::error::PithosError; use pithos_lib::fs::{FsError, append_files}; @@ -41,18 +41,23 @@ fn appends_to_version_1_0_archives_reject_version_1_1_processing_before_writing( let unique = ProcessingOptions::default() .with_key_mode(BlockKeyMode::Unique) .unwrap(); - assert!(matches!( - append_files( - &archive, - options().with_processing(unique), - std::slice::from_ref(&source) - ), - Err(FsError::Core { - source: PithosError::UnsupportedProcessingFlags(_), - .. - }) - )); - assert_eq!(std::fs::read(&archive).unwrap(), original); + let aes = ProcessingOptions::default() + .with_cipher(PayloadCipher::Aes256Gcm) + .unwrap(); + for processing in [unique, aes] { + assert!(matches!( + append_files( + &archive, + options().with_processing(processing), + std::slice::from_ref(&source) + ), + Err(FsError::Core { + source: PithosError::UnsupportedProcessingFlags(_), + .. + }) + )); + assert_eq!(std::fs::read(&archive).unwrap(), original); + } append_files(&archive, options(), &[source]).unwrap(); let mut contents = Vec::new(); diff --git a/crates/pithos_lib/tests/pieces.rs b/crates/pithos_lib/tests/pieces.rs index 7312c8a..8a59045 100644 --- a/crates/pithos_lib/tests/pieces.rs +++ b/crates/pithos_lib/tests/pieces.rs @@ -3,7 +3,7 @@ mod common; use common::util::{private_key, public_key}; use pithos_lib::archive::{ AccessKeys, AppendOptions, Archive, ArchivePath, BlockKeyMode, EntryKind, EntryMetadata, - OpenOptions, Piece, PieceEncoder, ProcessingOptions, compose, + OpenOptions, PayloadCipher, Piece, PieceEncoder, ProcessingOptions, compose, }; use pithos_lib::error::{DeserializationError, PithosError}; use pithos_lib::source::MemorySource; @@ -116,6 +116,33 @@ fn unique_key_pieces_store_every_block_and_compose_with_convergent_pieces() { assert_eq!(output, repeated.repeat(3)); } +#[test] +fn pieces_with_different_ciphers_and_key_modes_compose_into_one_file() { + let shared = content(9, BLOCK); + let chacha = ProcessingOptions::new(true, 0).unwrap(); + let aes = chacha.with_cipher(PayloadCipher::Aes256Gcm).unwrap(); + let unique_aes = aes.with_key_mode(BlockKeyMode::Unique).unwrap(); + let parts = vec![ + encode_with(1, &[shared.clone(), content(1, 500)].concat(), aes), + encode_with(2, &[content(2, BLOCK), shared.clone()].concat(), chacha), + encode_with(3, &[shared.clone(), shared.clone()].concat(), unique_aes), + ]; + // The ChaCha20-Poly1305 piece reads its shared block from the AES-256-GCM piece. + let archive = open(assemble(&parts), recipient()); + let mut output = Vec::new(); + archive.copy_to("object", &mut output).unwrap(); + let expected = [ + shared.clone(), + content(1, 500), + content(2, BLOCK), + shared.clone(), + shared.clone(), + shared, + ] + .concat(); + assert_eq!(output, expected); +} + #[test] fn the_composition_digest_guards_the_metadata_on_open() { let parts = vec![encode(1, &content(1, 1500))]; From 1ebb64d5f339686e5ef4843d69576b7b8b872454 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 16:10:28 +0200 Subject: [PATCH 064/166] chore: add the AES-256-GCM crates to the fuzz lockfile --- fuzz/Cargo.lock | 61 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index c132b71..adac8a6 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -18,6 +18,31 @@ dependencies = [ "inout", ] +[[package]] +name = "aes" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8eb277bec05f56a0e0591f155a484cbd0f4f07ff2905051a48c72f004f7ed58" +dependencies = [ + "cipher", + "cpubits", + "cpufeatures", +] + +[[package]] +name = "aes-gcm" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f2b8006a0c83f52b62ba44a97b58bf76fe2f70a329e588f67f89691d93d498f" +dependencies = [ + "aead", + "aes", + "cipher", + "ctr", + "ctutils", + "ghash", +] + [[package]] name = "ambient-authority" version = "0.0.2" @@ -261,6 +286,12 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "cpubits" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15b85f9c39137c3a891689859392b1bd49812121d0d61c9caf00d46ed5ce06ae" + [[package]] name = "cpufeatures" version = "0.3.0" @@ -300,6 +331,15 @@ dependencies = [ "rand_core", ] +[[package]] +name = "ctr" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baaca1c4b237092596f64d571e9db6ce4109c4ef9742e27590f1709594461f21" +dependencies = [ + "cipher", +] + [[package]] name = "ctutils" version = "0.4.2" @@ -553,6 +593,15 @@ dependencies = [ "rand_core", ] +[[package]] +name = "ghash" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eecf2d5dc9b66b732b97707a0210906b1d30523eb773193ab777c0c84b3e8d5" +dependencies = [ + "polyval", +] + [[package]] name = "hashbrown" version = "0.17.1" @@ -1021,6 +1070,7 @@ dependencies = [ name = "pithos_lib" version = "0.8.0" dependencies = [ + "aes-gcm", "blake2", "blake3", "byteorder", @@ -1072,6 +1122,17 @@ dependencies = [ "universal-hash", ] +[[package]] +name = "polyval" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0fa31d631f2b2cb2a544d0aa321ce847a94764d701ca2becc411138b93d49cd" +dependencies = [ + "cpubits", + "cpufeatures", + "universal-hash", +] + [[package]] name = "potential_utf" version = "0.1.5" From 155203181eed83b86ba5940b5a09f6b8f803ef5a Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 18:45:18 +0200 Subject: [PATCH 065/166] fix: limit direct block key use to ChaCha20-Poly1305 payloads in the spec --- spec/PITHOS_1.0.0_draft.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/spec/PITHOS_1.0.0_draft.md b/spec/PITHOS_1.0.0_draft.md index 38f99e1..6d9d9a3 100644 --- a/spec/PITHOS_1.0.0_draft.md +++ b/spec/PITHOS_1.0.0_draft.md @@ -868,7 +868,9 @@ additional authenticated data (AAD) is empty. By default the block key is convergent: the first 32 output bytes of `SHAKE256(plaintext)`, where `plaintext` is the exact block plaintext before compression or encryption. No label or length prefix is included. A block -payload with encryption enabled is encrypted with its block key. +payload with encryption enabled and ProcessingFlags bit 5 clear is encrypted +with its block key directly. With bit 5 set, the payload key is derived from the +block key as described below. In version 1.1, a block whose ProcessingFlags bit 4 is set has a unique key instead: 32 bytes generated uniformly at random with a cryptographically secure From 025e3480c62c44d93b8dff0bc11a37722e797464 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 18:45:19 +0200 Subject: [PATCH 066/166] test: cover compressed and appended version 1.1 blocks --- .../src/archive/reader_private_tests.rs | 82 +++++++++++++++++++ crates/pithos_lib/tests/append_workflow.rs | 3 +- 2 files changed, 84 insertions(+), 1 deletion(-) diff --git a/crates/pithos_lib/src/archive/reader_private_tests.rs b/crates/pithos_lib/src/archive/reader_private_tests.rs index d745ea8..ca5779f 100644 --- a/crates/pithos_lib/src/archive/reader_private_tests.rs +++ b/crates/pithos_lib/src/archive/reader_private_tests.rs @@ -1930,3 +1930,85 @@ fn ciphers_mix_per_descriptor_and_reused_blocks_keep_their_cipher() { assert_eq!(output, expected, "{name}"); } } + +#[test] +fn version_1_1_processing_round_trips_compressed_blocks() { + let content = b"compressible version 1.1 payload ".repeat(64); + for (processing, extra) in [ + (aes_processing(), 0x20), + (unique_processing(), 0x10), + ( + unique_processing() + .with_cipher(PayloadCipher::Aes256Gcm) + .unwrap(), + 0x30, + ), + ] { + let processing = ProcessingOptions::new(true, 3) + .unwrap() + .with_key_mode(processing.key_mode()) + .unwrap() + .with_cipher(processing.cipher()) + .unwrap(); + let (_temporary, path) = fixture_with_processing(&content, processing); + let blocks = decode_terminal_directory(&path).blocks; + assert!(!blocks.is_empty()); + for block in blocks.values() { + assert_eq!(block.flags.0, 0x08 | extra | 3); + assert!(block.stored_size < block.original_size); + } + let archive = open_path(&path, AccessKeys::new().with_key(private("recipient1"))); + let mut output = Vec::new(); + archive.copy_to("data", &mut output).unwrap(); + assert_eq!(output, content); + } +} + +#[test] +fn version_1_1_appends_reuse_ciphers_and_give_unique_blocks_fresh_identities() { + let content = b"block shared by the base archive and every append"; + let (temporary, path) = + fixture_with_processing(content, ProcessingOptions::new(true, 0).unwrap()); + let base_blocks = decode_terminal_directory(&path).blocks; + assert_eq!(base_blocks.len(), 1); + assert_eq!(base_blocks[0].flags.0, 0x08); + let mut identities = std::collections::HashSet::from([*base_blocks.keys().next().unwrap()]); + let names = ["aes", "unique", "unique-aes"]; + for (name, processing, new_flags) in [ + (names[0], aes_processing(), None), + (names[1], unique_processing(), Some(0x18)), + ( + names[2], + unique_processing() + .with_cipher(PayloadCipher::Aes256Gcm) + .unwrap(), + Some(0x38), + ), + ] { + let source = temporary.path().join(name); + std::fs::write(&source, content).unwrap(); + crate::fs::append_files( + &path, + crate::archive::AppendOptions::new(private("recipient1"), vec![public("recipient1")]) + .with_processing(processing), + &[source], + ) + .unwrap(); + let blocks = decode_terminal_directory(&path).blocks; + match new_flags { + // The content-derived block reuses the ancestor's ChaCha20-Poly1305 descriptor. + None => assert!(blocks.is_empty()), + Some(flags) => { + assert_eq!(blocks.len(), 1); + assert_eq!(blocks[0].flags.0, flags); + assert!(identities.insert(*blocks.keys().next().unwrap())); + } + } + } + let archive = open_path(&path, AccessKeys::new().with_key(private("recipient1"))); + for name in ["data"].into_iter().chain(names) { + let mut output = Vec::new(); + archive.copy_to(name, &mut output).unwrap(); + assert_eq!(output, content, "{name}"); + } +} diff --git a/crates/pithos_lib/tests/append_workflow.rs b/crates/pithos_lib/tests/append_workflow.rs index b48746b..f2fa3b0 100644 --- a/crates/pithos_lib/tests/append_workflow.rs +++ b/crates/pithos_lib/tests/append_workflow.rs @@ -44,7 +44,8 @@ fn appends_to_version_1_0_archives_reject_version_1_1_processing_before_writing( let aes = ProcessingOptions::default() .with_cipher(PayloadCipher::Aes256Gcm) .unwrap(); - for processing in [unique, aes] { + let both = unique.with_cipher(PayloadCipher::Aes256Gcm).unwrap(); + for processing in [unique, aes, both] { assert!(matches!( append_files( &archive, From fd1e87ee4d6caf2a8e1d693376af1389adfa905f Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 19:14:22 +0200 Subject: [PATCH 067/166] feat: write fixed 4 MiB blocks by default and keep FastCDC as an option --- README.md | 2 +- crates/pithos/src/main.rs | 16 +- crates/pithos_lib/benches/adapters.rs | 10 +- crates/pithos_lib/src/adapters/ro_crate.rs | 7 +- crates/pithos_lib/src/archive/append.rs | 11 +- crates/pithos_lib/src/archive/mod.rs | 2 +- crates/pithos_lib/src/archive/reader.rs | 5 +- .../src/archive/reader_private_tests.rs | 9 +- crates/pithos_lib/src/archive/snapshot.rs | 4 +- crates/pithos_lib/src/archive/writer.rs | 235 +++++++++++++++--- crates/pithos_lib/src/error.rs | 2 + crates/pithos_lib/src/fs/append.rs | 18 +- crates/pithos_lib/tests/append_workflow.rs | 28 ++- crates/pithos_lib/tests/common/append.rs | 10 +- crates/pithos_lib/tests/common/ro_crate.rs | 7 +- .../pithos_lib/tests/ro_crate_conversion.rs | 6 +- crates/pithos_lib/tests/ro_crate_directory.rs | 4 +- crates/pithos_lib/tests/ro_crate_zip.rs | 8 +- crates/pithos_lib/tests/streaming_memory.rs | 5 +- crates/pithos_lib/tests/writer_failures.rs | 6 +- crates/pithos_lib/tests/writer_processing.rs | 9 +- spec/PITHOS_1.0.0_draft.md | 22 +- 22 files changed, 322 insertions(+), 104 deletions(-) diff --git a/README.md b/README.md index 592b401..75a4141 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ For the complete command workflow, including extracting entries, see the [CLI gu ## What it provides - Encrypted archive entries with recipient-key access control and per-block integrity verification. -- Content-defined chunking, optional compression, and indexed reads of complete entries or byte ranges. +- Fixed 4 MiB blocks by default or optional content-defined chunking, optional compression, and indexed reads of complete entries or byte ranges. - Local filesystem operations that avoid symlink traversal and refuse to overwrite existing extracted files. - RO-Crate directory and ZIP conversion, plus Crypt4GH export for readable entries. diff --git a/crates/pithos/src/main.rs b/crates/pithos/src/main.rs index 48c9508..4aea576 100644 --- a/crates/pithos/src/main.rs +++ b/crates/pithos/src/main.rs @@ -6,8 +6,8 @@ use clap::{Parser, Subcommand, ValueEnum}; use pithos_lib::adapters::crypt4gh; use pithos_lib::adapters::crypt4gh::Crypt4GHError; use pithos_lib::archive::{ - AccessKeys, AppendDurability, AppendOptions, Archive, ArchiveWriter, CdcConfig, OpenOptions, - ProcessingOptions, WriteOptions, WriterError, + AccessKeys, AppendDurability, AppendOptions, Archive, ArchiveWriter, CdcConfig, Chunking, + OpenOptions, ProcessingOptions, WriteOptions, WriterError, }; use pithos_lib::crypto::{PrivateKey, PublicKey, generate_private_key}; use pithos_lib::error::PithosError; @@ -110,7 +110,7 @@ enum PithosCommands { #[arg(long)] reader_public_keys: Option>, // Iterate files and parse all keys */ - /// Set values for content-defined chunking + /// Use content-defined chunking with these sizes instead of fixed 4 MiB blocks #[arg(long="cdc", value_parser=parse_cdc_input, value_name = "MIN,AVG,MAX")] cdc: Option, /// Create a local, uncompressed, unencrypted base archive @@ -232,7 +232,7 @@ enum AppendCommands { /// Path to Pithos file #[arg(short, long, value_name = "PITHOS FILE")] file: PathBuf, - /// Set values for content-defined chunking + /// Use content-defined chunking with these sizes instead of fixed 4 MiB blocks #[arg(long = "cdc", value_parser = parse_cdc_input, value_name = "MIN,AVG,MAX")] cdc: Option, /// Durability after publishing the appended directory @@ -415,7 +415,8 @@ fn run() -> Result<(), PithosCliError> { let (options, processing) = if plain { ( - WriteOptions::base().with_cdc(cdc.unwrap_or_default()), + WriteOptions::base() + .with_chunking(cdc.map(Chunking::ContentDefined).unwrap_or_default()), ProcessingOptions::new(false, 0).expect("zero compression is valid"), ) } else { @@ -431,7 +432,8 @@ fn run() -> Result<(), PithosCliError> { .map(|path| load_public_key_from_pem(path)) .collect(); ( - WriteOptions::new(sender_key, reader_keys?).with_cdc(cdc.unwrap_or_default()), + WriteOptions::new(sender_key, reader_keys?) + .with_chunking(cdc.map(Chunking::ContentDefined).unwrap_or_default()), ProcessingOptions::default(), ) }; @@ -543,7 +545,7 @@ fn run() -> Result<(), PithosCliError> { append_files( &file, AppendOptions::new(sender_key, reader_keys) - .with_cdc(cdc.unwrap_or_default()) + .with_chunking(cdc.map(Chunking::ContentDefined).unwrap_or_default()) .with_durability(durability.into()), &files, )?; diff --git a/crates/pithos_lib/benches/adapters.rs b/crates/pithos_lib/benches/adapters.rs index 69c6aaf..045954d 100644 --- a/crates/pithos_lib/benches/adapters.rs +++ b/crates/pithos_lib/benches/adapters.rs @@ -6,8 +6,8 @@ use pithos_lib::adapters::ro_crate::{ LoadedRoCrate, read_ro_crate_directory, read_ro_crate_zip, write_ro_crate, }; use pithos_lib::archive::{ - AccessKeys, Archive, ArchivePath, ArchiveWriter, CdcConfig, EntryMetadata, OpenOptions, - ProcessingOptions, WriteOptions, + AccessKeys, Archive, ArchivePath, ArchiveWriter, CdcConfig, Chunking, EntryMetadata, + OpenOptions, ProcessingOptions, WriteOptions, }; use pithos_lib::crypto::{PrivateKey, PublicKey}; use pithos_lib::source::FileSource; @@ -69,7 +69,9 @@ fn write_archive_fixture(path: &Path, sender: PrivateKey, recipient: PublicKey) let payload = support::deterministic_bytes(1, LARGE_MEMBER_BYTES); let mut writer = ArchiveWriter::create( std::fs::File::create(path).unwrap(), - WriteOptions::new(sender, vec![recipient]).with_cdc(CdcConfig::new(64, 256, 1024).unwrap()), + WriteOptions::new(sender, vec![recipient]).with_chunking(Chunking::ContentDefined( + CdcConfig::new(64, 256, 1024).unwrap(), + )), ) .unwrap(); writer @@ -107,7 +109,7 @@ fn convert_ro_crate( ) -> u64 { let mut writer = ArchiveWriter::create( support::CountingSink::default(), - WriteOptions::new(sender, vec![recipient]).with_cdc(cdc), + WriteOptions::new(sender, vec![recipient]).with_chunking(Chunking::ContentDefined(cdc)), ) .unwrap(); write_ro_crate(&mut writer, loaded, ProcessingOptions::default()).unwrap(); diff --git a/crates/pithos_lib/src/adapters/ro_crate.rs b/crates/pithos_lib/src/adapters/ro_crate.rs index 955013b..8b564b5 100644 --- a/crates/pithos_lib/src/adapters/ro_crate.rs +++ b/crates/pithos_lib/src/adapters/ro_crate.rs @@ -1455,7 +1455,7 @@ fn inspect_ro_crate_zip_archive_manifest( #[cfg(test)] mod tests { use super::*; - use crate::archive::{CdcConfig, WriteOptions}; + use crate::archive::{CdcConfig, Chunking, WriteOptions}; use crate::crypto::PrivateKey; use std::cell::Cell; use std::io::{Cursor, SeekFrom}; @@ -1519,8 +1519,9 @@ mod tests { let sender = PrivateKey::generate(); let mut writer = ArchiveWriter::create( Vec::new(), - WriteOptions::new(sender.duplicate(), vec![sender.public_key()]) - .with_cdc(CdcConfig::new(64, 256, 1024).unwrap()), + WriteOptions::new(sender.duplicate(), vec![sender.public_key()]).with_chunking( + Chunking::ContentDefined(CdcConfig::new(64, 256, 1024).unwrap()), + ), ) .unwrap(); write_ro_crate_zip_archive( diff --git a/crates/pithos_lib/src/archive/append.rs b/crates/pithos_lib/src/archive/append.rs index fd8519a..c97ad02 100644 --- a/crates/pithos_lib/src/archive/append.rs +++ b/crates/pithos_lib/src/archive/append.rs @@ -1,4 +1,4 @@ -use super::{CdcConfig, ProcessingOptions, WriteOptions}; +use super::{Chunking, ProcessingOptions, WriteOptions}; use crate::crypto::{PrivateKey, PublicKey}; use crate::error::PithosError; @@ -16,7 +16,7 @@ pub struct AppendOptions { /// Private key used to open the existing archive and authorize the append. pub(crate) access_key: PrivateKey, pub(crate) recipients: Vec, - pub(crate) cdc: CdcConfig, + pub(crate) chunking: Chunking, pub(crate) durability: AppendDurability, pub(crate) processing: ProcessingOptions, } @@ -28,14 +28,15 @@ impl AppendOptions { Self { access_key, recipients, - cdc: CdcConfig::default(), + chunking: Chunking::default(), durability: AppendDurability::Flush, processing: ProcessingOptions::append_default(), } } - pub fn with_cdc(mut self, cdc: CdcConfig) -> Self { - self.cdc = cdc; + /// Selects how appended content is split into blocks. The default is [`Chunking::default`]. + pub fn with_chunking(mut self, chunking: Chunking) -> Self { + self.chunking = chunking; self } diff --git a/crates/pithos_lib/src/archive/mod.rs b/crates/pithos_lib/src/archive/mod.rs index ba229a5..6ef6637 100644 --- a/crates/pithos_lib/src/archive/mod.rs +++ b/crates/pithos_lib/src/archive/mod.rs @@ -58,7 +58,7 @@ pub(crate) fn decode_validated_directory( ) } pub use writer::{ - ArchiveWriter, BlockKeyMode, CdcConfig, CreateError, EntryMetadata, EntryReference, + ArchiveWriter, BlockKeyMode, CdcConfig, Chunking, CreateError, EntryMetadata, EntryReference, FinishError, IncompleteWriter, PayloadCipher, ProcessingOptions, WriteOptions, WriterError, WrittenEntry, }; diff --git a/crates/pithos_lib/src/archive/reader.rs b/crates/pithos_lib/src/archive/reader.rs index 5ca6ef0..7baf41c 100644 --- a/crates/pithos_lib/src/archive/reader.rs +++ b/crates/pithos_lib/src/archive/reader.rs @@ -67,6 +67,9 @@ impl DecodedDirectoryCounts { } } +/// The default limit for one decoded block, which also bounds fixed writer block sizes. +pub(crate) const DEFAULT_MAX_DECODED_BLOCK_BYTES: u64 = 64 * 1024 * 1024; + /// Limits enforced before directory data is retained or expensive metadata work begins. #[derive(Clone, Copy, Debug)] pub struct OpenLimits { @@ -96,7 +99,7 @@ impl Default for OpenLimits { max_accessible_block_references: 1_000_000, max_opaque_metadata_bytes: 64 * 1024 * 1024, max_stored_block_bytes: 64 * 1024 * 1024, - max_decoded_block_bytes: 64 * 1024 * 1024, + max_decoded_block_bytes: DEFAULT_MAX_DECODED_BLOCK_BYTES, } } } diff --git a/crates/pithos_lib/src/archive/reader_private_tests.rs b/crates/pithos_lib/src/archive/reader_private_tests.rs index ca5779f..eff817e 100644 --- a/crates/pithos_lib/src/archive/reader_private_tests.rs +++ b/crates/pithos_lib/src/archive/reader_private_tests.rs @@ -3,7 +3,7 @@ use super::reader::{ ExternalBlockResolver, OpenLimits, OpenOptions, }; use crate::archive::{ - ArchivePath, ArchiveWriter, BlockKeyMode, CdcConfig, EntryMetadata, PayloadCipher, + ArchivePath, ArchiveWriter, BlockKeyMode, CdcConfig, Chunking, EntryMetadata, PayloadCipher, ProcessingOptions, WriteOptions, }; use crate::crypto::{self, FileKey, PrivateKey, PublicKey}; @@ -77,7 +77,7 @@ fn fixture_with_options( let path = temp.path().join("archive.pith"); let options = WriteOptions::new(private("sender"), vec![public("recipient1")]); let options = match cdc { - Some(cdc) => options.with_cdc(cdc), + Some(cdc) => options.with_chunking(Chunking::ContentDefined(cdc)), None => options, }; let mut writer = ArchiveWriter::create(File::create(&path).unwrap(), options).unwrap(); @@ -1728,8 +1728,9 @@ fn fixture_with_processing( ) -> (tempfile::TempDir, PathBuf) { let temp = tempfile::tempdir().unwrap(); let path = temp.path().join("processing.pith"); - let options = WriteOptions::new(private("sender"), vec![public("recipient1")]) - .with_cdc(CdcConfig::new(64, 256, 1024).unwrap()); + let options = WriteOptions::new(private("sender"), vec![public("recipient1")]).with_chunking( + Chunking::ContentDefined(CdcConfig::new(64, 256, 1024).unwrap()), + ); let mut writer = ArchiveWriter::create(File::create(&path).unwrap(), options).unwrap(); writer .add_file( diff --git a/crates/pithos_lib/src/archive/snapshot.rs b/crates/pithos_lib/src/archive/snapshot.rs index 55a80ab..d5c9266 100644 --- a/crates/pithos_lib/src/archive/snapshot.rs +++ b/crates/pithos_lib/src/archive/snapshot.rs @@ -447,7 +447,7 @@ mod tests { Vec::new(), PrivateKey::generate(), vec![recipient.public_key()], - crate::archive::CdcConfig::default(), + crate::archive::Chunking::default(), snapshot, ) .unwrap() @@ -524,7 +524,7 @@ mod tests { Vec::new(), sender, vec![recipient.public_key()], - crate::archive::CdcConfig::default(), + crate::archive::Chunking::default(), snapshot, ) .unwrap(); diff --git a/crates/pithos_lib/src/archive/writer.rs b/crates/pithos_lib/src/archive/writer.rs index ce6dab7..47c0ee5 100644 --- a/crates/pithos_lib/src/archive/writer.rs +++ b/crates/pithos_lib/src/archive/writer.rs @@ -4,6 +4,7 @@ use crate::archive::path_validation::{ validate_directory_entry_hierarchy_complete, validate_directory_entry_hierarchy_with_snapshot, validate_new_candidate_with_snapshot, }; +use crate::archive::reader::DEFAULT_MAX_DECODED_BLOCK_BYTES; use crate::archive::types::Processing; use crate::archive::validation::validate_relationships; use crate::archive::{AppendSnapshot, ArchivePath, FileId, Span, validated_segment_from_directory}; @@ -78,6 +79,100 @@ impl Default for CdcConfig { } } +/// How streamed content is split into blocks. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum Chunking { + /// Blocks of exactly this many bytes. Only the last block of a file may be shorter. + /// The size must be between 1 byte and the default reader limit for one decoded block. + Fixed(usize), + /// FastCDC boundaries chosen from the content, so shifted content can still deduplicate. + ContentDefined(CdcConfig), +} + +impl Chunking { + /// The default fixed block size of 4 MiB. + pub const DEFAULT_BLOCK_SIZE: usize = 4 * 1024 * 1024; + + pub(crate) fn validate(self) -> Result<(), PithosError> { + match self { + Self::Fixed(size) => validate_block_size(size), + Self::ContentDefined(_) => Ok(()), + } + } +} + +impl Default for Chunking { + fn default() -> Self { + Self::Fixed(Self::DEFAULT_BLOCK_SIZE) + } +} + +pub(crate) fn validate_block_size(size: usize) -> Result<(), PithosError> { + if size == 0 || size as u64 > DEFAULT_MAX_DECODED_BLOCK_BYTES { + return Err(PithosError::InvalidBlockSize(size)); + } + Ok(()) +} + +/// Splits streamed content into block plaintexts. +enum Chunker { + Fixed { + content: R, + size: usize, + block: Zeroizing>, + }, + ContentDefined { + stream: StreamCDC, + block: Zeroizing>, + }, +} + +impl Chunker { + fn new(content: R, chunking: Chunking) -> Self { + match chunking { + Chunking::Fixed(size) => Self::Fixed { + content, + size, + block: Zeroizing::new(Vec::new()), + }, + Chunking::ContentDefined(cdc) => Self::ContentDefined { + stream: StreamCDC::with_level( + content, + cdc.min_size, + cdc.avg_size, + cdc.max_size, + Normalization::Level1, + ), + block: Zeroizing::new(Vec::new()), + }, + } + } + + /// Returns the next block, or `None` at the end of the content. + fn next_block(&mut self) -> Result, PithosError> { + match self { + Self::Fixed { + content, + size, + block, + } => { + // Reserving the whole block once keeps plaintext from being copied on growth. + block.clear(); + block.reserve_exact(*size); + content.by_ref().take(*size as u64).read_to_end(block)?; + Ok((!block.is_empty()).then_some(block.as_slice())) + } + Self::ContentDefined { stream, block } => match stream.next() { + Some(chunk) => { + *block = Zeroizing::new(chunk?.data); + Ok(Some(block.as_slice())) + } + None => Ok(None), + }, + } + } +} + /// How the key of each encrypted block is chosen. #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] pub enum BlockKeyMode { @@ -234,31 +329,33 @@ enum WriteMode { /// Creation options for a base or encrypted archive. pub struct WriteOptions { mode: WriteMode, - cdc: CdcConfig, + chunking: Chunking, } impl WriteOptions { pub fn new(sender: PrivateKey, recipients: Vec) -> Self { Self { mode: WriteMode::Encrypted { sender, recipients }, - cdc: CdcConfig::default(), + chunking: Chunking::default(), } } pub fn base() -> Self { Self { mode: WriteMode::Base, - cdc: CdcConfig::default(), + chunking: Chunking::default(), } } - pub fn with_cdc(mut self, cdc: CdcConfig) -> Self { - self.cdc = cdc; + /// Selects how content is split into blocks. The default is [`Chunking::default`]. + pub fn with_chunking(mut self, chunking: Chunking) -> Self { + self.chunking = chunking; self } /// Check all creation metadata before taking ownership of an output sink. pub fn validate(&self) -> Result<(), PithosError> { + self.chunking.validate()?; if let WriteMode::Encrypted { recipients, .. } = &self.mode { if recipients.is_empty() { return Err(PithosError::WriterRequiresRecipient); @@ -595,7 +692,7 @@ impl Write for CountingSink { pub struct ArchiveWriter { mode: ArchiveWriterMode, version: FormatVersion, - cdc: CdcConfig, + chunking: Chunking, sink: CountingSink, directory: Directory, poisoned: bool, @@ -632,7 +729,7 @@ impl ArchiveWriter { if let Err(error) = options.validate() { return Err(CreateError { error, sink }); } - let WriteOptions { mode, cdc } = options; + let WriteOptions { mode, chunking } = options; let (mode, encryption) = match mode { WriteMode::Base => (ArchiveWriterMode::Base, IndexMap::new()), WriteMode::Encrypted { sender, recipients } => { @@ -659,7 +756,7 @@ impl ArchiveWriter { Ok(Self { mode, version: FormatVersion::CURRENT, - cdc, + chunking, sink, directory, poisoned: false, @@ -678,10 +775,12 @@ impl ArchiveWriter { sink: W, sender: PrivateKey, recipients: Vec, - cdc: CdcConfig, + chunking: Chunking, snapshot: AppendSnapshot, ) -> Result { - WriteOptions::new(sender.duplicate(), recipients.clone()).validate()?; + WriteOptions::new(sender.duplicate(), recipients.clone()) + .with_chunking(chunking) + .validate()?; let sender = sender.into_dalek_static_secret(); let recipients = recipients .into_iter() @@ -698,7 +797,7 @@ impl ArchiveWriter { Ok(Self { mode: ArchiveWriterMode::Encrypted { sender }, version: snapshot.version(), - cdc, + chunking, sink: CountingSink { sink, offset: snapshot.archive_len(), @@ -916,22 +1015,17 @@ impl ArchiveWriter { } processing.validate_for(self.version)?; let mut delta = self.stage_entry(file_type, path, metadata, 0, None)?; - let mut stream = StreamCDC::with_level( - content, - self.cdc.min_size, - self.cdc.avg_size, - self.cdc.max_size, - Normalization::Level1, - ); + let mut chunker = Chunker::new(content, self.chunking); let mut size = 0u64; let mut block_references = HashMap::new(); - for chunk in &mut stream { - let chunk = match chunk { - Ok(chunk) => chunk, - Err(error) => return self.poison(error.into()), + loop { + let chunk_data = match chunker.next_block() { + Ok(Some(chunk_data)) => chunk_data, + Ok(None) => break, + Err(error) => return self.poison(error), }; - let chunk_data = Zeroizing::new(chunk.data); - size = match size.checked_add(chunk.length as u64) { + let chunk_len = chunk_data.len() as u64; + size = match size.checked_add(chunk_len) { Some(size) => size, None => return self.poison(PithosError::WriterSizeOverflow), }; @@ -941,7 +1035,7 @@ impl ArchiveWriter { }; let encoded = match self .runtime - .encode_block(&chunk_data, processing.flags(), nonce) + .encode_block(chunk_data, processing.flags(), nonce) { Ok(encoded) => encoded, Err(error) => return self.poison(error), @@ -960,11 +1054,11 @@ impl ArchiveWriter { return self.poison(error); } if let Some(existing) = self.directory.blocks.get(&hash) { - if existing.original_size != chunk.length as u64 { + if existing.original_size != chunk_len { return self.poison(PithosError::BlockIndexConflict { hash, existing_original_size: existing.original_size, - new_original_size: chunk.length as u64, + new_original_size: chunk_len, }); } continue; @@ -974,21 +1068,21 @@ impl ArchiveWriter { .as_ref() .and_then(|snapshot| snapshot.descriptor(crate::archive::types::BlockHash(hash))) { - if existing.original_size != chunk.length as u64 { + if existing.original_size != chunk_len { return self.poison(PithosError::BlockIndexConflict { hash, existing_original_size: existing.original_size, - new_original_size: chunk.length as u64, + new_original_size: chunk_len, }); } continue; } if let Some(existing) = delta.descriptors.get(&hash) { - if existing.original_size != chunk.length as u64 { + if existing.original_size != chunk_len { return self.poison(PithosError::BlockIndexConflict { hash, existing_original_size: existing.original_size, - new_original_size: chunk.length as u64, + new_original_size: chunk_len, }); } continue; @@ -999,7 +1093,7 @@ impl ArchiveWriter { Ok(size) => size, Err(_) => return self.poison(PithosError::WriterSizeOverflow), }, - original_size: chunk.length as u64, + original_size: chunk_len, flags: encoded.flags, location: BlockLocation::Local, }; @@ -1637,7 +1731,9 @@ mod tests { fn content_writer(failure: Option<(RuntimeOperation, usize)>) -> ArchiveWriter> { ArchiveWriter::with_test_runtime( Vec::new(), - deterministic_options().with_cdc(CdcConfig::new(64, 256, 1024).unwrap()), + deterministic_options().with_chunking(Chunking::ContentDefined( + CdcConfig::new(64, 256, 1024).unwrap(), + )), Box::new(TestRuntime::new(failure)), 0, ) @@ -1990,7 +2086,7 @@ mod tests { Vec::new(), PrivateKey::generate(), vec![sender.public_key()], - CdcConfig::default(), + Chunking::default(), snapshot, ) .unwrap(); @@ -2233,7 +2329,7 @@ mod tests { Vec::new(), sender, vec![recipient.public_key()], - CdcConfig::default(), + Chunking::default(), snapshot, ) .unwrap(); @@ -2317,7 +2413,7 @@ mod tests { Vec::new(), PrivateKey::generate(), vec![recipient], - CdcConfig::default(), + Chunking::default(), snapshot, ) .unwrap(); @@ -2370,4 +2466,71 @@ mod tests { Err(WriterError::Pithos(PithosError::FileIdExhausted)) )); } + + /// Yields one byte per read call. + struct ByteReader<'a>(&'a [u8]); + + impl Read for ByteReader<'_> { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + match (self.0.split_first(), buffer.first_mut()) { + (Some((first, rest)), Some(target)) => { + *target = *first; + self.0 = rest; + Ok(1) + } + _ => Ok(0), + } + } + } + + #[test] + fn fixed_blocks_ignore_read_sizes_and_end_with_one_short_block() { + fn blocks(content: impl Read) -> Vec<([u8; 32], u64)> { + let options = options().with_chunking(Chunking::Fixed(1000)); + let mut writer = ArchiveWriter::create(Vec::new(), options).unwrap(); + writer + .add_file( + ArchivePath::new("data").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(true, 0).unwrap(), + None, + content, + ) + .unwrap(); + let blocks = writer.directory.blocks.iter(); + blocks + .map(|(hash, block)| (*hash, block.original_size)) + .collect() + } + let content = (0..2500u32) + .map(|index| (index * 7) as u8) + .collect::>(); + for content in [&content[..], &content[..2000]] { + let expected = content + .chunks(1000) + .map(|chunk| (*blake3::hash(chunk).as_bytes(), chunk.len() as u64)) + .collect::>(); + assert_eq!(blocks(Cursor::new(content)), expected); + assert_eq!(blocks(ByteReader(content)), expected); + } + } + + #[test] + fn fixed_block_sizes_are_validated_before_header_output() { + assert_eq!( + WriteOptions::base().chunking, + Chunking::Fixed(4 * 1024 * 1024) + ); + let largest = 64 * 1024 * 1024; + for size in [0, largest + 1] { + let options = options().with_chunking(Chunking::Fixed(size)); + let error = ArchiveWriter::create(Vec::new(), options).err().unwrap(); + assert!( + matches!(error.error(), PithosError::InvalidBlockSize(actual) if *actual == size) + ); + assert!(error.into_incomplete().is_empty()); + } + let options = options().with_chunking(Chunking::Fixed(largest)); + assert!(ArchiveWriter::create(Vec::new(), options).is_ok()); + } } diff --git a/crates/pithos_lib/src/error.rs b/crates/pithos_lib/src/error.rs index cd035a7..de2d20d 100644 --- a/crates/pithos_lib/src/error.rs +++ b/crates/pithos_lib/src/error.rs @@ -176,6 +176,8 @@ pub enum PithosError { avg_size: usize, max_size: usize, }, + #[error("invalid fixed block size {0}")] + InvalidBlockSize(usize), #[error("writer is poisoned")] WriterPoisoned, #[error("streamed content size overflow")] diff --git a/crates/pithos_lib/src/fs/append.rs b/crates/pithos_lib/src/fs/append.rs index aadfb4c..d3845a8 100644 --- a/crates/pithos_lib/src/fs/append.rs +++ b/crates/pithos_lib/src/fs/append.rs @@ -179,8 +179,13 @@ fn append_files_impl( Arc::clone(&mutated), ); sink.seek(SeekFrom::Start(original_len))?; - let mut writer = - ArchiveWriter::append(sink, wrapping_sender, recipients, options.cdc, snapshot)?; + let mut writer = ArchiveWriter::append( + sink, + wrapping_sender, + recipients, + options.chunking, + snapshot, + )?; plan.manifest .ingest_planned(&plan.ids, &mut writer, options.processing) .map_err(writer_error)?; @@ -258,8 +263,13 @@ pub fn grant_readers( Arc::clone(&mutated), ); sink.seek(SeekFrom::Start(original_len))?; - let mut writer = - ArchiveWriter::append(sink, wrapping_sender, recipients, options.cdc, snapshot)?; + let mut writer = ArchiveWriter::append( + sink, + wrapping_sender, + recipients, + options.chunking, + snapshot, + )?; writer.grant_file_keys(&ids.iter().copied().map(FileId).collect::>())?; let mut sink = writer.finish().map_err(|error| error.into_parts().0)?; if options.durability == AppendDurability::SyncAll { diff --git a/crates/pithos_lib/tests/append_workflow.rs b/crates/pithos_lib/tests/append_workflow.rs index f2fa3b0..6b2fb01 100644 --- a/crates/pithos_lib/tests/append_workflow.rs +++ b/crates/pithos_lib/tests/append_workflow.rs @@ -1,10 +1,12 @@ mod common; -use common::append::{WITHHELD_ID, append, append_fixture, append_with_cdc, archive_with_entry}; +use common::append::{ + WITHHELD_ID, append, append_fixture, append_with_chunking, archive_with_entry, +}; use common::util::{open, private_key, public_key}; use pithos_lib::archive::{ - AccessKeys, AppendOptions, ArchivePath, ArchiveWriter, BlockKeyMode, CdcConfig, EntryMetadata, - PayloadCipher, ProcessingOptions, WriteOptions, + AccessKeys, AppendOptions, ArchivePath, ArchiveWriter, BlockKeyMode, CdcConfig, Chunking, + EntryMetadata, PayloadCipher, ProcessingOptions, WriteOptions, }; use pithos_lib::error::PithosError; use pithos_lib::fs::{FsError, append_files}; @@ -212,9 +214,9 @@ fn append_streams_multiblock_content_and_writes_one_child_directory_without_a_he &(0..8192u32).flat_map(u32::to_le_bytes).collect::>(), ); - append_with_cdc( + append_with_chunking( &fixture.archive, - CdcConfig::new(64, 256, 1024).unwrap(), + Chunking::ContentDefined(CdcConfig::new(64, 256, 1024).unwrap()), vec![source], ) .unwrap(); @@ -249,3 +251,19 @@ fn append_streams_multiblock_content_and_writes_one_child_directory_without_a_he .unwrap(); assert_eq!(contents.len(), 8192 * 4); } + +#[test] +fn appends_reject_invalid_fixed_block_sizes_before_writing() { + let temporary = tempfile::tempdir().unwrap(); + let fixture = append_fixture(&temporary); + let original = std::fs::read(&fixture.archive).unwrap(); + let source = fixture.append_source("appended.txt", b"appended payload"); + assert!(matches!( + append_with_chunking(&fixture.archive, Chunking::Fixed(0), vec![source]), + Err(FsError::Core { + source: PithosError::InvalidBlockSize(0), + .. + }) + )); + assert_eq!(std::fs::read(&fixture.archive).unwrap(), original); +} diff --git a/crates/pithos_lib/tests/common/append.rs b/crates/pithos_lib/tests/common/append.rs index 9036479..17b5d0d 100644 --- a/crates/pithos_lib/tests/common/append.rs +++ b/crates/pithos_lib/tests/common/append.rs @@ -3,7 +3,7 @@ use super::{ keys::{private_key, public_key}, }; use pithos_lib::archive::{ - AccessKeys, AppendOptions, Archive, ArchivePath, ArchiveWriter, CdcConfig, EntryMetadata, + AccessKeys, AppendOptions, Archive, ArchivePath, ArchiveWriter, Chunking, EntryMetadata, ProcessingOptions, WriteOptions, }; use pithos_lib::fs::{FsError, append_files, grant_readers}; @@ -95,13 +95,13 @@ pub fn append_fixture(temporary: &tempfile::TempDir) -> AppendFixture { #[allow(dead_code)] pub fn append(archive: &Path, sources: Vec) -> Result<(), FsError> { - append_with_cdc(archive, CdcConfig::default(), sources) + append_with_chunking(archive, Chunking::default(), sources) } #[allow(dead_code)] -pub fn append_with_cdc( +pub fn append_with_chunking( archive: &Path, - cdc: CdcConfig, + chunking: Chunking, sources: Vec, ) -> Result<(), FsError> { append_files( @@ -110,7 +110,7 @@ pub fn append_with_cdc( private_key("sender"), vec![public_key("sender"), public_key("recipient1")], ) - .with_cdc(cdc), + .with_chunking(chunking), &sources, ) .map(|_| ()) diff --git a/crates/pithos_lib/tests/common/ro_crate.rs b/crates/pithos_lib/tests/common/ro_crate.rs index a61f1dc..9de7939 100644 --- a/crates/pithos_lib/tests/common/ro_crate.rs +++ b/crates/pithos_lib/tests/common/ro_crate.rs @@ -1,7 +1,7 @@ use super::keys::{private_key, public_key}; use pithos_lib::adapters::ro_crate::{LoadedRoCrate, RoCrateError, write_ro_crate}; use pithos_lib::archive::{ - ArchivePath, ArchiveWriter, CdcConfig, EntryMetadata, ProcessingOptions, WriteOptions, + ArchivePath, ArchiveWriter, Chunking, EntryMetadata, ProcessingOptions, WriteOptions, }; use std::fs::{self, File}; use std::path::Path; @@ -84,10 +84,11 @@ pub fn write_raw_zip(path: &Path, entries: &[(&[u8], &[u8], u32)]) { } #[allow(dead_code)] -pub fn write_loaded(output: &Path, loaded: LoadedRoCrate, cdc: CdcConfig) { +pub fn write_loaded(output: &Path, loaded: LoadedRoCrate, chunking: Chunking) { let mut writer = ArchiveWriter::create( File::create(output).unwrap(), - WriteOptions::new(private_key("sender"), vec![public_key("recipient1")]).with_cdc(cdc), + WriteOptions::new(private_key("sender"), vec![public_key("recipient1")]) + .with_chunking(chunking), ) .unwrap(); write_ro_crate( diff --git a/crates/pithos_lib/tests/ro_crate_conversion.rs b/crates/pithos_lib/tests/ro_crate_conversion.rs index 6e1821b..1cd2175 100644 --- a/crates/pithos_lib/tests/ro_crate_conversion.rs +++ b/crates/pithos_lib/tests/ro_crate_conversion.rs @@ -3,7 +3,7 @@ mod common; use common::ro_crate::{metadata, write_loaded, write_raw_zip}; use common::util::{open, private_key}; use pithos_lib::adapters::ro_crate::{read_ro_crate_directory, read_ro_crate_zip}; -use pithos_lib::archive::{AccessKeys, ArchiveEntry, CdcConfig, EntryKind}; +use pithos_lib::archive::{AccessKeys, ArchiveEntry, Chunking, EntryKind}; use std::fs; use std::os::unix::fs::symlink; use std::path::Path; @@ -61,12 +61,12 @@ fn typed_directory_and_zip_conversion_have_semantic_parity_and_extract() { write_loaded( &directory_output, read_ro_crate_directory(&source).unwrap(), - CdcConfig::default(), + Chunking::default(), ); write_loaded( &zip_output, read_ro_crate_zip(&zip).unwrap(), - CdcConfig::default(), + Chunking::default(), ); assert_eq!( diff --git a/crates/pithos_lib/tests/ro_crate_directory.rs b/crates/pithos_lib/tests/ro_crate_directory.rs index 1a3fe8a..19027f1 100644 --- a/crates/pithos_lib/tests/ro_crate_directory.rs +++ b/crates/pithos_lib/tests/ro_crate_directory.rs @@ -5,7 +5,7 @@ use common::util::{open, private_key}; use pithos_lib::adapters::ro_crate::{ RO_CRATE_METADATA_FILE, RoCrateError, RoCrateSource, read_ro_crate_directory, }; -use pithos_lib::archive::{AccessKeys, CdcConfig, WriterError}; +use pithos_lib::archive::{AccessKeys, Chunking, WriterError}; use pithos_lib::error::PithosError; use rocraters::ro_crate::graph_vector::GraphVector; use rocraters::ro_crate::read::CrateReadError; @@ -32,7 +32,7 @@ fn loaded_directory_converts_from_retained_sources_after_path_replacement() { fs::write(source.join("replacement-only.bin"), b"replacement").unwrap(); let output = temporary.path().join("retained-directory.pith"); - write_loaded(&output, loaded, CdcConfig::default()); + write_loaded(&output, loaded, Chunking::default()); let archive = open( &output, AccessKeys::new().with_key(private_key("recipient1")), diff --git a/crates/pithos_lib/tests/ro_crate_zip.rs b/crates/pithos_lib/tests/ro_crate_zip.rs index 554eca7..41c237f 100644 --- a/crates/pithos_lib/tests/ro_crate_zip.rs +++ b/crates/pithos_lib/tests/ro_crate_zip.rs @@ -7,7 +7,7 @@ use common::util::{open, private_key}; use pithos_lib::adapters::ro_crate::{ RO_CRATE_METADATA_FILE, RoCrateError, RoCrateSource, read_ro_crate_zip, }; -use pithos_lib::archive::{AccessKeys, CdcConfig, EntryKind, WriterError}; +use pithos_lib::archive::{AccessKeys, CdcConfig, Chunking, EntryKind, WriterError}; use pithos_lib::error::PithosError; use std::fs; use std::io::{Seek, SeekFrom, Write}; @@ -47,7 +47,7 @@ fn loaded_zip_converts_from_retained_open_archive_after_path_replacement() { ); let output = temporary.path().join("retained-zip.pith"); - write_loaded(&output, loaded, CdcConfig::default()); + write_loaded(&output, loaded, Chunking::default()); let archive = open( &output, AccessKeys::new().with_key(private_key("recipient1")), @@ -208,7 +208,7 @@ fn typed_zip_conversion_streams_large_multiblock_member_and_round_trips() { write_loaded( &output, read_ro_crate_zip(&zip).unwrap(), - CdcConfig::new(64, 256, 1024).unwrap(), + Chunking::ContentDefined(CdcConfig::new(64, 256, 1024).unwrap()), ); let archive = open( &output, @@ -421,7 +421,7 @@ fn parser_preserves_a_synthetic_zip_fixture_graph_source_and_permissions() { assert!(!loaded.ro_crate().graph.is_empty()); let output = temporary.path().join("fixture.pith"); - write_loaded(&output, loaded, CdcConfig::default()); + write_loaded(&output, loaded, Chunking::default()); let archive = open( &output, AccessKeys::new().with_key(private_key("recipient1")), diff --git a/crates/pithos_lib/tests/streaming_memory.rs b/crates/pithos_lib/tests/streaming_memory.rs index a23cada..e69db59 100644 --- a/crates/pithos_lib/tests/streaming_memory.rs +++ b/crates/pithos_lib/tests/streaming_memory.rs @@ -1,6 +1,6 @@ use peak_alloc::PeakAlloc; use pithos_lib::archive::{ - ArchivePath, ArchiveWriter, CdcConfig, EntryMetadata, ProcessingOptions, WriteOptions, + ArchivePath, ArchiveWriter, CdcConfig, Chunking, EntryMetadata, ProcessingOptions, WriteOptions, }; use pithos_lib::crypto::PrivateKey; use std::io::{Read, Write}; @@ -58,7 +58,8 @@ fn streamed_peak(logical_bytes: u64) -> (u64, u64) { PEAK_ALLOC.reset_peak_usage(); let mut writer = ArchiveWriter::create( CountingSink::default(), - WriteOptions::new(sender, vec![recipient]).with_cdc(CdcConfig::DEFAULT), + WriteOptions::new(sender, vec![recipient]) + .with_chunking(Chunking::ContentDefined(CdcConfig::DEFAULT)), ) .unwrap(); writer diff --git a/crates/pithos_lib/tests/writer_failures.rs b/crates/pithos_lib/tests/writer_failures.rs index 036fb23..477abae 100644 --- a/crates/pithos_lib/tests/writer_failures.rs +++ b/crates/pithos_lib/tests/writer_failures.rs @@ -2,7 +2,7 @@ mod common; use common::writer::options; use pithos_lib::archive::{ - AccessKeys, Archive, ArchivePath, ArchiveWriter, CdcConfig, EntryKind, EntryMetadata, + AccessKeys, Archive, ArchivePath, ArchiveWriter, CdcConfig, Chunking, EntryKind, EntryMetadata, OpenOptions, ProcessingOptions, WriteOptions, WriterError, }; use pithos_lib::crypto::PrivateKey; @@ -241,7 +241,9 @@ fn input_failure_between_emitted_blocks_poisons_the_writer() { .collect(); let mut writer = ArchiveWriter::create( Vec::new(), - WriteOptions::new(sender, vec![recipient]).with_cdc(CdcConfig::new(64, 256, 1024).unwrap()), + WriteOptions::new(sender, vec![recipient]).with_chunking(Chunking::ContentDefined( + CdcConfig::new(64, 256, 1024).unwrap(), + )), ) .unwrap(); assert!( diff --git a/crates/pithos_lib/tests/writer_processing.rs b/crates/pithos_lib/tests/writer_processing.rs index 3818b48..f14fd29 100644 --- a/crates/pithos_lib/tests/writer_processing.rs +++ b/crates/pithos_lib/tests/writer_processing.rs @@ -1,8 +1,8 @@ mod common; use pithos_lib::archive::{ - AccessKeys, Archive, ArchivePath, ArchiveWriter, CdcConfig, EntryMetadata, OpenOptions, - ProcessingOptions, WriteOptions, + AccessKeys, Archive, ArchivePath, ArchiveWriter, CdcConfig, Chunking, EntryMetadata, + OpenOptions, ProcessingOptions, WriteOptions, }; use pithos_lib::crypto::PrivateKey; use pithos_lib::source::MemorySource; @@ -15,8 +15,9 @@ fn repeated_identical_chunks_round_trip_in_order() { let data = vec![0u8; 2 * 1024]; let mut writer = ArchiveWriter::create( Vec::new(), - WriteOptions::new(sender, vec![reader.public_key()]) - .with_cdc(CdcConfig::new(64, 256, 1024).unwrap()), + WriteOptions::new(sender, vec![reader.public_key()]).with_chunking( + Chunking::ContentDefined(CdcConfig::new(64, 256, 1024).unwrap()), + ), ) .unwrap(); writer diff --git a/spec/PITHOS_1.0.0_draft.md b/spec/PITHOS_1.0.0_draft.md index 6d9d9a3..aad7917 100644 --- a/spec/PITHOS_1.0.0_draft.md +++ b/spec/PITHOS_1.0.0_draft.md @@ -791,10 +791,11 @@ for each of its piece key IDs. ## 5. Content Processing -### 5.1 Content-Defined Chunking +### 5.1 Chunking -Writers MAY use content-defined chunking. Chunking parameters are writer guidance, -not a compatibility requirement; see Appendix A. +Writers choose block boundaries. Fixed-size blocks and content-defined chunking +are both valid. Chunking parameters are writer guidance, not a compatibility +requirement; see Appendix A. ### 5.2 Block Hashing @@ -973,7 +974,7 @@ Zstandard versions. 1. Write file header 2. Process files in correct directory order -3. Chunk content using content-defined chunking +3. Split content into blocks (fixed-size or content-defined) 4. Deduplicate blocks by hash (unique-key blocks never repeat a hash) 5. Write a directory, including its encryption sections when present 6. Validate complete structure @@ -1170,8 +1171,17 @@ Extensions MUST maintain backwards compatibility for reading. This appendix suggests writer choices only. It does not define archive conformance or reader behavior. -Recommended content-defined chunking parameters are a 64 KB minimum, 128 KB -average, 512 KB maximum, and 48-byte window. +**Recommended block size:** fixed 4 MiB (4,194,304 byte) blocks. Every block +of a file has this size except the last, which may be shorter. Block boundaries +then depend only on byte offsets, not on how the input was read. This suits +object storage and multipart uploads, where parts are written independently. +Content-defined chunking (for example FastCDC) is optional. It can find more +repeated blocks when content shifts, at the cost of variable block sizes. + +**Directory size:** each block adds one descriptor and one block-list entry, so +the directory grows with the block count. A 5 TiB file at 4 MiB blocks has +about 1.31 million blocks and roughly 150 MB of directory. Smaller blocks +increase this size in proportion. **Recommended compression mapping:** a writer may map ProcessingFlags values `1` through `7` to Zstandard levels `1`, `4`, `8`, `11`, `15`, `18`, and `22`, From 588f87d7528604977a505bd21b5a34ed405a1f5f Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 19:20:30 +0200 Subject: [PATCH 068/166] feat: split written piece content into fixed-size blocks --- crates/pithos_lib/src/archive/pieces.rs | 113 +++++++++++++++++++++++- crates/pithos_lib/src/error.rs | 2 + 2 files changed, 112 insertions(+), 3 deletions(-) diff --git a/crates/pithos_lib/src/archive/pieces.rs b/crates/pithos_lib/src/archive/pieces.rs index e76ee46..d5ed09b 100644 --- a/crates/pithos_lib/src/archive/pieces.rs +++ b/crates/pithos_lib/src/archive/pieces.rs @@ -6,7 +6,7 @@ use crate::archive::path_validation::validate_entry; use crate::archive::types::{ArchivePath, Processing}; -use crate::archive::writer::{EntryMetadata, ProcessingOptions}; +use crate::archive::writer::{Chunking, EntryMetadata, ProcessingOptions, validate_block_size}; use crate::block; use crate::crypto::{self, FileKey, PublicKey}; use crate::error::PithosError; @@ -58,8 +58,11 @@ pub struct Piece { pub(crate) grants: Vec<([u8; 32], Vec)>, } -/// Encodes the blocks of one piece. The caller chooses block boundaries and appends every -/// returned byte string, in order, to the piece's stored bytes. +/// Encodes the blocks of one piece. The caller appends every returned byte string, in order, +/// to the piece's stored bytes. +/// +/// [`PieceEncoder::write`] and [`PieceEncoder::flush`] split the content into fixed-size blocks, +/// which is the recommended use. [`PieceEncoder::push`] instead encodes caller-chosen blocks. pub struct PieceEncoder { key_id: u64, recipients: Vec<[u8; 32]>, @@ -69,6 +72,8 @@ pub struct PieceEncoder { seen: HashSet<[u8; 32]>, stored_len: u64, original_size: u64, + block_size: usize, + pending: Zeroizing>, } impl PieceEncoder { @@ -106,13 +111,65 @@ impl PieceEncoder { seen: HashSet::new(), stored_len: 0, original_size: 0, + block_size: Chunking::DEFAULT_BLOCK_SIZE, + pending: Zeroizing::new(Vec::new()), }) } + /// Sets the block size used by [`PieceEncoder::write`]. The default is 4 MiB. + pub fn with_block_size(mut self, size: usize) -> Result { + validate_block_size(size)?; + self.block_size = size; + Ok(self) + } + + /// Adds content and returns the stored bytes of every block it completes. + /// Blocks have the configured size no matter how the content is split across calls. + /// After the last write, call [`PieceEncoder::flush`] for the short final block. + pub fn write(&mut self, mut content: &[u8]) -> Result, PithosError> { + let mut stored = Vec::new(); + while !content.is_empty() { + if self.pending.is_empty() && content.len() >= self.block_size { + let (block, rest) = content.split_at(self.block_size); + stored.extend(self.encode(block)?); + content = rest; + continue; + } + // Reserving the whole block once keeps plaintext from being copied on growth. + self.pending.reserve_exact(self.block_size); + let take = (self.block_size - self.pending.len()).min(content.len()); + self.pending.extend_from_slice(&content[..take]); + content = &content[take..]; + if self.pending.len() == self.block_size { + stored.extend(self.flush()?); + } + } + Ok(stored) + } + + /// Encodes the bytes buffered by [`PieceEncoder::write`] as one block, if there are any. + pub fn flush(&mut self) -> Result, PithosError> { + if self.pending.is_empty() { + return Ok(Vec::new()); + } + let block = std::mem::take(&mut self.pending); + let stored = self.encode(&block); + self.pending = block; + self.pending.clear(); + stored + } + /// Encodes one non-empty block and returns `BLCK || payload` to append to the piece. /// With content-derived keys, a block that repeats an earlier block of this piece returns /// no bytes. With unique keys, every block is stored. pub fn push(&mut self, plaintext: &[u8]) -> Result, PithosError> { + if !self.pending.is_empty() { + return Err(PithosError::UnflushedPieceBytes); + } + self.encode(plaintext) + } + + fn encode(&mut self, plaintext: &[u8]) -> Result, PithosError> { if plaintext.is_empty() { return Err(PithosError::InvalidBlockDescriptor( "piece blocks must not be empty", @@ -143,7 +200,11 @@ impl PieceEncoder { } /// Seals the block list under a fresh piece key and grants that key to every recipient. + /// Fails if written bytes were not flushed. pub fn finish(self) -> Result { + if !self.pending.is_empty() { + return Err(PithosError::UnflushedPieceBytes); + } let piece_key = FileKey::from_bytes(StaticSecret::random().to_bytes()); let mut list = Zeroizing::new(Vec::new()); encode_decrypted_block_list(&self.entries, &mut *list)?; @@ -514,4 +575,50 @@ mod tests { )); } } + + #[test] + fn written_fragments_give_the_same_fixed_blocks_as_pushed_blocks() { + let content = (0..2500u32) + .map(|index| (index * 7) as u8) + .collect::>(); + let encoder = || { + let recipient = crate::crypto::PrivateKey::generate().public_key(); + let processing = ProcessingOptions::new(true, 0).unwrap(); + let encoder = PieceEncoder::new(1, vec![recipient], processing).unwrap(); + encoder.with_block_size(1000).unwrap() + }; + let mut pushed = encoder(); + for block in content.chunks(1000) { + pushed.push(block).unwrap(); + } + let expected = pushed.finish().unwrap().blocks; + assert_eq!(expected.len(), 3); + for fragment in [1, 7, 1000, 4096] { + let mut written = encoder(); + let mut stored = 0; + for part in content.chunks(fragment) { + stored += written.write(part).unwrap().len(); + } + assert!(matches!( + written.push(b"block"), + Err(PithosError::UnflushedPieceBytes) + )); + stored += written.flush().unwrap().len(); + assert!(written.flush().unwrap().is_empty()); + let piece = written.finish().unwrap(); + assert_eq!(piece.blocks, expected); + assert_eq!(piece.stored_len, stored as u64); + } + + let mut unflushed = encoder(); + unflushed.write(b"short").unwrap(); + assert!(matches!( + unflushed.finish(), + Err(PithosError::UnflushedPieceBytes) + )); + assert!(matches!( + encoder().with_block_size(0), + Err(PithosError::InvalidBlockSize(0)) + )); + } } diff --git a/crates/pithos_lib/src/error.rs b/crates/pithos_lib/src/error.rs index de2d20d..c72a7ae 100644 --- a/crates/pithos_lib/src/error.rs +++ b/crates/pithos_lib/src/error.rs @@ -89,6 +89,8 @@ pub enum PithosError { PieceKeyIdConflict(u64), #[error("invalid piece record")] InvalidPieceRecord, + #[error("the piece encoder holds written bytes that were not flushed")] + UnflushedPieceBytes, #[error("archive metadata does not match the expected digest")] MetadataDigestMismatch, #[error("duplicate encoded recipient file id")] From ee17916e1e01a4ab94b1ea9c18cff04e5091326c Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 19:37:11 +0200 Subject: [PATCH 069/166] feat: admit 5 TiB objects in the default open limits --- crates/pithos_lib/src/archive/pieces.rs | 9 ++--- crates/pithos_lib/src/archive/reader.rs | 11 +++--- crates/pithos_lib/src/format/directory.rs | 17 +++++++++ crates/pithos_lib/tests/pieces.rs | 44 +++++++++++++++++++++++ 4 files changed, 69 insertions(+), 12 deletions(-) diff --git a/crates/pithos_lib/src/archive/pieces.rs b/crates/pithos_lib/src/archive/pieces.rs index d5ed09b..893ef99 100644 --- a/crates/pithos_lib/src/archive/pieces.rs +++ b/crates/pithos_lib/src/archive/pieces.rs @@ -11,9 +11,7 @@ use crate::block; use crate::crypto::{self, FileKey, PublicKey}; use crate::error::PithosError; use crate::format::block::{BlockIndexEntry, BlockLocation, ProcessingFlags}; -use crate::format::directory::{ - Directory, DirectoryEntries, encode_directory, update_directory_crc, update_directory_len, -}; +use crate::format::directory::{Directory, DirectoryEntries, encode_complete_directory}; use crate::format::encryption::{ EncryptionSection, RecipientData, RecipientSection, encode_decrypted_recipient_list, }; @@ -537,10 +535,7 @@ pub fn compose( files.insert(0, path.as_str(), entry)?; let mut directory = Directory::new(None, files, encryption); directory.blocks = blocks; - update_directory_len(&mut directory)?; - update_directory_crc(&mut directory)?; - let mut bytes = Vec::new(); - encode_directory(&directory, &mut bytes)?; + let bytes = encode_complete_directory(&directory)?; Ok(Composition { piece_offsets, archive_len: checked_add(offset, bytes.len() as u64)?, diff --git a/crates/pithos_lib/src/archive/reader.rs b/crates/pithos_lib/src/archive/reader.rs index 7baf41c..5f3d6be 100644 --- a/crates/pithos_lib/src/archive/reader.rs +++ b/crates/pithos_lib/src/archive/reader.rs @@ -71,6 +71,7 @@ impl DecodedDirectoryCounts { pub(crate) const DEFAULT_MAX_DECODED_BLOCK_BYTES: u64 = 64 * 1024 * 1024; /// Limits enforced before directory data is retained or expensive metadata work begins. +/// The defaults admit one 5 TiB file stored in 4 MiB blocks. #[derive(Clone, Copy, Debug)] pub struct OpenLimits { pub max_directory_bytes: u64, @@ -89,15 +90,15 @@ pub struct OpenLimits { impl Default for OpenLimits { fn default() -> Self { Self { - max_directory_bytes: 64 * 1024 * 1024, - max_total_directory_bytes: 256 * 1024 * 1024, + max_directory_bytes: 256 * 1024 * 1024, + max_total_directory_bytes: 512 * 1024 * 1024, max_parent_directories: 1024, max_entries: 1_000_000, - max_descriptors: 1_000_000, + max_descriptors: 2_097_152, max_references: 1_000_000, max_relationships: 1_000_000, - max_accessible_block_references: 1_000_000, - max_opaque_metadata_bytes: 64 * 1024 * 1024, + max_accessible_block_references: 2_097_152, + max_opaque_metadata_bytes: 128 * 1024 * 1024, max_stored_block_bytes: 64 * 1024 * 1024, max_decoded_block_bytes: DEFAULT_MAX_DECODED_BLOCK_BYTES, } diff --git a/crates/pithos_lib/src/format/directory.rs b/crates/pithos_lib/src/format/directory.rs index 09742bc..4c7517c 100644 --- a/crates/pithos_lib/src/format/directory.rs +++ b/crates/pithos_lib/src/format/directory.rs @@ -564,6 +564,21 @@ pub(crate) fn decode_complete_directory_with_validation_and_budget( Ok(directory) } +/// Encodes a directory once and writes its real length and CRC into the encoded footer. +pub(crate) fn encode_complete_directory( + directory: &Directory, +) -> Result, SerializationError> { + let mut bytes = Vec::new(); + encode_directory(directory, &mut bytes)?; + let len = bytes.len(); + let dir_len = u64::try_from(len) + .map_err(|_| SerializationError::Other("length does not fit in u64".to_string()))?; + bytes[len - 12..len - 4].copy_from_slice(&dir_len.to_be_bytes()); + let crc32 = crc32fast::hash(&bytes[..len - 4]); + bytes[len - 4..].copy_from_slice(&crc32.to_be_bytes()); + Ok(bytes) +} + pub(crate) fn update_directory_len(directory: &mut Directory) -> Result<(), SerializationError> { let mut bytes = Vec::new(); encode_directory(directory, &mut bytes)?; @@ -671,10 +686,12 @@ mod tests { dir_len: 0, crc32: 0, }; + let complete = encode_complete_directory(&directory).unwrap(); update_directory_len(&mut directory).unwrap(); update_directory_crc(&mut directory).unwrap(); let mut bytes = Vec::new(); encode_directory(&directory, &mut bytes).unwrap(); + assert_eq!(complete, bytes); assert!(matches!( decode_complete_directory(&bytes, &DeserializationLimits::default()), diff --git a/crates/pithos_lib/tests/pieces.rs b/crates/pithos_lib/tests/pieces.rs index 8a59045..cb669dc 100644 --- a/crates/pithos_lib/tests/pieces.rs +++ b/crates/pithos_lib/tests/pieces.rs @@ -304,3 +304,47 @@ fn a_composed_file_exports_to_crypt4gh_with_a_fresh_data_key() { let header_len = 16 + u32::from_le_bytes(exported[16..20].try_into().unwrap()) as usize; assert_eq!(exported.len(), header_len + 12 + 1500 + 16); } + +/// One 5 TiB file at 4 MiB blocks has 1,310,720 blocks. Tiny blocks give the same directory +/// shape, and the default open limits must admit it. +#[test] +#[ignore = "slow in debug builds (about 5 minutes, 8 seconds with --release); run with --ignored"] +fn a_composition_with_the_block_count_of_5_tib_opens_with_default_limits() { + const BLOCKS: u32 = 1_310_720; + const PIECES: u32 = 10; + let per_piece = BLOCKS / PIECES; + let parts = (0..PIECES) + .map(|piece| { + let processing = ProcessingOptions::new(true, 0).unwrap(); + let mut encoder = PieceEncoder::new( + u64::from(piece) + 1, + vec![public_key("recipient1")], + processing, + ) + .unwrap(); + let mut stored = Vec::new(); + for block in piece * per_piece..(piece + 1) * per_piece { + stored.extend(encoder.push(&block.to_be_bytes()).unwrap()); + } + (stored, encoder.finish().unwrap()) + }) + .collect::>(); + let archive = open(assemble(&parts), recipient()); + let entry = archive.entry("object").unwrap().unwrap(); + assert!(matches!( + entry.kind, + EntryKind::File { + size, + available: true + } if size == u64::from(BLOCKS) * 4 + )); + let mut tail = Vec::new(); + let end = u64::from(BLOCKS) * 4; + archive + .copy_range_to("object", end - 8..end, &mut tail) + .unwrap(); + assert_eq!( + tail, + [(BLOCKS - 2).to_be_bytes(), (BLOCKS - 1).to_be_bytes()].concat() + ); +} From 7e3bd3d0d36a0006c17b64d62dfcf3e0cd5a7f52 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 20:02:52 +0200 Subject: [PATCH 070/166] feat: report the whole-file BLAKE3 of composed pieces --- crates/pithos_lib/src/archive/content_tree.rs | 264 ++++++++++++++++++ crates/pithos_lib/src/archive/mod.rs | 1 + crates/pithos_lib/src/archive/pieces.rs | 171 +++++++++++- crates/pithos_lib/src/error.rs | 4 + spec/PITHOS_1.0.0_draft.md | 10 + 5 files changed, 445 insertions(+), 5 deletions(-) create mode 100644 crates/pithos_lib/src/archive/content_tree.rs diff --git a/crates/pithos_lib/src/archive/content_tree.rs b/crates/pithos_lib/src/archive/content_tree.rs new file mode 100644 index 0000000..b1e62be --- /dev/null +++ b/crates/pithos_lib/src/archive/content_tree.rs @@ -0,0 +1,264 @@ +//! BLAKE3 subtree chaining values that let separately encoded pieces yield the file hash. + +use blake3::hazmat::{ + ChainingValue, HasherExt, Mode, left_subtree_len, max_subtree_len, merge_subtrees_non_root, + merge_subtrees_root, +}; +use blake3::{CHUNK_LEN, Hasher}; +use zeroize::Zeroizing; + +const CHUNK: u64 = CHUNK_LEN as u64; +/// Largest subtree hashed in one call. Larger subtrees are merged from these. +const UNIT_LEN: u64 = 64 * CHUNK; + +/// One subtree of the file's BLAKE3 tree, at an absolute plaintext offset. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct Subtree { + pub(crate) offset: u64, + pub(crate) len: u64, + pub(crate) value: ChainingValue, +} + +/// The aligned subtrees that cover one piece's plaintext, in order. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ContentTree { + pub(crate) offset: u64, + pub(crate) subtrees: Vec, + /// BLAKE3 of the piece alone. Kept only for a piece at offset 0. + pub(crate) root: Option<[u8; 32]>, +} + +impl ContentTree { + /// Checks that the subtrees are aligned and exactly cover `len` bytes from the offset. + /// Only the last subtree may be a short chunk. + pub(crate) fn is_valid(&self, len: u64) -> bool { + if !self.offset.is_multiple_of(CHUNK) || self.root.is_some() != (self.offset == 0) { + return false; + } + let mut next = self.offset; + for (index, subtree) in self.subtrees.iter().enumerate() { + let last = index + 1 == self.subtrees.len(); + let aligned = subtree.len >= CHUNK + && subtree.len.is_power_of_two() + && subtree.offset.is_multiple_of(subtree.len); + let short = last && subtree.len > 0 && subtree.len < CHUNK; + if subtree.offset != next || !(aligned || short) { + return false; + } + let Some(end) = next.checked_add(subtree.len) else { + return false; + }; + next = end; + } + self.offset.checked_add(len) == Some(next) + } +} + +/// Records the subtrees of plaintext that starts at a fixed absolute offset. +pub(crate) struct TreeHasher { + offset: u64, + next: u64, + pending: Zeroizing>, + subtrees: Vec, + root: Option, +} + +impl TreeHasher { + /// `offset` must be a multiple of the 1024-byte BLAKE3 chunk length. + pub(crate) fn new(offset: u64) -> Self { + debug_assert!(offset.is_multiple_of(CHUNK)); + Self { + offset, + next: offset, + pending: Zeroizing::new(Vec::new()), + subtrees: Vec::new(), + root: (offset == 0).then(Hasher::new), + } + } + + pub(crate) fn update(&mut self, mut bytes: &[u8]) { + if let Some(root) = &mut self.root { + root.update(bytes); + } + while !bytes.is_empty() { + // A subtree starting at `next` may hold at most `max_subtree_len(next)` bytes. + let unit = max_subtree_len(self.next).map_or(UNIT_LEN, |max| max.min(UNIT_LEN)); + let unit = unit as usize; + if self.pending.is_empty() && bytes.len() >= unit { + let (subtree, rest) = bytes.split_at(unit); + self.push(subtree); + bytes = rest; + continue; + } + let take = (unit - self.pending.len()).min(bytes.len()); + self.pending.extend_from_slice(&bytes[..take]); + bytes = &bytes[take..]; + if self.pending.len() == unit { + let pending = std::mem::take(&mut self.pending); + self.push(&pending); + self.pending = pending; + self.pending.clear(); + } + } + } + + /// Splits the buffered tail into the largest aligned subtrees, ending in one short chunk. + pub(crate) fn finish(mut self) -> ContentTree { + let pending = std::mem::take(&mut self.pending); + let mut rest = pending.as_slice(); + while !rest.is_empty() { + let len = if rest.len() < CHUNK_LEN { + rest.len() + } else { + 1 << rest.len().ilog2() + }; + let (subtree, tail) = rest.split_at(len); + self.push(subtree); + rest = tail; + } + ContentTree { + offset: self.offset, + subtrees: self.subtrees, + root: self.root.map(|root| *root.finalize().as_bytes()), + } + } + + /// Hashes one aligned subtree and merges equal aligned neighbours into their parent. + fn push(&mut self, bytes: &[u8]) { + let len = bytes.len() as u64; + let value = Hasher::new() + .set_input_offset(self.next) + .update(bytes) + .finalize_non_root(); + self.subtrees.push(Subtree { + offset: self.next, + len, + value, + }); + self.next += len; + while let [.., left, right] = self.subtrees.as_slice() { + let Some(parent_len) = left.len.checked_mul(2) else { + break; + }; + if left.len != right.len || !left.offset.is_multiple_of(parent_len) { + break; + } + let parent = Subtree { + offset: left.offset, + len: parent_len, + value: merge_subtrees_non_root(&left.value, &right.value, Mode::Hash), + }; + self.subtrees.truncate(self.subtrees.len() - 2); + self.subtrees.push(parent); + } + } +} + +/// Returns the BLAKE3 of the whole file when every piece has a tree, the trees start at the +/// running sum of the piece sizes from 0, and each tree exactly covers its piece. +pub(crate) fn file_hash<'a>( + pieces: impl IntoIterator, u64)>, +) -> Option<[u8; 32]> { + let mut subtrees = Vec::new(); + let mut total = 0u64; + let mut nonempty = Vec::new(); + for (tree, len) in pieces { + let tree = tree?; + if tree.offset != total || !tree.is_valid(len) { + return None; + } + if len > 0 { + nonempty.push(tree); + } + subtrees.extend_from_slice(&tree.subtrees); + total = total.checked_add(len)?; + } + match nonempty.as_slice() { + [] => return Some(*blake3::hash(&[]).as_bytes()), + // One piece holds the whole file, so its own hash is the file hash. + [tree] => return tree.root, + _ if total <= CHUNK => return None, + _ => {} + } + let left = left_subtree_len(total); + let left_value = resolve(&subtrees, 0, left)?; + let right_value = resolve(&subtrees, left, total - left)?; + Some(*merge_subtrees_root(&left_value, &right_value, Mode::Hash).as_bytes()) +} + +/// Finds or merges the chaining value of the tree node covering `len` bytes at `offset`. +fn resolve(subtrees: &[Subtree], offset: u64, len: u64) -> Option { + let index = subtrees + .partition_point(|subtree| subtree.offset <= offset) + .checked_sub(1)?; + let found = &subtrees[index]; + if found.offset == offset && found.len == len { + return Some(found.value); + } + if found.offset != offset || found.len > len || len <= CHUNK { + return None; + } + let left = left_subtree_len(len); + let left_value = resolve(subtrees, offset, left)?; + let right_value = resolve(subtrees, offset + left, len - left)?; + Some(merge_subtrees_non_root( + &left_value, + &right_value, + Mode::Hash, + )) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tree(offset: u64, content: &[u8], fragment: usize) -> ContentTree { + let mut hasher = TreeHasher::new(offset); + for part in content.chunks(fragment.max(1)) { + hasher.update(part); + } + hasher.finish() + } + + #[test] + fn split_trees_merge_to_the_plain_hash_at_every_boundary() { + let content = (0..300_000u32) + .map(|index| (index * 31 % 251) as u8) + .collect::>(); + let lengths = [0, 1, 1023, 1024, 1025, 2048, 3072, 65_536, 66_560, 200_000]; + for len in lengths { + for split in [0, 1024, 3072, 65_536, 70_656] { + let split = split.min(len / 1024 * 1024); + let first = tree(0, &content[..split], 7); + let second = tree(split as u64, &content[split..len], 4096); + assert!(first.is_valid(split as u64) && second.is_valid((len - split) as u64)); + let pieces = [ + (Some(&first), split as u64), + (Some(&second), (len - split) as u64), + ]; + assert_eq!( + file_hash(pieces), + Some(*blake3::hash(&content[..len]).as_bytes()), + "length {len}, split {split}" + ); + } + } + } + + #[test] + fn invalid_trees_are_detected() { + let content = vec![7u8; 5000]; + let valid = tree(0, &content, 5000); + assert!(valid.is_valid(5000)); + assert!(!valid.is_valid(4999)); + let mut short_inside = valid.clone(); + short_inside.subtrees[0].len = 100; + assert!(!short_inside.is_valid(5000)); + let mut misaligned = tree(1024, &content, 5000); + misaligned.subtrees[1].offset += 1024; + assert!(!misaligned.is_valid(5000)); + let mut rootless = valid; + rootless.root = None; + assert!(!rootless.is_valid(5000)); + } +} diff --git a/crates/pithos_lib/src/archive/mod.rs b/crates/pithos_lib/src/archive/mod.rs index 6ef6637..0d8f5b4 100644 --- a/crates/pithos_lib/src/archive/mod.rs +++ b/crates/pithos_lib/src/archive/mod.rs @@ -6,6 +6,7 @@ mod access; mod append; +mod content_tree; mod index; mod path_validation; mod pieces; diff --git a/crates/pithos_lib/src/archive/pieces.rs b/crates/pithos_lib/src/archive/pieces.rs index 893ef99..d0154f6 100644 --- a/crates/pithos_lib/src/archive/pieces.rs +++ b/crates/pithos_lib/src/archive/pieces.rs @@ -4,9 +4,12 @@ //! piece key granted to the recipients. The resulting [`Piece`] holds no secret, so it can be //! stored and later joined by [`compose`] into a version 1.1 archive with one data file. +use crate::archive::content_tree::{self, ContentTree, Subtree, TreeHasher}; use crate::archive::path_validation::validate_entry; use crate::archive::types::{ArchivePath, Processing}; -use crate::archive::writer::{Chunking, EntryMetadata, ProcessingOptions, validate_block_size}; +use crate::archive::writer::{ + BlockKeyMode, Chunking, EntryMetadata, ProcessingOptions, validate_block_size, +}; use crate::block; use crate::crypto::{self, FileKey, PublicKey}; use crate::error::PithosError; @@ -29,7 +32,8 @@ use x25519_dalek::{PublicKey as DalekPublicKey, StaticSecret}; use zeroize::Zeroizing; const PIECE_MAGIC: &[u8; 8] = b"PITHPIEC"; -const PIECE_RECORD_VERSION: u8 = 1; +/// Version 2 adds the optional content tree; version 1 records still decode. +const PIECE_RECORD_VERSION: u8 = 2; const BLOCK_MARKER_LEN: u64 = 4; /// Location and size of one encoded block inside its piece. @@ -54,6 +58,7 @@ pub struct Piece { pub(crate) sealed_list: Vec, pub(crate) sender: [u8; 32], pub(crate) grants: Vec<([u8; 32], Vec)>, + pub(crate) content: Option, } /// Encodes the blocks of one piece. The caller appends every returned byte string, in order, @@ -61,6 +66,11 @@ pub struct Piece { /// /// [`PieceEncoder::write`] and [`PieceEncoder::flush`] split the content into fixed-size blocks, /// which is the recommended use. [`PieceEncoder::push`] instead encodes caller-chosen blocks. +/// +/// The encoder also records BLAKE3 subtree chaining values of the piece plaintext, so that +/// [`Composition::content_hash`] can report the whole-file BLAKE3. They are plaintext +/// fingerprints like convergent block hashes, so with [`BlockKeyMode::Unique`] they are off +/// unless [`PieceEncoder::with_content_hash`] turns them on. pub struct PieceEncoder { key_id: u64, recipients: Vec<[u8; 32]>, @@ -72,6 +82,8 @@ pub struct PieceEncoder { original_size: u64, block_size: usize, pending: Zeroizing>, + content_offset: u64, + tree: Option, } impl PieceEncoder { @@ -111,9 +123,40 @@ impl PieceEncoder { original_size: 0, block_size: Chunking::DEFAULT_BLOCK_SIZE, pending: Zeroizing::new(Vec::new()), + content_offset: 0, + tree: (processing.key_mode() != BlockKeyMode::Unique).then(|| TreeHasher::new(0)), }) } + /// Sets the absolute file offset of this piece's first plaintext byte. The default is 0. + /// The offset must be a multiple of 1024, the BLAKE3 chunk length, and must be set before + /// any content is added. + pub fn with_content_offset(mut self, offset: u64) -> Result { + self.ensure_no_content()?; + if !offset.is_multiple_of(blake3::CHUNK_LEN as u64) { + return Err(PithosError::InvalidContentOffset(offset)); + } + self.content_offset = offset; + if self.tree.is_some() { + self.tree = Some(TreeHasher::new(offset)); + } + Ok(self) + } + + /// Turns the recorded subtree chaining values on or off before any content is added. + pub fn with_content_hash(mut self, record: bool) -> Result { + self.ensure_no_content()?; + self.tree = record.then(|| TreeHasher::new(self.content_offset)); + Ok(self) + } + + fn ensure_no_content(&self) -> Result<(), PithosError> { + if self.original_size != 0 || !self.pending.is_empty() { + return Err(PithosError::PieceContentStarted); + } + Ok(()) + } + /// Sets the block size used by [`PieceEncoder::write`]. The default is 4 MiB. pub fn with_block_size(mut self, size: usize) -> Result { validate_block_size(size)?; @@ -176,6 +219,10 @@ impl PieceEncoder { let encoded = block::encode(plaintext, self.flags, crypto::random_nonce())?; let original_size = plaintext.len() as u64; self.original_size = checked_add(self.original_size, original_size)?; + if let Some(tree) = &mut self.tree { + checked_add(self.content_offset, self.original_size)?; + tree.update(plaintext); + } self.entries .push((encoded.hash, *encoded.key.expose_for_protocol())); if !self.seen.insert(encoded.hash) { @@ -239,6 +286,7 @@ impl PieceEncoder { sealed_list, sender: sender_public, grants, + content: self.tree.map(TreeHasher::finish), }) } } @@ -288,15 +336,36 @@ impl Piece { write_uleb(&mut bytes, wrapped.len() as u64); bytes.extend_from_slice(wrapped); } + let Some(tree) = &self.content else { + bytes.push(0); + return bytes; + }; + bytes.push(1); + write_uleb(&mut bytes, tree.offset); + write_uleb(&mut bytes, tree.subtrees.len() as u64); + for subtree in &tree.subtrees { + write_uleb(&mut bytes, subtree.offset); + write_uleb(&mut bytes, subtree.len); + bytes.extend_from_slice(&subtree.value); + } + match tree.root { + Some(root) => { + bytes.push(1); + bytes.extend_from_slice(&root); + } + None => bytes.push(0), + } bytes } /// Decodes a record from [`Piece::to_bytes`] and checks that its blocks are contiguous. pub fn from_bytes(bytes: &[u8]) -> Result { let mut reader = Cursor::new(bytes); - if read_vec(&mut reader, PIECE_MAGIC.len())? != PIECE_MAGIC - || read_vec(&mut reader, 1)? != [PIECE_RECORD_VERSION] - { + if read_vec(&mut reader, PIECE_MAGIC.len())? != PIECE_MAGIC { + return Err(PithosError::InvalidPieceRecord); + } + let version = read_vec(&mut reader, 1)?[0]; + if !(1..=PIECE_RECORD_VERSION).contains(&version) { return Err(PithosError::InvalidPieceRecord); } let key_id = read_uleb(&mut reader)?; @@ -331,6 +400,10 @@ impl Piece { let len = read_count(&mut reader, 1)?; grants.push((recipient, read_vec(&mut reader, len)?)); } + let content = match version { + 1 => None, + _ => read_content_tree(&mut reader, original_size)?, + }; if offset != stored_len || reader.position() != bytes.len() as u64 || key_id == 0 @@ -346,10 +419,46 @@ impl Piece { sealed_list, sender, grants, + content, }) } } +fn read_content_tree( + reader: &mut Cursor<&[u8]>, + original_size: u64, +) -> Result, PithosError> { + match read_vec(reader, 1)?[0] { + 0 => return Ok(None), + 1 => {} + _ => return Err(PithosError::InvalidPieceRecord), + } + let offset = read_uleb(reader)?; + let count = read_count(reader, 2 + 32)?; + let mut subtrees = Vec::with_capacity(count); + for _ in 0..count { + subtrees.push(Subtree { + offset: read_uleb(reader)?, + len: read_uleb(reader)?, + value: read_array(reader)?, + }); + } + let root = match read_vec(reader, 1)?[0] { + 0 => None, + 1 => Some(read_array(reader)?), + _ => return Err(PithosError::InvalidPieceRecord), + }; + let tree = ContentTree { + offset, + subtrees, + root, + }; + if !tree.is_valid(original_size) { + return Err(PithosError::InvalidPieceRecord); + } + Ok(Some(tree)) +} + fn write_uleb(bytes: &mut Vec, value: u64) { bytes .write_varint(value) @@ -405,6 +514,7 @@ pub struct Composition { piece_offsets: Vec, directory: Vec, archive_len: u64, + content_hash: Option<[u8; 32]>, } impl Composition { @@ -428,6 +538,16 @@ impl Composition { self.archive_len } + /// The BLAKE3 hash of the whole composed file, or `None` when the pieces do not allow it. + /// + /// It is known when every piece recorded subtree chaining values, the recorded offsets + /// are the running sums of the piece sizes from 0, and each piece's values cover it + /// exactly. The value is only as trustworthy as the stored piece records, which hold no + /// proof of the plaintext. A full read of the composed file confirms it. + pub fn content_hash(&self) -> Option<[u8; 32]> { + self.content_hash + } + /// The digest [`crate::archive::Archive::metadata_digest`] reports for this archive. pub fn metadata_digest(&self) -> [u8; 32] { crate::archive::metadata_digest(&[*blake3::hash(&self.directory).as_bytes()]) @@ -537,6 +657,11 @@ pub fn compose( directory.blocks = blocks; let bytes = encode_complete_directory(&directory)?; Ok(Composition { + content_hash: content_tree::file_hash( + pieces + .iter() + .map(|piece| (piece.content.as_ref(), piece.original_size)), + ), piece_offsets, archive_len: checked_add(offset, bytes.len() as u64)?, directory: bytes, @@ -571,6 +696,42 @@ mod tests { } } + #[test] + fn version_1_records_decode_without_a_content_tree_and_bad_trees_are_rejected() { + let recipient = crate::crypto::PrivateKey::generate().public_key(); + let processing = ProcessingOptions::new(true, 0).unwrap(); + let encoder = PieceEncoder::new(1, vec![recipient], processing).unwrap(); + let mut encoder = encoder.with_content_hash(false).unwrap(); + encoder.push(b"version one block").unwrap(); + let piece = encoder.finish().unwrap(); + let mut version_1 = piece.to_bytes(); + assert_eq!(version_1.pop(), Some(0)); + version_1[PIECE_MAGIC.len()] = 1; + assert_eq!(Piece::from_bytes(&version_1).unwrap(), piece); + + let encoder = PieceEncoder::new(1, vec![recipient], processing).unwrap(); + let mut encoder = encoder.with_content_offset(2048).unwrap(); + encoder.push(&[3; 3000]).unwrap(); + let piece = encoder.finish().unwrap(); + assert_eq!(Piece::from_bytes(&piece.to_bytes()).unwrap(), piece); + let changes: [fn(&mut ContentTree); 4] = [ + |tree| tree.offset += 1024, + |tree| tree.subtrees[0].len += 1, + |tree| tree.root = Some([0; 32]), + |tree| { + tree.subtrees.pop(); + }, + ]; + for change in changes { + let mut invalid = piece.clone(); + change(invalid.content.as_mut().unwrap()); + assert!(matches!( + Piece::from_bytes(&invalid.to_bytes()), + Err(PithosError::InvalidPieceRecord) + )); + } + } + #[test] fn written_fragments_give_the_same_fixed_blocks_as_pushed_blocks() { let content = (0..2500u32) diff --git a/crates/pithos_lib/src/error.rs b/crates/pithos_lib/src/error.rs index c72a7ae..cc541e3 100644 --- a/crates/pithos_lib/src/error.rs +++ b/crates/pithos_lib/src/error.rs @@ -91,6 +91,10 @@ pub enum PithosError { InvalidPieceRecord, #[error("the piece encoder holds written bytes that were not flushed")] UnflushedPieceBytes, + #[error("content offset {0} is not a multiple of 1024")] + InvalidContentOffset(u64), + #[error("piece content options must be set before any content is added")] + PieceContentStarted, #[error("archive metadata does not match the expected digest")] MetadataDigestMismatch, #[error("duplicate encoded recipient file id")] diff --git a/spec/PITHOS_1.0.0_draft.md b/spec/PITHOS_1.0.0_draft.md index aad7917..4451380 100644 --- a/spec/PITHOS_1.0.0_draft.md +++ b/spec/PITHOS_1.0.0_draft.md @@ -1197,6 +1197,16 @@ Directory. Because convergent block keys are equal for equal plaintext, a block that repeats in two parts needs only one effective descriptor. Unique-key blocks never repeat. +**Whole-file hash of joined parts:** a writer that knows a part's absolute file +offset (a multiple of 1024 bytes) may also keep the BLAKE3 chaining values of +the aligned subtrees that cover that part. The joining step can merge them into +the BLAKE3 hash of the whole file without any key. This works only when the +recorded offsets match the actual part sizes. The result is only as trustworthy +as the stored part records; a full read of the file confirms it. Chaining values +are plaintext fingerprints, like convergent block hashes, so a writer that uses +unique-key blocks should not keep them unless the owner allows it. This record +is not part of the archive format. + ## Appendix B. Conformance Examples and Vectors This appendix is informative. The cited sections remain authoritative. Complete From 7076f6e980f1698ca4c4835a4c21faf0d7abac49 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 20:02:52 +0200 Subject: [PATCH 071/166] test: cover content hashes of composed pieces --- crates/pithos_lib/tests/pieces.rs | 114 ++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/crates/pithos_lib/tests/pieces.rs b/crates/pithos_lib/tests/pieces.rs index cb669dc..51a008a 100644 --- a/crates/pithos_lib/tests/pieces.rs +++ b/crates/pithos_lib/tests/pieces.rs @@ -348,3 +348,117 @@ fn a_composition_with_the_block_count_of_5_tib_opens_with_default_limits() { [(BLOCKS - 2).to_be_bytes(), (BLOCKS - 1).to_be_bytes()].concat() ); } + +/// Encodes one part at `offset` in fragments that do not match blocks or BLAKE3 chunks, and +/// returns its record after a storage round trip. +fn hashed_piece(key_id: u64, offset: u64, part: &[u8], processing: ProcessingOptions) -> Piece { + let mut encoder = PieceEncoder::new(key_id, vec![public_key("recipient1")], processing) + .unwrap() + .with_block_size(BLOCK) + .unwrap() + .with_content_offset(offset) + .unwrap(); + for fragment in part.chunks(777) { + encoder.write(fragment).unwrap(); + } + encoder.flush().unwrap(); + Piece::from_bytes(&encoder.finish().unwrap().to_bytes()).unwrap() +} + +/// Encodes consecutive parts of `file` with the given sizes at their running offsets. +fn hashed_parts(file: &[u8], sizes: &[usize]) -> Vec { + let mut offset = 0; + let processing = ProcessingOptions::new(true, 3).unwrap(); + let mut pieces = Vec::new(); + for (index, size) in sizes.iter().enumerate() { + let part = &file[offset..offset + size]; + pieces.push(hashed_piece( + index as u64 + 1, + offset as u64, + part, + processing, + )); + offset += size; + } + assert_eq!(offset, file.len()); + pieces +} + +fn content_hash(pieces: &[Piece]) -> Option<[u8; 32]> { + let path = ArchivePath::new("object").unwrap(); + compose(path, EntryMetadata::new(0, 0, 0o644), pieces) + .unwrap() + .content_hash() +} + +#[test] +fn content_hash_of_aligned_parts_is_the_blake3_of_the_file() { + let file = content(5, 700_000); + let part = 200 * 1024; + let equal = [part, part, part, 700_000 - 3 * part]; + let varied = [3072, 1024, 70 * 1024, 0, 5120, 128 * 1024, 488_032]; + for sizes in [&equal[..], &varied[..]] { + let pieces = hashed_parts(&file, sizes); + assert_eq!(content_hash(&pieces), Some(*blake3::hash(&file).as_bytes())); + } +} + +#[test] +fn content_hash_of_one_piece_and_of_an_empty_file() { + let file = content(6, 131_079); + for len in [0, 1, 500, 1024, 1025, 65_536, 131_079] { + let pieces = hashed_parts(&file[..len], &[len]); + let expected = *blake3::hash(&file[..len]).as_bytes(); + assert_eq!(content_hash(&pieces), Some(expected), "{len}"); + } + assert_eq!(content_hash(&[]), Some(*blake3::hash(b"").as_bytes())); +} + +#[test] +fn content_hash_is_unknown_for_wrong_offsets_or_missing_values() { + let file = content(7, 5000); + let processing = ProcessingOptions::new(true, 0).unwrap(); + let first = hashed_piece(1, 0, &file[..1024], processing); + let wrong = hashed_piece(2, 2048, &file[1024..], processing); + assert_eq!(content_hash(&[first.clone(), wrong]), None); + let second = hashed_piece(2, 1024, &file[1024..], processing); + assert_eq!(content_hash(std::slice::from_ref(&second)), None); + assert!(content_hash(&[first.clone(), second]).is_some()); + + let encoder = PieceEncoder::new(2, vec![public_key("recipient1")], processing).unwrap(); + let mut encoder = encoder.with_content_hash(false).unwrap(); + encoder.push(&file[1024..]).unwrap(); + assert_eq!(content_hash(&[first, encoder.finish().unwrap()]), None); + + let encoder = || PieceEncoder::new(1, vec![public_key("recipient1")], processing).unwrap(); + assert!(matches!( + encoder().with_content_offset(1000), + Err(PithosError::InvalidContentOffset(1000)) + )); + let mut started = encoder(); + started.write(b"content").unwrap(); + assert!(matches!( + started.with_content_offset(0), + Err(PithosError::PieceContentStarted) + )); +} + +#[test] +fn unique_key_pieces_record_no_content_hash_unless_asked() { + let file = content(8, 3000); + let unique = ProcessingOptions::new(true, 0) + .unwrap() + .with_key_mode(BlockKeyMode::Unique) + .unwrap(); + assert_eq!(content_hash(&[hashed_piece(1, 0, &file, unique)]), None); + let mut encoder = PieceEncoder::new(1, vec![public_key("recipient1")], unique) + .unwrap() + .with_content_hash(true) + .unwrap(); + encoder.push(&file).unwrap(); + let piece = Piece::from_bytes(&encoder.finish().unwrap().to_bytes()).unwrap(); + assert_eq!( + content_hash(&[piece]), + Some(*blake3::hash(&file).as_bytes()) + ); +} From 2ee4a0d978559b20b1b464c5c0b96621be48bbfd Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 20:16:52 +0200 Subject: [PATCH 072/166] chore: raise the minimum Rust version to 1.89 --- .github/workflows/ci.yaml | 6 +++--- Cargo.lock | 4 ++-- Cargo.toml | 2 +- crates/pithos_lib/README.md | 2 +- crates/pithos_lib/benches/README.md | 4 ++-- fuzz/Cargo.lock | 4 ++-- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5ed8a9c..eba3bd7 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -36,11 +36,11 @@ jobs: - name: Check out repository uses: actions/checkout@v4 - - name: Install Rust 1.88.0 - uses: dtolnay/rust-toolchain@1.88.0 + - name: Install Rust 1.89.0 + uses: dtolnay/rust-toolchain@1.89.0 - name: Cache Rust build artifacts uses: Swatinem/rust-cache@v2 - name: Run MSRV workspace tests - run: cargo +1.88.0 test --locked --workspace --all-features + run: cargo +1.89.0 test --locked --workspace --all-features diff --git a/Cargo.lock b/Cargo.lock index 358b68e..bd8d4d7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -20,9 +20,9 @@ dependencies = [ [[package]] name = "aes" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8eb277bec05f56a0e0591f155a484cbd0f4f07ff2905051a48c72f004f7ed58" +checksum = "35f0f96ce78e38c3dc6d8948aa8163d06385be74000f3c7a95bf1eef35d3ea32" dependencies = [ "cipher", "cpubits", diff --git a/Cargo.toml b/Cargo.toml index 5bf1cc5..af9273b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,7 @@ resolver = "2" version = "0.8.0" authors = ["Jannis Schlegel ", "Sebastian Beyvers "] edition = "2024" -rust-version = "1.88" +rust-version = "1.89" repository = "https://github.com/arunaengine/pithos" license = "MIT OR Apache-2.0" diff --git a/crates/pithos_lib/README.md b/crates/pithos_lib/README.md index 55c529a..19577db 100644 --- a/crates/pithos_lib/README.md +++ b/crates/pithos_lib/README.md @@ -4,7 +4,7 @@ ## Installation -Add the crate to an application using Rust 1.88 or newer: +Add the crate to an application using Rust 1.89 or newer: ```toml [dependencies] diff --git a/crates/pithos_lib/benches/README.md b/crates/pithos_lib/benches/README.md index c5e2db3..ebe5166 100644 --- a/crates/pithos_lib/benches/README.md +++ b/crates/pithos_lib/benches/README.md @@ -69,7 +69,7 @@ On the baseline revision, save each Criterion baseline: ```bash for bench in archive_path block_pipeline archive_io append adapters; do - cargo +1.88.0 bench --locked -p pithos_lib --bench "$bench" -- --save-baseline before + cargo +1.89.0 bench --locked -p pithos_lib --bench "$bench" -- --save-baseline before done ``` @@ -77,7 +77,7 @@ Then switch to the candidate revision and compare against it: ```bash for bench in archive_path block_pipeline archive_io append adapters; do - cargo +1.88.0 bench --locked -p pithos_lib --bench "$bench" -- --baseline before + cargo +1.89.0 bench --locked -p pithos_lib --bench "$bench" -- --baseline before done ``` diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index adac8a6..08fe60b 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -20,9 +20,9 @@ dependencies = [ [[package]] name = "aes" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8eb277bec05f56a0e0591f155a484cbd0f4f07ff2905051a48c72f004f7ed58" +checksum = "35f0f96ce78e38c3dc6d8948aa8163d06385be74000f3c7a95bf1eef35d3ea32" dependencies = [ "cipher", "cpubits", From e455971a09465935fd39ae800dba8d533e6f3548 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 20:40:01 +0200 Subject: [PATCH 073/166] fix: refuse block size changes after piece content started --- crates/pithos_lib/src/archive/pieces.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/crates/pithos_lib/src/archive/pieces.rs b/crates/pithos_lib/src/archive/pieces.rs index d0154f6..64db9a5 100644 --- a/crates/pithos_lib/src/archive/pieces.rs +++ b/crates/pithos_lib/src/archive/pieces.rs @@ -158,7 +158,9 @@ impl PieceEncoder { } /// Sets the block size used by [`PieceEncoder::write`]. The default is 4 MiB. + /// Fails once content was added, because buffered bytes assume the old size. pub fn with_block_size(mut self, size: usize) -> Result { + self.ensure_no_content()?; validate_block_size(size)?; self.block_size = size; Ok(self) @@ -176,9 +178,10 @@ impl PieceEncoder { content = rest; continue; } - // Reserving the whole block once keeps plaintext from being copied on growth. - self.pending.reserve_exact(self.block_size); - let take = (self.block_size - self.pending.len()).min(content.len()); + let missing = self.block_size - self.pending.len(); + // Reserving the rest of the block once keeps plaintext from being copied on growth. + self.pending.reserve_exact(missing); + let take = missing.min(content.len()); self.pending.extend_from_slice(&content[..take]); content = &content[take..]; if self.pending.len() == self.block_size { @@ -776,5 +779,11 @@ mod tests { encoder().with_block_size(0), Err(PithosError::InvalidBlockSize(0)) )); + let mut started = encoder(); + started.write(b"pending").unwrap(); + assert!(matches!( + started.with_block_size(4), + Err(PithosError::PieceContentStarted) + )); } } From db71e7308f9511ab99065b8bb34bd6835af74fc9 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 20:40:02 +0200 Subject: [PATCH 074/166] fix: describe short blocks at the end of every piece in the spec --- spec/PITHOS_1.0.0_draft.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/spec/PITHOS_1.0.0_draft.md b/spec/PITHOS_1.0.0_draft.md index 4451380..f53c568 100644 --- a/spec/PITHOS_1.0.0_draft.md +++ b/spec/PITHOS_1.0.0_draft.md @@ -1172,9 +1172,13 @@ This appendix suggests writer choices only. It does not define archive conformance or reader behavior. **Recommended block size:** fixed 4 MiB (4,194,304 byte) blocks. Every block -of a file has this size except the last, which may be shorter. Block boundaries -then depend only on byte offsets, not on how the input was read. This suits -object storage and multipart uploads, where parts are written independently. +of an independently encoded part has this size except its last, which may be +shorter. A file written in one pass therefore has at most one short block, at +its end. A file joined from pieces has one possible short block at the end of +each piece: two 5 MiB pieces give blocks of 4, 1, 4 and 1 MiB. Block boundaries +depend only on byte offsets within a part, not on how the input was read. This +suits object storage and multipart uploads, where parts are written +independently. Content-defined chunking (for example FastCDC) is optional. It can find more repeated blocks when content shifts, at the cost of variable block sizes. From 38225b9631a65a79ca0ad239c721eae105b06be7 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 20:45:51 +0200 Subject: [PATCH 075/166] refactor: hold opened archive state in an archive view --- crates/pithos_lib/src/archive/mod.rs | 2 + crates/pithos_lib/src/archive/reader.rs | 145 +++++++++--------------- crates/pithos_lib/src/archive/view.rs | 105 +++++++++++++++++ 3 files changed, 159 insertions(+), 93 deletions(-) create mode 100644 crates/pithos_lib/src/archive/view.rs diff --git a/crates/pithos_lib/src/archive/mod.rs b/crates/pithos_lib/src/archive/mod.rs index 0d8f5b4..6ee6591 100644 --- a/crates/pithos_lib/src/archive/mod.rs +++ b/crates/pithos_lib/src/archive/mod.rs @@ -17,6 +17,7 @@ mod reader_private_tests; mod snapshot; mod types; mod validation; +mod view; mod writer; pub(crate) use access::{AccessProvenance, ResolvedAccess}; @@ -36,6 +37,7 @@ pub(crate) use snapshot::AppendSnapshot; pub use types::{ArchivePath, ExternalLocation}; pub(crate) use types::{FileId, Span}; pub(crate) use validation::validated_segment_from_directory; +pub use view::ArchiveView; /// The metadata digest: BLAKE3 over each directory's BLAKE3 hash, from base to terminal. pub(crate) fn metadata_digest(directory_hashes: &[[u8; 32]]) -> [u8; 32] { diff --git a/crates/pithos_lib/src/archive/reader.rs b/crates/pithos_lib/src/archive/reader.rs index 5f3d6be..fe52143 100644 --- a/crates/pithos_lib/src/archive/reader.rs +++ b/crates/pithos_lib/src/archive/reader.rs @@ -1,11 +1,10 @@ use super::{ - AccessProvenance, AppendSnapshot, FileId, ResolvedAccess, Span, build_effective_index, - decode_validated_directory, validated_segment_from_directory, + AccessProvenance, AppendSnapshot, ArchiveView, FileId, ResolvedAccess, Span, + build_effective_index, decode_validated_directory, validated_segment_from_directory, }; use crate::archive::index::ArchiveIndex; use crate::archive::types::{ - ArchivePath, BlockHash, BlockLocation, ContentState, Entry, ExternalLocation, ReadRange, - ValidatedSegment, + BlockHash, BlockLocation, ContentState, Entry, ExternalLocation, ReadRange, ValidatedSegment, }; use crate::archive::validation::IndexLimits; use crate::block; @@ -162,7 +161,7 @@ pub trait ExternalBlockAccessPolicy: Send + Sync { } #[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum ContentAvailability { +pub(super) enum ContentAvailability { Available, MissingAccess, Unsupported(ArchiveFeature), @@ -284,18 +283,7 @@ pub struct Archive { source: S, external: E, external_access_policy: Option>, - archive_len: u64, - version: FormatVersion, - metadata_digest: [u8; 32], - terminal_directory: Span, - index: ArchiveIndex, - segments: Vec, - index_limits: IndexLimits, - access: ResolvedAccess, - #[cfg(feature = "crypt4gh")] - access_keys: AccessKeys, - limits: OpenLimits, - content_availability: BTreeMap, + view: ArchiveView, } impl Archive @@ -449,68 +437,51 @@ where source, external: options.external, external_access_policy: options.external_access_policy, - archive_len, - version, - metadata_digest, - terminal_directory: Span::new(terminal_start, terminal_len)?, - index, - segments, - index_limits, - access, - #[cfg(feature = "crypt4gh")] - access_keys: options.keys, - limits: options.limits, - content_availability, + view: ArchiveView { + archive_len, + version, + metadata_digest, + terminal_directory: Span::new(terminal_start, terminal_len)?, + index, + segments, + index_limits, + access, + #[cfg(feature = "crypt4gh")] + access_keys: options.keys, + limits: options.limits, + content_availability, + }, }) } /// BLAKE3 over the hashes of every directory, from the base to the terminal directory. /// Keeping it in trusted storage lets a later open detect changed metadata. pub fn metadata_digest(&self) -> [u8; 32] { - self.metadata_digest + self.view.metadata_digest() } pub fn entries(&self) -> impl ExactSizeIterator + '_ { - self.index - .entries() - .map(|entry| archive_entry(&self.index, entry, &self.content_availability)) + self.view.entries() + } + + /// The validated metadata, for planning reads without this reader's source. + pub fn view(&self) -> &ArchiveView { + &self.view } /// Consumes the reader and transfers its validated state to append/grant planning. pub(crate) fn into_append_snapshot(self) -> AppendSnapshot { - let Self { - archive_len, - version, - terminal_directory, - index, - segments, - index_limits, - access, - .. - } = self; - AppendSnapshot::new( - archive_len, - version, - terminal_directory, - index, - segments, - index_limits, - access, - ) + self.view.into_append_snapshot() } pub fn entry(&self, path: &str) -> Result, PithosError> { - let path = ArchivePath::new(path)?; - Ok(self - .index - .entry_at_path(&path) - .map(|entry| archive_entry(&self.index, entry, &self.content_availability))) + self.view.entry(path) } pub fn copy_to(&self, path: &str, sink: &mut W) -> Result<(), PithosError> { - let (id, _) = self.content_id(path)?; + let (id, _) = self.view.content_id(path)?; self.require_content_available(id)?; - self.copy_plan(id, self.index.full_file_plan(id)?, sink) + self.copy_plan(id, self.view.index.full_file_plan(id)?, sink) } pub fn copy_range_to( @@ -519,10 +490,10 @@ where range: Range, sink: &mut W, ) -> Result<(), PithosError> { - let (id, size) = self.content_id(path)?; + let (id, size) = self.view.content_id(path)?; let range = ReadRange::new(range, size)?; self.require_content_available(id)?; - self.copy_plan(id, self.index.range_plan(id, range)?, sink) + self.copy_plan(id, self.view.index.range_plan(id, range)?, sink) } #[cfg(feature = "crypt4gh")] @@ -531,15 +502,19 @@ where path: &str, operation: impl FnOnce(FileId, &PrivateKey, &FileKey) -> Result, ) -> Result> { - let (id, _) = self.content_id(path).map_err(ContentOperationError::Core)?; + let (id, _) = self + .view + .content_id(path) + .map_err(ContentOperationError::Core)?; self.require_content_available(id) .map_err(ContentOperationError::Core)?; let fresh_key; - let (key, key_owner) = match self.access.key(id) { + let (key, key_owner) = match self.view.access.key(id) { Some(key) => (key, id), None => { // A file sealed in pieces has no file key, so the export gets a fresh one. let first_piece = self + .view .access .grant_keys(id) .and_then(|keys| keys.first().map(|(key_id, _)| *key_id)) @@ -550,11 +525,13 @@ where } }; let provenance = self + .view .access .provenance(key_owner) .ok_or(PithosError::ContentUnavailable) .map_err(ContentOperationError::Core)?; let reader = self + .view .access_keys .0 .get(provenance.access_key) @@ -572,6 +549,7 @@ where self.require_content_available(id) .map_err(ContentOperationError::Core)?; let plan = self + .view .index .full_file_plan(id) .map_err(ContentOperationError::Core)?; @@ -584,29 +562,8 @@ where Ok(()) } - fn content_id(&self, path: &str) -> Result<(FileId, u64), PithosError> { - let path = ArchivePath::new(path)?; - let entry = self - .index - .entry_at_path(&path) - .ok_or_else(|| PithosError::FileNotFound(path.as_str().to_owned()))?; - let content = entry.entry.content().ok_or_else(|| { - PithosError::InvalidBlockDataState("only data/metadata entries have content".into()) - })?; - Ok((entry.id, content.size)) - } - pub(crate) fn require_content_available(&self, id: FileId) -> Result<(), PithosError> { - match self.content_availability.get(&id).copied() { - Some(ContentAvailability::Available) => Ok(()), - Some(ContentAvailability::MissingAccess) => Err(PithosError::ContentUnavailable), - Some(ContentAvailability::Unsupported(feature)) => { - Err(PithosError::UnsupportedFeature(feature)) - } - None => Err(PithosError::InvalidBlockDataState( - "only data/metadata entries have content".into(), - )), - } + self.view.require_content_available(id) } fn copy_plan( @@ -627,17 +584,17 @@ where id: FileId, planned: &super::planning::PlannedBlock, ) -> Result>, PithosError> { - if planned.descriptor.stored_size > self.limits.max_stored_block_bytes { + if planned.descriptor.stored_size > self.view.limits.max_stored_block_bytes { return Err(PithosError::LimitExceeded { field: "stored block", - limit: self.limits.max_stored_block_bytes, + limit: self.view.limits.max_stored_block_bytes, actual: planned.descriptor.stored_size, }); } - if planned.descriptor.original_size > self.limits.max_decoded_block_bytes { + if planned.descriptor.original_size > self.view.limits.max_decoded_block_bytes { return Err(PithosError::LimitExceeded { field: "decoded block", - limit: self.limits.max_decoded_block_bytes, + limit: self.view.limits.max_decoded_block_bytes, actual: planned.descriptor.original_size, }); } @@ -674,7 +631,8 @@ where )?, location, expected_len, - self.limits + self.view + .limits .max_stored_block_bytes .checked_add(4) .ok_or_else(|| { @@ -699,6 +657,7 @@ where location: FormatBlockLocation::Local, }; let key = self + .view .access .block_key(id, planned.hash) .ok_or(PithosError::ContentUnavailable)?; @@ -708,8 +667,8 @@ where planned.hash.0, &meta, block::Limits { - max_stored_bytes: self.limits.max_stored_block_bytes, - max_decoded_bytes: self.limits.max_decoded_block_bytes, + max_stored_bytes: self.view.limits.max_stored_block_bytes, + max_decoded_bytes: self.view.limits.max_decoded_block_bytes, }, ) } @@ -742,7 +701,7 @@ fn entry_kind( } } -fn archive_entry( +pub(super) fn archive_entry( index: &ArchiveIndex, entry: &super::index::IndexedEntry, content_availability: &BTreeMap, diff --git a/crates/pithos_lib/src/archive/view.rs b/crates/pithos_lib/src/archive/view.rs new file mode 100644 index 0000000..3f38c57 --- /dev/null +++ b/crates/pithos_lib/src/archive/view.rs @@ -0,0 +1,105 @@ +use super::access::ResolvedAccess; +use super::index::ArchiveIndex; +#[cfg(feature = "crypt4gh")] +use super::reader::AccessKeys; +use super::reader::{ArchiveEntry, ContentAvailability, OpenLimits, archive_entry}; +use super::snapshot::AppendSnapshot; +use super::types::{ArchivePath, FileId, Span, ValidatedSegment}; +use super::validation::IndexLimits; +use crate::error::PithosError; +use crate::format::header::FormatVersion; +use std::collections::BTreeMap; + +/// Validated archive metadata without a byte source. +/// +/// A view is the result of opening an archive. It lists entries and plans content reads, but +/// holds no source: callers fetch block bytes themselves and pass them back for decoding. +pub struct ArchiveView { + pub(super) archive_len: u64, + pub(super) version: FormatVersion, + pub(super) metadata_digest: [u8; 32], + pub(super) terminal_directory: Span, + pub(super) index: ArchiveIndex, + pub(super) segments: Vec, + pub(super) index_limits: IndexLimits, + pub(super) access: ResolvedAccess, + #[cfg(feature = "crypt4gh")] + pub(super) access_keys: AccessKeys, + pub(super) limits: OpenLimits, + pub(super) content_availability: BTreeMap, +} + +impl ArchiveView { + /// BLAKE3 over the hashes of every directory, from the base to the terminal directory. + /// Keeping it in trusted storage lets a later open detect changed metadata. + pub fn metadata_digest(&self) -> [u8; 32] { + self.metadata_digest + } + + /// The format version stored in the file header: `0x0100` for 1.0, `0x0101` for 1.1. + pub fn version(&self) -> u16 { + self.version.wire() + } + + pub fn entries(&self) -> impl ExactSizeIterator + '_ { + self.index + .entries() + .map(|entry| archive_entry(&self.index, entry, &self.content_availability)) + } + + pub fn entry(&self, path: &str) -> Result, PithosError> { + let path = ArchivePath::new(path)?; + Ok(self + .index + .entry_at_path(&path) + .map(|entry| archive_entry(&self.index, entry, &self.content_availability))) + } + + /// Transfers the validated state to append and grant planning. + pub(crate) fn into_append_snapshot(self) -> AppendSnapshot { + let Self { + archive_len, + version, + terminal_directory, + index, + segments, + index_limits, + access, + .. + } = self; + AppendSnapshot::new( + archive_len, + version, + terminal_directory, + index, + segments, + index_limits, + access, + ) + } + + pub(crate) fn content_id(&self, path: &str) -> Result<(FileId, u64), PithosError> { + let path = ArchivePath::new(path)?; + let entry = self + .index + .entry_at_path(&path) + .ok_or_else(|| PithosError::FileNotFound(path.as_str().to_owned()))?; + let content = entry.entry.content().ok_or_else(|| { + PithosError::InvalidBlockDataState("only data/metadata entries have content".into()) + })?; + Ok((entry.id, content.size)) + } + + pub(crate) fn require_content_available(&self, id: FileId) -> Result<(), PithosError> { + match self.content_availability.get(&id).copied() { + Some(ContentAvailability::Available) => Ok(()), + Some(ContentAvailability::MissingAccess) => Err(PithosError::ContentUnavailable), + Some(ContentAvailability::Unsupported(feature)) => { + Err(PithosError::UnsupportedFeature(feature)) + } + None => Err(PithosError::InvalidBlockDataState( + "only data/metadata entries have content".into(), + )), + } + } +} From fccf3759b48eb5fdac74596f9a2b07396ffc85cc Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 20:52:53 +0200 Subject: [PATCH 076/166] chore: update the agent guide for format 1.1 and Rust 1.89 --- AGENTS.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index c7d9725..2f6264d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,7 +2,7 @@ ## Boundaries -- This is a Rust 2024 workspace (`crates/*`) with MSRV Rust 1.88. `pithos_lib` and the `pithos` CLI each declare their own version; `pithos_pyo3` is an unpublished, nonfunctional stub. +- This is a Rust 2024 workspace (`crates/*`) with MSRV Rust 1.89. `pithos_lib` and the `pithos` CLI each declare their own version; `pithos_pyo3` is an unpublished, nonfunctional stub. - `pithos_lib` is the implementation: `archive/` is the public reader/writer API; private `format/` is the only wire-format authority; `crypto.rs` and `block.rs` isolate keys and transforms. - Keep core archive/format code independent of host files and presentation formats. Linux-only filesystem work belongs in `fs/`; RO-Crate and Crypt4GH presentation work belongs in `adapters/`. Preserve adapter-specific errors and host/path/member context instead of expanding core `PithosError` or exposing secrets. - `pithos` is the Clap wrapper. Keep CLI parsing, output staging, and filesystem presentation in `crates/pithos`; put archive behavior in `pithos_lib`. @@ -10,15 +10,15 @@ ## Compatibility And Tests -- This branch implements the Pithos 1.0 draft wire rules; Cargo package versioning is separate. Do not expose old `model`, `helpers`, or wire-record paths. +- This branch implements the Pithos 1.1 draft wire rules and still reads every 1.0 archive; Cargo package versioning is separate. Do not expose old `model`, `helpers`, or wire-record paths. - Treat changes under `src/format/`, archive validation/indexing, encryption, compression, flags, indexes, or directory layout as on-disk compatibility changes. Do not casually normalize the currently inconsistent `.pto`, `.pith`, and `.pithos` extensions. - `pithos_lib` integration tests use committed fixtures and test-only PEM keys in `crates/pithos_lib/tests/data/`; reuse helpers from `tests/common/`. - Put wire-format unit coverage in `src/format/`, reader-internal coverage in `src/archive/reader_private_tests.rs`, and public extraction/range coverage in `crates/pithos_lib/tests/reader.rs`. Keep RO-Crate tests in the `ro_crate_{directory,zip,conversion}` integration targets. ## Verification -- CI parity: `cargo +stable test --locked --workspace --all-features --lib --bins --tests`; MSRV parity: `cargo +1.88.0 test --locked --workspace --all-features`. +- CI parity: `cargo +stable test --locked --workspace --all-features --lib --bins --tests`; MSRV parity: `cargo +1.89.0 test --locked --workspace --all-features`. - Before finishing Rust changes, run `cargo +stable fmt --all -- --check` and `cargo +stable clippy --locked --workspace --all-targets --all-features -- -D warnings` when practical. - Focused library tests use integration targets, for example `cargo test -p pithos_lib --test reader`; focused RO-Crate coverage is `cargo test -p pithos_lib --test ro_crate_directory` (or `ro_crate_zip` / `ro_crate_conversion`). CLI smoke check: `cargo +stable run --locked -p pithos -- --help`. -- Release/package changes also require `python3 .github/release/check.py contracts` and `python3 .github/release/check.py package check pithos_lib` (and `pithos` when affected). The package policy is a ratchet: `package update` needs `--allow-increase` to raise a budget. +- Release/package changes also require `cargo deny --locked check`, the same check for `fuzz/Cargo.toml`, and `cargo +stable package -p pithos_lib --locked`, as in `.github/workflows/tag_publish.yaml`. - Run benchmark baselines and candidates locally on the same machine with the same toolchain, locked dependencies, power settings, and workload environment variables; GitHub-hosted runners are not comparable benchmark evidence. From 39befd3e475149cf875c71bf1e7976b241360927 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 21:15:54 +0200 Subject: [PATCH 077/166] refactor: move post-chain open validation into the opener module --- crates/pithos_lib/src/archive/mod.rs | 2 +- crates/pithos_lib/src/archive/opener.rs | 129 ++++++++++++++++++ crates/pithos_lib/src/archive/reader.rs | 166 +++++++----------------- 3 files changed, 180 insertions(+), 117 deletions(-) create mode 100644 crates/pithos_lib/src/archive/opener.rs diff --git a/crates/pithos_lib/src/archive/mod.rs b/crates/pithos_lib/src/archive/mod.rs index 6ee6591..2661229 100644 --- a/crates/pithos_lib/src/archive/mod.rs +++ b/crates/pithos_lib/src/archive/mod.rs @@ -8,6 +8,7 @@ mod access; mod append; mod content_tree; mod index; +mod opener; mod path_validation; mod pieces; mod planning; @@ -22,7 +23,6 @@ mod writer; pub(crate) use access::{AccessProvenance, ResolvedAccess}; pub use append::{AppendDurability, AppendObservation, AppendOptions}; -pub(crate) use index::build_effective_index; pub(crate) use path_validation::{ validate_directory_entries, validate_new_candidate, validate_symlink_target, }; diff --git a/crates/pithos_lib/src/archive/opener.rs b/crates/pithos_lib/src/archive/opener.rs new file mode 100644 index 0000000..6b23a32 --- /dev/null +++ b/crates/pithos_lib/src/archive/opener.rs @@ -0,0 +1,129 @@ +use super::access::ResolvedAccess; +use super::index::build_effective_index; +use super::reader::{ + AccessKeys, OpenLimits, classify_content_availability, resolve_block_lists, resolve_recipients, + validate_piece_keys, +}; +use super::types::{FileId, Span, ValidatedSegment}; +use super::validated_segment_from_directory; +use super::validation::IndexLimits; +use super::view::ArchiveView; +use crate::error::PithosError; +use crate::format::directory::Directory; +use crate::format::header::FormatVersion; +use std::collections::HashMap; + +/// The open settings that matter for metadata. External resolution is a read-time concern. +pub(super) struct OpenSettings { + pub(super) limits: OpenLimits, + pub(super) keys: AccessKeys, + pub(super) expected_metadata_digest: Option<[u8; 32]>, + pub(super) external_enabled: bool, +} + +/// Directories decoded so far, from the terminal one back toward the base. +pub(super) struct DecodedChain { + pub(super) version: FormatVersion, + pub(super) terminal: Span, + pub(super) raw: Vec<(Directory, Span)>, + pub(super) remaining_block_references: u64, + pub(super) directory_hashes: Vec<[u8; 32]>, +} + +/// Runs every metadata check after the whole chain is decoded and builds the view. +pub(super) fn complete_open( + archive_len: u64, + settings: &mut OpenSettings, + chain: DecodedChain, +) -> Result { + let DecodedChain { + version, + terminal, + mut raw, + mut remaining_block_references, + mut directory_hashes, + } = chain; + let limits = settings.limits; + raw.reverse(); + directory_hashes.reverse(); + let metadata_digest = crate::archive::metadata_digest(&directory_hashes); + if settings + .expected_metadata_digest + .is_some_and(|expected| expected != metadata_digest) + { + return Err(PithosError::MetadataDigestMismatch); + } + let maximum_piece_key = validate_piece_keys(version, &raw)?; + + let mut first_grants = HashMap::new(); + for (directory, _) in &raw { + for (sender, section) in &directory.encryption { + for (recipient, recipient_section) in §ion.recipients { + let pair = (*sender, *recipient); + if let Some(first) = first_grants.get(&pair) + && *first != &recipient_section.recipient_data + { + return Err(PithosError::ConflictingRecipientGrant); + } + first_grants + .entry(pair) + .or_insert(&recipient_section.recipient_data); + } + } + } + + let mut access = ResolvedAccess::new(); + let mut recovery_order = 0usize; + for (segment, (directory, _)) in raw.iter().enumerate() { + resolve_recipients( + version, + directory, + &settings.keys, + segment, + &mut recovery_order, + &mut access, + limits, + )?; + } + + let mut segments: Vec = Vec::new(); + for (segment_index, (directory, span)) in raw.into_iter().enumerate() { + let mut directory = directory; + resolve_block_lists( + &mut directory, + &mut access, + &mut remaining_block_references, + limits, + )?; + let parent = segment_index + .checked_sub(1) + .map(|index| segments[index].span); + segments.push(validated_segment_from_directory( + version, &directory, span, parent, + )?); + } + let index_limits = IndexLimits { + max_entries: limits.max_entries, + max_descriptors: limits.max_descriptors, + max_references: limits.max_references, + max_relationships: limits.max_relationships, + max_segments: limits.max_parent_directories.saturating_add(1), + }; + let mut index = build_effective_index(&segments, archive_len, index_limits)?; + index.cover_piece_keys(maximum_piece_key.map(FileId)); + let content_availability = classify_content_availability(&index, settings.external_enabled)?; + Ok(ArchiveView { + archive_len, + version, + metadata_digest, + terminal_directory: terminal, + index, + segments, + index_limits, + access, + #[cfg(feature = "crypt4gh")] + access_keys: std::mem::take(&mut settings.keys), + limits, + content_availability, + }) +} diff --git a/crates/pithos_lib/src/archive/reader.rs b/crates/pithos_lib/src/archive/reader.rs index fe52143..60489fe 100644 --- a/crates/pithos_lib/src/archive/reader.rs +++ b/crates/pithos_lib/src/archive/reader.rs @@ -1,12 +1,10 @@ -use super::{ - AccessProvenance, AppendSnapshot, ArchiveView, FileId, ResolvedAccess, Span, - build_effective_index, decode_validated_directory, validated_segment_from_directory, -}; +use super::decode_validated_directory; +use super::opener::{DecodedChain, OpenSettings, complete_open}; +use super::{AccessProvenance, AppendSnapshot, ArchiveView, FileId, ResolvedAccess, Span}; use crate::archive::index::ArchiveIndex; use crate::archive::types::{ - BlockHash, BlockLocation, ContentState, Entry, ExternalLocation, ReadRange, ValidatedSegment, + BlockHash, BlockLocation, ContentState, Entry, ExternalLocation, ReadRange, }; -use crate::archive::validation::IndexLimits; use crate::block; #[cfg(feature = "crypt4gh")] use crate::crypto::FileKey; @@ -21,7 +19,7 @@ use crate::format::file_entry::{BlockDataEntry, BlockDataState}; use crate::format::header::{FileHeader, FormatVersion}; use crate::format::limits::DeserializationLimits; use crate::source::ArchiveSource; -use std::collections::{BTreeMap, HashMap, HashSet}; +use std::collections::{BTreeMap, HashSet}; use std::io::Write; use std::sync::Arc; @@ -37,7 +35,7 @@ use x25519_dalek::PublicKey as DalekPublicKey; use zeroize::Zeroizing; #[derive(Default)] -struct DecodedDirectoryCounts { +pub(super) struct DecodedDirectoryCounts { entries: u64, descriptors: u64, references: u64, @@ -46,7 +44,7 @@ struct DecodedDirectoryCounts { } impl DecodedDirectoryCounts { - fn record(&mut self, directory: &Directory) { + pub(super) fn record(&mut self, directory: &Directory) { self.entries += directory.files.len() as u64; self.descriptors += directory.blocks.len() as u64; self.references += directory @@ -244,6 +242,20 @@ impl OpenOptions { self.external_access_policy = Some(policy); self } + + /// Splits off the read-time external resolver and access policy. + pub(super) fn into_parts( + self, + ) -> (OpenSettings, E, Option>) { + let settings = OpenSettings { + limits: self.limits, + keys: self.keys, + expected_metadata_digest: self.expected_metadata_digest, + external_enabled: self.external_resolver_supplied + && self.external_access_policy.is_some(), + }; + (settings, self.external, self.external_access_policy) + } } /// Public, secret-free entry shape returned by an opened archive. @@ -294,6 +306,8 @@ where /// Opens, frames, validates, resolves access metadata, and indexes an archive. pub fn open(source: S, options: OpenOptions) -> Result { let archive_len = source.len()?; + let (mut settings, external, external_access_policy) = options.into_parts(); + let limits = settings.limits; let mut header = [0; FileHeader::ENCODED_LEN]; source.read_exact_at(0, &mut header)?; let header = crate::format::header::decode_header(&mut header.as_slice())?; @@ -304,21 +318,21 @@ where }, )?; - let (terminal_start, terminal_len) = terminal_span(&source, archive_len, options.limits)?; + let (terminal_start, terminal_len) = terminal_span(&source, archive_len, limits)?; let mut raw = Vec::new(); let mut total_directory_bytes = 0u64; let mut next = Some((terminal_start, terminal_len)); let mut child_start = archive_len; let mut visited = HashSet::new(); let mut decoded = DecodedDirectoryCounts::default(); - let mut remaining_block_references = options.limits.max_accessible_block_references; + let mut remaining_block_references = limits.max_accessible_block_references; let mut directory_hashes = Vec::new(); while let Some((start, len)) = next { - validate_directory_len(len, options.limits)?; - if raw.len() as u64 > options.limits.max_parent_directories { + validate_directory_len(len, limits)?; + if raw.len() as u64 > limits.max_parent_directories { return Err(PithosError::LimitExceeded { field: "parent directories", - limit: options.limits.max_parent_directories, + limit: limits.max_parent_directories, actual: raw.len() as u64, }); } @@ -333,13 +347,13 @@ where .checked_add(len) .ok_or(PithosError::LimitExceeded { field: "total directory bytes", - limit: options.limits.max_total_directory_bytes, + limit: limits.max_total_directory_bytes, actual: u64::MAX, })?; - if total_directory_bytes > options.limits.max_total_directory_bytes { + if total_directory_bytes > limits.max_total_directory_bytes { return Err(PithosError::LimitExceeded { field: "total directory bytes", - limit: options.limits.max_total_directory_bytes, + limit: limits.max_total_directory_bytes, actual: total_directory_bytes, }); } @@ -347,7 +361,7 @@ where directory_hashes.push(*blake3::hash(&bytes).as_bytes()); let directory = decode_validated_directory( &bytes, - &remaining_deserialization_limits(options.limits, &decoded), + &remaining_deserialization_limits(limits, &decoded), &mut remaining_block_references, )?; decoded.record(&directory); @@ -355,102 +369,24 @@ where child_start = start; raw.push((directory, span)); } - raw.reverse(); - directory_hashes.reverse(); - let metadata_digest = crate::archive::metadata_digest(&directory_hashes); - if options - .expected_metadata_digest - .is_some_and(|expected| expected != metadata_digest) - { - return Err(PithosError::MetadataDigestMismatch); - } - let maximum_piece_key = validate_piece_keys(version, &raw)?; - - let mut first_grants = HashMap::new(); - for (directory, _) in &raw { - for (sender, section) in &directory.encryption { - for (recipient, recipient_section) in §ion.recipients { - let pair = (*sender, *recipient); - if let Some(first) = first_grants.get(&pair) - && *first != &recipient_section.recipient_data - { - return Err(PithosError::ConflictingRecipientGrant); - } - first_grants - .entry(pair) - .or_insert(&recipient_section.recipient_data); - } - } - } - - let mut access = ResolvedAccess::new(); - let mut recovery_order = 0usize; - for (segment, (directory, _)) in raw.iter().enumerate() { - resolve_recipients( - version, - directory, - &options.keys, - segment, - &mut recovery_order, - &mut access, - options.limits, - )?; - } - - let mut segments: Vec = Vec::new(); - for (segment_index, (directory, span)) in raw.into_iter().enumerate() { - let mut directory = directory; - resolve_block_lists( - &mut directory, - &mut access, - &mut remaining_block_references, - options.limits, - )?; - let parent = segment_index - .checked_sub(1) - .map(|index| segments[index].span); - segments.push(validated_segment_from_directory( - version, &directory, span, parent, - )?); - } - let index_limits = IndexLimits { - max_entries: options.limits.max_entries, - max_descriptors: options.limits.max_descriptors, - max_references: options.limits.max_references, - max_relationships: options.limits.max_relationships, - max_segments: options.limits.max_parent_directories.saturating_add(1), + let chain = DecodedChain { + version, + terminal: Span::new(terminal_start, terminal_len)?, + raw, + remaining_block_references, + directory_hashes, }; - let mut index = build_effective_index(&segments, archive_len, index_limits)?; - index.cover_piece_keys(maximum_piece_key.map(FileId)); - let content_availability = classify_content_availability( - &index, - options.external_resolver_supplied, - options.external_access_policy.is_some(), - )?; - for span in index.local_block_spans() { + let view = complete_open(archive_len, &mut settings, chain)?; + for span in view.index.local_block_spans() { let mut marker = [0; 4]; source.read_exact_at(span.start(), &mut marker)?; crate::format::block::decode_block_marker(&mut marker.as_slice())?; } - Ok(Self { source, - external: options.external, - external_access_policy: options.external_access_policy, - view: ArchiveView { - archive_len, - version, - metadata_digest, - terminal_directory: Span::new(terminal_start, terminal_len)?, - index, - segments, - index_limits, - access, - #[cfg(feature = "crypt4gh")] - access_keys: options.keys, - limits: options.limits, - content_availability, - }, + external, + external_access_policy, + view, }) } @@ -728,12 +664,10 @@ pub(super) fn archive_entry( } } -fn classify_content_availability( +pub(super) fn classify_content_availability( index: &ArchiveIndex, - external_resolver_supplied: bool, - external_access_policy_supplied: bool, + external_supported: bool, ) -> Result, PithosError> { - let external_supported = external_resolver_supplied && external_access_policy_supplied; let mut availability = BTreeMap::new(); for entry in index.entries() { let Some(content) = entry.entry.content() else { @@ -776,7 +710,7 @@ fn deserialization_limits(limits: OpenLimits) -> DeserializationLimits { } } -fn remaining_deserialization_limits( +pub(super) fn remaining_deserialization_limits( limits: OpenLimits, decoded: &DecodedDirectoryCounts, ) -> DeserializationLimits { @@ -817,7 +751,7 @@ fn terminal_span( Ok((start, len)) } -fn validate_directory_len(len: u64, limits: OpenLimits) -> Result<(), PithosError> { +pub(super) fn validate_directory_len(len: u64, limits: OpenLimits) -> Result<(), PithosError> { if len < 25 { return Err(PithosError::DirectoryLengthMismatch { expected: 25, @@ -854,7 +788,7 @@ fn read_source( Ok(bytes) } -fn resolve_recipients( +pub(super) fn resolve_recipients( version: FormatVersion, directory: &Directory, keys: &AccessKeys, @@ -924,7 +858,7 @@ fn resolve_recipients( Ok(()) } -fn resolve_block_lists( +pub(super) fn resolve_block_lists( directory: &mut Directory, access: &mut ResolvedAccess, remaining_block_references: &mut u64, @@ -995,7 +929,7 @@ fn resolve_block_lists( /// Checks the version 1.1 piece rules across the selected chain. Returns the largest piece /// key id, which new file ids must stay above. -fn validate_piece_keys( +pub(super) fn validate_piece_keys( version: FormatVersion, raw: &[(Directory, Span)], ) -> Result, PithosError> { From e1e02fba1e08125ea2594cd3b90aa3fb9f0b37ce Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 21:21:37 +0200 Subject: [PATCH 078/166] feat: add a sans-I/O archive opener --- crates/pithos_lib/src/archive/mod.rs | 1 + crates/pithos_lib/src/archive/opener.rs | 294 +++++++++++++++++- crates/pithos_lib/src/archive/reader.rs | 108 +------ crates/pithos_lib/src/error.rs | 6 + crates/pithos_lib/tests/source_equivalence.rs | 5 +- 5 files changed, 303 insertions(+), 111 deletions(-) diff --git a/crates/pithos_lib/src/archive/mod.rs b/crates/pithos_lib/src/archive/mod.rs index 2661229..04593ed 100644 --- a/crates/pithos_lib/src/archive/mod.rs +++ b/crates/pithos_lib/src/archive/mod.rs @@ -23,6 +23,7 @@ mod writer; pub(crate) use access::{AccessProvenance, ResolvedAccess}; pub use append::{AppendDurability, AppendObservation, AppendOptions}; +pub use opener::{ArchiveOpener, ReadRequest}; pub(crate) use path_validation::{ validate_directory_entries, validate_new_candidate, validate_symlink_target, }; diff --git a/crates/pithos_lib/src/archive/opener.rs b/crates/pithos_lib/src/archive/opener.rs index 6b23a32..1684fc9 100644 --- a/crates/pithos_lib/src/archive/opener.rs +++ b/crates/pithos_lib/src/archive/opener.rs @@ -1,17 +1,40 @@ use super::access::ResolvedAccess; use super::index::build_effective_index; use super::reader::{ - AccessKeys, OpenLimits, classify_content_availability, resolve_block_lists, resolve_recipients, - validate_piece_keys, + AccessKeys, DecodedDirectoryCounts, OpenLimits, OpenOptions, classify_content_availability, + remaining_deserialization_limits, resolve_block_lists, resolve_recipients, + validate_directory_len, validate_piece_keys, }; use super::types::{FileId, Span, ValidatedSegment}; -use super::validated_segment_from_directory; use super::validation::IndexLimits; use super::view::ArchiveView; +use super::{decode_validated_directory, validated_segment_from_directory}; use crate::error::PithosError; use crate::format::directory::Directory; -use crate::format::header::FormatVersion; -use std::collections::HashMap; +use crate::format::header::{FileHeader, FormatVersion}; +use std::collections::{HashMap, HashSet}; + +/// One exact byte range the opener needs next. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ReadRequest { + offset: u64, + len: u64, + sequence: u64, +} + +impl ReadRequest { + pub fn offset(&self) -> u64 { + self.offset + } + + pub fn len(&self) -> u64 { + self.len + } + + pub fn is_empty(&self) -> bool { + self.len == 0 + } +} /// The open settings that matter for metadata. External resolution is a read-time concern. pub(super) struct OpenSettings { @@ -21,17 +44,264 @@ pub(super) struct OpenSettings { pub(super) external_enabled: bool, } +/// Opens an archive without performing I/O. +/// +/// The opener hands out one read request at a time: the header, the footer, then each +/// directory from the terminal one back to the base. The caller reads exactly the requested +/// bytes and passes them to [`ArchiveOpener::feed`]. Limits are checked before a request is +/// issued. Opening validates metadata only; block markers are checked when a block is read. +/// +/// ```no_run +/// # fn read(offset: u64, len: u64) -> Vec { unimplemented!() } +/// # fn main() -> Result<(), pithos_lib::error::PithosError> { +/// use pithos_lib::archive::{ArchiveOpener, OpenOptions}; +/// +/// let mut opener = ArchiveOpener::new(4096, OpenOptions::default())?; +/// while let Some(request) = opener.request() { +/// let bytes = read(request.offset(), request.len()); +/// opener.feed(request, &bytes)?; +/// } +/// let view = opener.finish()?; +/// # let _ = view; +/// # Ok(()) +/// # } +/// ``` +pub struct ArchiveOpener { + archive_len: u64, + settings: OpenSettings, + next_sequence: u64, + pending: Option, + phase: Phase, +} + +enum Phase { + Header, + Footer(FormatVersion), + Directories(Box), + Opened(Box), + Failed, +} + /// Directories decoded so far, from the terminal one back toward the base. -pub(super) struct DecodedChain { - pub(super) version: FormatVersion, - pub(super) terminal: Span, - pub(super) raw: Vec<(Directory, Span)>, - pub(super) remaining_block_references: u64, - pub(super) directory_hashes: Vec<[u8; 32]>, +struct DecodedChain { + version: FormatVersion, + terminal: Span, + raw: Vec<(Directory, Span)>, + remaining_block_references: u64, + directory_hashes: Vec<[u8; 32]>, +} + +/// A decoded chain plus the bookkeeping for the next directory request. +struct Chain { + decoded_chain: DecodedChain, + pending: Span, + total_directory_bytes: u64, + child_start: u64, + visited: HashSet, + decoded: DecodedDirectoryCounts, +} + +impl ArchiveOpener { + /// Starts opening an archive of `archive_len` bytes. Only the keys, limits and expected + /// metadata digest of `options` are used, plus whether external resolution is enabled. + pub fn new(archive_len: u64, options: OpenOptions) -> Result { + let (settings, _, _) = options.into_parts(); + Self::with_settings(archive_len, settings) + } + + pub(super) fn with_settings( + archive_len: u64, + settings: OpenSettings, + ) -> Result { + if archive_len < FileHeader::ENCODED_LEN as u64 { + return Err(PithosError::InvalidDirectoryRange { + operation: "read file header", + }); + } + let mut opener = Self { + archive_len, + settings, + next_sequence: 0, + pending: None, + phase: Phase::Header, + }; + opener.issue(0, FileHeader::ENCODED_LEN as u64); + Ok(opener) + } + + /// The outstanding request, or `None` once the archive is opened or opening failed. + pub fn request(&self) -> Option { + self.pending + } + + /// Accepts the bytes for the outstanding request. + /// + /// A response to another request or of the wrong length is rejected and the request stays + /// outstanding. Any other error ends the open. + pub fn feed(&mut self, request: ReadRequest, response: &[u8]) -> Result<(), PithosError> { + if self.pending != Some(request) { + return Err(PithosError::UnexpectedReadResponse); + } + if response.len() as u64 != request.len { + return Err(PithosError::ReadResponseLength { + expected: request.len, + actual: response.len() as u64, + }); + } + self.pending = None; + let phase = std::mem::replace(&mut self.phase, Phase::Failed); + match self.advance(phase, response) { + Ok(phase) => { + self.phase = phase; + Ok(()) + } + Err(error) => { + self.pending = None; + Err(error) + } + } + } + + /// Returns the opened view once every request has been answered. + pub fn finish(self) -> Result { + match self.phase { + Phase::Opened(view) => Ok(*view), + _ => Err(PithosError::OpenIncomplete), + } + } + + fn issue(&mut self, offset: u64, len: u64) { + self.pending = Some(ReadRequest { + offset, + len, + sequence: self.next_sequence, + }); + self.next_sequence += 1; + } + + fn advance(&mut self, phase: Phase, response: &[u8]) -> Result { + match phase { + Phase::Header => { + let header = crate::format::header::decode_header(&mut &response[..])?; + let version = FormatVersion::from_wire(header.version).ok_or( + PithosError::UnsupportedFileVersion { + supported: FormatVersion::CURRENT.wire(), + actual: header.version, + }, + )?; + let footer = + self.archive_len + .checked_sub(12) + .ok_or(PithosError::InvalidDirectoryRange { + operation: "read directory footer", + })?; + self.issue(footer, 12); + Ok(Phase::Footer(version)) + } + Phase::Footer(version) => { + let len = u64::from_be_bytes(response[..8].try_into().expect("12-byte footer")); + validate_directory_len(len, self.settings.limits)?; + let start = self.archive_len.checked_sub(len).ok_or( + PithosError::InvalidDirectoryRange { + operation: "validate terminal directory", + }, + )?; + let terminal = Span::new(start, len)?; + let mut chain = Box::new(Chain { + decoded_chain: DecodedChain { + version, + terminal, + raw: Vec::new(), + remaining_block_references: self + .settings + .limits + .max_accessible_block_references, + directory_hashes: Vec::new(), + }, + pending: terminal, + total_directory_bytes: 0, + child_start: self.archive_len, + visited: HashSet::new(), + decoded: DecodedDirectoryCounts::default(), + }); + chain.admit(terminal, self.settings.limits)?; + self.issue(start, len); + Ok(Phase::Directories(chain)) + } + Phase::Directories(mut chain) => { + let limits = remaining_deserialization_limits(self.settings.limits, &chain.decoded); + let decoded_chain = &mut chain.decoded_chain; + let hash = blake3::hash(response); + decoded_chain.directory_hashes.push(*hash.as_bytes()); + let directory = decode_validated_directory( + response, + &limits, + &mut decoded_chain.remaining_block_references, + )?; + let next = directory.parent_directory_offset; + let span = chain.pending; + chain.decoded.record(&directory); + chain.decoded_chain.raw.push((directory, span)); + chain.child_start = span.start(); + match next { + Some((start, len)) => { + let span = Span::new(start, len)?; + chain.admit(span, self.settings.limits)?; + self.issue(start, len); + Ok(Phase::Directories(chain)) + } + None => { + let view = complete_open( + self.archive_len, + &mut self.settings, + chain.decoded_chain, + )?; + Ok(Phase::Opened(Box::new(view))) + } + } + } + Phase::Opened(_) | Phase::Failed => Err(PithosError::UnexpectedReadResponse), + } + } +} + +impl Chain { + /// Checks a directory against the chain rules and limits before it is requested. + fn admit(&mut self, span: Span, limits: OpenLimits) -> Result<(), PithosError> { + validate_directory_len(span.len(), limits)?; + if self.decoded_chain.raw.len() as u64 > limits.max_parent_directories { + return Err(PithosError::LimitExceeded { + field: "parent directories", + limit: limits.max_parent_directories, + actual: self.decoded_chain.raw.len() as u64, + }); + } + if span.end() > self.child_start || !self.visited.insert(span) { + return Err(PithosError::InvalidDirectoryChain { + operation: "validate parent ordering", + }); + } + self.total_directory_bytes = self.total_directory_bytes.checked_add(span.len()).ok_or( + PithosError::LimitExceeded { + field: "total directory bytes", + limit: limits.max_total_directory_bytes, + actual: u64::MAX, + }, + )?; + if self.total_directory_bytes > limits.max_total_directory_bytes { + return Err(PithosError::LimitExceeded { + field: "total directory bytes", + limit: limits.max_total_directory_bytes, + actual: self.total_directory_bytes, + }); + } + self.pending = span; + Ok(()) + } } /// Runs every metadata check after the whole chain is decoded and builds the view. -pub(super) fn complete_open( +fn complete_open( archive_len: u64, settings: &mut OpenSettings, chain: DecodedChain, diff --git a/crates/pithos_lib/src/archive/reader.rs b/crates/pithos_lib/src/archive/reader.rs index 60489fe..5d2ac34 100644 --- a/crates/pithos_lib/src/archive/reader.rs +++ b/crates/pithos_lib/src/archive/reader.rs @@ -1,5 +1,4 @@ -use super::decode_validated_directory; -use super::opener::{DecodedChain, OpenSettings, complete_open}; +use super::opener::{ArchiveOpener, OpenSettings}; use super::{AccessProvenance, AppendSnapshot, ArchiveView, FileId, ResolvedAccess, Span}; use crate::archive::index::ArchiveIndex; use crate::archive::types::{ @@ -16,7 +15,7 @@ use crate::format::block::{ use crate::format::directory::Directory; use crate::format::encryption::RecipientData; use crate::format::file_entry::{BlockDataEntry, BlockDataState}; -use crate::format::header::{FileHeader, FormatVersion}; +use crate::format::header::FormatVersion; use crate::format::limits::DeserializationLimits; use crate::source::ArchiveSource; use std::collections::{BTreeMap, HashSet}; @@ -306,77 +305,18 @@ where /// Opens, frames, validates, resolves access metadata, and indexes an archive. pub fn open(source: S, options: OpenOptions) -> Result { let archive_len = source.len()?; - let (mut settings, external, external_access_policy) = options.into_parts(); - let limits = settings.limits; - let mut header = [0; FileHeader::ENCODED_LEN]; - source.read_exact_at(0, &mut header)?; - let header = crate::format::header::decode_header(&mut header.as_slice())?; - let version = FormatVersion::from_wire(header.version).ok_or( - PithosError::UnsupportedFileVersion { - supported: FormatVersion::CURRENT.wire(), - actual: header.version, - }, - )?; - - let (terminal_start, terminal_len) = terminal_span(&source, archive_len, limits)?; - let mut raw = Vec::new(); - let mut total_directory_bytes = 0u64; - let mut next = Some((terminal_start, terminal_len)); - let mut child_start = archive_len; - let mut visited = HashSet::new(); - let mut decoded = DecodedDirectoryCounts::default(); - let mut remaining_block_references = limits.max_accessible_block_references; - let mut directory_hashes = Vec::new(); - while let Some((start, len)) = next { - validate_directory_len(len, limits)?; - if raw.len() as u64 > limits.max_parent_directories { - return Err(PithosError::LimitExceeded { - field: "parent directories", - limit: limits.max_parent_directories, - actual: raw.len() as u64, - }); - } - let span = Span::new(start, len)?; - if span.end() > child_start || !visited.insert((start, len)) { - return Err(PithosError::InvalidDirectoryChain { - operation: "validate parent ordering", - }); - } - total_directory_bytes = - total_directory_bytes - .checked_add(len) - .ok_or(PithosError::LimitExceeded { - field: "total directory bytes", - limit: limits.max_total_directory_bytes, - actual: u64::MAX, - })?; - if total_directory_bytes > limits.max_total_directory_bytes { - return Err(PithosError::LimitExceeded { - field: "total directory bytes", - limit: limits.max_total_directory_bytes, - actual: total_directory_bytes, - }); - } - let bytes = Zeroizing::new(read_source(&source, start, len, "directory")?); - directory_hashes.push(*blake3::hash(&bytes).as_bytes()); - let directory = decode_validated_directory( - &bytes, - &remaining_deserialization_limits(limits, &decoded), - &mut remaining_block_references, - )?; - decoded.record(&directory); - next = directory.parent_directory_offset; - child_start = start; - raw.push((directory, span)); + let (settings, external, external_access_policy) = options.into_parts(); + let mut opener = ArchiveOpener::with_settings(archive_len, settings)?; + while let Some(request) = opener.request() { + let response = Zeroizing::new(read_source( + &source, + request.offset(), + request.len(), + "directory", + )?); + opener.feed(request, &response)?; } - let chain = DecodedChain { - version, - terminal: Span::new(terminal_start, terminal_len)?, - raw, - remaining_block_references, - directory_hashes, - }; - let view = complete_open(archive_len, &mut settings, chain)?; + let view = opener.finish()?; for span in view.index.local_block_spans() { let mut marker = [0; 4]; source.read_exact_at(span.start(), &mut marker)?; @@ -729,28 +669,6 @@ pub(super) fn remaining_deserialization_limits( remaining } -fn terminal_span( - source: &S, - archive_len: u64, - limits: OpenLimits, -) -> Result<(u64, u64), PithosError> { - if archive_len < 12 { - return Err(PithosError::InvalidDirectoryRange { - operation: "read directory footer", - }); - } - let mut footer = [0; 12]; - source.read_exact_at(archive_len - 12, &mut footer)?; - let len = u64::from_be_bytes(footer[..8].try_into().expect("fixed footer length")); - validate_directory_len(len, limits)?; - let start = archive_len - .checked_sub(len) - .ok_or(PithosError::InvalidDirectoryRange { - operation: "validate terminal directory", - })?; - Ok((start, len)) -} - pub(super) fn validate_directory_len(len: u64, limits: OpenLimits) -> Result<(), PithosError> { if len < 25 { return Err(PithosError::DirectoryLengthMismatch { diff --git a/crates/pithos_lib/src/error.rs b/crates/pithos_lib/src/error.rs index cc541e3..98a00fa 100644 --- a/crates/pithos_lib/src/error.rs +++ b/crates/pithos_lib/src/error.rs @@ -221,4 +221,10 @@ pub enum PithosError { MissingManifestSource, #[error("append plan entry count does not match manifest")] AppendPlanLengthMismatch, + #[error("read response does not answer the outstanding request")] + UnexpectedReadResponse, + #[error("read response length mismatch: expected {expected}, got {actual}")] + ReadResponseLength { expected: u64, actual: u64 }, + #[error("archive open has requests left")] + OpenIncomplete, } diff --git a/crates/pithos_lib/tests/source_equivalence.rs b/crates/pithos_lib/tests/source_equivalence.rs index c5b6a46..9d2ffe8 100644 --- a/crates/pithos_lib/tests/source_equivalence.rs +++ b/crates/pithos_lib/tests/source_equivalence.rs @@ -164,10 +164,7 @@ fn fragmented_source_is_equivalent_and_source_errors_keep_context() { Ok(_) => panic!("truncated source must fail"), Err(error) => error, }; - assert!(matches!( - error, - PithosError::Source(SourceError::UnexpectedEof { offset: 0, .. }) - )); + assert!(matches!(error, PithosError::InvalidDirectoryRange { .. })); } #[test] From 8c712c1bbe539d105bbddc27dd954ab03327700b Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 21:34:19 +0200 Subject: [PATCH 079/166] feat: plan reads as lazy block tokens and decode them from one read --- crates/pithos_lib/src/archive/index.rs | 13 +- crates/pithos_lib/src/archive/mod.rs | 27 ++- crates/pithos_lib/src/archive/planning.rs | 208 ++++++++++++++++------ crates/pithos_lib/src/archive/reader.rs | 122 +++---------- crates/pithos_lib/src/archive/types.rs | 4 + crates/pithos_lib/src/archive/view.rs | 79 +++++++- 6 files changed, 282 insertions(+), 171 deletions(-) diff --git a/crates/pithos_lib/src/archive/index.rs b/crates/pithos_lib/src/archive/index.rs index 2175320..388ead7 100644 --- a/crates/pithos_lib/src/archive/index.rs +++ b/crates/pithos_lib/src/archive/index.rs @@ -1,7 +1,6 @@ -use crate::archive::planning::{ReadPlan, full_file_plan, range_plan}; use crate::archive::types::{ - ArchivePath, BlockDescriptor, BlockHash, ContentState, Entry, FileId, ReadRange, RelationId, - SegmentEntry, Span, ValidatedSegment, + ArchivePath, BlockDescriptor, BlockHash, ContentState, Entry, FileId, RelationId, SegmentEntry, + Span, ValidatedSegment, }; use crate::archive::validation::{IndexLimits, validate_aggregate, validate_entry}; use crate::error::PithosError; @@ -93,14 +92,6 @@ impl ArchiveIndex { ) { (self.entries, self.descriptors, self.maximum_id) } - - pub(crate) fn full_file_plan(&self, id: FileId) -> Result { - full_file_plan(self, id) - } - - pub(crate) fn range_plan(&self, id: FileId, range: ReadRange) -> Result { - range_plan(self, id, range) - } } /// Builds a new effective index. Inputs are only borrowed and are unchanged on every error. diff --git a/crates/pithos_lib/src/archive/mod.rs b/crates/pithos_lib/src/archive/mod.rs index 04593ed..71413f4 100644 --- a/crates/pithos_lib/src/archive/mod.rs +++ b/crates/pithos_lib/src/archive/mod.rs @@ -28,6 +28,7 @@ pub(crate) use path_validation::{ validate_directory_entries, validate_new_candidate, validate_symlink_target, }; pub use pieces::{Composition, Piece, PieceEncoder, compose}; +pub use planning::{BlockRequest, PlannedBlock, ReadPlan}; #[cfg(feature = "crypt4gh")] pub(crate) use reader::ContentOperationError; pub use reader::{ @@ -71,6 +72,7 @@ pub use writer::{ mod tests { use super::access::{AccessProvenance, ResolvedAccess}; use super::index::build_effective_index; + use super::planning::ReadPlan; use super::types::*; use super::validation::IndexLimits; use crate::error::PithosError; @@ -96,6 +98,13 @@ mod tests { } } + fn test_limits() -> crate::block::Limits { + crate::block::Limits { + max_stored_bytes: 1024, + max_decoded_bytes: 1024, + } + } + fn local_descriptor(span: Span) -> BlockDescriptor { BlockDescriptor { stored_size: span.len().saturating_sub(4), @@ -409,7 +418,12 @@ mod tests { Entry::File(_) )); assert!(matches!( - index.full_file_plan(FileId(1)), + ReadPlan::new( + &index, + FileId(1), + ReadRange::new(0..10, 10).unwrap(), + test_limits() + ), Err(crate::error::PithosError::ContentUnavailable) )); } @@ -430,10 +444,13 @@ mod tests { value.descriptors = vec![(first, descriptor(4)), (second, descriptor(6))]; let index = build_effective_index(&[value], 1_000, IndexLimits::default()).unwrap(); let range = ReadRange::new(3..7, 10).unwrap(); - let plan = index.range_plan(FileId(1), range).unwrap(); - assert_eq!(plan.blocks.len(), 2); - assert_eq!(plan.blocks[0].output, 3..4); - assert_eq!(plan.blocks[1].output, 0..3); + let plan = ReadPlan::new(&index, FileId(1), range, test_limits()) + .unwrap() + .collect::, _>>() + .unwrap(); + assert_eq!(plan.len(), 2); + assert_eq!(plan[0].output(), 3..4); + assert_eq!(plan[1].output(), 0..3); } #[test] diff --git a/crates/pithos_lib/src/archive/planning.rs b/crates/pithos_lib/src/archive/planning.rs index 1e9ea07..7b8c617 100644 --- a/crates/pithos_lib/src/archive/planning.rs +++ b/crates/pithos_lib/src/archive/planning.rs @@ -1,85 +1,175 @@ use crate::archive::index::ArchiveIndex; -use crate::archive::types::{BlockDescriptor, BlockHash, ContentState, Entry, FileId, ReadRange}; +use crate::archive::types::{ + BlockDescriptor, BlockHash, BlockLocation, ContentState, Entry, ExternalLocation, FileId, + ReadRange, +}; +use crate::block; use crate::error::PithosError; use std::ops::Range; -/// A block must be completely verified before its `output` slice is released. +/// One block of a planned read. +/// +/// The token keeps the file, block identity and descriptor needed to decode the block. A block +/// must be completely verified before its `output` slice is released. #[derive(Clone, Debug, Eq, PartialEq)] -pub(crate) struct PlannedBlock { +pub struct PlannedBlock { + pub(crate) file: FileId, pub(crate) hash: BlockHash, pub(crate) descriptor: BlockDescriptor, pub(crate) output: Range, } +/// The stored bytes of one planned block: `BLCK` followed by the payload. #[derive(Clone, Debug, Eq, PartialEq)] -pub(crate) struct ReadPlan { - pub(crate) file_size: u64, - pub(crate) blocks: Vec, +pub enum BlockRequest { + /// A span of this archive. + Local { offset: u64, len: u64 }, + /// An opaque location that only an external resolver can read. + External { + location: ExternalLocation, + len: u64, + }, } -pub(crate) fn full_file_plan(index: &ArchiveIndex, id: FileId) -> Result { - let content = content_entry(index, id)?; - let range = ReadRange::new(0..content.size, content.size)?; - range_plan(index, id, range) +impl PlannedBlock { + /// Where to fetch the block. One request covers the marker and the payload. + pub fn request(&self) -> BlockRequest { + let len = self.framed_len(); + match &self.descriptor.location { + BlockLocation::Local(span) => BlockRequest::Local { + offset: span.start(), + len, + }, + BlockLocation::External(location) => BlockRequest::External { + location: location.clone(), + len, + }, + } + } + + /// The part of the decoded block that belongs to the planned range. + pub fn output(&self) -> Range { + self.output.clone() + } + + /// The marker plus the payload. Plans check the stored size limit, so this cannot overflow. + pub(crate) fn framed_len(&self) -> u64 { + self.descriptor.stored_size.saturating_add(4) + } } -pub(crate) fn range_plan( - index: &ArchiveIndex, - id: FileId, +/// A lazy read plan: the blocks that intersect a range, in file order. +/// +/// File order can differ from storage order because blocks are shared, so offsets of +/// consecutive blocks need not increase. Each block is checked against the block limits before +/// it is yielded. The plan ends after its first error. +pub struct ReadPlan<'a> { + index: &'a ArchiveIndex, + file: FileId, + references: std::slice::Iter<'a, BlockHash>, + cursor: u64, range: ReadRange, -) -> Result { - let content = content_entry(index, id)?; - let ContentState::Available(references) = &content.content else { - return Err(PithosError::ContentUnavailable); - }; - let mut blocks = Vec::new(); - let mut cursor = 0u64; - for hash in references.iter() { - let descriptor = index + limits: block::Limits, +} + +impl<'a> ReadPlan<'a> { + pub(crate) fn new( + index: &'a ArchiveIndex, + file: FileId, + range: ReadRange, + limits: block::Limits, + ) -> Result { + let content = index + .entry(file) + .and_then(|entry| match &entry.entry { + Entry::File(content) | Entry::Metadata(content) => Some(content), + Entry::Directory(_) | Entry::Symlink { .. } => None, + }) + .ok_or_else(|| { + PithosError::InvalidBlockDataState("only data/metadata entries have content".into()) + })?; + let ContentState::Available(references) = &content.content else { + return Err(PithosError::ContentUnavailable); + }; + let references = if range.start() < range.end() { + references.as_slice() + } else { + &[] + }; + Ok(Self { + index, + file, + references: references.iter(), + cursor: 0, + range, + limits, + }) + } + + /// Plans the block at the cursor, or returns `None` when it lies before the range. + fn step(&mut self, hash: BlockHash) -> Result, PithosError> { + let descriptor = self + .index .descriptor(hash) .ok_or(PithosError::MissingBlockDescriptor)?; - let end = cursor.checked_add(descriptor.original_size).ok_or( - PithosError::AccessibleFileSizeMismatch { - expected: content.size, - actual: u64::MAX, + let start = self.cursor; + let end = start.checked_add(descriptor.original_size).ok_or( + PithosError::InvalidDirectoryRange { + operation: "sum block sizes", }, )?; - if end > range.start() && cursor < range.end() { - let output_start = range.start().saturating_sub(cursor); - let output_end = range.end().min(end) - cursor; - blocks.push(PlannedBlock { - hash, - descriptor: descriptor.clone(), - output: usize::try_from(output_start).map_err(|_| { - PithosError::InvalidDirectoryRange { - operation: "convert range index", - } - })?..usize::try_from(output_end).map_err(|_| { - PithosError::InvalidDirectoryRange { - operation: "convert range index", - } - })?, + self.cursor = end; + if start >= self.range.end() { + self.references = [].iter(); + return Ok(None); + } + if end <= self.range.start() { + return Ok(None); + } + if descriptor.stored_size > self.limits.max_stored_bytes { + return Err(PithosError::LimitExceeded { + field: "stored block", + limit: self.limits.max_stored_bytes, + actual: descriptor.stored_size, + }); + } + if descriptor.original_size > self.limits.max_decoded_bytes { + return Err(PithosError::LimitExceeded { + field: "decoded block", + limit: self.limits.max_decoded_bytes, + actual: descriptor.original_size, }); } - cursor = end; + let convert = |value: u64| { + usize::try_from(value).map_err(|_| PithosError::InvalidDirectoryRange { + operation: "convert range index", + }) + }; + let output_start = convert(self.range.start().saturating_sub(start))?; + let output_end = convert(self.range.end().min(end) - start)?; + Ok(Some(PlannedBlock { + file: self.file, + hash, + descriptor: descriptor.clone(), + output: output_start..output_end, + })) } - Ok(ReadPlan { - file_size: content.size, - blocks, - }) } -fn content_entry( - index: &ArchiveIndex, - id: FileId, -) -> Result<&crate::archive::types::ContentEntry, PithosError> { - index - .entry(id) - .and_then(|entry| match &entry.entry { - Entry::File(content) | Entry::Metadata(content) => Some(content), - Entry::Directory(_) | Entry::Symlink { .. } => None, - }) - .ok_or_else(|| { - PithosError::InvalidBlockDataState("only data/metadata entries have content".into()) - }) +impl Iterator for ReadPlan<'_> { + type Item = Result; + + fn next(&mut self) -> Option { + while let Some(&hash) = self.references.next() { + match self.step(hash) { + Ok(Some(block)) => return Some(Ok(block)), + Ok(None) => {} + Err(error) => { + self.references = [].iter(); + return Some(Err(error)); + } + } + } + None + } } diff --git a/crates/pithos_lib/src/archive/reader.rs b/crates/pithos_lib/src/archive/reader.rs index 5d2ac34..2d85bb6 100644 --- a/crates/pithos_lib/src/archive/reader.rs +++ b/crates/pithos_lib/src/archive/reader.rs @@ -1,17 +1,14 @@ use super::opener::{ArchiveOpener, OpenSettings}; +use super::planning::{BlockRequest, PlannedBlock, ReadPlan}; use super::{AccessProvenance, AppendSnapshot, ArchiveView, FileId, ResolvedAccess, Span}; use crate::archive::index::ArchiveIndex; use crate::archive::types::{ BlockHash, BlockLocation, ContentState, Entry, ExternalLocation, ReadRange, }; -use crate::block; #[cfg(feature = "crypt4gh")] use crate::crypto::FileKey; use crate::crypto::{self, PrivateKey}; use crate::error::PithosError; -use crate::format::block::{ - BlockIndexEntry, BlockLocation as FormatBlockLocation, ProcessingFlags, -}; use crate::format::directory::Directory; use crate::format::encryption::RecipientData; use crate::format::file_entry::{BlockDataEntry, BlockDataState}; @@ -355,9 +352,9 @@ where } pub fn copy_to(&self, path: &str, sink: &mut W) -> Result<(), PithosError> { - let (id, _) = self.view.content_id(path)?; + let (id, size) = self.view.content_id(path)?; self.require_content_available(id)?; - self.copy_plan(id, self.view.index.full_file_plan(id)?, sink) + self.copy_plan(self.view.plan(id, ReadRange::new(0..size, size)?)?, sink) } pub fn copy_range_to( @@ -366,10 +363,7 @@ where range: Range, sink: &mut W, ) -> Result<(), PithosError> { - let (id, size) = self.view.content_id(path)?; - let range = ReadRange::new(range, size)?; - self.require_content_available(id)?; - self.copy_plan(id, self.view.index.range_plan(id, range)?, sink) + self.copy_plan(self.view.plan_range(path, range)?, sink) } #[cfg(feature = "crypt4gh")] @@ -424,14 +418,16 @@ where ) -> Result<(), ContentOperationError> { self.require_content_available(id) .map_err(ContentOperationError::Core)?; - let plan = self + let size = self .view - .index - .full_file_plan(id) + .content_size(id) + .map_err(ContentOperationError::Core)?; + let plan = ReadRange::new(0..size, size) + .and_then(|range| self.view.plan(id, range)) .map_err(ContentOperationError::Core)?; - for block in plan.blocks { - let plaintext = self - .verified_block(id, &block) + for block in plan { + let plaintext = block + .and_then(|block| self.verified_block(&block)) .map_err(ContentOperationError::Core)?; operation(plaintext).map_err(ContentOperationError::Callback)?; } @@ -444,69 +440,28 @@ where fn copy_plan( &self, - id: FileId, - plan: super::planning::ReadPlan, + plan: ReadPlan<'_>, sink: &mut W, ) -> Result<(), PithosError> { - for block in plan.blocks { - let plaintext = self.verified_block(id, &block)?; - sink.write_all(&plaintext[block.output])?; + for block in plan { + let block = block?; + let plaintext = self.verified_block(&block)?; + sink.write_all(&plaintext[block.output()])?; } Ok(()) } - fn verified_block( - &self, - id: FileId, - planned: &super::planning::PlannedBlock, - ) -> Result>, PithosError> { - if planned.descriptor.stored_size > self.view.limits.max_stored_block_bytes { - return Err(PithosError::LimitExceeded { - field: "stored block", - limit: self.view.limits.max_stored_block_bytes, - actual: planned.descriptor.stored_size, - }); - } - if planned.descriptor.original_size > self.view.limits.max_decoded_block_bytes { - return Err(PithosError::LimitExceeded { - field: "decoded block", - limit: self.view.limits.max_decoded_block_bytes, - actual: planned.descriptor.original_size, - }); - } - let stored = match &planned.descriptor.location { - BlockLocation::Local(span) => { - let mut marker = [0; 4]; - self.source.read_exact_at(span.start(), &mut marker)?; - crate::format::block::decode_block_marker(&mut marker.as_slice())?; - let payload_start = - span.start() - .checked_add(4) - .ok_or(PithosError::InvalidDirectoryRange { - operation: "read block payload", - })?; - read_source( - &self.source, - payload_start, - planned.descriptor.stored_size, - "block", - )? - } - BlockLocation::External(location) => { - let expected_len = - planned - .descriptor - .stored_size - .checked_add(4) - .ok_or_else(|| { - PithosError::ExternalBlockFraming("response size overflow".into()) - })?; + /// Fetches one planned block with a single read and verifies it. + fn verified_block(&self, block: &PlannedBlock) -> Result>, PithosError> { + let stored = match block.request() { + BlockRequest::Local { offset, len } => read_source(&self.source, offset, len, "block")?, + BlockRequest::External { location, len } => { let response = self.external.resolve( self.external_access_policy.as_deref().ok_or( PithosError::UnsupportedFeature(ArchiveFeature::ExternalStorage), )?, - location, - expected_len, + &location, + len, self.view .limits .max_stored_block_bytes @@ -515,38 +470,15 @@ where PithosError::ExternalBlockFraming("response policy overflow".into()) })?, )?; - if response.len() as u64 != expected_len { + if response.len() as u64 != len { return Err(PithosError::ExternalBlockFraming( "response does not match expected size".into(), )); } - let (mut marker, stored) = response.split_at(4); - crate::format::block::decode_block_marker(&mut marker)?; - stored.to_vec() + response } }; - let meta = BlockIndexEntry { - offset: 0, - stored_size: planned.descriptor.stored_size, - original_size: planned.descriptor.original_size, - flags: ProcessingFlags::from_byte(planned.descriptor.processing.to_byte()), - location: FormatBlockLocation::Local, - }; - let key = self - .view - .access - .block_key(id, planned.hash) - .ok_or(PithosError::ContentUnavailable)?; - block::verify( - stored, - key, - planned.hash.0, - &meta, - block::Limits { - max_stored_bytes: self.view.limits.max_stored_block_bytes, - max_decoded_bytes: self.view.limits.max_decoded_block_bytes, - }, - ) + self.view.decode_block(block, &Zeroizing::new(stored)) } } diff --git a/crates/pithos_lib/src/archive/types.rs b/crates/pithos_lib/src/archive/types.rs index 2a1fb00..aad6d9e 100644 --- a/crates/pithos_lib/src/archive/types.rs +++ b/crates/pithos_lib/src/archive/types.rs @@ -263,6 +263,10 @@ impl BlockReferences { pub(crate) fn iter(&self) -> impl ExactSizeIterator + '_ { self.0.iter().copied() } + + pub(crate) fn as_slice(&self) -> &[BlockHash] { + &self.0 + } } #[derive(Clone, Debug, Eq, PartialEq)] diff --git a/crates/pithos_lib/src/archive/view.rs b/crates/pithos_lib/src/archive/view.rs index 3f38c57..8499f48 100644 --- a/crates/pithos_lib/src/archive/view.rs +++ b/crates/pithos_lib/src/archive/view.rs @@ -1,14 +1,19 @@ use super::access::ResolvedAccess; use super::index::ArchiveIndex; +use super::planning::{PlannedBlock, ReadPlan}; #[cfg(feature = "crypt4gh")] use super::reader::AccessKeys; use super::reader::{ArchiveEntry, ContentAvailability, OpenLimits, archive_entry}; use super::snapshot::AppendSnapshot; -use super::types::{ArchivePath, FileId, Span, ValidatedSegment}; +use super::types::{ArchivePath, FileId, ReadRange, Span, ValidatedSegment}; use super::validation::IndexLimits; +use crate::block; use crate::error::PithosError; +use crate::format::block::{BlockIndexEntry, BlockLocation, ProcessingFlags}; use crate::format::header::FormatVersion; use std::collections::BTreeMap; +use std::ops::Range; +use zeroize::Zeroizing; /// Validated archive metadata without a byte source. /// @@ -55,6 +60,67 @@ impl ArchiveView { .map(|entry| archive_entry(&self.index, entry, &self.content_availability))) } + /// Plans a read of `range` within the content of `path`. + /// + /// Fails before any block is planned when the content is unavailable or needs an + /// unsupported feature. + pub fn plan_range(&self, path: &str, range: Range) -> Result, PithosError> { + let (id, size) = self.content_id(path)?; + let range = ReadRange::new(range, size)?; + self.require_content_available(id)?; + self.plan(id, range) + } + + pub(crate) fn plan(&self, id: FileId, range: ReadRange) -> Result, PithosError> { + ReadPlan::new(&self.index, id, range, self.block_limits()) + } + + /// Decodes one planned block from its stored bytes, `BLCK` followed by the payload. + /// + /// Checks the marker, the sizes, decryption, decompression and the block identity, and + /// returns the whole verified block. This is CPU work without I/O, so callers may run it + /// on a blocking pool. + pub fn decode_block( + &self, + block: &PlannedBlock, + stored: &[u8], + ) -> Result>, PithosError> { + let expected = block.framed_len(); + if stored.len() as u64 != expected { + return Err(PithosError::BlockSizeMismatch { + expected, + actual: stored.len() as u64, + }); + } + let (mut marker, payload) = stored.split_at(4); + crate::format::block::decode_block_marker(&mut marker)?; + let key = self + .access + .block_key(block.file, block.hash) + .ok_or(PithosError::ContentUnavailable)?; + let meta = BlockIndexEntry { + offset: 0, + stored_size: block.descriptor.stored_size, + original_size: block.descriptor.original_size, + flags: ProcessingFlags::from_byte(block.descriptor.processing.to_byte()), + location: BlockLocation::Local, + }; + block::verify( + Zeroizing::new(payload.to_vec()), + key, + block.hash.0, + &meta, + self.block_limits(), + ) + } + + pub(crate) fn block_limits(&self) -> block::Limits { + block::Limits { + max_stored_bytes: self.limits.max_stored_block_bytes, + max_decoded_bytes: self.limits.max_decoded_block_bytes, + } + } + /// Transfers the validated state to append and grant planning. pub(crate) fn into_append_snapshot(self) -> AppendSnapshot { let Self { @@ -90,6 +156,17 @@ impl ArchiveView { Ok((entry.id, content.size)) } + #[cfg(feature = "crypt4gh")] + pub(crate) fn content_size(&self, id: FileId) -> Result { + self.index + .entry(id) + .and_then(|entry| entry.entry.content()) + .map(|content| content.size) + .ok_or_else(|| { + PithosError::InvalidBlockDataState("only data/metadata entries have content".into()) + }) + } + pub(crate) fn require_content_available(&self, id: FileId) -> Result<(), PithosError> { match self.content_availability.get(&id).copied() { Some(ContentAvailability::Available) => Ok(()), From 4247cdb2960b9943ea701d5e25f66e241587f35a Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 21:40:30 +0200 Subject: [PATCH 080/166] feat: check block markers when blocks are read instead of at open --- crates/pithos_lib/src/archive/index.rs | 6 ---- crates/pithos_lib/src/archive/reader.rs | 5 --- .../src/archive/reader_private_tests.rs | 31 ++++++++----------- crates/pithos_lib/tests/block_pipeline.rs | 18 ++++++----- spec/PITHOS_1.0.0_draft.md | 13 ++++++-- 5 files changed, 35 insertions(+), 38 deletions(-) diff --git a/crates/pithos_lib/src/archive/index.rs b/crates/pithos_lib/src/archive/index.rs index 388ead7..b18cc86 100644 --- a/crates/pithos_lib/src/archive/index.rs +++ b/crates/pithos_lib/src/archive/index.rs @@ -26,7 +26,6 @@ pub(crate) struct ArchiveIndex { descriptors: IndexMap, relationships: BTreeMap>, segment_spans: Vec, - local_spans: Vec, maximum_id: Option, } @@ -68,10 +67,6 @@ impl ArchiveIndex { self.relationships.get(&id).map(Arc::as_ref) } - pub(crate) fn local_block_spans(&self) -> impl Iterator + '_ { - self.local_spans.iter().copied() - } - #[cfg(test)] pub(crate) fn maximum_id(&self) -> Option { self.maximum_id @@ -207,7 +202,6 @@ pub(crate) fn build_effective_index( descriptors, relationships, segment_spans, - local_spans, maximum_id, }) } diff --git a/crates/pithos_lib/src/archive/reader.rs b/crates/pithos_lib/src/archive/reader.rs index 2d85bb6..7382cc2 100644 --- a/crates/pithos_lib/src/archive/reader.rs +++ b/crates/pithos_lib/src/archive/reader.rs @@ -314,11 +314,6 @@ where opener.feed(request, &response)?; } let view = opener.finish()?; - for span in view.index.local_block_spans() { - let mut marker = [0; 4]; - source.read_exact_at(span.start(), &mut marker)?; - crate::format::block::decode_block_marker(&mut marker.as_slice())?; - } Ok(Self { source, external, diff --git a/crates/pithos_lib/src/archive/reader_private_tests.rs b/crates/pithos_lib/src/archive/reader_private_tests.rs index eff817e..db8afbb 100644 --- a/crates/pithos_lib/src/archive/reader_private_tests.rs +++ b/crates/pithos_lib/src/archive/reader_private_tests.rs @@ -1120,7 +1120,7 @@ fn decrypted_encrypted_block_list_accepts_a_non_minimal_count() { } #[test] -fn archive_open_validates_local_markers_without_reading_payloads_or_access_keys() { +fn archive_open_reads_no_block_bytes() { let (_temporary, path) = fixture_with("marker validation payload", 0); let (offset, stored_size) = first_block(&path); let bytes = Arc::<[u8]>::from(std::fs::read(&path).unwrap()); @@ -1130,27 +1130,21 @@ fn archive_open_validates_local_markers_without_reading_payloads_or_access_keys( bytes, reads: Arc::clone(&reads), }, - OpenOptions::default(), + OpenOptions::default().with_access_keys(AccessKeys::new().with_key(private("recipient1"))), ) .unwrap(); - let reads = reads.lock().unwrap(); - assert!( - reads.contains(&(offset, 4)), - "local marker was not read at open" - ); - let payload_start = offset + 4; - let payload_end = payload_start + stored_size; + let block_end = offset + 4 + stored_size; assert!( - reads.iter().all(|(start, len)| { + reads.lock().unwrap().iter().all(|(start, len)| { let end = start.saturating_add(*len as u64); - end <= payload_start || *start >= payload_end + end <= offset || *start >= block_end }), - "archive open read local payload bytes" + "archive open read block bytes" ); } #[test] -fn archive_open_rejects_missing_or_changed_local_block_markers_without_access_keys() { +fn missing_or_changed_local_block_markers_fail_when_the_block_is_read() { for missing in [false, true] { let (_temporary, path) = fixture_with("invalid marker payload", 0); if missing { @@ -1166,11 +1160,12 @@ fn archive_open_rejects_missing_or_changed_local_block_markers_without_access_ke std::fs::write(&path, bytes).unwrap(); } - let error = match open_without_keys(&path) { - Ok(_) => panic!("archive with an invalid local block marker opened"), - Err(error) => error, - }; - assert!(error.to_string().contains("block marker")); + open_without_keys(&path).unwrap(); + let archive = open_path(&path, AccessKeys::new().with_key(private("recipient1"))); + let mut sink = RecordingSink(Vec::new()); + let error = archive.copy_to("data", &mut sink).unwrap_err(); + assert!(error.to_string().contains("block marker"), "{error}"); + assert!(sink.0.is_empty()); } } diff --git a/crates/pithos_lib/tests/block_pipeline.rs b/crates/pithos_lib/tests/block_pipeline.rs index 79e2bf8..382829e 100644 --- a/crates/pithos_lib/tests/block_pipeline.rs +++ b/crates/pithos_lib/tests/block_pipeline.rs @@ -1,11 +1,11 @@ mod common; -use common::util::fixture; -use pithos_lib::archive::{Archive, OpenOptions}; +use common::util::{fixture, private_key}; +use pithos_lib::archive::{AccessKeys, Archive, OpenOptions}; use pithos_lib::source::FileSource; #[test] -fn public_archive_rejects_a_bad_block_marker_during_keyless_open() { +fn public_archive_rejects_a_bad_block_marker_when_the_block_is_read() { let temporary = tempfile::tempdir().unwrap(); let path = fixture(&temporary, "recipient1"); let mut bytes = std::fs::read(&path).unwrap(); @@ -15,8 +15,12 @@ fn public_archive_rejects_a_bad_block_marker_during_keyless_open() { .unwrap(); bytes[marker] ^= 1; std::fs::write(&path, bytes).unwrap(); - assert!( - Archive::open(FileSource::open(&path).unwrap(), OpenOptions::default()).is_err(), - "bad local marker was accepted during keyless open" - ); + let keys = AccessKeys::new().with_key(private_key("recipient1")); + let archive = Archive::open( + FileSource::open(&path).unwrap(), + OpenOptions::default().with_access_keys(keys), + ) + .expect("opening validates metadata only"); + let error = archive.copy_to("data", &mut Vec::new()).unwrap_err(); + assert!(error.to_string().contains("block marker"), "{error}"); } diff --git a/spec/PITHOS_1.0.0_draft.md b/spec/PITHOS_1.0.0_draft.md index f53c568..6a5eb2c 100644 --- a/spec/PITHOS_1.0.0_draft.md +++ b/spec/PITHOS_1.0.0_draft.md @@ -331,6 +331,11 @@ search payload bytes for `BLCK`. Recovery tools MAY treat `BLCK` as an untrusted candidate only: the marker alone provides no length, flags, or identity. +Opening an archive validates its metadata and reads no block bytes. A reader +MUST check a local block's `BLCK` marker when it reads that block, before it +uses the payload, and MUST reject the block if the marker is wrong. A reader +SHOULD fetch the marker and the payload in one read of `4 + stored_size` bytes. + #### 4.2.6 External Block Resolution External resolution is an optional capability and MUST remain disabled until a @@ -965,10 +970,14 @@ Zstandard versions. ### 6.1 Reading Operations 1. Read and validate file header -2. Locate and validate the terminal Directory using the direct lookup in Section 4.3.1 +2. Locate and validate the terminal Directory using the direct lookup in Section 4.3.1, then each parent Directory of the selected chain 3. Validate directory ordering 4. Build the effective block-descriptor mapping -5. Extract files by reading referenced blocks +5. Extract files by reading referenced blocks, checking each local block's marker as it is read (Section 4.2.5) + +Steps 1 to 4 open the archive. They validate metadata only and read no block +bytes, so the number of reads at open depends on the length of the chain, not on +the number of blocks. ### 6.2 Writing Operations From 3abd2c2e1ca57b1bb60884d690b4ab587e796d07 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 21:47:11 +0200 Subject: [PATCH 081/166] feat: coalesce contiguous local block reads into bounded batches --- crates/pithos_lib/src/archive/mod.rs | 192 +++++++++++++++++++++- crates/pithos_lib/src/archive/planning.rs | 99 +++++++++++ 2 files changed, 288 insertions(+), 3 deletions(-) diff --git a/crates/pithos_lib/src/archive/mod.rs b/crates/pithos_lib/src/archive/mod.rs index 71413f4..41e4833 100644 --- a/crates/pithos_lib/src/archive/mod.rs +++ b/crates/pithos_lib/src/archive/mod.rs @@ -28,7 +28,7 @@ pub(crate) use path_validation::{ validate_directory_entries, validate_new_candidate, validate_symlink_target, }; pub use pieces::{Composition, Piece, PieceEncoder, compose}; -pub use planning::{BlockRequest, PlannedBlock, ReadPlan}; +pub use planning::{BlockBatch, BlockBatches, BlockRequest, PlannedBlock, ReadPlan}; #[cfg(feature = "crypt4gh")] pub(crate) use reader::ContentOperationError; pub use reader::{ @@ -71,8 +71,8 @@ pub use writer::{ #[cfg(test)] mod tests { use super::access::{AccessProvenance, ResolvedAccess}; - use super::index::build_effective_index; - use super::planning::ReadPlan; + use super::index::{self, build_effective_index}; + use super::planning::{BlockBatch, BlockRequest, PlannedBlock, ReadPlan}; use super::types::*; use super::validation::IndexLimits; use crate::error::PithosError; @@ -453,6 +453,192 @@ mod tests { assert_eq!(plan[1].output(), 0..3); } + /// One file over `count` contiguous local blocks of one plaintext byte, read in `order`. + fn local_file(count: usize, order: &[usize]) -> index::ArchiveIndex { + let hash = |block: usize| { + let mut bytes = [0; 32]; + bytes[..8].copy_from_slice(&(block as u64).to_be_bytes()); + BlockHash(bytes) + }; + let start = 6 + 5 * count as u64; + let mut value = segment_at( + start, + None, + vec![SegmentEntry { + id: FileId(1), + path: ArchivePath::new("data").unwrap(), + entry: Entry::File(ContentEntry { + metadata: metadata(), + size: order.len() as u64, + content: ContentState::Available(BlockReferences::new( + order.iter().map(|block| hash(*block)).collect(), + )), + }), + }], + ); + value.descriptors = (0..count) + .map(|block| { + let span = Span::new(6 + 5 * block as u64, 5).unwrap(); + (hash(block), local_descriptor(span)) + }) + .collect(); + build_effective_index(&[value], start + 10, IndexLimits::default()).unwrap() + } + + fn local_offsets(blocks: &[PlannedBlock]) -> Vec { + blocks + .iter() + .map(|block| match block.request() { + BlockRequest::Local { offset, len: 5 } => offset, + request => panic!("unexpected request {request:?}"), + }) + .collect() + } + + #[test] + fn lazy_plans_yield_only_the_blocks_of_a_range_over_many_blocks() { + let count = 10_000; + let index = local_file(count, &(0..count).collect::>()); + let size = count as u64; + let mut whole = ReadPlan::new( + &index, + FileId(1), + ReadRange::new(0..size, size).unwrap(), + test_limits(), + ) + .unwrap(); + let first = whole.next().unwrap().unwrap(); + assert_eq!(local_offsets(&[first]), [6]); + assert_eq!(whole.count(), count - 1); + + let range = ReadRange::new(4_000..4_003, size).unwrap(); + let blocks = ReadPlan::new(&index, FileId(1), range, test_limits()) + .unwrap() + .collect::, _>>() + .unwrap(); + assert_eq!(local_offsets(&blocks), [20_006, 20_011, 20_016]); + assert!(blocks.iter().all(|block| block.output() == (0..1))); + let empty = ReadRange::new(5..5, size).unwrap(); + assert_eq!( + ReadPlan::new(&index, FileId(1), empty, test_limits()) + .unwrap() + .count(), + 0 + ); + } + + #[test] + fn plans_check_block_limits_before_yielding_a_block() { + let index = local_file(3, &[0, 1, 2]); + let limits = crate::block::Limits { + max_stored_bytes: 0, + max_decoded_bytes: 1024, + }; + let range = ReadRange::new(1..3, 3).unwrap(); + let mut plan = ReadPlan::new(&index, FileId(1), range, limits).unwrap(); + assert!(matches!( + plan.next(), + Some(Err(PithosError::LimitExceeded { + field: "stored block", + .. + })) + )); + assert!(plan.next().is_none()); + } + + #[test] + fn batches_join_contiguous_local_spans_up_to_the_byte_bound() { + let index = local_file(5, &[0, 1, 2, 3, 4]); + let range = ReadRange::new(0..5, 5).unwrap(); + let plan = ReadPlan::new(&index, FileId(1), range, test_limits()).unwrap(); + let batches = plan.batches(12).collect::, _>>().unwrap(); + assert_eq!( + batches.iter().map(BlockBatch::request).collect::>(), + [ + BlockRequest::Local { offset: 6, len: 10 }, + BlockRequest::Local { + offset: 16, + len: 10 + }, + BlockRequest::Local { offset: 26, len: 5 }, + ] + ); + let response = (0..10).collect::>(); + let parts = batches[0].split(&response).unwrap().collect::>(); + assert_eq!(parts[0].1, &response[..5]); + assert_eq!(parts[1].1, &response[5..]); + assert_eq!(parts[1].0, &batches[0].blocks()[1]); + assert!(matches!( + batches[0].split(&response[1..]), + Err(PithosError::BlockSizeMismatch { + expected: 10, + actual: 9 + }) + )); + let single = ReadPlan::new(&index, FileId(1), range, test_limits()).unwrap(); + assert_eq!(single.batches(4).count(), 5); + } + + #[test] + fn shared_blocks_keep_file_order_and_only_join_contiguous_spans() { + let index = local_file(3, &[2, 0, 1, 2, 2]); + let range = ReadRange::new(0..5, 5).unwrap(); + let blocks = ReadPlan::new(&index, FileId(1), range, test_limits()) + .unwrap() + .collect::, _>>() + .unwrap(); + assert_eq!(local_offsets(&blocks), [16, 6, 11, 16, 16]); + let plan = ReadPlan::new(&index, FileId(1), range, test_limits()).unwrap(); + let batches = plan + .batches(u64::MAX) + .map(|batch| batch.unwrap().request()) + .collect::>(); + assert_eq!( + batches, + [ + BlockRequest::Local { offset: 16, len: 5 }, + BlockRequest::Local { offset: 6, len: 15 }, + BlockRequest::Local { offset: 16, len: 5 }, + ] + ); + } + + #[test] + fn external_blocks_are_a_distinct_request_kind_and_never_join_a_batch() { + let first = BlockHash([1; 32]); + let second = BlockHash([2; 32]); + let mut value = segment(vec![SegmentEntry { + id: FileId(1), + path: ArchivePath::new("data").unwrap(), + entry: Entry::File(ContentEntry { + metadata: metadata(), + size: 10, + content: ContentState::Available(BlockReferences::new(vec![first, second])), + }), + }]); + value.descriptors = vec![(first, descriptor(4)), (second, descriptor(6))]; + let index = build_effective_index(&[value], 1_000, IndexLimits::default()).unwrap(); + let range = ReadRange::new(0..10, 10).unwrap(); + let plan = ReadPlan::new(&index, FileId(1), range, test_limits()).unwrap(); + let batches = plan + .batches(u64::MAX) + .map(|batch| batch.unwrap().request()) + .collect::>(); + assert_eq!( + batches, + [ + BlockRequest::External { + location: ExternalLocation::new("opaque"), + len: 8 + }, + BlockRequest::External { + location: ExternalLocation::new("opaque"), + len: 10 + }, + ] + ); + } + #[test] fn resolved_access_rejects_conflicts_and_keeps_deterministic_provenance() { let mut access = ResolvedAccess::new(); diff --git a/crates/pithos_lib/src/archive/planning.rs b/crates/pithos_lib/src/archive/planning.rs index 7b8c617..e6e347e 100644 --- a/crates/pithos_lib/src/archive/planning.rs +++ b/crates/pithos_lib/src/archive/planning.rs @@ -52,6 +52,13 @@ impl PlannedBlock { self.output.clone() } + fn local_span(&self) -> Option<(u64, u64)> { + match &self.descriptor.location { + BlockLocation::Local(span) => Some((span.start(), self.framed_len())), + BlockLocation::External(_) => None, + } + } + /// The marker plus the payload. Plans check the stored size limit, so this cannot overflow. pub(crate) fn framed_len(&self) -> u64 { self.descriptor.stored_size.saturating_add(4) @@ -173,3 +180,95 @@ impl Iterator for ReadPlan<'_> { None } } + +impl<'a> ReadPlan<'a> { + /// Groups consecutive local blocks whose spans are contiguous in storage into one request + /// of at most `max_bytes`. A larger block and every external block form their own batch. + pub fn batches(self, max_bytes: u64) -> BlockBatches<'a> { + BlockBatches { + plan: self, + max_bytes, + pending: None, + } + } +} + +/// Consecutive planned blocks that one request can fetch. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct BlockBatch { + blocks: Vec, +} + +impl BlockBatch { + /// One request covering every block of the batch. + pub fn request(&self) -> BlockRequest { + let mut request = self.blocks[0].request(); + if let BlockRequest::Local { len, .. } = &mut request { + *len = self.blocks.iter().map(PlannedBlock::framed_len).sum(); + } + request + } + + pub fn blocks(&self) -> &[PlannedBlock] { + &self.blocks + } + + /// Pairs each block with its stored bytes from a response to [`BlockBatch::request`]. + pub fn split<'b>( + &'b self, + response: &'b [u8], + ) -> Result, PithosError> { + let expected = self.blocks.iter().map(PlannedBlock::framed_len).sum(); + if response.len() as u64 != expected { + return Err(PithosError::BlockSizeMismatch { + expected, + actual: response.len() as u64, + }); + } + let mut rest = response; + Ok(self.blocks.iter().map(move |block| { + let (stored, tail) = rest.split_at(block.framed_len() as usize); + rest = tail; + (block, stored) + })) + } +} + +/// The batches of a read plan, in file order. See [`ReadPlan::batches`]. +pub struct BlockBatches<'a> { + plan: ReadPlan<'a>, + max_bytes: u64, + pending: Option>, +} + +impl Iterator for BlockBatches<'_> { + type Item = Result; + + fn next(&mut self) -> Option { + let first = match self.pending.take().or_else(|| self.plan.next())? { + Ok(block) => block, + Err(error) => return Some(Err(error)), + }; + let Some((offset, len)) = first.local_span() else { + return Some(Ok(BlockBatch { + blocks: vec![first], + })); + }; + let (mut end, mut total) = (offset + len, len); + let mut blocks = vec![first]; + for next in self.plan.by_ref() { + match next.as_ref().map(PlannedBlock::local_span) { + Ok(Some((offset, len))) if offset == end && total + len <= self.max_bytes => { + end += len; + total += len; + blocks.extend(next.ok()); + } + _ => { + self.pending = Some(next); + break; + } + } + } + Some(Ok(BlockBatch { blocks })) + } +} From a4085a7bbc257f795d8bf56930d5c4cc55e75989 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 21:56:48 +0200 Subject: [PATCH 082/166] test: cover the archive opener request sequence, feeds, limits and read counts --- .../src/archive/reader_private_tests.rs | 31 ++ crates/pithos_lib/tests/opener.rs | 276 ++++++++++++++++++ 2 files changed, 307 insertions(+) create mode 100644 crates/pithos_lib/tests/opener.rs diff --git a/crates/pithos_lib/src/archive/reader_private_tests.rs b/crates/pithos_lib/src/archive/reader_private_tests.rs index db8afbb..1593996 100644 --- a/crates/pithos_lib/src/archive/reader_private_tests.rs +++ b/crates/pithos_lib/src/archive/reader_private_tests.rs @@ -1169,6 +1169,37 @@ fn missing_or_changed_local_block_markers_fail_when_the_block_is_read() { } } +#[test] +fn an_expected_digest_mismatch_fails_before_any_metadata_is_decrypted() { + let (_temporary, path) = fixture(); + rewrite_terminal_directory(&path, |directory| { + for section in directory.encryption.values_mut() { + for recipient in section.recipients.values_mut() { + let RecipientData::Encrypted(bytes) = &mut recipient.recipient_data else { + panic!("fixture grants are encrypted"); + }; + *bytes.last_mut().unwrap() ^= 1; + } + } + }); + let digest = open_without_keys(&path).unwrap().metadata_digest(); + let open = |digest| { + Archive::open( + MemorySource::new(Arc::<[u8]>::from(std::fs::read(&path).unwrap())), + OpenOptions::default() + .with_access_keys(AccessKeys::new().with_key(private("recipient1"))) + .with_expected_metadata_digest(digest), + ) + }; + assert!(matches!(open(digest), Err(PithosError::Crypt(_)))); + let mut wrong = digest; + wrong[0] ^= 1; + assert!(matches!( + open(wrong), + Err(PithosError::MetadataDigestMismatch) + )); +} + #[test] fn encrypted_local_descriptor_stored_size_has_a_28_byte_minimum() { for stored_size in [0, 27] { diff --git a/crates/pithos_lib/tests/opener.rs b/crates/pithos_lib/tests/opener.rs new file mode 100644 index 0000000..2b16521 --- /dev/null +++ b/crates/pithos_lib/tests/opener.rs @@ -0,0 +1,276 @@ +mod common; + +use common::append::append_fixture; +use common::util::{fixture, private_key}; +use pithos_lib::archive::{ + AccessKeys, Archive, ArchiveOpener, ArchivePath, ArchiveView, ArchiveWriter, BlockRequest, + Chunking, EntryMetadata, OpenLimits, OpenOptions, ProcessingOptions, WriteOptions, +}; +use pithos_lib::error::PithosError; +use pithos_lib::source::{ArchiveSource, MemorySource, SourceError}; +use std::sync::{Arc, Mutex}; + +fn recipient() -> OpenOptions { + OpenOptions::default().with_access_keys(AccessKeys::new().with_key(private_key("recipient1"))) +} + +/// Drives an opener over `bytes` and records every request. +fn drive( + bytes: &[u8], + options: OpenOptions, +) -> (Result, Vec<(u64, u64)>) { + let mut requests = Vec::new(); + let result = ArchiveOpener::new(bytes.len() as u64, options).and_then(|mut opener| { + while let Some(request) = opener.request() { + requests.push((request.offset(), request.len())); + let start = request.offset() as usize; + opener.feed(request, &bytes[start..start + request.len() as usize])?; + } + opener.finish() + }); + (result, requests) +} + +fn terminal_len(bytes: &[u8]) -> u64 { + u64::from_be_bytes(bytes[bytes.len() - 12..bytes.len() - 4].try_into().unwrap()) +} + +#[test] +fn a_single_segment_archive_requests_header_footer_and_directory() { + let temporary = tempfile::tempdir().unwrap(); + let bytes = std::fs::read(fixture(&temporary, "recipient1")).unwrap(); + let len = bytes.len() as u64; + let (view, requests) = drive(&bytes, recipient()); + let directory = terminal_len(&bytes); + assert_eq!( + requests, + [(0, 6), (len - 12, 12), (len - directory, directory)] + ); + let view = view.unwrap(); + assert_eq!(view.version(), 0x0101); + assert_eq!(view.entries().len(), 1); + assert!(view.entry("data").unwrap().is_some()); +} + +#[test] +fn an_appended_archive_requests_each_directory_from_terminal_to_base() { + let temporary = tempfile::tempdir().unwrap(); + let bytes = std::fs::read(append_fixture(&temporary).archive).unwrap(); + let len = bytes.len() as u64; + let (view, requests) = drive(&bytes, recipient()); + let terminal = terminal_len(&bytes); + assert_eq!(requests.len(), 4); + assert_eq!( + &requests[..3], + [(0, 6), (len - 12, 12), (len - terminal, terminal)] + ); + let (parent_start, parent_len) = requests[3]; + assert!(parent_start + parent_len <= len - terminal); + let view = view.unwrap(); + assert_eq!(view.entries().len(), 5); + let archive = Archive::open(MemorySource::new(bytes), recipient()).unwrap(); + assert_eq!(view.metadata_digest(), archive.metadata_digest()); +} + +#[test] +fn feed_rejects_wrong_lengths_unknown_and_repeated_responses() { + let temporary = tempfile::tempdir().unwrap(); + let bytes = std::fs::read(fixture(&temporary, "recipient1")).unwrap(); + let len = bytes.len() as u64; + let mut opener = ArchiveOpener::new(len, recipient()).unwrap(); + let header = opener.request().unwrap(); + assert!(matches!( + opener.feed(header, &bytes[..5]), + Err(PithosError::ReadResponseLength { + expected: 6, + actual: 5 + }) + )); + assert!(matches!( + opener.feed(header, &bytes[..7]), + Err(PithosError::ReadResponseLength { .. }) + )); + assert_eq!(opener.request(), Some(header)); + opener.feed(header, &bytes[..6]).unwrap(); + assert!(matches!( + opener.feed(header, &bytes[..6]), + Err(PithosError::UnexpectedReadResponse) + )); + + let mut other = ArchiveOpener::new(len + 1, OpenOptions::default()).unwrap(); + let other_header = other.request().unwrap(); + other.feed(other_header, &bytes[..6]).unwrap(); + let foreign = other.request().unwrap(); + assert!(matches!( + opener.feed(foreign, &bytes[..12]), + Err(PithosError::UnexpectedReadResponse) + )); + let footer = opener.request().unwrap(); + assert_eq!((footer.offset(), footer.len()), (len - 12, 12)); + let unfinished = ArchiveOpener::new(len, OpenOptions::default()).unwrap(); + assert!(matches!( + unfinished.finish(), + Err(PithosError::OpenIncomplete) + )); + + opener.feed(footer, &bytes[bytes.len() - 12..]).unwrap(); + let directory = opener.request().unwrap(); + let start = directory.offset() as usize; + opener.feed(directory, &bytes[start..]).unwrap(); + assert_eq!(opener.request(), None); + assert!(matches!( + opener.feed(directory, &bytes[start..]), + Err(PithosError::UnexpectedReadResponse) + )); + assert_eq!(opener.finish().unwrap().entries().len(), 1); +} + +#[test] +fn limits_are_checked_before_a_directory_is_requested() { + let temporary = tempfile::tempdir().unwrap(); + let bytes = std::fs::read(append_fixture(&temporary).archive).unwrap(); + let terminal = terminal_len(&bytes); + let cases = [ + ( + OpenLimits { + max_directory_bytes: terminal - 1, + ..OpenLimits::default() + }, + 2, + "directory", + ), + ( + OpenLimits { + max_total_directory_bytes: terminal, + ..OpenLimits::default() + }, + 3, + "total directory bytes", + ), + ( + OpenLimits { + max_parent_directories: 0, + ..OpenLimits::default() + }, + 3, + "parent directories", + ), + ]; + for (limits, issued, expected) in cases { + let (result, requests) = drive(&bytes, recipient().with_limits(limits)); + assert!( + matches!(result, Err(PithosError::LimitExceeded { field, .. }) if field == expected), + "{expected}" + ); + assert_eq!(requests.len(), issued, "{expected}"); + } + assert!(matches!( + ArchiveOpener::new(5, OpenOptions::default()), + Err(PithosError::InvalidDirectoryRange { .. }) + )); +} + +struct RecordingSource { + bytes: Arc<[u8]>, + reads: Arc>>, +} + +impl ArchiveSource for RecordingSource { + fn len(&self) -> Result { + Ok(self.bytes.len() as u64) + } + + fn read_exact_at(&self, offset: u64, output: &mut [u8]) -> Result<(), SourceError> { + self.reads.lock().unwrap().push((offset, output.len())); + MemorySource::new(Arc::clone(&self.bytes)).read_exact_at(offset, output) + } +} + +/// An archive with `count` distinct 16-byte blocks in one file. +fn many_blocks(count: usize) -> Vec { + let content = (0..count as u64) + .flat_map(|block| [block.to_be_bytes(), (!block).to_be_bytes()]) + .flatten() + .collect::>(); + let sender = private_key("sender"); + let options = WriteOptions::new(sender.duplicate(), vec![sender.public_key()]) + .with_chunking(Chunking::Fixed(16)); + let mut writer = ArchiveWriter::create(Vec::new(), options).unwrap(); + writer + .add_file( + ArchivePath::new("data").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(false, 0).unwrap(), + Some(content.len() as u64), + std::io::Cursor::new(content), + ) + .unwrap(); + writer.finish().unwrap() +} + +#[test] +fn open_reads_do_not_depend_on_the_block_count_and_reads_fetch_each_block_once() { + let count = 1_000; + let bytes = Arc::<[u8]>::from(many_blocks(count)); + let reads = Arc::new(Mutex::new(Vec::new())); + let archive = Archive::open( + RecordingSource { + bytes: Arc::clone(&bytes), + reads: Arc::clone(&reads), + }, + OpenOptions::default().with_access_keys(AccessKeys::new().with_key(private_key("sender"))), + ) + .unwrap(); + assert_eq!(reads.lock().unwrap().len(), 3); + + let mut output = Vec::new(); + archive.copy_range_to("data", 32..48, &mut output).unwrap(); + assert_eq!(output, [2u64.to_be_bytes(), (!2u64).to_be_bytes()].concat()); + assert_eq!(reads.lock().unwrap().len(), 4); + assert_eq!(reads.lock().unwrap()[3].1, 4 + 16); + + archive.copy_to("data", &mut Vec::new()).unwrap(); + assert_eq!(reads.lock().unwrap().len(), 4 + count); +} + +#[test] +fn a_view_decodes_blocks_from_bytes_the_caller_fetched() { + let bytes = many_blocks(64); + let (view, _) = drive( + &bytes, + OpenOptions::default().with_access_keys(AccessKeys::new().with_key(private_key("sender"))), + ); + let view = view.unwrap(); + let fetch = |request: BlockRequest| match request { + BlockRequest::Local { offset, len } => &bytes[offset as usize..(offset + len) as usize], + BlockRequest::External { .. } => panic!("unexpected external block"), + }; + let mut output = Vec::new(); + for batch in view.plan_range("data", 8..40).unwrap().batches(1024) { + let batch = batch.unwrap(); + assert_eq!(batch.blocks().len(), 3); + for (block, stored) in batch.split(fetch(batch.request())).unwrap() { + output.extend_from_slice(&view.decode_block(block, stored).unwrap()[block.output()]); + } + } + let content = (0..3u64) + .flat_map(|block| [block.to_be_bytes(), (!block).to_be_bytes()]) + .flatten() + .collect::>(); + assert_eq!(output, &content[8..40]); + + let block = view + .plan_range("data", 0..1) + .unwrap() + .next() + .unwrap() + .unwrap(); + let mut stored = fetch(block.request()).to_vec(); + assert!(matches!( + view.decode_block(&block, &stored[1..]), + Err(PithosError::BlockSizeMismatch { .. }) + )); + stored[0] ^= 1; + let error = view.decode_block(&block, &stored).unwrap_err(); + assert!(error.to_string().contains("block marker"), "{error}"); +} From 1b0e1e04170773267ad9c770500afcde7dcbb2ae Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 22:05:53 +0200 Subject: [PATCH 083/166] fix: admit the largest encrypted block in the default stored block limit --- crates/pithos_lib/src/archive/reader.rs | 6 +++- crates/pithos_lib/tests/writer_processing.rs | 34 ++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/crates/pithos_lib/src/archive/reader.rs b/crates/pithos_lib/src/archive/reader.rs index 7382cc2..b2160fa 100644 --- a/crates/pithos_lib/src/archive/reader.rs +++ b/crates/pithos_lib/src/archive/reader.rs @@ -92,7 +92,11 @@ impl Default for OpenLimits { max_relationships: 1_000_000, max_accessible_block_references: 2_097_152, max_opaque_metadata_bytes: 128 * 1024 * 1024, - max_stored_block_bytes: 64 * 1024 * 1024, + // The worst case of a block at the decoded limit: the zstd bound plus nonce and tag. + max_stored_block_bytes: zstd::zstd_safe::compress_bound( + DEFAULT_MAX_DECODED_BLOCK_BYTES as usize, + ) as u64 + + 28, max_decoded_block_bytes: DEFAULT_MAX_DECODED_BLOCK_BYTES, } } diff --git a/crates/pithos_lib/tests/writer_processing.rs b/crates/pithos_lib/tests/writer_processing.rs index f14fd29..dec38c5 100644 --- a/crates/pithos_lib/tests/writer_processing.rs +++ b/crates/pithos_lib/tests/writer_processing.rs @@ -119,3 +119,37 @@ fn short_writes_are_counted_and_completed_without_saturating_offsets() { archive.copy_to("data", &mut copied).unwrap(); assert_eq!(copied, b"short writes"); } + +#[test] +fn the_largest_fixed_block_reads_with_default_limits() { + let size = 64 * 1024 * 1024; + let mut data = vec![0; size]; + blake3::Hasher::new() + .update(b"incompressible block") + .finalize_xof() + .fill(&mut data); + let sender = PrivateKey::generate(); + let reader = sender.duplicate(); + let mut writer = ArchiveWriter::create( + Vec::new(), + WriteOptions::new(sender, vec![reader.public_key()]).with_chunking(Chunking::Fixed(size)), + ) + .unwrap(); + writer + .add_file( + ArchivePath::new("large.bin").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(true, 0).unwrap(), + Some(size as u64), + Cursor::new(&data), + ) + .unwrap(); + let archive = Archive::open( + MemorySource::new(writer.finish().unwrap()), + OpenOptions::default().with_access_keys(AccessKeys::new().with_key(reader)), + ) + .unwrap(); + let mut output = Vec::new(); + archive.copy_to("large.bin", &mut output).unwrap(); + assert!(output == data, "the large block did not round trip"); +} From e55229d9e5ca49af95a8bb1dc672aeae5c44d043 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 22:31:30 +0200 Subject: [PATCH 084/166] fix: resolve block list pieces without extra copies of the list --- crates/pithos_lib/src/archive/access.rs | 13 ++- crates/pithos_lib/src/archive/reader.rs | 31 +++---- crates/pithos_lib/src/format/file_entry.rs | 46 +++++++-- crates/pithos_lib/tests/open_memory.rs | 103 +++++++++++++++++++++ crates/pithos_lib/tests/pieces.rs | 6 +- 5 files changed, 164 insertions(+), 35 deletions(-) create mode 100644 crates/pithos_lib/tests/open_memory.rs diff --git a/crates/pithos_lib/src/archive/access.rs b/crates/pithos_lib/src/archive/access.rs index a79bcfd..c5402db 100644 --- a/crates/pithos_lib/src/archive/access.rs +++ b/crates/pithos_lib/src/archive/access.rs @@ -84,13 +84,12 @@ impl ResolvedAccess { id: FileId, entries: impl IntoIterator, ) { - self.block_keys.insert( - id, - entries - .into_iter() - .map(|(hash, key)| (hash, BlockKey::from_protocol(key))) - .collect(), - ); + // Inserting one by one avoids the sorted copy that collecting into a map would make. + let mut keys = BTreeMap::new(); + for (hash, key) in entries { + keys.insert(hash, BlockKey::from_protocol(key)); + } + self.block_keys.insert(id, keys); } pub(crate) fn block_key( diff --git a/crates/pithos_lib/src/archive/reader.rs b/crates/pithos_lib/src/archive/reader.rs index b2160fa..166cf33 100644 --- a/crates/pithos_lib/src/archive/reader.rs +++ b/crates/pithos_lib/src/archive/reader.rs @@ -715,7 +715,7 @@ pub(super) fn resolve_block_lists( ) -> Result<(), PithosError> { directory.files.try_for_each_mut(|id, file| { let file_id = FileId(id); - match &file.block_data { + match &mut file.block_data { BlockDataState::Decrypted(entries) => { record_block_keys(access, file_id, entries); } @@ -726,6 +726,10 @@ pub(super) fn resolve_block_lists( { return Ok(()); } + let key_ids = pieces.iter().map(|piece| FileId(piece.key_id)).collect(); + let pieces = std::mem::take(pieces); + // Each sealed piece is dropped after decryption and each plaintext after + // decoding, so only the final list and one piece buffer are held. let mut entries = Zeroizing::new(Vec::new()); for piece in pieces { let key = access @@ -734,25 +738,16 @@ pub(super) fn resolve_block_lists( let mut decoded_limits = deserialization_limits(limits); decoded_limits.max_block_references = *remaining_block_references; let plaintext = crypto::open_file_block_list(key, &piece.sealed)?; - let piece_entries = - crate::format::file_entry::decode_decrypted_block_list_with_budget( - &plaintext, - &decoded_limits, - remaining_block_references, - )?; - entries.try_reserve(piece_entries.len()).map_err(|_| { - PithosError::AllocationFailed { - field: "block list pieces", - size: piece_entries.len() as u64, - } - })?; - entries.extend_from_slice(&piece_entries); + drop(piece); + crate::format::file_entry::append_decrypted_block_list( + &plaintext, + &decoded_limits, + remaining_block_references, + &mut entries, + )?; } crate::format::file_entry::validate_unique_block_references(&entries)?; - access.insert_pieces( - file_id, - pieces.iter().map(|piece| FileId(piece.key_id)).collect(), - ); + access.insert_pieces(file_id, key_ids); record_block_keys(access, file_id, &entries); file.block_data = BlockDataState::Decrypted(entries); } diff --git a/crates/pithos_lib/src/format/file_entry.rs b/crates/pithos_lib/src/format/file_entry.rs index c75457a..8e0ff63 100644 --- a/crates/pithos_lib/src/format/file_entry.rs +++ b/crates/pithos_lib/src/format/file_entry.rs @@ -83,17 +83,19 @@ impl BlockDataState { } } +/// Rejects a hash listed with two different keys. Sorting indices keeps the extra memory at +/// one index per entry instead of a copy of the list. pub(crate) fn validate_unique_block_references( entries: &[BlockDataEntry], ) -> Result<(), PithosError> { - let mut keys = HashMap::with_capacity(entries.len()); - for (hash, key) in entries { - if keys - .insert(*hash, *key) - .is_some_and(|existing| existing != *key) - { - return Err(PithosError::DuplicateBlockReference); - } + let mut order = (0..entries.len()).collect::>(); + order.sort_unstable_by(|left, right| entries[*left].0.cmp(&entries[*right].0)); + let conflict = order.windows(2).any(|pair| { + let (left, right) = (&entries[pair[0]], &entries[pair[1]]); + left.0 == right.0 && left.1 != right.1 + }); + if conflict { + return Err(PithosError::DuplicateBlockReference); } Ok(()) } @@ -369,6 +371,34 @@ fn decode_decrypted_block_list_reader_with_budget( Ok(entries) } +/// Appends one decrypted block list to `entries`. The caller checks the combined list for +/// conflicting keys. +pub(crate) fn append_decrypted_block_list( + bytes: &[u8], + limits: &DeserializationLimits, + remaining_block_references: &mut u64, + entries: &mut Vec, +) -> Result<(), DeserializationError> { + let mut reader = std::io::Cursor::new(bytes); + let count = bounded_len( + reader.read_varint::()?, + limits.max_block_references.min(*remaining_block_references), + "block references", + )?; + *remaining_block_references -= count as u64; + reserve(entries, count, "block references")?; + for _ in 0..count { + let mut entry = ([0; 32], [0; 32]); + reader.read_exact(&mut entry.0)?; + reader.read_exact(&mut entry.1)?; + entries.push(entry); + } + if reader.position() != bytes.len() as u64 { + return Err(DeserializationError::InvalidLength); + } + Ok(()) +} + pub(crate) fn decode_decrypted_block_list_with_budget( bytes: &[u8], limits: &DeserializationLimits, diff --git a/crates/pithos_lib/tests/open_memory.rs b/crates/pithos_lib/tests/open_memory.rs new file mode 100644 index 0000000..1d7fefc --- /dev/null +++ b/crates/pithos_lib/tests/open_memory.rs @@ -0,0 +1,103 @@ +//! Peak memory while opening a composed file. This target holds a single test, so the counting +//! allocator sees no concurrent allocations from other tests. + +mod common; + +use common::util::{private_key, public_key}; +use pithos_lib::archive::{ + AccessKeys, Archive, ArchivePath, EntryMetadata, OpenOptions, PieceEncoder, ProcessingOptions, + compose, +}; +use pithos_lib::source::MemorySource; +use std::alloc::{GlobalAlloc, Layout, System}; +use std::sync::atomic::{AtomicUsize, Ordering}; + +struct Counting; + +static CURRENT: AtomicUsize = AtomicUsize::new(0); +static PEAK: AtomicUsize = AtomicUsize::new(0); + +fn grow(bytes: usize) { + let now = CURRENT.fetch_add(bytes, Ordering::Relaxed) + bytes; + PEAK.fetch_max(now, Ordering::Relaxed); +} + +unsafe impl GlobalAlloc for Counting { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + let pointer = unsafe { System.alloc(layout) }; + if !pointer.is_null() { + grow(layout.size()); + } + pointer + } + + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + let pointer = unsafe { System.alloc_zeroed(layout) }; + if !pointer.is_null() { + grow(layout.size()); + } + pointer + } + + unsafe fn dealloc(&self, pointer: *mut u8, layout: Layout) { + unsafe { System.dealloc(pointer, layout) }; + CURRENT.fetch_sub(layout.size(), Ordering::Relaxed); + } + + unsafe fn realloc(&self, pointer: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + let moved = unsafe { System.realloc(pointer, layout, new_size) }; + if !moved.is_null() { + // A moving reallocation briefly holds both buffers. + grow(new_size); + CURRENT.fetch_sub(layout.size(), Ordering::Relaxed); + } + moved + } +} + +#[global_allocator] +static ALLOCATOR: Counting = Counting; + +/// One composed file in a single piece that references one 4-byte block `references` times. +/// Its block list is far larger than everything else in the directory. +fn one_piece(references: u32) -> Vec { + let processing = ProcessingOptions::new(true, 0).unwrap(); + let mut encoder = PieceEncoder::new(1, vec![public_key("recipient1")], processing).unwrap(); + let mut stored = Vec::new(); + for _ in 0..references { + stored.extend(encoder.push(b"same").unwrap()); + } + let piece = encoder.finish().unwrap(); + let composition = compose( + ArchivePath::new("object").unwrap(), + EntryMetadata::new(0, 0, 0o644), + &[piece], + ) + .unwrap(); + let mut archive = composition.header().to_vec(); + archive.extend_from_slice(&stored); + archive.extend_from_slice(composition.directory()); + archive +} + +/// Opening holds the directory response plus at most two lists at once: the sealed list and its +/// plaintext, then the plaintext and the decoded list. That is about three list sizes. Decoding +/// into a separate list, copying it into an aggregate list and collecting the keys into a map +/// through a sorted copy needed more than five. +#[test] +fn opening_one_large_piece_holds_no_extra_copies_of_its_block_list() { + let references = 100_000; + let source = MemorySource::new(one_piece(references)); + let options = OpenOptions::default() + .with_access_keys(AccessKeys::new().with_key(private_key("recipient1"))); + let baseline = CURRENT.load(Ordering::Relaxed); + PEAK.store(baseline, Ordering::Relaxed); + let archive = Archive::open(source, options).unwrap(); + let peak = PEAK.load(Ordering::Relaxed) - baseline; + let list = references as usize * 64; + assert_eq!(archive.entries().len(), 1); + assert!( + 2 * peak < 7 * list, + "peak {peak} bytes for a {list}-byte list" + ); +} diff --git a/crates/pithos_lib/tests/pieces.rs b/crates/pithos_lib/tests/pieces.rs index 51a008a..17dc2a8 100644 --- a/crates/pithos_lib/tests/pieces.rs +++ b/crates/pithos_lib/tests/pieces.rs @@ -306,12 +306,14 @@ fn a_composed_file_exports_to_crypt4gh_with_a_fresh_data_key() { } /// One 5 TiB file at 4 MiB blocks has 1,310,720 blocks. Tiny blocks give the same directory -/// shape, and the default open limits must admit it. +/// shape, and the default open limits must admit it. A single piece holds the whole list, as +/// when one upload part covers the object. Offsets are smaller than real ones, so the real +/// directory is about 8 bytes per block larger. #[test] #[ignore = "slow in debug builds (about 5 minutes, 8 seconds with --release); run with --ignored"] fn a_composition_with_the_block_count_of_5_tib_opens_with_default_limits() { const BLOCKS: u32 = 1_310_720; - const PIECES: u32 = 10; + const PIECES: u32 = 1; let per_piece = BLOCKS / PIECES; let parts = (0..PIECES) .map(|piece| { From 894c011695fcb4e4c24ebace60b759827a2f8823 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 22:41:28 +0200 Subject: [PATCH 085/166] feat: let the opener release each directory response after decoding --- crates/pithos_lib/src/archive/opener.rs | 20 ++++++++++++++------ crates/pithos_lib/src/archive/reader.rs | 9 ++------- crates/pithos_lib/tests/open_memory.rs | 10 +++++----- crates/pithos_lib/tests/opener.rs | 25 +++++++++++++++---------- 4 files changed, 36 insertions(+), 28 deletions(-) diff --git a/crates/pithos_lib/src/archive/opener.rs b/crates/pithos_lib/src/archive/opener.rs index 1684fc9..115f064 100644 --- a/crates/pithos_lib/src/archive/opener.rs +++ b/crates/pithos_lib/src/archive/opener.rs @@ -13,6 +13,7 @@ use crate::error::PithosError; use crate::format::directory::Directory; use crate::format::header::{FileHeader, FormatVersion}; use std::collections::{HashMap, HashSet}; +use zeroize::Zeroizing; /// One exact byte range the opener needs next. #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -59,7 +60,7 @@ pub(super) struct OpenSettings { /// let mut opener = ArchiveOpener::new(4096, OpenOptions::default())?; /// while let Some(request) = opener.request() { /// let bytes = read(request.offset(), request.len()); -/// opener.feed(request, &bytes)?; +/// opener.feed(request, bytes)?; /// } /// let view = opener.finish()?; /// # let _ = view; @@ -137,8 +138,10 @@ impl ArchiveOpener { /// Accepts the bytes for the outstanding request. /// /// A response to another request or of the wrong length is rejected and the request stays - /// outstanding. Any other error ends the open. - pub fn feed(&mut self, request: ReadRequest, response: &[u8]) -> Result<(), PithosError> { + /// outstanding. Any other error ends the open. The opener takes the response so it can + /// release a directory's bytes as soon as they are decoded. + pub fn feed(&mut self, request: ReadRequest, response: Vec) -> Result<(), PithosError> { + let response = Zeroizing::new(response); if self.pending != Some(request) { return Err(PithosError::UnexpectedReadResponse); } @@ -179,7 +182,11 @@ impl ArchiveOpener { self.next_sequence += 1; } - fn advance(&mut self, phase: Phase, response: &[u8]) -> Result { + fn advance( + &mut self, + phase: Phase, + response: Zeroizing>, + ) -> Result { match phase { Phase::Header => { let header = crate::format::header::decode_header(&mut &response[..])?; @@ -231,13 +238,14 @@ impl ArchiveOpener { Phase::Directories(mut chain) => { let limits = remaining_deserialization_limits(self.settings.limits, &chain.decoded); let decoded_chain = &mut chain.decoded_chain; - let hash = blake3::hash(response); + let hash = blake3::hash(&response); decoded_chain.directory_hashes.push(*hash.as_bytes()); let directory = decode_validated_directory( - response, + &response, &limits, &mut decoded_chain.remaining_block_references, )?; + drop(response); let next = directory.parent_directory_offset; let span = chain.pending; chain.decoded.record(&directory); diff --git a/crates/pithos_lib/src/archive/reader.rs b/crates/pithos_lib/src/archive/reader.rs index 166cf33..3730df5 100644 --- a/crates/pithos_lib/src/archive/reader.rs +++ b/crates/pithos_lib/src/archive/reader.rs @@ -309,13 +309,8 @@ where let (settings, external, external_access_policy) = options.into_parts(); let mut opener = ArchiveOpener::with_settings(archive_len, settings)?; while let Some(request) = opener.request() { - let response = Zeroizing::new(read_source( - &source, - request.offset(), - request.len(), - "directory", - )?); - opener.feed(request, &response)?; + let response = read_source(&source, request.offset(), request.len(), "directory")?; + opener.feed(request, response)?; } let view = opener.finish()?; Ok(Self { diff --git a/crates/pithos_lib/tests/open_memory.rs b/crates/pithos_lib/tests/open_memory.rs index 1d7fefc..1b2cb25 100644 --- a/crates/pithos_lib/tests/open_memory.rs +++ b/crates/pithos_lib/tests/open_memory.rs @@ -80,10 +80,10 @@ fn one_piece(references: u32) -> Vec { archive } -/// Opening holds the directory response plus at most two lists at once: the sealed list and its -/// plaintext, then the plaintext and the decoded list. That is about three list sizes. Decoding -/// into a separate list, copying it into an aggregate list and collecting the keys into a map -/// through a sorted copy needed more than five. +/// Opening holds at most two list-sized buffers at once: the directory response and the +/// decoded sealed list, then the sealed list and its plaintext, then the plaintext and the +/// decoded list. Keeping the response, copying the decoded list and collecting the keys into a +/// map through a sorted copy needed more than five. #[test] fn opening_one_large_piece_holds_no_extra_copies_of_its_block_list() { let references = 100_000; @@ -97,7 +97,7 @@ fn opening_one_large_piece_holds_no_extra_copies_of_its_block_list() { let list = references as usize * 64; assert_eq!(archive.entries().len(), 1); assert!( - 2 * peak < 7 * list, + 2 * peak < 5 * list, "peak {peak} bytes for a {list}-byte list" ); } diff --git a/crates/pithos_lib/tests/opener.rs b/crates/pithos_lib/tests/opener.rs index 2b16521..65ed166 100644 --- a/crates/pithos_lib/tests/opener.rs +++ b/crates/pithos_lib/tests/opener.rs @@ -24,7 +24,10 @@ fn drive( while let Some(request) = opener.request() { requests.push((request.offset(), request.len())); let start = request.offset() as usize; - opener.feed(request, &bytes[start..start + request.len() as usize])?; + opener.feed( + request, + bytes[start..start + request.len() as usize].to_vec(), + )?; } opener.finish() }); @@ -80,29 +83,29 @@ fn feed_rejects_wrong_lengths_unknown_and_repeated_responses() { let mut opener = ArchiveOpener::new(len, recipient()).unwrap(); let header = opener.request().unwrap(); assert!(matches!( - opener.feed(header, &bytes[..5]), + opener.feed(header, bytes[..5].to_vec()), Err(PithosError::ReadResponseLength { expected: 6, actual: 5 }) )); assert!(matches!( - opener.feed(header, &bytes[..7]), + opener.feed(header, bytes[..7].to_vec()), Err(PithosError::ReadResponseLength { .. }) )); assert_eq!(opener.request(), Some(header)); - opener.feed(header, &bytes[..6]).unwrap(); + opener.feed(header, bytes[..6].to_vec()).unwrap(); assert!(matches!( - opener.feed(header, &bytes[..6]), + opener.feed(header, bytes[..6].to_vec()), Err(PithosError::UnexpectedReadResponse) )); let mut other = ArchiveOpener::new(len + 1, OpenOptions::default()).unwrap(); let other_header = other.request().unwrap(); - other.feed(other_header, &bytes[..6]).unwrap(); + other.feed(other_header, bytes[..6].to_vec()).unwrap(); let foreign = other.request().unwrap(); assert!(matches!( - opener.feed(foreign, &bytes[..12]), + opener.feed(foreign, bytes[..12].to_vec()), Err(PithosError::UnexpectedReadResponse) )); let footer = opener.request().unwrap(); @@ -113,13 +116,15 @@ fn feed_rejects_wrong_lengths_unknown_and_repeated_responses() { Err(PithosError::OpenIncomplete) )); - opener.feed(footer, &bytes[bytes.len() - 12..]).unwrap(); + opener + .feed(footer, bytes[bytes.len() - 12..].to_vec()) + .unwrap(); let directory = opener.request().unwrap(); let start = directory.offset() as usize; - opener.feed(directory, &bytes[start..]).unwrap(); + opener.feed(directory, bytes[start..].to_vec()).unwrap(); assert_eq!(opener.request(), None); assert!(matches!( - opener.feed(directory, &bytes[start..]), + opener.feed(directory, bytes[start..].to_vec()), Err(PithosError::UnexpectedReadResponse) )); assert_eq!(opener.finish().unwrap().entries().len(), 1); From 55f046fbfa0e257a110f0688527ff9370ad1645e Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 22:48:56 +0200 Subject: [PATCH 086/166] test: require the opener, view and plan types to be Send and Sync --- crates/pithos_lib/tests/public_api.rs | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/crates/pithos_lib/tests/public_api.rs b/crates/pithos_lib/tests/public_api.rs index 6bb6d0c..770724f 100644 --- a/crates/pithos_lib/tests/public_api.rs +++ b/crates/pithos_lib/tests/public_api.rs @@ -1,10 +1,11 @@ use pithos_lib::adapters::{crypt4gh::Crypt4GHError, ro_crate::RoCrateError}; use pithos_lib::archive::{ AccessKeys, AppendDurability, AppendObservation, AppendOptions, Archive, ArchiveFeature, - ArchivePath, ArchiveReference, ArchiveWriter, CdcConfig, CreateError, EntryKind, EntryMetadata, - EntryReference, ExternalBlockAccessPolicy, ExternalBlockResolver, ExternalLocation, - FinishError, IncompleteWriter, NoExternalBlocks, OpenLimits, OpenOptions, ProcessingOptions, - WriteOptions, WriterError, WrittenEntry, + ArchiveOpener, ArchivePath, ArchiveReference, ArchiveView, ArchiveWriter, BlockBatch, + BlockRequest, CdcConfig, CreateError, EntryKind, EntryMetadata, EntryReference, + ExternalBlockAccessPolicy, ExternalBlockResolver, ExternalLocation, FinishError, + IncompleteWriter, NoExternalBlocks, OpenLimits, OpenOptions, PlannedBlock, ProcessingOptions, + ReadPlan, ReadRequest, WriteOptions, WriterError, WrittenEntry, }; use pithos_lib::crypto::{CryptoError, PrivateKey, PublicKey}; use pithos_lib::error::PithosError; @@ -47,6 +48,13 @@ fn standard_archive_configurations_are_send_and_sync() { assert_send_sync::>(); assert_send_sync::>(); assert_send_sync::(); + assert_send_sync::(); + assert_send_sync::(); + assert_send_sync::(); + assert_send_sync::>(); + assert_send_sync::(); + assert_send_sync::(); + assert_send_sync::(); assert_send_sync::>(); let _ = OpenOptions::default().with_access_keys(AccessKeys::new()); let _ = OpenOptions::default() From 88b3c23f60e47edd618188963d7b4bf95b54be12 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 22:49:40 +0200 Subject: [PATCH 087/166] refactor: serialize the writer directory once at finish --- crates/pithos_lib/src/archive/writer.rs | 10 ++++++---- crates/pithos_lib/src/format/directory.rs | 7 +++---- crates/pithos_lib/src/fs/append.rs | 2 +- crates/pithos_lib/tests/writer_failures.rs | 8 +++++--- 4 files changed, 15 insertions(+), 12 deletions(-) diff --git a/crates/pithos_lib/src/archive/writer.rs b/crates/pithos_lib/src/archive/writer.rs index 47c0ee5..bc39bfb 100644 --- a/crates/pithos_lib/src/archive/writer.rs +++ b/crates/pithos_lib/src/archive/writer.rs @@ -15,6 +15,7 @@ use crate::error::PithosError; use crate::format::block::{BlockHeader, BlockIndexEntry, BlockLocation, ProcessingFlags}; use crate::format::directory::{Directory, DirectoryEntries}; use crate::format::encryption::{EncryptionSection, RecipientData}; +use crate::format::error::SerializationError; use crate::format::file_entry::{BlockDataState, FileEntry, FileType, Reference}; use crate::format::header::FormatVersion; use crate::format::{directory, header}; @@ -1419,10 +1420,9 @@ impl ArchiveWriter { self.validate_required_access_records()?; self.seal_recipient_lists()?; self.validate_publishable()?; - directory::update_directory_len(&mut self.directory)?; - directory::update_directory_crc(&mut self.directory)?; + let bytes = directory::encode_complete_directory(&self.directory)?; if let Some(snapshot) = &self.append_snapshot { - let span = Span::new(self.sink.offset, self.directory.dir_len)?; + let span = Span::new(self.sink.offset, bytes.len() as u64)?; let child = validated_segment_from_directory( self.version, &self.directory, @@ -1431,7 +1431,9 @@ impl ArchiveWriter { )?; snapshot.validate_prospective_child(child)?; } - directory::encode_directory(&self.directory, &mut self.sink)?; + self.sink + .write_all(&bytes) + .map_err(SerializationError::from)?; self.sink.flush()?; Ok(()) })(); diff --git a/crates/pithos_lib/src/format/directory.rs b/crates/pithos_lib/src/format/directory.rs index 4c7517c..0c49f1b 100644 --- a/crates/pithos_lib/src/format/directory.rs +++ b/crates/pithos_lib/src/format/directory.rs @@ -14,7 +14,6 @@ use crate::format::primitives::{ bounded_len, decode_string, encode_string, reserve, write_len_prefix, }; use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt}; -use crc32fast::Hasher; use indexmap::IndexMap; use integer_encoding::{VarIntReader, VarIntWriter}; use std::cmp::Ordering; @@ -579,6 +578,7 @@ pub(crate) fn encode_complete_directory( Ok(bytes) } +#[cfg(test)] pub(crate) fn update_directory_len(directory: &mut Directory) -> Result<(), SerializationError> { let mut bytes = Vec::new(); encode_directory(directory, &mut bytes)?; @@ -587,12 +587,11 @@ pub(crate) fn update_directory_len(directory: &mut Directory) -> Result<(), Seri Ok(()) } +#[cfg(test)] pub(crate) fn update_directory_crc(directory: &mut Directory) -> Result<(), SerializationError> { let mut bytes = Vec::new(); encode_directory(directory, &mut bytes)?; - let mut hasher = Hasher::new(); - hasher.update(&bytes[..bytes.len() - 4]); - directory.crc32 = hasher.finalize(); + directory.crc32 = crc32fast::hash(&bytes[..bytes.len() - 4]); Ok(()) } diff --git a/crates/pithos_lib/src/fs/append.rs b/crates/pithos_lib/src/fs/append.rs index d3845a8..3d707a8 100644 --- a/crates/pithos_lib/src/fs/append.rs +++ b/crates/pithos_lib/src/fs/append.rs @@ -492,7 +492,7 @@ impl Write for AppendSink { #[cfg(test)] if accepted == bytes.len() { let is_marker = bytes == b"BLCK"; - let is_directory = bytes == b"PITHOSDR"; + let is_directory = bytes.starts_with(b"PITHOSDR"); let after_payload = self.after_block_marker && !is_marker; self.after_block_marker = is_marker; let fail = match self.failure { diff --git a/crates/pithos_lib/tests/writer_failures.rs b/crates/pithos_lib/tests/writer_failures.rs index 477abae..9d12a78 100644 --- a/crates/pithos_lib/tests/writer_failures.rs +++ b/crates/pithos_lib/tests/writer_failures.rs @@ -367,11 +367,13 @@ struct FailDuringDirectory { impl Write for FailDuringDirectory { fn write(&mut self, bytes: &[u8]) -> io::Result { self.writes += 1; - if self.writes == 5 { + if self.writes == 4 { return Err(io::Error::other("mid-directory failure")); } - self.bytes.extend_from_slice(bytes); - Ok(bytes.len()) + // The third write starts the directory; accept only part of it. + let accepted = if self.writes == 3 { 8 } else { bytes.len() }; + self.bytes.extend_from_slice(&bytes[..accepted]); + Ok(accepted) } fn flush(&mut self) -> io::Result<()> { Ok(()) From ccdf52a2296f76be23bfc613f3683bee65130a7e Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 22:49:40 +0200 Subject: [PATCH 088/166] feat: add cipher, unique key and block size options to the CLI --- crates/pithos/README.md | 15 +++ crates/pithos/src/main.rs | 103 ++++++++++++++--- crates/pithos/tests/append.rs | 52 +++++++++ crates/pithos/tests/create.rs | 144 ++++++++++++++++++++++++ crates/pithos_lib/src/archive/append.rs | 5 + 5 files changed, 303 insertions(+), 16 deletions(-) diff --git a/crates/pithos/README.md b/crates/pithos/README.md index 4141caf..ed81b07 100644 --- a/crates/pithos/README.md +++ b/crates/pithos/README.md @@ -46,6 +46,21 @@ pithos read data plain.pith report.txt `read data` also accepts `--ranges START:END,...` for half-open byte ranges. Use `read info` to inspect an entry and `read directory` to inspect every entry's metadata. +## Block and encryption options + +`create` and `append files` accept these options: + +- `--block-size BYTES` sets the size of fixed blocks. The default is 4 MiB (4194304 bytes). It cannot be combined with `--cdc`. +- `--cdc MIN,AVG,MAX` uses content-defined chunking instead of fixed blocks. +- `--cipher chacha20-poly1305|aes-256-gcm` selects the cipher for new encrypted blocks. The default is `chacha20-poly1305`. +- `--unique-keys` gives every block its own random key. Equal blocks are then stored again instead of once, so their equality stays hidden. + +`--unique-keys` and `--cipher aes-256-gcm` need encryption, so they fail with `--plain`. When appending, they also need a version 1.1 archive. + +```bash +pithos --secret-key keys/owner.sec.pem --public-keys keys/owner.pub.pem --output private.pith create --cipher aes-256-gcm --unique-keys --block-size 1048576 input +``` + ## More operations - `append files` adds filesystem input to an existing archive. diff --git a/crates/pithos/src/main.rs b/crates/pithos/src/main.rs index 4aea576..185e486 100644 --- a/crates/pithos/src/main.rs +++ b/crates/pithos/src/main.rs @@ -6,8 +6,8 @@ use clap::{Parser, Subcommand, ValueEnum}; use pithos_lib::adapters::crypt4gh; use pithos_lib::adapters::crypt4gh::Crypt4GHError; use pithos_lib::archive::{ - AccessKeys, AppendDurability, AppendOptions, Archive, ArchiveWriter, CdcConfig, Chunking, - OpenOptions, ProcessingOptions, WriteOptions, WriterError, + AccessKeys, AppendDurability, AppendOptions, Archive, ArchiveWriter, BlockKeyMode, CdcConfig, + Chunking, OpenOptions, PayloadCipher, ProcessingOptions, WriteOptions, WriterError, }; use pithos_lib::crypto::{PrivateKey, PublicKey, generate_private_key}; use pithos_lib::error::PithosError; @@ -57,6 +57,24 @@ impl From for AppendDurability { } } +#[derive(Clone, Copy, Default, ValueEnum)] +enum CliCipher { + #[default] + #[value(name = "chacha20-poly1305")] + ChaCha20Poly1305, + #[value(name = "aes-256-gcm")] + Aes256Gcm, +} + +impl From for PayloadCipher { + fn from(value: CliCipher) -> Self { + match value { + CliCipher::ChaCha20Poly1305 => Self::ChaCha20Poly1305, + CliCipher::Aes256Gcm => Self::Aes256Gcm, + } + } +} + #[derive(Parser)] #[command(author, version, about, long_about = None)] struct Cli { @@ -110,9 +128,18 @@ enum PithosCommands { #[arg(long)] reader_public_keys: Option>, // Iterate files and parse all keys */ - /// Use content-defined chunking with these sizes instead of fixed 4 MiB blocks + /// Use content-defined chunking with these sizes instead of fixed blocks #[arg(long="cdc", value_parser=parse_cdc_input, value_name = "MIN,AVG,MAX")] cdc: Option, + /// Size of fixed blocks in bytes + #[arg(long, value_name = "BYTES", default_value_t = Chunking::DEFAULT_BLOCK_SIZE, conflicts_with = "cdc")] + block_size: usize, + /// Cipher for encrypted blocks; needs an encrypted archive for aes-256-gcm + #[arg(long, value_enum, default_value_t)] + cipher: CliCipher, + /// Encrypt every block with its own random key, without deduplication; needs encryption + #[arg(long)] + unique_keys: bool, /// Create a local, uncompressed, unencrypted base archive #[arg(long, conflicts_with_all = ["secret_key", "public_keys"])] plain: bool, @@ -232,9 +259,18 @@ enum AppendCommands { /// Path to Pithos file #[arg(short, long, value_name = "PITHOS FILE")] file: PathBuf, - /// Use content-defined chunking with these sizes instead of fixed 4 MiB blocks + /// Use content-defined chunking with these sizes instead of fixed blocks #[arg(long = "cdc", value_parser = parse_cdc_input, value_name = "MIN,AVG,MAX")] cdc: Option, + /// Size of fixed blocks in bytes + #[arg(long, value_name = "BYTES", default_value_t = Chunking::DEFAULT_BLOCK_SIZE, conflicts_with = "cdc")] + block_size: usize, + /// Cipher for new encrypted blocks; aes-256-gcm needs a version 1.1 archive + #[arg(long, value_enum, default_value_t)] + cipher: CliCipher, + /// Encrypt every new block with its own random key, without deduplication; needs version 1.1 + #[arg(long)] + unique_keys: bool, /// Durability after publishing the appended directory #[arg(long, value_enum, default_value_t)] durability: CliAppendDurability, @@ -281,6 +317,33 @@ pub enum PithosCliError { }, } +fn chunking(cdc: Option, block_size: usize) -> Chunking { + cdc.map_or(Chunking::Fixed(block_size), Chunking::ContentDefined) +} + +/// Applies the key mode and cipher options; the library rejects them without encryption. +fn block_processing( + processing: ProcessingOptions, + cipher: CliCipher, + unique_keys: bool, +) -> Result { + let key_mode = if unique_keys { + BlockKeyMode::Unique + } else { + BlockKeyMode::ContentDerived + }; + let processing = processing.with_key_mode(key_mode).map_err(|error| { + PithosCliError::InvalidArgumentError(format!( + "--unique-keys needs an encrypted archive: {error}" + )) + })?; + processing.with_cipher(cipher.into()).map_err(|error| { + PithosCliError::InvalidArgumentError(format!( + "--cipher needs an encrypted archive: {error}" + )) + }) +} + #[tracing::instrument(level = "trace", skip())] fn run() -> Result<(), PithosCliError> { // Parse CLI parameter input @@ -401,7 +464,14 @@ fn run() -> Result<(), PithosCliError> { } } }, - PithosCommands::Create { cdc, plain, files } => { + PithosCommands::Create { + cdc, + block_size, + cipher, + unique_keys, + plain, + files, + } => { if plain && (cli.secret_key.is_some() || cli.public_keys.is_some()) { return Err(PithosCliError::InvalidArgumentError( "--plain conflicts with --secret-key and --public-keys".into(), @@ -413,10 +483,10 @@ fn run() -> Result<(), PithosCliError> { )); } + let chunking = chunking(cdc, block_size); let (options, processing) = if plain { ( - WriteOptions::base() - .with_chunking(cdc.map(Chunking::ContentDefined).unwrap_or_default()), + WriteOptions::base().with_chunking(chunking), ProcessingOptions::new(false, 0).expect("zero compression is valid"), ) } else { @@ -432,12 +502,12 @@ fn run() -> Result<(), PithosCliError> { .map(|path| load_public_key_from_pem(path)) .collect(); ( - WriteOptions::new(sender_key, reader_keys?) - .with_chunking(cdc.map(Chunking::ContentDefined).unwrap_or_default()), + WriteOptions::new(sender_key, reader_keys?).with_chunking(chunking), ProcessingOptions::default(), ) }; options.validate()?; + let processing = block_processing(processing, cipher, unique_keys)?; let manifest = build_input_manifest(&files)?; let output_path = match cli.output { @@ -532,6 +602,9 @@ fn run() -> Result<(), PithosCliError> { AppendCommands::Files { file, cdc, + block_size, + cipher, + unique_keys, durability, files, } => { @@ -542,13 +615,11 @@ fn run() -> Result<(), PithosCliError> { } let sender_key = required_private_key(&cli.secret_key)?; let reader_keys = required_recipient_keys(&cli.public_keys)?; - append_files( - &file, - AppendOptions::new(sender_key, reader_keys) - .with_chunking(cdc.map(Chunking::ContentDefined).unwrap_or_default()) - .with_durability(durability.into()), - &files, - )?; + let options = AppendOptions::new(sender_key, reader_keys) + .with_chunking(chunking(cdc, block_size)) + .with_durability(durability.into()); + let processing = block_processing(options.processing(), cipher, unique_keys)?; + append_files(&file, options.with_processing(processing), &files)?; } }, PithosCommands::Export { file, path, .. } => { diff --git a/crates/pithos/tests/append.rs b/crates/pithos/tests/append.rs index d9a8a61..5b85a82 100644 --- a/crates/pithos/tests/append.rs +++ b/crates/pithos/tests/append.rs @@ -258,3 +258,55 @@ fn append_cli_rejects_a_non_archive_target_without_panicking_or_mutating_archive assert_eq!(fs::read(&target).unwrap(), target_original); let _ = fs::remove_dir_all(temporary); } + +#[test] +fn append_files_block_options_round_trip_and_conflicts_leave_the_archive() { + let temporary = temporary(); + let archive = create_archive(&temporary); + let input = temporary.join("options.txt"); + fs::write(&input, b"appended with block options").unwrap(); + + let mut conflict = sender_command(); + add_recipient(&mut conflict, "sender"); + let original = fs::read(&archive).unwrap(); + let output = conflict + .args(["append", "files", "--file"]) + .arg(&archive) + .args(["--block-size", "1024", "--cdc", "64,256,1024"]) + .arg(&input) + .output() + .unwrap(); + assert!(!output.status.success()); + assert!(stderr(&output).contains("cannot be used with")); + assert_eq!(fs::read(&archive).unwrap(), original); + + let mut append = sender_command(); + add_recipient(&mut append, "sender"); + add_recipient(&mut append, "recipient1"); + let output = append + .args(["append", "files", "--file"]) + .arg(&archive) + .args([ + "--block-size", + "8", + "--unique-keys", + "--cipher", + "aes-256-gcm", + ]) + .arg(&input) + .output() + .unwrap(); + assert!(output.status.success(), "{}", stderr(&output)); + + let read = command() + .arg("--secret-key") + .arg(workspace_file("keys/recipient1_private.pem")) + .args(["read", "data"]) + .arg(&archive) + .arg("options.txt") + .output() + .unwrap(); + assert!(read.status.success(), "{}", stderr(&read)); + assert_eq!(read.stdout, b"appended with block options"); + let _ = fs::remove_dir_all(temporary); +} diff --git a/crates/pithos/tests/create.rs b/crates/pithos/tests/create.rs index 030330b..38bcd41 100644 --- a/crates/pithos/tests/create.rs +++ b/crates/pithos/tests/create.rs @@ -500,3 +500,147 @@ fn create_rejects_unreadable_input_before_publication_when_permissions_apply() { fs::set_permissions(&input, fs::Permissions::from_mode(0o644)).unwrap(); let _ = fs::remove_dir_all(temporary); } + +fn read_back(archive: &std::path::Path, path: &str) -> Vec { + let output = command() + .arg("--secret-key") + .arg(workspace_file("keys/recipient1_private.pem")) + .args(["read", "data"]) + .arg(archive) + .arg(path) + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + output.stdout +} + +#[test] +fn create_block_options_round_trip() { + let temporary = temporary(); + let input = temporary.join("input.bin"); + let content: Vec = (0..3000_u32).map(|value| (value % 251) as u8).collect(); + fs::write(&input, &content).unwrap(); + for (index, options) in [ + vec!["--cipher", "chacha20-poly1305"], + vec!["--cipher", "aes-256-gcm"], + vec!["--unique-keys"], + vec!["--block-size", "1024"], + vec![ + "--unique-keys", + "--cipher", + "aes-256-gcm", + "--block-size", + "512", + ], + ] + .into_iter() + .enumerate() + { + let archive = temporary.join(format!("options-{index}.pith")); + let output = create_command(&archive, &input) + .args(&options) + .output() + .unwrap(); + assert!( + output.status.success(), + "{options:?}: {}", + String::from_utf8_lossy(&output.stderr) + ); + assert_eq!(read_back(&archive, "input.bin"), content, "{options:?}"); + } + let _ = fs::remove_dir_all(temporary); +} + +#[test] +fn block_size_splits_and_unique_keys_store_equal_blocks() { + let temporary = temporary(); + let distinct = temporary.join("distinct.bin"); + let content: Vec = (0..4_u8).flat_map(|value| [value; 1024]).collect(); + fs::write(&distinct, &content).unwrap(); + let markers = |block_size: Option<&str>| { + let archive = temporary.join(format!("plain-{block_size:?}.pith")); + let mut create = command(); + create + .arg("--output") + .arg(&archive) + .args(["create", "--plain"]); + if let Some(size) = block_size { + create.args(["--block-size", size]); + } + assert!(create.arg(&distinct).status().unwrap().success()); + let bytes = fs::read(&archive).unwrap(); + bytes.windows(4).filter(|window| window == b"BLCK").count() + }; + assert_eq!(markers(None), 1); + assert_eq!(markers(Some("1024")), 4); + + let equal = temporary.join("equal.bin"); + fs::write(&equal, [7_u8; 4096]).unwrap(); + let archive_len = |unique: bool| { + let archive = temporary.join(format!("equal-{unique}.pith")); + let mut create = create_command(&archive, &equal); + create.args(["--block-size", "1024"]); + if unique { + create.arg("--unique-keys"); + } + assert!(create.status().unwrap().success()); + assert_eq!(read_back(&archive, "equal.bin"), [7_u8; 4096]); + fs::metadata(&archive).unwrap().len() + }; + // Three more stored blocks, each with a marker and at least 28 payload bytes. + assert!(archive_len(true) >= archive_len(false) + 3 * 32); + let _ = fs::remove_dir_all(temporary); +} + +#[test] +fn invalid_block_options_fail_without_output() { + let temporary = temporary(); + let input = temporary.join("input.txt"); + fs::write(&input, b"input").unwrap(); + for (index, (options, message)) in [ + ( + vec!["--plain", "--unique-keys"], + "--unique-keys needs an encrypted archive", + ), + ( + vec!["--plain", "--cipher", "aes-256-gcm"], + "--cipher needs an encrypted archive", + ), + ( + vec!["--plain", "--block-size", "0"], + "invalid fixed block size", + ), + (vec!["--block-size", "0"], "invalid fixed block size"), + ( + vec!["--block-size", "1024", "--cdc", "64,256,1024"], + "cannot be used with", + ), + (vec!["--cipher", "des"], "invalid value"), + ] + .into_iter() + .enumerate() + { + let output = temporary.join(format!("invalid-{index}.pith")); + let mut create = if options.contains(&"--plain") { + let mut create = command(); + create + .arg("--output") + .arg(&output) + .arg("create") + .arg(&input); + create + } else { + create_command(&output, &input) + }; + let result = create.args(&options).output().unwrap(); + let stderr = String::from_utf8_lossy(&result.stderr); + assert!(!result.status.success(), "{options:?}"); + assert!(stderr.contains(message), "{options:?}: {stderr}"); + assert!(!output.exists(), "{options:?}"); + } + let _ = fs::remove_dir_all(temporary); +} diff --git a/crates/pithos_lib/src/archive/append.rs b/crates/pithos_lib/src/archive/append.rs index c97ad02..364b846 100644 --- a/crates/pithos_lib/src/archive/append.rs +++ b/crates/pithos_lib/src/archive/append.rs @@ -52,6 +52,11 @@ impl AppendOptions { self } + /// The processing used for appended content. + pub fn processing(&self) -> ProcessingOptions { + self.processing + } + pub(crate) fn validate_recipients(&self) -> Result<(), PithosError> { WriteOptions::new(self.access_key.duplicate(), self.recipients.clone()).validate() } From 3dfc01a7ef5bb18fe9ceb3721c288feb58cf96ab Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 22:49:40 +0200 Subject: [PATCH 089/166] fix: validate an append before buffering the encoded directory --- crates/pithos_lib/src/archive/writer.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/crates/pithos_lib/src/archive/writer.rs b/crates/pithos_lib/src/archive/writer.rs index bc39bfb..e43badb 100644 --- a/crates/pithos_lib/src/archive/writer.rs +++ b/crates/pithos_lib/src/archive/writer.rs @@ -1420,9 +1420,14 @@ impl ArchiveWriter { self.validate_required_access_records()?; self.seal_recipient_lists()?; self.validate_publishable()?; - let bytes = directory::encode_complete_directory(&self.directory)?; if let Some(snapshot) = &self.append_snapshot { - let span = Span::new(self.sink.offset, bytes.len() as u64)?; + // Count the directory length without keeping its bytes during validation. + let mut counter = CountingSink { + sink: io::sink(), + offset: 0, + }; + directory::encode_directory(&self.directory, &mut counter)?; + let span = Span::new(self.sink.offset, counter.offset)?; let child = validated_segment_from_directory( self.version, &self.directory, @@ -1431,6 +1436,7 @@ impl ArchiveWriter { )?; snapshot.validate_prospective_child(child)?; } + let bytes = directory::encode_complete_directory(&self.directory)?; self.sink .write_all(&bytes) .map_err(SerializationError::from)?; From f3778a3c8680ae4e23be5f8f122abab5ac87d94f Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 22:49:41 +0200 Subject: [PATCH 090/166] test: fail the append directory hook after a partial directory write --- crates/pithos_lib/src/fs/append.rs | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/crates/pithos_lib/src/fs/append.rs b/crates/pithos_lib/src/fs/append.rs index 3d707a8..93e1acd 100644 --- a/crates/pithos_lib/src/fs/append.rs +++ b/crates/pithos_lib/src/fs/append.rs @@ -450,6 +450,8 @@ struct AppendSink { failure: Option, #[cfg(test)] after_block_marker: bool, + #[cfg(test)] + directory_started: bool, } impl AppendSink { @@ -465,6 +467,8 @@ impl AppendSink { failure, #[cfg(test)] after_block_marker: false, + #[cfg(test)] + directory_started: false, } } @@ -485,6 +489,23 @@ impl AppendSink { impl Write for AppendSink { fn write(&mut self, bytes: &[u8]) -> io::Result { + #[cfg(test)] + let bytes = if self.failure == Some(AppendFailurePoint::DuringDirectory) { + if self.directory_started { + return Err(io::Error::other( + "injected append failure after 8 directory bytes", + )); + } + // Accept only the directory marker, so the next write fails inside the directory. + self.directory_started = bytes.starts_with(b"PITHOSDR"); + if self.directory_started { + &bytes[..8] + } else { + bytes + } + } else { + bytes + }; let accepted = self.file.write(bytes)?; if accepted > 0 { self.mutated.store(true, Ordering::Relaxed); @@ -492,7 +513,6 @@ impl Write for AppendSink { #[cfg(test)] if accepted == bytes.len() { let is_marker = bytes == b"BLCK"; - let is_directory = bytes.starts_with(b"PITHOSDR"); let after_payload = self.after_block_marker && !is_marker; self.after_block_marker = is_marker; let fail = match self.failure { @@ -503,7 +523,6 @@ impl Write for AppendSink { | AppendFailurePoint::RollbackFlush | AppendFailurePoint::MetadataQuery, ) if after_payload => true, - Some(AppendFailurePoint::DuringDirectory) if is_directory => true, _ => false, }; if fail { @@ -758,6 +777,9 @@ mod tests { ), "unexpected error at {point:?}: {error:?}" ); + if point == AppendFailurePoint::DuringDirectory { + assert!(format!("{error:?}").contains("after 8 directory bytes")); + } assert_eq!( std::fs::read(&fixture.archive).unwrap(), original, From 33055cb32bc64668602e3953858fc06a4e29b365 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 22:49:41 +0200 Subject: [PATCH 091/166] test: check stored block flags and version 1.0 appends in the CLI --- crates/pithos/tests/append.rs | 75 +++++++++++++++++++++++++++ crates/pithos/tests/common/mod.rs | 86 +++++++++++++++++++++++++++++++ crates/pithos/tests/create.rs | 18 +++++++ 3 files changed, 179 insertions(+) create mode 100644 crates/pithos/tests/common/mod.rs diff --git a/crates/pithos/tests/append.rs b/crates/pithos/tests/append.rs index 5b85a82..c5eb08e 100644 --- a/crates/pithos/tests/append.rs +++ b/crates/pithos/tests/append.rs @@ -1,3 +1,5 @@ +mod common; + use std::fs; use std::path::{Path, PathBuf}; use std::process::{Command, Output}; @@ -297,6 +299,11 @@ fn append_files_block_options_round_trip_and_conflicts_leave_the_archive() { .output() .unwrap(); assert!(output.status.success(), "{}", stderr(&output)); + let flags = common::block_flags(&fs::read(&archive).unwrap()); + assert!(!flags.is_empty()); + for flag in flags { + assert_eq!(flag & 0x30, 0x30, "flags {flag:#04x}"); + } let read = command() .arg("--secret-key") @@ -310,3 +317,71 @@ fn append_files_block_options_round_trip_and_conflicts_leave_the_archive() { assert_eq!(read.stdout, b"appended with block options"); let _ = fs::remove_dir_all(temporary); } + +#[test] +fn append_files_rejects_invalid_block_options_without_mutation() { + let temporary = temporary(); + let input = temporary.join("input.txt"); + fs::write(&input, b"rejected payload").unwrap(); + let base = temporary.join("base.txt"); + fs::write(&base, b"version 1.0 base").unwrap(); + let legacy = temporary.join("legacy.pith"); + assert!( + command() + .arg("--output") + .arg(&legacy) + .args(["create", "--plain"]) + .arg(&base) + .status() + .unwrap() + .success() + ); + // A base archive has no grants, so only its header version differs from 1.0. + let mut bytes = fs::read(&legacy).unwrap(); + bytes[4..6].copy_from_slice(&[0x01, 0x00]); + fs::write(&legacy, &bytes).unwrap(); + let current = create_archive(&temporary); + + for (archive, options, message) in [ + (&legacy, vec!["--unique-keys"], "require format version 1.1"), + ( + &legacy, + vec!["--cipher", "aes-256-gcm"], + "require format version 1.1", + ), + ( + ¤t, + vec!["--block-size", "0"], + "invalid fixed block size", + ), + ] { + let original = fs::read(archive).unwrap(); + let mut append = sender_command(); + add_recipient(&mut append, "recipient1"); + let output = append + .args(["append", "files", "--file"]) + .arg(archive) + .args(&options) + .arg(&input) + .output() + .unwrap(); + let message_text = stderr(&output); + assert!(!output.status.success(), "{options:?}"); + assert!( + message_text.contains(message), + "{options:?}: {message_text}" + ); + assert_eq!(fs::read(archive).unwrap(), original, "{options:?}"); + } + + let mut append = sender_command(); + add_recipient(&mut append, "recipient1"); + let output = append + .args(["append", "files", "--file"]) + .arg(&legacy) + .arg(&input) + .output() + .unwrap(); + assert!(output.status.success(), "{}", stderr(&output)); + let _ = fs::remove_dir_all(temporary); +} diff --git a/crates/pithos/tests/common/mod.rs b/crates/pithos/tests/common/mod.rs new file mode 100644 index 0000000..78dc2ab --- /dev/null +++ b/crates/pithos/tests/common/mod.rs @@ -0,0 +1,86 @@ +//! A minimal decoder for the block descriptors of a terminal directory. + +/// Returns the processing flags of every block descriptor in the terminal directory. +pub fn block_flags(archive: &[u8]) -> Vec { + let footer = &archive[archive.len() - 12..archive.len() - 4]; + let directory_len = u64::from_be_bytes(footer.try_into().unwrap()) as usize; + let mut reader = Reader(&archive[archive.len() - directory_len..]); + assert_eq!(reader.take(8), b"PITHOSDR"); + if reader.byte() == 1 { + reader.varint(); + reader.varint(); + } + for _ in 0..reader.varint() { + reader.varint(); + reader.string(); + reader.byte(); + match reader.byte() { + 0 => reader.string(), + 1 => { + let count = reader.varint(); + reader.take(64 * count); + } + 2 => { + for _ in 0..reader.varint() { + reader.varint(); + reader.string(); + } + } + tag => panic!("unknown block list tag {tag}"), + } + for _ in 0..4 { + reader.varint(); + } + for _ in 0..reader.varint() { + reader.varint(); + reader.varint(); + } + if reader.byte() == 1 { + reader.string(); + } + } + (0..reader.varint()) + .map(|_| { + reader.take(32); + for _ in 0..3 { + reader.varint(); + } + let flags = reader.byte(); + if reader.byte() == 1 { + reader.string(); + } + flags + }) + .collect() +} + +struct Reader<'a>(&'a [u8]); + +impl<'a> Reader<'a> { + fn take(&mut self, len: usize) -> &'a [u8] { + let (head, tail) = self.0.split_at(len); + self.0 = tail; + head + } + + fn byte(&mut self) -> u8 { + self.take(1)[0] + } + + fn varint(&mut self) -> usize { + let mut value = 0; + for shift in (0..64).step_by(7) { + let byte = self.byte(); + value |= usize::from(byte & 0x7f) << shift; + if byte & 0x80 == 0 { + return value; + } + } + panic!("varint is too long") + } + + fn string(&mut self) { + let len = self.varint(); + self.take(len); + } +} diff --git a/crates/pithos/tests/create.rs b/crates/pithos/tests/create.rs index 38bcd41..0beea67 100644 --- a/crates/pithos/tests/create.rs +++ b/crates/pithos/tests/create.rs @@ -1,3 +1,5 @@ +mod common; + use std::fs; use std::process::Command; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -551,6 +553,22 @@ fn create_block_options_round_trip() { String::from_utf8_lossy(&output.stderr) ); assert_eq!(read_back(&archive, "input.bin"), content, "{options:?}"); + let aes = if options.contains(&"aes-256-gcm") { + 0x20 + } else { + 0 + }; + let unique = if options.contains(&"--unique-keys") { + 0x10 + } else { + 0 + }; + let expected = aes | unique; + let flags = common::block_flags(&fs::read(&archive).unwrap()); + assert!(!flags.is_empty()); + for flag in flags { + assert_eq!(flag & 0x30, expected, "{options:?}: flags {flag:#04x}"); + } } let _ = fs::remove_dir_all(temporary); } From ac7550d868f0fc8b6905a413714ae56a8d0e7aca Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 22:49:42 +0200 Subject: [PATCH 092/166] test: add a conformance matrix for ciphers, key modes, flags and 1.0 grants --- crates/pithos_lib/tests/conformance.rs | 398 +++++++++++++++++++++++++ 1 file changed, 398 insertions(+) create mode 100644 crates/pithos_lib/tests/conformance.rs diff --git a/crates/pithos_lib/tests/conformance.rs b/crates/pithos_lib/tests/conformance.rs new file mode 100644 index 0000000..0985cb7 --- /dev/null +++ b/crates/pithos_lib/tests/conformance.rs @@ -0,0 +1,398 @@ +//! Conformance matrix for the version 1.1 rules, using only the public API. + +mod common; + +use common::util::{private_key, public_key}; +use pithos_lib::archive::{ + AccessKeys, AppendOptions, Archive, ArchivePath, ArchiveWriter, BlockKeyMode, Chunking, + EntryKind, EntryMetadata, OpenOptions, PayloadCipher, Piece, PieceEncoder, ProcessingOptions, + WriteOptions, compose, +}; +use pithos_lib::error::{DeserializationError, PithosError}; +use pithos_lib::fs::{append_files, grant_readers}; +use pithos_lib::source::MemorySource; +use std::io::Cursor; +use std::path::{Path, PathBuf}; + +const BLOCK: usize = 1024; +/// `BLCK`, a 12-byte nonce, one uncompressed block and a 16-byte tag. +const STORED_BLOCK: usize = 4 + 12 + BLOCK + 16; + +fn content(seed: u8, len: usize) -> Vec { + (0..len) + .map(|index| (index as u8).wrapping_mul(31) ^ seed ^ (index >> 8) as u8) + .collect() +} + +fn keys(names: &[&str]) -> AccessKeys { + names.iter().fold(AccessKeys::new(), |keys, name| { + keys.with_key(private_key(name)) + }) +} + +fn open(bytes: Vec, keys: AccessKeys) -> Archive { + Archive::open( + MemorySource::new(bytes), + OpenOptions::default().with_access_keys(keys), + ) + .unwrap() +} + +fn read(archive: &Archive, path: &str) -> Vec { + let mut output = Vec::new(); + archive.copy_to(path, &mut output).unwrap(); + output +} + +fn available(archive: &Archive, path: &str) -> bool { + match archive.entry(path).unwrap().unwrap().kind { + EntryKind::File { available, .. } | EntryKind::Metadata { available, .. } => available, + other => panic!("{path} is not content: {other:?}"), + } +} + +fn encrypted(level: u8) -> ProcessingOptions { + ProcessingOptions::new(true, level).unwrap() +} + +fn aes(options: ProcessingOptions) -> ProcessingOptions { + options.with_cipher(PayloadCipher::Aes256Gcm).unwrap() +} + +fn unique(options: ProcessingOptions) -> ProcessingOptions { + options.with_key_mode(BlockKeyMode::Unique).unwrap() +} + +/// Writes an encrypted archive in 1 KiB blocks, one file per `(path, processing, content)`. +fn write_archive(files: &[(&str, ProcessingOptions, &[u8])]) -> Vec { + let options = WriteOptions::new(private_key("sender"), vec![public_key("recipient1")]) + .with_chunking(Chunking::Fixed(BLOCK)); + let mut writer = ArchiveWriter::create(Vec::new(), options).unwrap(); + for (path, processing, bytes) in files { + writer + .add_file( + ArchivePath::new(path).unwrap(), + EntryMetadata::new(0, 0, 0o644), + *processing, + None, + Cursor::new(bytes), + ) + .unwrap(); + } + writer.finish().unwrap() +} + +/// A plain base archive with one file. Its descriptor is easy to patch. +fn plain_archive(path: &str, bytes: &[u8]) -> Vec { + let mut writer = ArchiveWriter::create(Vec::new(), WriteOptions::base()).unwrap(); + writer + .add_file( + ArchivePath::new(path).unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(false, 0).unwrap(), + None, + Cursor::new(bytes), + ) + .unwrap(); + writer.finish().unwrap() +} + +fn set_version(bytes: &mut [u8], minor: u8) { + assert_eq!(&bytes[..5], b"PITH\x01"); + bytes[5] = minor; +} + +fn terminal_directory_start(bytes: &[u8]) -> usize { + let footer = &bytes[bytes.len() - 12..]; + bytes.len() - u64::from_be_bytes(footer[..8].try_into().unwrap()) as usize +} + +fn read_uleb(bytes: &[u8], position: &mut usize) -> u64 { + let mut value = 0; + for shift in (0..64).step_by(7) { + let byte = bytes[*position]; + *position += 1; + value |= u64::from(byte & 0x7f) << shift; + if byte & 0x80 == 0 { + return value; + } + } + panic!("unterminated ULEB128"); +} + +/// Replaces the flags of the descriptor for `hash` in the terminal directory and refreshes +/// the CRC. The descriptor follows the block-list copy of the hash, so it is the last one. +fn patch_flags(bytes: &mut [u8], hash: [u8; 32], flags: u8) { + let start = terminal_directory_start(bytes); + let directory = &bytes[start..]; + let found = directory + .windows(32) + .rposition(|window| window == hash) + .unwrap(); + let mut position = start + found + 32; + for _ in 0..3 { + read_uleb(bytes, &mut position); + } + bytes[position] = flags; + refresh_crc(bytes); +} + +fn refresh_crc(bytes: &mut [u8]) { + let start = terminal_directory_start(bytes); + let crc_at = bytes.len() - 4; + let crc = crc32fast::hash(&bytes[start..crc_at]); + bytes[crc_at..].copy_from_slice(&crc.to_be_bytes()); +} + +fn piece(key_id: u64, recipients: &[&str], processing: ProcessingOptions, part: &[u8]) -> Part { + let recipients = recipients.iter().map(|name| public_key(name)).collect(); + let encoder = PieceEncoder::new(key_id, recipients, processing).unwrap(); + encode_piece(encoder.with_block_size(1000).unwrap(), part) +} + +/// Writes `part` in fragments that match neither blocks nor BLAKE3 chunks. +fn encode_piece(mut encoder: PieceEncoder, part: &[u8]) -> Part { + let mut stored = Vec::new(); + for fragment in part.chunks(333) { + stored.extend(encoder.write(fragment).unwrap()); + } + stored.extend(encoder.flush().unwrap()); + Part { + stored, + piece: encoder.finish().unwrap(), + } +} + +struct Part { + stored: Vec, + piece: Piece, +} + +/// Composes the parts from their stored records only, as a node without keys would. +fn assemble(parts: &[Part]) -> (Vec, Option<[u8; 32]>) { + let pieces = parts + .iter() + .map(|part| Piece::from_bytes(&part.piece.to_bytes()).unwrap()) + .collect::>(); + let composition = compose( + ArchivePath::new("object").unwrap(), + EntryMetadata::new(0, 0, 0o644), + &pieces, + ) + .unwrap(); + let mut archive = composition.header().to_vec(); + for part in parts { + archive.extend_from_slice(&part.stored); + } + archive.extend_from_slice(composition.directory()); + assert_eq!(archive.len() as u64, composition.archive_len()); + (archive, composition.content_hash()) +} + +fn write_temporary(directory: &Path, name: &str, bytes: &[u8]) -> PathBuf { + let path = directory.join(name); + std::fs::write(&path, bytes).unwrap(); + path +} + +#[test] +fn one_archive_and_one_file_mix_both_ciphers() { + let shared = content(1, BLOCK); + let chacha_file = [shared.clone(), content(2, BLOCK)].concat(); + let aes_file = [shared.clone(), content(3, BLOCK)].concat(); + let bytes = write_archive(&[ + ("chacha", encrypted(0), &chacha_file), + ("aes", aes(encrypted(0)), &aes_file), + ]); + // The AES-256-GCM file reuses the ChaCha20-Poly1305 descriptor of the shared block, so + // the archive is one stored block smaller than with distinct content. + let distinct = write_archive(&[ + ("chacha", encrypted(0), &chacha_file), + ( + "aes", + aes(encrypted(0)), + &[content(4, BLOCK), content(3, BLOCK)].concat(), + ), + ]); + assert!(distinct.len() >= bytes.len() + STORED_BLOCK); + + let archive = open(bytes.clone(), keys(&["recipient1"])); + assert_eq!(read(&archive, "chacha"), chacha_file); + assert_eq!(read(&archive, "aes"), aes_file); + let mut range = Vec::new(); + archive + .copy_range_to("aes", 1000..1100, &mut range) + .unwrap(); + assert_eq!(range, aes_file[1000..1100]); + + // An AES-256-GCM append reuses the base block that is sealed with ChaCha20-Poly1305: + // the appended segment holds only the new short block and the directory. + let temporary = tempfile::tempdir().unwrap(); + let path = write_temporary(temporary.path(), "archive.pith", &bytes); + let appended = [shared, content(5, 300)].concat(); + let source = write_temporary(temporary.path(), "appended", &appended); + append_files( + &path, + AppendOptions::new(private_key("sender"), vec![public_key("recipient1")]) + .with_chunking(Chunking::Fixed(BLOCK)) + .with_processing(aes(encrypted(0))), + &[source], + ) + .unwrap(); + let grown = std::fs::read(&path).unwrap(); + assert!(grown.len() < bytes.len() + STORED_BLOCK + (4 + 12 + 300 + 16)); + let archive = open(grown, keys(&["recipient1"])); + assert_eq!(read(&archive, "chacha"), chacha_file); + assert_eq!(read(&archive, "aes"), aes_file); + assert_eq!(read(&archive, "appended"), appended); +} + +#[test] +fn unique_keys_with_aes_store_every_block_and_read_back() { + let repeated = content(6, BLOCK); + let file = [ + repeated.clone(), + repeated.clone(), + content(7, BLOCK), + repeated, + content(8, 100), + ] + .concat(); + let short_block = 4 + 12 + 100 + 16; + let convergent = write_archive(&[("first", aes(encrypted(0)), &file)]); + let processing = unique(aes(encrypted(0))); + let one = write_archive(&[("first", processing, &file)]); + let two = write_archive(&[("first", processing, &file), ("second", processing, &file)]); + // No repeat is deduplicated, within a file or across files. + assert!(one.len() >= convergent.len() + 2 * STORED_BLOCK); + assert!(two.len() >= one.len() + 4 * STORED_BLOCK + short_block); + + let compressed = unique(aes(encrypted(3))); + let three = write_archive(&[("first", compressed, &file), ("second", processing, &file)]); + for bytes in [two, three] { + let archive = open(bytes, keys(&["recipient1"])); + for path in ["first", "second"] { + assert_eq!(read(&archive, path), file); + let mut range = Vec::new(); + archive.copy_range_to(path, 1000..3100, &mut range).unwrap(); + assert_eq!(range, file[1000..3100]); + } + } +} + +#[test] +fn invalid_flag_combinations_are_rejected_in_both_versions() { + let plaintext = content(9, 40); + let hash = *blake3::hash(&plaintext).as_bytes(); + let patched = |minor: u8, flags: u8| { + let mut bytes = plain_archive("data", &plaintext); + set_version(&mut bytes, minor); + patch_flags(&mut bytes, hash, flags); + Archive::open(MemorySource::new(bytes), OpenOptions::default()) + }; + for minor in [0, 1] { + // Bit 4 or 5 without encryption, and the reserved bits 6 and 7. + for flags in [0x10, 0x20, 0x30, 0x13, 0x40, 0x80, 0xc8] { + assert!( + matches!( + patched(minor, flags), + Err(PithosError::ProcessingRequiresEncryption(_) + | PithosError::UnsupportedProcessingFlags(_) + | PithosError::Deserialization( + DeserializationError::InvalidProcessingFlags(_) + )) + ), + "version 1.{minor} accepted {flags:#04x}" + ); + } + for flags in [0x00, 0x03, 0x08] { + assert!(patched(minor, flags).is_ok(), "{minor} {flags:#04x}"); + } + } + // Either bit with encryption is valid only in version 1.1. + for flags in [0x18, 0x28, 0x38, 0x3b] { + assert!(matches!( + patched(0, flags), + Err(PithosError::UnsupportedProcessingFlags(_)) + )); + // The payload is plain, so decryption fails before any output. + let archive = patched(1, flags).unwrap(); + let mut output = Vec::new(); + assert!(archive.copy_to("data", &mut output).is_err()); + assert!(output.is_empty()); + } + + // Block lists sealed in pieces are valid only in version 1.1. + let (mut bytes, _) = assemble(&[piece(1, &["recipient1"], encrypted(0), b"piece")]); + set_version(&mut bytes, 0); + assert!(matches!( + Archive::open(MemorySource::new(bytes), OpenOptions::default()), + Err(PithosError::UnsupportedBlockListPieces) + )); + + let plain = ProcessingOptions::new(false, 0).unwrap(); + assert!(matches!( + plain.with_key_mode(BlockKeyMode::Unique), + Err(PithosError::ProcessingRequiresEncryption(0x10)) + )); + assert!(matches!( + plain.with_cipher(PayloadCipher::Aes256Gcm), + Err(PithosError::ProcessingRequiresEncryption(0x20)) + )); +} + +#[test] +fn version_1_0_archives_keep_the_1_0_grant_rules() { + let temporary = tempfile::tempdir().unwrap(); + let mut base = plain_archive("base.txt", b"base payload"); + set_version(&mut base, 0); + let path = write_temporary(temporary.path(), "legacy.pith", &base); + let source = write_temporary(temporary.path(), "appended.txt", b"appended payload"); + append_files( + &path, + AppendOptions::new(private_key("sender"), vec![public_key("recipient1")]), + &[source], + ) + .unwrap(); + grant_readers( + &path, + AppendOptions::new(private_key("recipient1"), vec![public_key("recipient2")]), + &[1], + ) + .unwrap(); + let mut bytes = std::fs::read(&path).unwrap(); + assert_eq!(&bytes[..6], b"PITH\x01\x00"); + for reader in ["recipient1", "recipient2"] { + let archive = open(bytes.clone(), keys(&[reader])); + assert_eq!(read(&archive, "base.txt"), b"base payload"); + assert_eq!(read(&archive, "appended.txt"), b"appended payload"); + } + + // Under the version 1.1 rules the same grants derive other keys and open nothing. + set_version(&mut bytes, 1); + for reader in ["recipient1", "recipient2"] { + assert_no_access(&bytes, reader, "appended.txt"); + } + + // The reverse holds for a version 1.1 grant read under the version 1.0 rules. + let mut current = write_archive(&[("data", encrypted(0), b"current payload")]); + assert_eq!(¤t[..6], b"PITH\x01\x01"); + assert_eq!( + read(&open(current.clone(), keys(&["recipient1"])), "data"), + b"current payload" + ); + set_version(&mut current, 0); + assert_no_access(¤t, "recipient1", "data"); +} + +/// A grant that fails authentication for its recipient either fails the open or leaves the +/// content unavailable. Both outcomes release nothing. +fn assert_no_access(bytes: &[u8], reader: &str, path: &str) { + match Archive::open( + MemorySource::new(bytes.to_vec()), + OpenOptions::default().with_access_keys(keys(&[reader])), + ) { + Ok(archive) => assert!(!available(&archive, path)), + Err(error) => assert!(matches!(error, PithosError::Crypt(_)), "{error:?}"), + } +} From 85ad8a92d2e60fec8c8d384e425d39f579bb08cc Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 22:49:42 +0200 Subject: [PATCH 093/166] test: cover content hash boundaries, block limits, piece grants and digests --- crates/pithos_lib/tests/conformance.rs | 191 ++++++++++++++++++++++++- 1 file changed, 189 insertions(+), 2 deletions(-) diff --git a/crates/pithos_lib/tests/conformance.rs b/crates/pithos_lib/tests/conformance.rs index 0985cb7..0ced21a 100644 --- a/crates/pithos_lib/tests/conformance.rs +++ b/crates/pithos_lib/tests/conformance.rs @@ -5,8 +5,8 @@ mod common; use common::util::{private_key, public_key}; use pithos_lib::archive::{ AccessKeys, AppendOptions, Archive, ArchivePath, ArchiveWriter, BlockKeyMode, Chunking, - EntryKind, EntryMetadata, OpenOptions, PayloadCipher, Piece, PieceEncoder, ProcessingOptions, - WriteOptions, compose, + EntryKind, EntryMetadata, OpenLimits, OpenOptions, PayloadCipher, Piece, PieceEncoder, + ProcessingOptions, WriteOptions, compose, }; use pithos_lib::error::{DeserializationError, PithosError}; use pithos_lib::fs::{append_files, grant_readers}; @@ -396,3 +396,190 @@ fn assert_no_access(bytes: &[u8], reader: &str, path: &str) { Err(error) => assert!(matches!(error, PithosError::Crypt(_)), "{error:?}"), } } + +fn recipient() -> Vec { + vec![public_key("recipient1")] +} + +#[test] +fn composed_content_hash_matches_the_full_read_at_piece_boundaries() { + let file = content(10, 9000); + let cases: [&[usize]; 6] = [ + &[2048, 4096, 1024], + &[3072, 10], + &[1024, 0, 1024], + &[2048, 0], + &[999], + &[], + ]; + for sizes in cases { + let mut offset = 0; + let mut parts = Vec::new(); + for (index, size) in sizes.iter().enumerate() { + let encoder = PieceEncoder::new(index as u64 + 1, recipient(), encrypted(0)) + .unwrap() + .with_block_size(1000) + .unwrap(); + let encoder = encoder.with_content_offset(offset as u64).unwrap(); + parts.push(encode_piece(encoder, &file[offset..offset + size])); + offset += size; + } + let (bytes, hash) = assemble(&parts); + let output = read(&open(bytes, keys(&["recipient1"])), "object"); + assert_eq!(output, file[..offset]); + assert_eq!(hash, Some(*blake3::hash(&output).as_bytes()), "{sizes:?}"); + } + + // A part that ends inside a BLAKE3 chunk leaves no valid offset for the next part. + let first = &file[..1500]; + let second = &file[1500..2500]; + let encoder = || PieceEncoder::new(2, recipient(), encrypted(0)).unwrap(); + assert!(matches!( + encoder().with_content_offset(1500), + Err(PithosError::InvalidContentOffset(1500)) + )); + for recorded in [1024, 2048] { + let later = encoder().with_content_offset(recorded).unwrap(); + let parts = [ + piece(1, &["recipient1"], encrypted(0), first), + encode_piece(later, second), + ]; + let (bytes, hash) = assemble(&parts); + assert_eq!(hash, None); + assert_eq!( + read(&open(bytes, keys(&["recipient1"])), "object"), + file[..2500] + ); + } +} + +#[test] +fn oversized_blocks_are_rejected_by_open_limits() { + let file = content(11, 1000); + let (composed, _) = assemble(&[piece(1, &["recipient1"], encrypted(0), &file)]); + let written = write_archive(&[("object", encrypted(0), &file)]); + // One block: 1000 plaintext bytes, stored as a nonce, the ciphertext and a tag. + let stored = 12 + 1000 + 16; + for bytes in [composed, written] { + let attempt = |limits: OpenLimits| { + let mut output = Vec::new(); + let result = Archive::open( + MemorySource::new(bytes.clone()), + OpenOptions::default() + .with_limits(limits) + .with_access_keys(keys(&["recipient1"])), + ) + .and_then(|archive| archive.copy_to("object", &mut output)); + (result, output) + }; + let limited = |stored_bytes: u64, decoded_bytes: u64| OpenLimits { + max_stored_block_bytes: stored_bytes, + max_decoded_block_bytes: decoded_bytes, + ..OpenLimits::default() + }; + for limits in [limited(stored - 1, 1000), limited(stored, 999)] { + let (result, output) = attempt(limits); + assert!(matches!(result, Err(PithosError::LimitExceeded { .. }))); + assert!(output.is_empty()); + } + let (result, output) = attempt(limited(stored, 1000)); + result.unwrap(); + assert_eq!(output, file); + } +} + +#[test] +fn a_piece_without_a_grant_leaves_the_whole_file_unavailable() { + let first = content(12, 2500); + let second = content(13, 1500); + let parts = [ + piece(1, &["recipient1", "recipient2"], encrypted(3), &first), + piece(2, &["recipient1"], aes(encrypted(0)), &second), + ]; + let (bytes, _) = assemble(&parts); + let archive = open(bytes.clone(), keys(&["recipient2"])); + assert!(!available(&archive, "object")); + let mut output = Vec::new(); + assert!(archive.copy_to("object", &mut output).is_err()); + assert!( + archive + .copy_range_to("object", 0..100, &mut output) + .is_err() + ); + assert!(output.is_empty()); + + let archive = open(bytes, keys(&["recipient2", "recipient1"])); + assert_eq!(read(&archive, "object"), [first, second].concat()); +} + +#[test] +fn metadata_digest_covers_every_directory_of_the_chain() { + let temporary = tempfile::tempdir().unwrap(); + let base = write_archive(&[("data", encrypted(2), &content(14, 3000))]); + let base_start = terminal_directory_start(&base); + let base_hash = *blake3::hash(&base[base_start..]).as_bytes(); + let first = *blake3::hash(&base_hash).as_bytes(); + assert_eq!( + open(base.clone(), AccessKeys::new()).metadata_digest(), + first + ); + + // The digest covers only directories, so it is the same under both versions. + let mut legacy = plain_archive("data", b"plain"); + let plain_digest = open(legacy.clone(), AccessKeys::new()).metadata_digest(); + set_version(&mut legacy, 0); + assert_eq!( + open(legacy, AccessKeys::new()).metadata_digest(), + plain_digest + ); + + let path = write_temporary(temporary.path(), "archive.pith", &base); + let source = write_temporary(temporary.path(), "appended", b"appended payload"); + append_files( + &path, + AppendOptions::new(private_key("sender"), vec![public_key("recipient1")]), + &[source], + ) + .unwrap(); + let appended = std::fs::read(&path).unwrap(); + let terminal = *blake3::hash(&appended[terminal_directory_start(&appended)..]).as_bytes(); + let second = *blake3::hash(&[base_hash, terminal].concat()).as_bytes(); + assert_ne!(second, first); + + let with_digest = |digest: [u8; 32], keys: AccessKeys| { + Archive::open( + MemorySource::new(appended.clone()), + OpenOptions::default() + .with_access_keys(keys) + .with_expected_metadata_digest(digest), + ) + }; + let archive = with_digest(second, keys(&["recipient1"])).unwrap(); + assert_eq!(archive.metadata_digest(), second); + assert_eq!(read(&archive, "appended"), b"appended payload"); + for keys in [AccessKeys::new(), keys(&["recipient1"])] { + assert!(matches!( + with_digest(first, keys), + Err(PithosError::MetadataDigestMismatch) + )); + } + + // Changing one metadata byte, with a fresh CRC, changes the digest. + let mut changed = appended.clone(); + let start = terminal_directory_start(&changed); + let name = start + + changed[start..] + .windows(8) + .position(|window| window == b"appended") + .unwrap(); + changed[name] = b'A'; + refresh_crc(&mut changed); + assert!(Archive::open(MemorySource::new(changed.clone()), OpenOptions::default()).is_ok()); + assert!(matches!( + Archive::open( + MemorySource::new(changed), + OpenOptions::default().with_expected_metadata_digest(second), + ), + Err(PithosError::MetadataDigestMismatch) + )); +} From 3609d9881ab4336fa4f1f54b59cec7581d4dd872 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 22:49:42 +0200 Subject: [PATCH 094/166] fix: complete the version 1.1 tables and rules in the spec --- spec/PITHOS_1.0.0_draft.md | 84 ++++++++++++++++++++++++++------------ 1 file changed, 59 insertions(+), 25 deletions(-) diff --git a/spec/PITHOS_1.0.0_draft.md b/spec/PITHOS_1.0.0_draft.md index 6a5eb2c..8331750 100644 --- a/spec/PITHOS_1.0.0_draft.md +++ b/spec/PITHOS_1.0.0_draft.md @@ -9,7 +9,7 @@ This document specifies the Pithos file format using the key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" as described in [RFC 2119](https://www.rfc-editor.org/rfc/rfc2119). -Pithos is an append-only archive format designed for efficient storage and sharing of scientific data. It combines content-defined deduplication, convergent encryption, and flexible metadata support optimized for object storage systems. +Pithos is an append-only archive format designed for efficient storage and sharing of scientific data. It combines block-level deduplication, convergent encryption, and flexible metadata support optimized for object storage systems. **Illustrative data model.** Rust declarations in this document illustrate the data model only; implementations may use different declarations. Normative @@ -83,7 +83,7 @@ to a version 1.0 archive uses the version 1.0 rules. ## 2. Core Design Principles 1. **Append-only architecture**: New data and metadata MUST be appended, never modifying existing content -2. **Content-addressed storage**: All blocks MUST be identified by Blake3 hashes enabling deduplication +2. **Content-addressed storage**: All blocks MUST be identified by BLAKE3 block hashes; convergent blocks enable deduplication 3. **Encrypted recipient grants**: Encrypted recipient data protects its file-key grants from parties that cannot decrypt it 4. **Flexible metadata**: Metadata MUST be stored as regular files with special type markers @@ -421,8 +421,8 @@ Normal reading MUST reject trailing bytes, truncation, underflow, a false marker Each Directory contains the entries introduced by its segment. To construct the effective archive, readers merge the selected chain from the base Directory to the terminal Directory. An append adds file entries, block -descriptors, relationship definitions, and recipient grants. Version 1.0 has -no deletion, replacement, or tombstone. +descriptors, relationship definitions, and recipient grants. Neither version +has deletion, replacement, or tombstone. File IDs and paths MUST be unique across the selected chain. Writers assign file ID 0 to the first file and assign each later file ID as the current maximum ID plus @@ -621,7 +621,7 @@ The directory file sequence is a vector. Each record is encoded as follows: | FileEntry body field | Bytes stored in the file | | --- | --- | | `file_type` | FileType encoded form above | -| `block_data` | Tag `00` and byte vector, or tag `01` and vector of `block_hash[32] || block_key[32]` tuples | +| `block_data` | Tag `00` and byte vector; tag `01` and vector of `block_hash[32] || block_key[32]` tuples; or, in version 1.1, tag `02` and vector of ULEB128 `key_id` plus byte-vector items (Section 4.4.2) | | `created` | ULEB128 `u64` | | `modified` | ULEB128 `u64` | | `file_size` | ULEB128 `u64` | @@ -638,14 +638,15 @@ of the FileEntry body. | `file_type` | `block_data` | `file_size` | `symlink_target` | | --- | --- | --- | --- | | `Directory` | MUST be `Decrypted` with an empty list | MUST be `0` | MUST be absent (`00`) | -| `Data` | Any state permitted by the archive version | MUST equal the sum of referenced effective descriptors' `original_size` values when the block list is available | MUST be absent (`00`) | -| `Metadata` | Any state permitted by the archive version | MUST equal the sum of referenced effective descriptors' `original_size` values when the block list is available | MUST be absent (`00`) | +| `Data` | `Encrypted` or `Decrypted`; in version 1.1 also `Pieces` | MUST equal the sum of referenced effective descriptors' `original_size` values when the block list is available | MUST be absent (`00`) | +| `Metadata` | `Encrypted` or `Decrypted`; in version 1.1 also `Pieces` | MUST equal the sum of referenced effective descriptors' `original_size` values when the block list is available | MUST be absent (`00`) | | `Symlink` | MUST be `Decrypted` with an empty list | MUST be `0` | MUST be present (`01`) | Readers MUST reject a combination that violates this table. For encrypted block lists, the content-size validation for `Data` and `Metadata` occurs once the block list is available; until then, the content is unavailable as specified in -Section 8.2. +Section 8.2. A block list sealed in pieces is available once every piece is +decrypted. `permissions` stores the low 12 bits of a POSIX mode, in the range `0o0000..=0o7777`; file-type bits are not stored. Readers MUST reject a value @@ -858,13 +859,16 @@ until its required transforms, size, and hash have all been verified. | Repetition in one file | `a: [(H, K), (H, K)]` | Valid. `a` reconstructs as the plaintext for `H` followed by itself. | | Conflicting key | `a: [(H, K), (H, L)]` | Invalid. The file is rejected. | | Later-segment size conflict | Base Directory has `H` with `original_size: 4`; an appended Directory has `H` with `original_size: 5` | Invalid. The appended Directory is rejected; no effective descriptor is selected from it. | +| Repetition across pieces | `a` sealed in pieces `[(H, K)]` and `[(H, K)]` | Valid. The whole sequence is `[(H, K), (H, K)]`. | +| Conflicting key across pieces | `a` sealed in pieces `[(H, K)]` and `[(H, L)]` | Invalid. The rule applies to the whole sequence; the file is rejected. | +| Unique keys and equal plaintext | `a: [(H, K), (J, L)]`; both blocks set bit 4, have equal plaintext, and have random keys `K` and `L` | Valid. `H` and `J` differ because each is keyed by its own block key; each has its own descriptor. | ### 5.3 Convergent Encryption Encryption is optional. An implementation that supports encryption MUST use X25519, SHAKE256, and ChaCha20-Poly1305 as specified here. Version 1.1 adds -BLAKE3 key derivation and, for blocks that set ProcessingFlags bit 5, -AES-256-GCM. +HKDF-SHA256 for recipient grants, BLAKE3 key derivation and, for blocks that set +ProcessingFlags bit 5, AES-256-GCM. All encryption keys are 32 bytes. ChaCha20-Poly1305 and AES-256-GCM each use a 12-byte nonce and produce a 16-byte authentication tag. Every encrypted value is stored as @@ -960,7 +964,7 @@ A writer that stores the plaintext payload rather than compressed data MUST encode compression value `0`. Conforming encoders need not produce byte-identical Zstandard output. Appendix B -will provide decode-direction vectors containing stored compressed bytes, +provides decode-direction vectors containing stored compressed bytes, `original_size`, the expected plaintext, and its expected 32-byte BLAKE3 hash. Those vectors MUST decode to the expected plaintext with different supported Zstandard versions. @@ -981,12 +985,28 @@ the number of blocks. ### 6.2 Writing Operations -1. Write file header +1. Write the file header: version 1.1 for a new archive. An append writes no + header and keeps the version of the archive it extends (Section 1.3). 2. Process files in correct directory order 3. Split content into blocks (fixed-size or content-defined) -4. Deduplicate blocks by hash (unique-key blocks never repeat a hash) -5. Write a directory, including its encryption sections when present -6. Validate complete structure +4. Choose the ProcessingFlags of each new block. In version 1.1 an encrypted + block MAY also set bit 4 (unique key) or bit 5 (AES-256-GCM), or both. An + append to a version 1.0 archive MUST NOT set either bit and SHOULD reject + such a request before writing any bytes. +5. Deduplicate convergent blocks by hash. A reused descriptor keeps its own + flags, including its cipher. Unique-key blocks are never deduplicated. +6. Store each file's block list decrypted, encrypted under a file key, or, in + version 1.1, sealed in pieces under piece keys (Section 4.4.2). +7. Write a directory, including its encryption sections when present. A + recipient needs a grant for the file key or for every piece key of each file + it may read (Section 4.5.3). +8. Validate complete structure + +A version 1.1 writer MAY also join independently written pieces without their +keys: it writes the header, the stored blocks of every piece in order, and one +Directory. That Directory has one file record whose `Pieces` state lists the +sealed piece lists in order, one descriptor for each distinct block hash, and +the unchanged piece grants (Appendix A). ### 6.3 Directory Tree Operations @@ -1001,7 +1021,7 @@ When archiving directory trees: Directory CRC-32 detects accidental corruption of the serialized Directory bytes it covers; it is not authentication. An application that keeps the metadata digest (Section 4.3.4) in trusted storage gains metadata integrity for its own -later reads; the digest does not authenticate archive origin to anyone else. Pithos 1.0 provides no archive-wide origin +later reads; the digest does not authenticate archive origin to anyone else. Neither version provides archive-wide origin authentication or metadata integrity: an unauthenticated Directory can replace both a block hash and its referenced content. AEAD authenticates each encrypted value's ciphertext, but its empty AAD does not bind that value to its surrounding @@ -1105,19 +1125,19 @@ uncompressed or unencrypted bytes. | Unique-key local: local, encryption `1`, bit 4 `1` | Listed | Readable only with block-encryption and unique-block-key capability and any encrypted-list capability needed to obtain its block key; otherwise unavailable | | AES-256-GCM local: local, encryption `1`, bit 5 `1` | Listed | Readable only with block-encryption and AES-256-GCM capability and any encrypted-list capability needed to obtain its block key; otherwise unavailable | | External: BlockLocation `External` | Listed | Readable only with external-storage capability and every capability required by its flags and data states; otherwise unavailable | -| Combined: compression and encryption, at either location | Listed | Readable only when every indicated capability is supported; otherwise unavailable | +| Combined: any combination of compression, encryption, unique key, and AES-256-GCM, at either location | Listed | Readable only when every indicated capability is supported; otherwise unavailable | -For the table, an encrypted BlockDataState requires encrypted-block-list -capability, and an encrypted RecipientData required to obtain its file key or -piece keys requires encrypted-recipient-list capability. Content sealed in -pieces is unavailable until the key of every piece is available. A content read that needs either -unsupported list form is unavailable even when its block flags themselves are -otherwise supported. +For the table, an encrypted BlockDataState, which includes a `Pieces` state, +requires encrypted-block-list capability, and an encrypted RecipientData +required to obtain its file key or piece keys requires encrypted-recipient-list +capability. Content sealed in pieces is unavailable until the key of every piece +is available. A content read that needs either unsupported list form is +unavailable even when its block flags themselves are otherwise supported. Readers MUST reject an unsupported header version and any unknown tag rather than list the archive, because the structure of such input is not known. -Pithos 1.0 intentionally permits decrypted block and recipient list variants +Both versions intentionally permit decrypted block and recipient list variants and an empty Directory `encryption` vector. ### 8.3 Platform-Specific Considerations @@ -1143,10 +1163,16 @@ Non-POSIX platforms MAY retain them as metadata without an ACL mapping. ## 9. Future Extensions The format reserves space for future extensions: +- Header versions other than `0x0100` and `0x0101` - FileType values 4-255 - ProcessingFlags bits 6-7 +- Relationship IDs 10 through 999, for future standard relationships - Custom relationship types starting at 1000 - BlockDataState tags `03` through `ff` +- BlockLocation and RecipientData tags `02` through `ff` + +Custom relationship types are valid as defined in Section 4.4.4. Readers of this +version MUST reject every other value listed here (Sections 3.1 and 8.2). A paged block index for very large files is a planned extension. It would let a reader fetch only the descriptors and block keys a byte range needs. Version 1.1 @@ -1168,7 +1194,15 @@ Extensions MUST maintain backwards compatibility for reading. - Version 1.0: `0x0100` - Version 1.1: `0x0101` -### 10.3 Default Values +### 10.3 Key-Derivation Context Strings + +Version 1.1 uses these ASCII strings, without a terminator: + +- `pithos 1.1 block identity`: block identity subkey (Section 5.2) +- `pithos 1.1 aes-256-gcm payload`: AES-256-GCM payload key (Section 5.3) +- `pithos 1.1 recipient grant`: HKDF info prefix of a grant key (Section 5.3) + +### 10.4 Default Values - Current timestamp: Unix seconds since epoch - Default file permissions: `0o644` From 000800a531d7d70c5f545ccce1f6504148a7b0f7 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 22:49:43 +0200 Subject: [PATCH 095/166] fix: index the version 1.1 vectors and conformance cases in the spec --- spec/PITHOS_1.0.0_draft.md | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/spec/PITHOS_1.0.0_draft.md b/spec/PITHOS_1.0.0_draft.md index 8331750..99e8ad0 100644 --- a/spec/PITHOS_1.0.0_draft.md +++ b/spec/PITHOS_1.0.0_draft.md @@ -1266,9 +1266,10 @@ Hex offsets are zero-based file offsets. Each hex line contains at most 16 bytes `0000:` is an offset, not encoded data. Integers without a fixed-width suffix use shortest ULEB128 in these deterministic vectors, although Section 3.1 also permits bounded non-minimal forms. `u64be` and `u32be` are big-endian. The -canonical archives are version 1.0 archives, which every reader accepts, so their -header is always -`50 49 54 48 01 00` (`PITH`, version `0x0100`); the other magic values are +canonical archives other than CV-PIECES-HELLO-766 are version 1.0 archives, which +every reader accepts, so their header is +`50 49 54 48 01 00` (`PITH`, version `0x0100`). CV-PIECES-HELLO-766 is a version +1.1 archive with header `50 49 54 48 01 01`; the other magic values are `42 4c 43 4b` (`BLCK`) and `50 49 54 48 4f 53 44 52` (`PITHOSDR`). CRC values use Section 4.3's CRC-32/ISO-HDLC definition. As a check on an @@ -1582,11 +1583,21 @@ archive. ### B.5 Validation Index +Entries marked *case* are conformance cases without fixed bytes. A test builds +each one with a writer, or by patching the header, a flags byte, or a Directory +byte and then refreshing the CRC. + | Condition class | Authoritative section | Vector ID | Required result | | --- | --- | --- | --- | | Header, integer, tags, and flags | 3.1, 4.1, 4.2.3-4.2.4 | AV-ULEB-NONMINIMAL, RV-ULEB, RV-FLAGS, RV-UNKNOWN-TAG | Accept or reject archive as stated | +| Version 1.1 processing flags | 1.3, 4.2.3, 6.2 | RV-FLAGS; *case:* bit 4 or 5 without bit 3, in either version; *case:* bit 4 or 5 with bit 3 in a version 1.0 archive; *case:* reserved bit 6 or 7; *case:* an append to a version 1.0 archive that requests bit 4 or 5 | Reject archive; the append fails before writing any bytes | | Directory framing, CRC, and chain | 4.3, 4.3.1, 4.3.2 | CV-BASE-EMPTY-146, CV-APPEND-EMPTY-28, RV-PARENT, RV-UNDERFLOW, RV-TRAILING, RV-CRC, RV-CROSS-SIZE | Valid archive or reject archive as stated | +| Metadata digest | 4.3.4 | *case:* digest of one Directory and of a two-Directory chain; *case:* the same digest under both header versions; *case:* an outdated expected digest, with and without keys; *case:* one changed Directory byte with a refreshed CRC | Accept the matching digest; reject archive on a mismatch | | Entries and paths | 4.3.3, 4.4.1, 4.4.3 | CV-LOCAL-HELLO-279, RV-DUPLICATES, RV-PATH, RV-SYMLINK, RV-FILETYPE, RV-PERMISSIONS | Valid archive or reject archive as stated | | Block locations and extents | 4.2.2, 4.2.5, 4.2.6, 8.2 | CV-LOCAL-HELLO-279, RV-EXTENT, RV-SHORT-ENCRYPTED, RV-EXTERNAL | Valid archive, reject archive, content read fails before output, or content unavailable as stated | +| Resource limits | 3.2 | *case:* a block whose `stored_size` or `original_size` is one byte above a configured limit, in a written and in a composed archive | Fail before any output; a limit equal to the size reads the block | | Content transforms and hashes | 5.2, 5.3, 5.4 | PV-ZSTD-HELLO, PV-ZSTD-TEXT, PV-RECIPIENT-WRAP-01, PV-RECIPIENT-WRAP-11, PV-UNIQUE-KEY-HELLO, PV-AES-GCM-HELLO | Decode/decrypt to stated output | -| Version 1.1 pieces and grants | 1.3, 4.4.2, 5.3 | CV-PIECES-HELLO-766, PV-RECIPIENT-WRAP-11 | Valid archive; content readable only with the granted key | +| Ciphers and key modes | 5.2, 5.3 | PV-UNIQUE-KEY-HELLO, PV-AES-GCM-HELLO; *case:* both ciphers in one archive and in one file, where a reused descriptor keeps its cipher across files and appends; *case:* unique keys with AES-256-GCM, with and without compression, where every repeated block is stored again | Read every file and range as written | +| Version 1.1 pieces and grants | 1.3, 4.4.2, 5.3, 8.2 | CV-PIECES-HELLO-766, PV-RECIPIENT-WRAP-11; *case:* tag `02` in a version 1.0 archive; *case:* a reader granted only some of a file's piece keys | Valid archive; content readable only with every piece key; reject tag `02` in version 1.0; a partly granted file is unavailable | +| Version 1.0 compatibility | 1.3, 5.3 | CV-BASE-EMPTY-146, CV-LOCAL-HELLO-279, PV-RECIPIENT-WRAP-01; *case:* an append and a reader grant on a version 1.0 archive keep header `0x0100` and the version 1.0 grant construction | Readable under the version 1.0 rules; the same grants open nothing under the version 1.1 rules, and the reverse | +| Whole-file hash of joined parts (informative) | Appendix A | *case:* aligned parts, a short last part, empty parts, one part, and an empty file; *case:* a part that ends inside a 1024-byte chunk | The hash equals the BLAKE3 of the full read; no hash for the misaligned part | From aeabf4f6d30647177e7e46485199c24e0fd71c6f Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 22:49:43 +0200 Subject: [PATCH 096/166] fix: fail the open on an addressed grant that fails authentication in the spec and tests --- crates/pithos_lib/tests/conformance.rs | 154 +++++++++++++++++++++++-- spec/PITHOS_1.0.0_draft.md | 24 +++- 2 files changed, 165 insertions(+), 13 deletions(-) diff --git a/crates/pithos_lib/tests/conformance.rs b/crates/pithos_lib/tests/conformance.rs index 0ced21a..1335286 100644 --- a/crates/pithos_lib/tests/conformance.rs +++ b/crates/pithos_lib/tests/conformance.rs @@ -8,6 +8,7 @@ use pithos_lib::archive::{ EntryKind, EntryMetadata, OpenLimits, OpenOptions, PayloadCipher, Piece, PieceEncoder, ProcessingOptions, WriteOptions, compose, }; +use pithos_lib::crypto::CryptoError; use pithos_lib::error::{DeserializationError, PithosError}; use pithos_lib::fs::{append_files, grant_readers}; use pithos_lib::source::MemorySource; @@ -137,6 +138,84 @@ fn patch_flags(bytes: &mut [u8], hash: [u8; 32], flags: u8) { refresh_crc(bytes); } +/// Encrypted grant ranges of the terminal directory, found by walking its encoded form +/// (Section 4.3). +struct Layout { + grants: Vec>, +} + +fn layout(bytes: &[u8]) -> Layout { + let skip_bytes = |at: &mut usize| { + let len = read_uleb(bytes, at) as usize; + *at += len; + }; + let skip_option = |at: &mut usize| { + *at += 1; + if bytes[*at - 1] == 1 { + skip_bytes(at); + } + }; + let mut at = terminal_directory_start(bytes) + 8; + at += 1; + if bytes[at - 1] == 1 { + read_uleb(bytes, &mut at); + read_uleb(bytes, &mut at); + } + for _ in 0..read_uleb(bytes, &mut at) { + read_uleb(bytes, &mut at); + skip_bytes(&mut at); + at += 2; + match bytes[at - 1] { + 0 => skip_bytes(&mut at), + 1 => at += 64 * read_uleb(bytes, &mut at) as usize, + _ => { + for _ in 0..read_uleb(bytes, &mut at) { + read_uleb(bytes, &mut at); + skip_bytes(&mut at); + } + } + } + for _ in 0..4 { + read_uleb(bytes, &mut at); + } + for _ in 0..2 * read_uleb(bytes, &mut at) { + read_uleb(bytes, &mut at); + } + skip_option(&mut at); + } + for _ in 0..read_uleb(bytes, &mut at) { + at += 32; + for _ in 0..3 { + read_uleb(bytes, &mut at); + } + at += 1; + skip_option(&mut at); + } + for _ in 0..read_uleb(bytes, &mut at) { + read_uleb(bytes, &mut at); + skip_bytes(&mut at); + } + let mut grants = Vec::new(); + for _ in 0..read_uleb(bytes, &mut at) { + at += 32; + for _ in 0..read_uleb(bytes, &mut at) { + at += 33; + if bytes[at - 1] == 0 { + let len = read_uleb(bytes, &mut at) as usize; + grants.push(at..at + len); + at += len; + } else { + for _ in 0..read_uleb(bytes, &mut at) { + read_uleb(bytes, &mut at); + at += 32; + } + } + } + } + assert_eq!(at + 12, bytes.len(), "the walk must end at the footer"); + Layout { grants } +} + fn refresh_crc(bytes: &mut [u8]) { let start = terminal_directory_start(bytes); let crc_at = bytes.len() - 4; @@ -368,10 +447,10 @@ fn version_1_0_archives_keep_the_1_0_grant_rules() { assert_eq!(read(&archive, "appended.txt"), b"appended payload"); } - // Under the version 1.1 rules the same grants derive other keys and open nothing. + // Under the version 1.1 rules the same grants derive other keys and fail authentication. set_version(&mut bytes, 1); for reader in ["recipient1", "recipient2"] { - assert_no_access(&bytes, reader, "appended.txt"); + assert_grant_rejected(&bytes, reader); } // The reverse holds for a version 1.1 grant read under the version 1.0 rules. @@ -382,18 +461,73 @@ fn version_1_0_archives_keep_the_1_0_grant_rules() { b"current payload" ); set_version(&mut current, 0); - assert_no_access(¤t, "recipient1", "data"); + assert_grant_rejected(¤t, "recipient1"); } -/// A grant that fails authentication for its recipient either fails the open or leaves the -/// content unavailable. Both outcomes release nothing. -fn assert_no_access(bytes: &[u8], reader: &str, path: &str) { - match Archive::open( +/// A grant addressed to the reader that fails authentication fails the open. +fn assert_grant_rejected(bytes: &[u8], reader: &str) { + let result = Archive::open( MemorySource::new(bytes.to_vec()), OpenOptions::default().with_access_keys(keys(&[reader])), - ) { - Ok(archive) => assert!(!available(&archive, path)), - Err(error) => assert!(matches!(error, PithosError::Crypt(_)), "{error:?}"), + ); + assert!( + matches!( + result, + Err(PithosError::Crypt(CryptoError::AuthenticationFailed)) + ), + "{reader}" + ); +} + +/// Without a grant the open succeeds, and the content is unavailable and releases nothing. +fn assert_missing_grant(bytes: &[u8], reader: &str, path: &str) { + let archive = open(bytes.to_vec(), keys(&[reader])); + assert!(!available(&archive, path)); + let mut output = Vec::new(); + assert!(archive.copy_to(path, &mut output).is_err()); + assert!(archive.copy_range_to(path, 0..1, &mut output).is_err()); + assert!(output.is_empty()); +} + +#[test] +fn an_addressed_grant_that_fails_authentication_fails_the_open() { + let temporary = tempfile::tempdir().unwrap(); + let base = write_archive(&[("data", encrypted(0), b"base payload")]); + let path = write_temporary(temporary.path(), "archive.pith", &base); + let source = write_temporary(temporary.path(), "appended", b"appended payload"); + append_files( + &path, + AppendOptions::new(private_key("sender"), vec![public_key("recipient1")]), + &[source], + ) + .unwrap(); + let mut bytes = std::fs::read(&path).unwrap(); + for reader in ["recipient1", "sender"] { + let archive = open(bytes.clone(), keys(&[reader])); + assert_eq!(read(&archive, "data"), b"base payload"); + assert_eq!(read(&archive, "appended"), b"appended payload"); + } + + // The base grants still give access to `data`, but the changed appended grants fail. + // The sender key reaches them through sender-key recovery. + let grants = layout(&bytes).grants; + assert!(!grants.is_empty()); + for grant in grants { + bytes[grant.end - 1] ^= 1; + } + refresh_crc(&mut bytes); + for reader in ["recipient1", "sender"] { + assert_grant_rejected(&bytes, reader); + } + assert!(matches!( + Archive::open( + MemorySource::new(bytes.clone()), + OpenOptions::default().with_access_keys(keys(&["recipient2", "recipient1"])), + ), + Err(PithosError::Crypt(CryptoError::AuthenticationFailed)) + )); + for path in ["data", "appended"] { + assert_missing_grant(&bytes, "recipient2", path); } } diff --git a/spec/PITHOS_1.0.0_draft.md b/spec/PITHOS_1.0.0_draft.md index 99e8ad0..8533856 100644 --- a/spec/PITHOS_1.0.0_draft.md +++ b/spec/PITHOS_1.0.0_draft.md @@ -933,6 +933,15 @@ deliberately reuse a nonce under the same key. It MUST authenticate and decrypt an encrypted value successfully before using its plaintext or releasing output derived from it. +**Grant authentication.** A recipient record is addressed to a reader's private +key when its recipient public key is that key's public key. When that public key +is also the sender public key of an EncryptionSection, every record in that +section is addressed to it as well (sender-key recovery). A reader MUST try +every encrypted record addressed to one of its keys. If any of them fails +authentication, the reader MUST fail the open, even when another record grants +the same or other keys. A key for which no record exists only makes the content +it would unlock unavailable (Section 8.2). + **Nonce scope of block payloads.** A ChaCha20-Poly1305 payload uses the block key as its AEAD key, and an AES-256-GCM payload uses the derived payload key, so the two ciphers never share an AEAD key. A unique key seals exactly one payload. A @@ -1035,6 +1044,12 @@ recipient counts, ciphertext lengths, and the segment timing of grants are visible. `RecipientData::Decrypted` exposes its file-key grants and provides no grant confidentiality. +An addressed grant that fails authentication makes the open fail (Section 5.3). +This has an availability cost: with untrusted metadata, anyone who can append to +or change the archive can add a forged grant addressed to a reader and so prevent +that reader from opening unrelated content. Applications that keep the metadata +digest (Section 4.3.4) detect such changes. + Version 1.0 recipient wrapping uses a raw static X25519 shared secret directly as its AEAD key, with no KDF, label, or AAD. That construction has no domain separation, and each static sender-recipient key pair has one nonce-collision @@ -1131,8 +1146,10 @@ For the table, an encrypted BlockDataState, which includes a `Pieces` state, requires encrypted-block-list capability, and an encrypted RecipientData required to obtain its file key or piece keys requires encrypted-recipient-list capability. Content sealed in pieces is unavailable until the key of every piece -is available. A content read that needs either unsupported list form is -unavailable even when its block flags themselves are otherwise supported. +is available. A missing grant only makes content unavailable; a grant addressed +to the reader that fails authentication makes the open fail (Section 5.3). A +content read that needs either unsupported list form is unavailable even when +its block flags themselves are otherwise supported. Readers MUST reject an unsupported header version and any unknown tag rather than list the archive, because the structure of such input is not known. @@ -1599,5 +1616,6 @@ byte and then refreshing the CRC. | Content transforms and hashes | 5.2, 5.3, 5.4 | PV-ZSTD-HELLO, PV-ZSTD-TEXT, PV-RECIPIENT-WRAP-01, PV-RECIPIENT-WRAP-11, PV-UNIQUE-KEY-HELLO, PV-AES-GCM-HELLO | Decode/decrypt to stated output | | Ciphers and key modes | 5.2, 5.3 | PV-UNIQUE-KEY-HELLO, PV-AES-GCM-HELLO; *case:* both ciphers in one archive and in one file, where a reused descriptor keeps its cipher across files and appends; *case:* unique keys with AES-256-GCM, with and without compression, where every repeated block is stored again | Read every file and range as written | | Version 1.1 pieces and grants | 1.3, 4.4.2, 5.3, 8.2 | CV-PIECES-HELLO-766, PV-RECIPIENT-WRAP-11; *case:* tag `02` in a version 1.0 archive; *case:* a reader granted only some of a file's piece keys | Valid archive; content readable only with every piece key; reject tag `02` in version 1.0; a partly granted file is unavailable | -| Version 1.0 compatibility | 1.3, 5.3 | CV-BASE-EMPTY-146, CV-LOCAL-HELLO-279, PV-RECIPIENT-WRAP-01; *case:* an append and a reader grant on a version 1.0 archive keep header `0x0100` and the version 1.0 grant construction | Readable under the version 1.0 rules; the same grants open nothing under the version 1.1 rules, and the reverse | +| Version 1.0 compatibility | 1.3, 5.3 | CV-BASE-EMPTY-146, CV-LOCAL-HELLO-279, PV-RECIPIENT-WRAP-01; *case:* an append and a reader grant on a version 1.0 archive keep header `0x0100` and the version 1.0 grant construction | Readable under the version 1.0 rules; under the version 1.1 rules the same grants fail authentication and the open fails, and the reverse | +| Grant authentication | 5.3, 7, 8.2 | *case:* one changed ciphertext byte in a grant addressed to the reader, while an older grant still gives access, for a recipient key and for a sender key through sender-key recovery; *case:* a reader with no grant | The open fails for an addressed grant that fails authentication; with no grant the open succeeds, the content is unavailable, and reads fail without output | | Whole-file hash of joined parts (informative) | Appendix A | *case:* aligned parts, a short last part, empty parts, one part, and an empty file; *case:* a part that ends inside a 1024-byte chunk | The hash equals the BLAKE3 of the full read; no hash for the misaligned part | From 05d5e5c0af79a04b644940ca9d6dc10146055b5c Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 22:49:43 +0200 Subject: [PATCH 097/166] test: prove compression in the unique key and AES-256-GCM case --- crates/pithos_lib/tests/conformance.rs | 34 +++++++++++++++++++++----- 1 file changed, 28 insertions(+), 6 deletions(-) diff --git a/crates/pithos_lib/tests/conformance.rs b/crates/pithos_lib/tests/conformance.rs index 1335286..a7655fa 100644 --- a/crates/pithos_lib/tests/conformance.rs +++ b/crates/pithos_lib/tests/conformance.rs @@ -138,9 +138,10 @@ fn patch_flags(bytes: &mut [u8], hash: [u8; 32], flags: u8) { refresh_crc(bytes); } -/// Encrypted grant ranges of the terminal directory, found by walking its encoded form -/// (Section 4.3). +/// Descriptor flags and encrypted grant ranges of the terminal directory, found by walking +/// its encoded form (Section 4.3). struct Layout { + flags: Vec, grants: Vec>, } @@ -183,11 +184,13 @@ fn layout(bytes: &[u8]) -> Layout { } skip_option(&mut at); } + let mut flags = Vec::new(); for _ in 0..read_uleb(bytes, &mut at) { at += 32; for _ in 0..3 { read_uleb(bytes, &mut at); } + flags.push(bytes[at]); at += 1; skip_option(&mut at); } @@ -213,7 +216,7 @@ fn layout(bytes: &[u8]) -> Layout { } } assert_eq!(at + 12, bytes.len(), "the walk must end at the footer"); - Layout { grants } + Layout { flags, grants } } fn refresh_crc(bytes: &mut [u8]) { @@ -346,12 +349,31 @@ fn unique_keys_with_aes_store_every_block_and_read_back() { assert!(one.len() >= convergent.len() + 2 * STORED_BLOCK); assert!(two.len() >= one.len() + 4 * STORED_BLOCK + short_block); + // Compressible blocks keep compression level 3 next to the unique-key and AES bits. + let text = |seed: u8, len: usize| { + let phrase = b"pithos stores compressible text. "; + (0..len) + .map(|index| phrase[index % phrase.len()] ^ seed) + .collect::>() + }; + let compressible = [ + text(1, BLOCK), + text(1, BLOCK), + text(2, BLOCK), + text(1, BLOCK), + text(3, 100), + ] + .concat(); let compressed = unique(aes(encrypted(3))); - let three = write_archive(&[("first", compressed, &file), ("second", processing, &file)]); - for bytes in [two, three] { + let three = write_archive(&[ + ("first", compressed, &compressible), + ("second", compressed, &compressible), + ]); + assert_eq!(layout(&three).flags, [0x3b; 10]); + for (bytes, file) in [(two, &file), (three, &compressible)] { let archive = open(bytes, keys(&["recipient1"])); for path in ["first", "second"] { - assert_eq!(read(&archive, path), file); + assert_eq!(read(&archive, path), *file); let mut range = Vec::new(); archive.copy_range_to(path, 1000..3100, &mut range).unwrap(); assert_eq!(range, file[1000..3100]); From d0dfebe900d2fcb936eb5e58e8303b56e01d5114 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 22:49:44 +0200 Subject: [PATCH 098/166] test: compare exact block regions instead of archive sizes --- crates/pithos_lib/tests/conformance.rs | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/crates/pithos_lib/tests/conformance.rs b/crates/pithos_lib/tests/conformance.rs index a7655fa..fe03ac3 100644 --- a/crates/pithos_lib/tests/conformance.rs +++ b/crates/pithos_lib/tests/conformance.rs @@ -108,6 +108,11 @@ fn terminal_directory_start(bytes: &[u8]) -> usize { bytes.len() - u64::from_be_bytes(footer[..8].try_into().unwrap()) as usize } +/// Bytes of a single-segment archive between the 6-byte header and its directory. +fn block_region(bytes: &[u8]) -> usize { + terminal_directory_start(bytes) - 6 +} + fn read_uleb(bytes: &[u8], position: &mut usize) -> u64 { let mut value = 0; for shift in (0..64).step_by(7) { @@ -287,7 +292,7 @@ fn one_archive_and_one_file_mix_both_ciphers() { ("aes", aes(encrypted(0)), &aes_file), ]); // The AES-256-GCM file reuses the ChaCha20-Poly1305 descriptor of the shared block, so - // the archive is one stored block smaller than with distinct content. + // the archive stores one block fewer than with distinct content. let distinct = write_archive(&[ ("chacha", encrypted(0), &chacha_file), ( @@ -296,7 +301,9 @@ fn one_archive_and_one_file_mix_both_ciphers() { &[content(4, BLOCK), content(3, BLOCK)].concat(), ), ]); - assert!(distinct.len() >= bytes.len() + STORED_BLOCK); + assert_eq!(block_region(&bytes), 3 * STORED_BLOCK); + assert_eq!(layout(&bytes).flags, [0x08, 0x08, 0x28]); + assert_eq!(block_region(&distinct), 4 * STORED_BLOCK); let archive = open(bytes.clone(), keys(&["recipient1"])); assert_eq!(read(&archive, "chacha"), chacha_file); @@ -322,7 +329,11 @@ fn one_archive_and_one_file_mix_both_ciphers() { ) .unwrap(); let grown = std::fs::read(&path).unwrap(); - assert!(grown.len() < bytes.len() + STORED_BLOCK + (4 + 12 + 300 + 16)); + assert_eq!( + terminal_directory_start(&grown) - bytes.len(), + 4 + 12 + 300 + 16 + ); + assert_eq!(layout(&grown).flags, [0x28]); let archive = open(grown, keys(&["recipient1"])); assert_eq!(read(&archive, "chacha"), chacha_file); assert_eq!(read(&archive, "aes"), aes_file); @@ -345,9 +356,12 @@ fn unique_keys_with_aes_store_every_block_and_read_back() { let processing = unique(aes(encrypted(0))); let one = write_archive(&[("first", processing, &file)]); let two = write_archive(&[("first", processing, &file), ("second", processing, &file)]); - // No repeat is deduplicated, within a file or across files. - assert!(one.len() >= convergent.len() + 2 * STORED_BLOCK); - assert!(two.len() >= one.len() + 4 * STORED_BLOCK + short_block); + // The convergent file stores its repeated block once. Unique keys store every block, + // within a file and across files, including the short last blocks. + assert_eq!(block_region(&convergent), 2 * STORED_BLOCK + short_block); + assert_eq!(block_region(&one), 4 * STORED_BLOCK + short_block); + assert_eq!(block_region(&two), 2 * (4 * STORED_BLOCK + short_block)); + assert_eq!(layout(&two).flags, [0x38; 10]); // Compressible blocks keep compression level 3 next to the unique-key and AES bits. let text = |seed: u8, len: usize| { From 03f351e56dc44e227ebb589fd516a6f3e4b93bdd Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 22:49:44 +0200 Subject: [PATCH 099/166] fix: limit the spec grant requirement to encrypted block lists --- spec/PITHOS_1.0.0_draft.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/spec/PITHOS_1.0.0_draft.md b/spec/PITHOS_1.0.0_draft.md index 8533856..335a29b 100644 --- a/spec/PITHOS_1.0.0_draft.md +++ b/spec/PITHOS_1.0.0_draft.md @@ -1006,9 +1006,10 @@ the number of blocks. flags, including its cipher. Unique-key blocks are never deduplicated. 6. Store each file's block list decrypted, encrypted under a file key, or, in version 1.1, sealed in pieces under piece keys (Section 4.4.2). -7. Write a directory, including its encryption sections when present. A - recipient needs a grant for the file key or for every piece key of each file - it may read (Section 4.5.3). +7. Write a directory, including its encryption sections when present. For + encrypted block lists, a recipient needs a grant for the file key or for + every piece key of each file it may read (Section 4.5.3). A decrypted block + list needs no file-key grant. 8. Validate complete structure A version 1.1 writer MAY also join independently written pieces without their From ed09c4066971dde26715c1dc231ab209d67ab89a Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sat, 3 Oct 2026 22:49:44 +0200 Subject: [PATCH 100/166] fix: require version 1.0 appends to reject new flags before writing --- spec/PITHOS_1.0.0_draft.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/PITHOS_1.0.0_draft.md b/spec/PITHOS_1.0.0_draft.md index 335a29b..e92e614 100644 --- a/spec/PITHOS_1.0.0_draft.md +++ b/spec/PITHOS_1.0.0_draft.md @@ -1000,7 +1000,7 @@ the number of blocks. 3. Split content into blocks (fixed-size or content-defined) 4. Choose the ProcessingFlags of each new block. In version 1.1 an encrypted block MAY also set bit 4 (unique key) or bit 5 (AES-256-GCM), or both. An - append to a version 1.0 archive MUST NOT set either bit and SHOULD reject + append to a version 1.0 archive MUST NOT set either bit and MUST reject such a request before writing any bytes. 5. Deduplicate convergent blocks by hash. A reused descriptor keeps its own flags, including its cipher. Unique-key blocks are never deduplicated. From 8e663e6421c226456db0c32bc1a295004d805e32 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 00:50:09 +0200 Subject: [PATCH 101/166] fix: reserve decrypted key lists once and wipe every buffer they leave --- crates/pithos_lib/src/archive/reader.rs | 32 +++- .../src/archive/reader_private_tests.rs | 4 +- crates/pithos_lib/src/crypto.rs | 26 ++- crates/pithos_lib/src/format/encryption.rs | 47 ++++- crates/pithos_lib/src/format/file_entry.rs | 164 +++++++++++++----- crates/pithos_lib/src/format/primitives.rs | 41 +++++ 6 files changed, 250 insertions(+), 64 deletions(-) diff --git a/crates/pithos_lib/src/archive/reader.rs b/crates/pithos_lib/src/archive/reader.rs index 3730df5..4c7d301 100644 --- a/crates/pithos_lib/src/archive/reader.rs +++ b/crates/pithos_lib/src/archive/reader.rs @@ -13,7 +13,7 @@ use crate::format::directory::Directory; use crate::format::encryption::RecipientData; use crate::format::file_entry::{BlockDataEntry, BlockDataState}; use crate::format::header::FormatVersion; -use crate::format::limits::DeserializationLimits; +use crate::format::limits::{DeserializationError, DeserializationLimits}; use crate::source::ArchiveSource; use std::collections::{BTreeMap, HashSet}; use std::io::Write; @@ -723,17 +723,37 @@ pub(super) fn resolve_block_lists( } let key_ids = pieces.iter().map(|piece| FileId(piece.key_id)).collect(); let pieces = std::mem::take(pieces); - // Each sealed piece is dropped after decryption and each plaintext after - // decoding, so only the final list and one piece buffer are held. + // The room for every piece is reserved once, so the keys never move to a new + // buffer. Each sealed piece and each plaintext is dropped after decoding. + let capacity = pieces.iter().try_fold(0u64, |total, piece| { + total.checked_add(crate::format::file_entry::sealed_block_list_capacity( + piece.sealed.len(), + ) as u64) + }); + let capacity = match capacity { + Some(capacity) if capacity <= *remaining_block_references => capacity, + actual => { + return Err(DeserializationError::LimitExceeded { + field: "block references", + limit: *remaining_block_references, + actual: actual.unwrap_or(u64::MAX), + } + .into()); + } + }; let mut entries = Zeroizing::new(Vec::new()); + crate::format::primitives::reserve_secret( + &mut entries, + capacity as usize, + "block references", + )?; for piece in pieces { let key = access .key(FileId(piece.key_id)) .expect("every piece key was checked above"); let mut decoded_limits = deserialization_limits(limits); decoded_limits.max_block_references = *remaining_block_references; - let plaintext = crypto::open_file_block_list(key, &piece.sealed)?; - drop(piece); + let plaintext = crypto::open_file_block_list(key, piece.sealed)?; crate::format::file_entry::append_decrypted_block_list( &plaintext, &decoded_limits, @@ -752,7 +772,7 @@ pub(super) fn resolve_block_lists( }; let mut decoded_limits = deserialization_limits(limits); decoded_limits.max_block_references = *remaining_block_references; - let plaintext = crypto::open_file_block_list(file_key, bytes)?; + let plaintext = crypto::open_file_block_list(file_key, std::mem::take(bytes))?; let entries = crate::format::file_entry::decode_decrypted_block_list_with_budget( &plaintext, &decoded_limits, diff --git a/crates/pithos_lib/src/archive/reader_private_tests.rs b/crates/pithos_lib/src/archive/reader_private_tests.rs index 1593996..4f604a9 100644 --- a/crates/pithos_lib/src/archive/reader_private_tests.rs +++ b/crates/pithos_lib/src/archive/reader_private_tests.rs @@ -1101,7 +1101,7 @@ fn decrypted_encrypted_block_list_accepts_a_non_minimal_count() { let BlockDataState::Encrypted(encrypted) = &mut file.block_data else { panic!("writer block list was not encrypted"); }; - let plaintext = crypto::open_file_block_list(&file_key, encrypted).unwrap(); + let plaintext = crypto::open_file_block_list(&file_key, encrypted.clone()).unwrap(); assert_eq!(plaintext[0], 1); let mut non_minimal = vec![0x81, 0x00]; non_minimal.extend_from_slice(&plaintext[1..]); @@ -1578,7 +1578,7 @@ fn split_into_pieces(path: &Path, key_ids: &[u64], granted: &[u64]) { let BlockDataState::Encrypted(sealed) = &entry.block_data else { panic!("fixture block list was not encrypted"); }; - let plaintext = crypto::open_file_block_list(&file_key, sealed).unwrap(); + let plaintext = crypto::open_file_block_list(&file_key, sealed.clone()).unwrap(); let entries = crate::format::file_entry::decode_decrypted_block_list_with_budget( &plaintext, &DeserializationLimits::default(), diff --git a/crates/pithos_lib/src/crypto.rs b/crates/pithos_lib/src/crypto.rs index b7bde72..d3b75b2 100644 --- a/crates/pithos_lib/src/crypto.rs +++ b/crates/pithos_lib/src/crypto.rs @@ -8,7 +8,7 @@ use crate::format::header::FormatVersion; use aes_gcm::Aes256Gcm; use chacha20poly1305::{ ChaCha20Poly1305, Key, KeyInit, Nonce, - aead::{self, Aead, AeadCore, Generate, Payload, consts::U12}, + aead::{self, Aead, AeadCore, AeadInOut, Generate, Payload, consts::U12}, }; use digest::{ExtendableOutput, Update, XofReader}; use hkdf::Hkdf; @@ -373,11 +373,31 @@ pub(crate) fn seal_file_block_list_with_nonce( seal_empty_aad(key.expose_for_protocol(), plaintext, nonce) } +/// Opens a sealed block list inside its own buffer, so the plaintext needs no second buffer. pub(crate) fn open_file_block_list( key: &FileKey, - payload: &[u8], + payload: Vec, ) -> Result>, CryptoError> { - open_empty_aad(key.expose_for_protocol(), payload) + if payload.len() < 15 { + return Err(CryptoError::EncryptedPayloadTooShort); + } + let len = payload + .len() + .checked_sub(28) + .ok_or(CryptoError::AuthenticationFailed)?; + let cipher = ChaCha20Poly1305::new_from_slice(key.expose_for_protocol()) + .map_err(|_| CryptoError::CipherInitialization)?; + let mut buffer = Zeroizing::new(payload); + let (nonce, sealed) = buffer.split_at_mut(12); + let (message, tag) = sealed.split_at_mut(len); + let nonce: [u8; 12] = (&*nonce).try_into().expect("12-byte nonce"); + let tag = aead::Tag::::try_from(&*tag).expect("16-byte tag"); + cipher + .decrypt_inout_detached(&Nonce::from(nonce), b"", message.into(), &tag) + .map_err(|_| CryptoError::AuthenticationFailed)?; + buffer.copy_within(12..12 + len, 0); + buffer.truncate(len); + Ok(buffer) } pub(crate) fn wrap_recipient_list_with_nonce( diff --git a/crates/pithos_lib/src/format/encryption.rs b/crates/pithos_lib/src/format/encryption.rs index 6c44673..f28385a 100644 --- a/crates/pithos_lib/src/format/encryption.rs +++ b/crates/pithos_lib/src/format/encryption.rs @@ -2,7 +2,7 @@ use crate::crypto::{self, SharedSecret}; use crate::error::PithosError; use crate::format::error::SerializationError; use crate::format::limits::{DeserializationError, DeserializationLimits}; -use crate::format::primitives::{bounded_len, reserve, write_len_prefix}; +use crate::format::primitives::{bounded_len, reserve, reserve_secret, write_len_prefix}; use indexmap::IndexMap; use integer_encoding::{VarIntReader, VarIntWriter}; use std::collections::HashMap; @@ -81,7 +81,7 @@ impl RecipientData { )); } RecipientData::Decrypted(entries) => { - let mut data_bytes = Zeroizing::new(Vec::with_capacity(1 + entries.len() * 40)); + let mut data_bytes = Zeroizing::new(Vec::with_capacity(10 + entries.len() * 42)); encode_decrypted_recipient_list(entries, &mut *data_bytes)?; let encrypted_data = crypto::wrap_recipient_list_with_nonce(&shared_key, &data_bytes, nonce)?; @@ -196,15 +196,25 @@ pub(crate) fn decode_decrypted_recipient_list_reader( limits.max_collection_entries, "recipient keys", )?; + read_recipient_entries(reader, count) +} + +/// Reads `count` entries straight into reserved slots, so keys leave no temporary copies. +fn read_recipient_entries( + reader: &mut R, + count: usize, +) -> Result { let mut entries = Zeroizing::new(Vec::new()); - reserve(&mut entries, count, "recipient keys")?; - let mut keys = HashMap::with_capacity(count); - for _ in 0..count { + reserve_secret(&mut entries, count, "recipient keys")?; + // The map holds positions rather than keys, so it keeps no secret copies. + let mut positions = HashMap::with_capacity(count); + for position in 0..count { let id = reader.read_varint::()?; entries.push((id, [0; 32])); - reader.read_exact(&mut entries.last_mut().expect("just pushed file key").1)?; - let key = entries.last().expect("just pushed file key").1; - if keys.insert(id, key).is_some_and(|existing| existing != key) { + reader.read_exact(&mut entries[position].1)?; + if let Some(previous) = positions.insert(id, position) + && entries[previous].1 != entries[position].1 + { return Err(DeserializationError::DuplicateRecipientFileId); } } @@ -216,7 +226,17 @@ pub(crate) fn decode_decrypted_recipient_list( limits: &DeserializationLimits, ) -> Result { let mut reader = std::io::Cursor::new(bytes); - let entries = decode_decrypted_recipient_list_reader(&mut reader, limits)?; + let count = bounded_len( + reader.read_varint::()?, + limits.max_collection_entries, + "recipient keys", + )?; + // Each entry takes at least 33 bytes, so a larger count fails before any allocation. + let available = bytes.len() as u64 - reader.position(); + if (count as u64).saturating_mul(33) > available { + return Err(DeserializationError::InvalidLength); + } + let entries = read_recipient_entries(&mut reader, count)?; if reader.position() != bytes.len() as u64 { return Err(DeserializationError::InvalidLength); } @@ -249,5 +269,14 @@ mod tests { decode_decrypted_recipient_list(&conflicting, &limits), Err(DeserializationError::DuplicateRecipientFileId) )); + + // A count that the remaining bytes cannot hold fails before any allocation. + let mut claimed = Vec::new(); + claimed.write_varint(1_000u64).unwrap(); + claimed.extend_from_slice(&[7; 33]); + assert!(matches!( + decode_decrypted_recipient_list(&claimed, &limits), + Err(DeserializationError::InvalidLength) + )); } } diff --git a/crates/pithos_lib/src/format/file_entry.rs b/crates/pithos_lib/src/format/file_entry.rs index 8e0ff63..cdbcc77 100644 --- a/crates/pithos_lib/src/format/file_entry.rs +++ b/crates/pithos_lib/src/format/file_entry.rs @@ -2,9 +2,10 @@ use crate::crypto::{self, FileKey}; use crate::error::PithosError; use crate::format::error::SerializationError; use crate::format::limits::{DeserializationError, DeserializationLimits}; -use crate::format::primitives::{bounded_len, decode_string, reserve, write_len_prefix}; +use crate::format::primitives::{ + bounded_len, decode_string, reserve, reserve_secret, write_len_prefix, +}; use integer_encoding::{VarIntReader, VarIntWriter}; -use std::collections::HashMap; use std::fmt::{Display, Formatter}; use std::io::{Read, Write}; use zeroize::Zeroizing; @@ -72,7 +73,7 @@ impl BlockDataState { )); } BlockDataState::Decrypted(entries) => { - let mut data_bytes = Zeroizing::new(Vec::with_capacity(1 + entries.len() * 64)); + let mut data_bytes = Zeroizing::new(Vec::with_capacity(10 + entries.len() * 64)); encode_decrypted_block_list(entries, &mut *data_bytes)?; let encrypted_data = crypto::seal_file_block_list_with_nonce(key, &data_bytes, nonce)?; @@ -88,16 +89,19 @@ impl BlockDataState { pub(crate) fn validate_unique_block_references( entries: &[BlockDataEntry], ) -> Result<(), PithosError> { + if has_conflicting_keys(entries) { + return Err(PithosError::DuplicateBlockReference); + } + Ok(()) +} + +fn has_conflicting_keys(entries: &[BlockDataEntry]) -> bool { let mut order = (0..entries.len()).collect::>(); order.sort_unstable_by(|left, right| entries[*left].0.cmp(&entries[*right].0)); - let conflict = order.windows(2).any(|pair| { + order.windows(2).any(|pair| { let (left, right) = (&entries[pair[0]], &entries[pair[1]]); left.0 == right.0 && left.1 != right.1 - }); - if conflict { - return Err(PithosError::DuplicateBlockReference); - } - Ok(()) + }) } #[derive(Debug, Clone, PartialEq, Eq)] @@ -353,65 +357,82 @@ fn decode_decrypted_block_list_reader_with_budget( )?; *remaining_block_references -= count as u64; let mut entries = Zeroizing::new(Vec::new()); - reserve(&mut entries, count, "block references")?; - let mut keys = HashMap::with_capacity(count); - for _ in 0..count { - let mut hash = [0; 32]; - reader.read_exact(&mut hash)?; - let mut key = [0; 32]; - reader.read_exact(&mut key)?; - if keys - .insert(hash, key) - .is_some_and(|existing| existing != key) - { - return Err(DeserializationError::DuplicateBlockReference); - } - entries.push((hash, key)); + read_block_entries(reader, count, &mut entries)?; + if has_conflicting_keys(&entries) { + return Err(DeserializationError::DuplicateBlockReference); } Ok(entries) } -/// Appends one decrypted block list to `entries`. The caller checks the combined list for -/// conflicting keys. -pub(crate) fn append_decrypted_block_list( - bytes: &[u8], +/// The nonce and tag around a sealed block list. +const SEALED_LIST_OVERHEAD: usize = 28; + +/// The number of entries a sealed list of `sealed_len` bytes holds. Each entry takes 64 bytes +/// after a count of one to ten bytes, so this is exact for every well-formed list. +pub(crate) fn sealed_block_list_capacity(sealed_len: usize) -> usize { + sealed_len.saturating_sub(SEALED_LIST_OVERHEAD + 1) / BLOCK_ENTRY_LEN +} + +const BLOCK_ENTRY_LEN: usize = 64; + +/// Reads the entry count of a decrypted list and checks it against the bytes that follow +/// before anything is allocated. +fn read_list_count( + reader: &mut std::io::Cursor<&[u8]>, limits: &DeserializationLimits, remaining_block_references: &mut u64, - entries: &mut Vec, -) -> Result<(), DeserializationError> { - let mut reader = std::io::Cursor::new(bytes); +) -> Result { let count = bounded_len( reader.read_varint::()?, limits.max_block_references.min(*remaining_block_references), "block references", )?; + let available = reader.get_ref().len() as u64 - reader.position(); + if (count as u64).checked_mul(BLOCK_ENTRY_LEN as u64) != Some(available) { + return Err(DeserializationError::InvalidLength); + } *remaining_block_references -= count as u64; - reserve(entries, count, "block references")?; + Ok(count) +} + +/// Reads `count` entries straight into reserved slots, so keys leave no temporary copies. +fn read_block_entries( + reader: &mut R, + count: usize, + entries: &mut Zeroizing>, +) -> Result<(), DeserializationError> { + reserve_secret(entries, count, "block references")?; for _ in 0..count { - let mut entry = ([0; 32], [0; 32]); + entries.push(([0; 32], [0; 32])); + let entry = entries.last_mut().expect("just pushed block entry"); reader.read_exact(&mut entry.0)?; reader.read_exact(&mut entry.1)?; - entries.push(entry); - } - if reader.position() != bytes.len() as u64 { - return Err(DeserializationError::InvalidLength); } Ok(()) } +/// Appends one decrypted block list to `entries`. The caller reserves the room for every +/// piece first and checks the combined list for conflicting keys. +pub(crate) fn append_decrypted_block_list( + bytes: &[u8], + limits: &DeserializationLimits, + remaining_block_references: &mut u64, + entries: &mut Zeroizing>, +) -> Result<(), DeserializationError> { + let mut reader = std::io::Cursor::new(bytes); + let count = read_list_count(&mut reader, limits, remaining_block_references)?; + read_block_entries(&mut reader, count, entries) +} + pub(crate) fn decode_decrypted_block_list_with_budget( bytes: &[u8], limits: &DeserializationLimits, remaining_block_references: &mut u64, ) -> Result>, DeserializationError> { - let mut reader = std::io::Cursor::new(bytes); - let entries = decode_decrypted_block_list_reader_with_budget( - &mut reader, - limits, - remaining_block_references, - )?; - if reader.position() != bytes.len() as u64 { - return Err(DeserializationError::InvalidLength); + let mut entries = Zeroizing::new(Vec::new()); + append_decrypted_block_list(bytes, limits, remaining_block_references, &mut entries)?; + if has_conflicting_keys(&entries) { + return Err(DeserializationError::DuplicateBlockReference); } Ok(entries) } @@ -490,6 +511,61 @@ mod tests { ); } + fn sealed_list(count: usize) -> Vec { + let entries = (0..count) + .map(|index| ([index as u8; 32], [1; 32])) + .collect::>(); + let mut plaintext = Vec::new(); + encode_decrypted_block_list(&entries, &mut plaintext).unwrap(); + crypto::seal_file_block_list_with_nonce(&FileKey::from_bytes([2; 32]), &plaintext, [3; 12]) + .unwrap() + } + + #[test] + fn piece_lists_fill_one_reservation_without_moving() { + for count in [0, 1, 127, 128, 300] { + assert_eq!(sealed_block_list_capacity(sealed_list(count).len()), count); + } + let limits = DeserializationLimits::default(); + let mut budget = limits.max_block_references; + let lists = [sealed_list(127), sealed_list(130)]; + let capacity = lists + .iter() + .map(|sealed| sealed_block_list_capacity(sealed.len())) + .sum(); + let mut entries = Zeroizing::new(Vec::new()); + reserve_secret(&mut entries, capacity, "block references").unwrap(); + let buffer = entries.as_ptr(); + for sealed in &lists { + let plaintext = + crypto::open_file_block_list(&FileKey::from_bytes([2; 32]), sealed.clone()) + .unwrap(); + append_decrypted_block_list(&plaintext, &limits, &mut budget, &mut entries).unwrap(); + } + assert_eq!(entries.len(), 257); + assert_eq!(entries.as_ptr(), buffer); + } + + #[test] + fn list_counts_must_match_the_remaining_bytes_before_allocation() { + let limits = DeserializationLimits::default(); + let mut budget = limits.max_block_references; + let mut claimed = Vec::new(); + claimed.write_varint(1_000_000u64).unwrap(); + claimed.extend_from_slice(&[0; 64]); + assert!(matches!( + decode_decrypted_block_list_with_budget(&claimed, &limits, &mut budget), + Err(DeserializationError::InvalidLength) + )); + assert_eq!(budget, limits.max_block_references); + let mut entries = Zeroizing::new(Vec::new()); + assert!(matches!( + append_decrypted_block_list(&claimed, &limits, &mut budget, &mut entries), + Err(DeserializationError::InvalidLength) + )); + assert_eq!(entries.capacity(), 0); + } + #[test] fn permission_field_uses_u32_varint_width() { assert!( diff --git a/crates/pithos_lib/src/format/primitives.rs b/crates/pithos_lib/src/format/primitives.rs index 7a6f79b..6a282b3 100644 --- a/crates/pithos_lib/src/format/primitives.rs +++ b/crates/pithos_lib/src/format/primitives.rs @@ -2,6 +2,7 @@ use crate::format::error::SerializationError; use crate::format::limits::{DeserializationError, DeserializationLimits}; use integer_encoding::{VarIntReader, VarIntWriter}; use std::io::{Read, Write}; +use zeroize::{Zeroize, Zeroizing}; pub(crate) fn write_len_prefix( writer: &mut W, @@ -51,6 +52,30 @@ pub(crate) fn reserve( }) } +/// Reserves room for `additional` secret items. A plain reallocation would free the old buffer +/// without wiping it, so a full buffer moves into a new one and the old one is wiped. +pub(crate) fn reserve_secret( + output: &mut Zeroizing>, + additional: usize, + field: &'static str, +) -> Result<(), DeserializationError> { + let failed = || DeserializationError::AllocationFailed { + field, + size: additional as u64, + }; + let needed = output.len().checked_add(additional).ok_or_else(failed)?; + if needed <= output.capacity() { + return Ok(()); + } + let mut grown = Zeroizing::new(Vec::new()); + grown + .try_reserve_exact(needed.max(output.capacity().saturating_mul(2))) + .map_err(|_| failed())?; + grown.extend_from_slice(output); + *output = grown; + Ok(()) +} + pub(crate) fn decode_string( reader: &mut R, limits: &DeserializationLimits, @@ -71,6 +96,22 @@ pub(crate) fn decode_string( mod tests { use super::*; + #[test] + fn secret_reservation_keeps_room_and_moves_a_full_buffer_once() { + let mut secrets = Zeroizing::new(vec![[7u8; 32]]); + reserve_secret(&mut secrets, 3, "secrets").unwrap(); + assert_eq!(secrets.as_slice(), &[[7; 32]]); + assert!(secrets.capacity() >= 4); + let buffer = secrets.as_ptr(); + reserve_secret(&mut secrets, 3, "secrets").unwrap(); + secrets.extend_from_slice(&[[8; 32]; 3]); + assert_eq!(secrets.as_ptr(), buffer); + assert!(matches!( + reserve_secret(&mut secrets, usize::MAX, "secrets"), + Err(DeserializationError::AllocationFailed { .. }) + )); + } + #[test] fn typed_uleb128_accepts_bounded_non_minimal_values() { for (mut encoded, expected) in [ From 3eb698eec966383cf0436b44b39c0b630a3b9fb8 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 00:50:10 +0200 Subject: [PATCH 102/166] fix: keep secret keys out of growing buffers and maps in the writer --- crates/pithos_lib/src/archive/access.rs | 11 ++++--- crates/pithos_lib/src/archive/content_tree.rs | 5 ++- crates/pithos_lib/src/archive/pieces.rs | 4 ++- crates/pithos_lib/src/archive/writer.rs | 33 ++++++++----------- 4 files changed, 27 insertions(+), 26 deletions(-) diff --git a/crates/pithos_lib/src/archive/access.rs b/crates/pithos_lib/src/archive/access.rs index c5402db..e56c270 100644 --- a/crates/pithos_lib/src/archive/access.rs +++ b/crates/pithos_lib/src/archive/access.rs @@ -5,7 +5,8 @@ use std::collections::BTreeMap; /// Recovered file secrets have one crate-private owner and never enter an index. pub(crate) struct ResolvedAccess { - keys: BTreeMap, + /// Boxed keys stay in place when the map rearranges its nodes, so no stale copy remains. + keys: BTreeMap>, block_keys: BTreeMap>, provenance: BTreeMap, /// Piece key ids of each file whose block list was sealed in pieces. @@ -48,7 +49,7 @@ impl ResolvedAccess { Ok(()) } None => { - self.keys.insert(id, FileKey::from_protocol(key)); + self.keys.insert(id, Box::new(FileKey::from_protocol(key))); self.provenance.insert(id, provenance); Ok(()) } @@ -56,7 +57,7 @@ impl ResolvedAccess { } pub(crate) fn key(&self, id: FileId) -> Option<&FileKey> { - self.keys.get(&id) + self.keys.get(&id).map(Box::as_ref) } #[cfg(feature = "crypt4gh")] @@ -73,9 +74,9 @@ impl ResolvedAccess { match self.pieces.get(&id) { Some(key_ids) => key_ids .iter() - .map(|key_id| self.keys.get(key_id).map(|key| (*key_id, key))) + .map(|key_id| self.key(*key_id).map(|key| (*key_id, key))) .collect(), - None => self.keys.get(&id).map(|key| vec![(id, key)]), + None => self.key(id).map(|key| vec![(id, key)]), } } diff --git a/crates/pithos_lib/src/archive/content_tree.rs b/crates/pithos_lib/src/archive/content_tree.rs index b1e62be..e362aca 100644 --- a/crates/pithos_lib/src/archive/content_tree.rs +++ b/crates/pithos_lib/src/archive/content_tree.rs @@ -90,7 +90,10 @@ impl TreeHasher { bytes = rest; continue; } - let take = (unit - self.pending.len()).min(bytes.len()); + // Reserving a whole unit once keeps plaintext from being copied on growth. + let filled = self.pending.len(); + self.pending.reserve_exact(UNIT_LEN as usize - filled); + let take = (unit - filled).min(bytes.len()); self.pending.extend_from_slice(&bytes[..take]); bytes = &bytes[take..]; if self.pending.len() == unit { diff --git a/crates/pithos_lib/src/archive/pieces.rs b/crates/pithos_lib/src/archive/pieces.rs index 64db9a5..334dfc1 100644 --- a/crates/pithos_lib/src/archive/pieces.rs +++ b/crates/pithos_lib/src/archive/pieces.rs @@ -226,6 +226,7 @@ impl PieceEncoder { checked_add(self.content_offset, self.original_size)?; tree.update(plaintext); } + crate::format::primitives::reserve_secret(&mut self.entries, 1, "block references")?; self.entries .push((encoded.hash, *encoded.key.expose_for_protocol())); if !self.seen.insert(encoded.hash) { @@ -254,7 +255,8 @@ impl PieceEncoder { return Err(PithosError::UnflushedPieceBytes); } let piece_key = FileKey::from_bytes(StaticSecret::random().to_bytes()); - let mut list = Zeroizing::new(Vec::new()); + // An exact capacity keeps the plaintext list from moving to a new buffer while written. + let mut list = Zeroizing::new(Vec::with_capacity(10 + self.entries.len() * 64)); encode_decrypted_block_list(&self.entries, &mut *list)?; let sealed_list = crypto::seal_file_block_list_with_nonce(&piece_key, &list, crypto::random_nonce())?; diff --git a/crates/pithos_lib/src/archive/writer.rs b/crates/pithos_lib/src/archive/writer.rs index e43badb..3c11efa 100644 --- a/crates/pithos_lib/src/archive/writer.rs +++ b/crates/pithos_lib/src/archive/writer.rs @@ -580,13 +580,13 @@ impl Reservable for Vec { } } -impl Reservable for Zeroizing> { +impl Reservable for Zeroizing> { fn reserve_capacity( &mut self, additional: usize, field: &'static str, ) -> Result<(), PithosError> { - self.try_reserve(additional) + crate::format::primitives::reserve_secret(self, additional, field) .map_err(|_| allocation_failed(field, additional)) } } @@ -647,8 +647,11 @@ fn allocation_failed(field: &'static str, size: usize) -> PithosError { } } +/// Appends one block reference. `positions` maps each hash to its first position rather than +/// to its key, so the map keeps no secret copies. fn append_block_reference( entry: &mut FileEntry, + positions: &mut HashMap<[u8; 32], usize>, hash: [u8; 32], key: &crate::crypto::BlockKey, ) -> Result<(), PithosError> { @@ -657,7 +660,12 @@ fn append_block_reference( "block data already/still encrypted".into(), )); }; + reserve(positions, 1, "block references")?; reserve(references, 1, "entry block references")?; + let position = *positions.entry(hash).or_insert(references.len()); + if position < references.len() && references[position].1 != *key.expose_for_protocol() { + return Err(PithosError::DuplicateBlockReference); + } references.push((hash, *key.expose_for_protocol())); Ok(()) } @@ -1042,16 +1050,9 @@ impl ArchiveWriter { Err(error) => return self.poison(error), }; let hash = encoded.hash; - if let Err(error) = reserve(&mut block_references, 1, "block references") { - return self.poison(error); - } - if block_references - .insert(hash, *encoded.key.expose_for_protocol()) - .is_some_and(|existing| existing != *encoded.key.expose_for_protocol()) + if let Err(error) = + append_block_reference(&mut delta.entry, &mut block_references, hash, &encoded.key) { - return self.poison(PithosError::DuplicateBlockReference); - } - if let Err(error) = append_block_reference(&mut delta.entry, hash, &encoded.key) { return self.poison(error); } if let Some(existing) = self.directory.blocks.get(&hash) { @@ -1293,14 +1294,8 @@ impl ArchiveWriter { let BlockDataState::Decrypted(references) = &delta.entry.block_data else { return Err(PithosError::WriterUnsealedBlockList); }; - let mut keys = HashMap::with_capacity(references.len()); - let actual = references.iter().try_fold(0u64, |total, (hash, key)| { - if keys - .insert(*hash, *key) - .is_some_and(|existing| existing != *key) - { - return Err(PithosError::DuplicateBlockReference); - } + crate::format::file_entry::validate_unique_block_references(references)?; + let actual = references.iter().try_fold(0u64, |total, (hash, _)| { let descriptor_size = delta .descriptors .get(hash) From dd420465ba99f0b906f55583f28d80f8fa82f777 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 00:50:10 +0200 Subject: [PATCH 103/166] fix: reject read requests issued by another opener --- crates/pithos_lib/src/archive/opener.rs | 18 ++++++++++--- crates/pithos_lib/tests/opener.rs | 35 +++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/crates/pithos_lib/src/archive/opener.rs b/crates/pithos_lib/src/archive/opener.rs index 115f064..8738ce4 100644 --- a/crates/pithos_lib/src/archive/opener.rs +++ b/crates/pithos_lib/src/archive/opener.rs @@ -13,16 +13,22 @@ use crate::error::PithosError; use crate::format::directory::Directory; use crate::format::header::{FileHeader, FormatVersion}; use std::collections::{HashMap, HashSet}; +use std::sync::atomic::{AtomicU64, Ordering}; use zeroize::Zeroizing; -/// One exact byte range the opener needs next. +/// One exact byte range the opener needs next. It belongs to the opener that issued it, so +/// another opener rejects it even when the range is the same. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct ReadRequest { + opener: u64, offset: u64, len: u64, sequence: u64, } +/// The source of opener identities. Every opener of this process gets a distinct value. +static NEXT_OPENER: AtomicU64 = AtomicU64::new(0); + impl ReadRequest { pub fn offset(&self) -> u64 { self.offset @@ -68,6 +74,7 @@ pub(super) struct OpenSettings { /// # } /// ``` pub struct ArchiveOpener { + identity: u64, archive_len: u64, settings: OpenSettings, next_sequence: u64, @@ -120,6 +127,7 @@ impl ArchiveOpener { }); } let mut opener = Self { + identity: NEXT_OPENER.fetch_add(1, Ordering::Relaxed), archive_len, settings, next_sequence: 0, @@ -137,9 +145,10 @@ impl ArchiveOpener { /// Accepts the bytes for the outstanding request. /// - /// A response to another request or of the wrong length is rejected and the request stays - /// outstanding. Any other error ends the open. The opener takes the response so it can - /// release a directory's bytes as soon as they are decoded. + /// A response to another request, including any request of another opener, or of the wrong + /// length is rejected and the request stays outstanding. Any other error ends the open. The + /// opener takes the response so it can release a directory's bytes as soon as they are + /// decoded. pub fn feed(&mut self, request: ReadRequest, response: Vec) -> Result<(), PithosError> { let response = Zeroizing::new(response); if self.pending != Some(request) { @@ -175,6 +184,7 @@ impl ArchiveOpener { fn issue(&mut self, offset: u64, len: u64) { self.pending = Some(ReadRequest { + opener: self.identity, offset, len, sequence: self.next_sequence, diff --git a/crates/pithos_lib/tests/opener.rs b/crates/pithos_lib/tests/opener.rs index 65ed166..1fe65e4 100644 --- a/crates/pithos_lib/tests/opener.rs +++ b/crates/pithos_lib/tests/opener.rs @@ -130,6 +130,41 @@ fn feed_rejects_wrong_lengths_unknown_and_repeated_responses() { assert_eq!(opener.finish().unwrap().entries().len(), 1); } +#[test] +fn feed_rejects_header_and_same_span_footer_requests_of_other_openers() { + let temporary = tempfile::tempdir().unwrap(); + let bytes = std::fs::read(fixture(&temporary, "recipient1")).unwrap(); + let len = bytes.len() as u64; + let mut opener = ArchiveOpener::new(len, recipient()).unwrap(); + let mut other = ArchiveOpener::new(len, recipient()).unwrap(); + let header = opener.request().unwrap(); + let foreign_header = other.request().unwrap(); + assert_eq!( + (foreign_header.offset(), foreign_header.len()), + (header.offset(), header.len()) + ); + assert!(matches!( + opener.feed(foreign_header, bytes[..6].to_vec()), + Err(PithosError::UnexpectedReadResponse) + )); + opener.feed(header, bytes[..6].to_vec()).unwrap(); + other.feed(foreign_header, bytes[..6].to_vec()).unwrap(); + + let footer = opener.request().unwrap(); + let foreign_footer = other.request().unwrap(); + assert_eq!( + (foreign_footer.offset(), foreign_footer.len()), + (footer.offset(), footer.len()) + ); + let tail = bytes[bytes.len() - 12..].to_vec(); + assert!(matches!( + opener.feed(foreign_footer, tail.clone()), + Err(PithosError::UnexpectedReadResponse) + )); + assert_eq!(opener.request(), Some(footer)); + opener.feed(footer, tail).unwrap(); +} + #[test] fn limits_are_checked_before_a_directory_is_requested() { let temporary = tempfile::tempdir().unwrap(); From 41708062650b87733ab0ca0de2c1c39058d6fcce Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 00:50:10 +0200 Subject: [PATCH 104/166] fix: check the decoding view's block limits before copying a block --- crates/pithos_lib/src/archive/planning.rs | 37 ++++++++------ crates/pithos_lib/src/archive/view.rs | 9 ++-- crates/pithos_lib/tests/opener.rs | 59 +++++++++++++++++++++++ 3 files changed, 87 insertions(+), 18 deletions(-) diff --git a/crates/pithos_lib/src/archive/planning.rs b/crates/pithos_lib/src/archive/planning.rs index e6e347e..25a40bc 100644 --- a/crates/pithos_lib/src/archive/planning.rs +++ b/crates/pithos_lib/src/archive/planning.rs @@ -133,20 +133,7 @@ impl<'a> ReadPlan<'a> { if end <= self.range.start() { return Ok(None); } - if descriptor.stored_size > self.limits.max_stored_bytes { - return Err(PithosError::LimitExceeded { - field: "stored block", - limit: self.limits.max_stored_bytes, - actual: descriptor.stored_size, - }); - } - if descriptor.original_size > self.limits.max_decoded_bytes { - return Err(PithosError::LimitExceeded { - field: "decoded block", - limit: self.limits.max_decoded_bytes, - actual: descriptor.original_size, - }); - } + check_block_limits(descriptor, self.limits)?; let convert = |value: u64| { usize::try_from(value).map_err(|_| PithosError::InvalidDirectoryRange { operation: "convert range index", @@ -163,6 +150,28 @@ impl<'a> ReadPlan<'a> { } } +/// Checks a block against the stored and decoded size limits. +pub(crate) fn check_block_limits( + descriptor: &BlockDescriptor, + limits: block::Limits, +) -> Result<(), PithosError> { + if descriptor.stored_size > limits.max_stored_bytes { + return Err(PithosError::LimitExceeded { + field: "stored block", + limit: limits.max_stored_bytes, + actual: descriptor.stored_size, + }); + } + if descriptor.original_size > limits.max_decoded_bytes { + return Err(PithosError::LimitExceeded { + field: "decoded block", + limit: limits.max_decoded_bytes, + actual: descriptor.original_size, + }); + } + Ok(()) +} + impl Iterator for ReadPlan<'_> { type Item = Result; diff --git a/crates/pithos_lib/src/archive/view.rs b/crates/pithos_lib/src/archive/view.rs index 8499f48..0cc3a5d 100644 --- a/crates/pithos_lib/src/archive/view.rs +++ b/crates/pithos_lib/src/archive/view.rs @@ -1,6 +1,6 @@ use super::access::ResolvedAccess; use super::index::ArchiveIndex; -use super::planning::{PlannedBlock, ReadPlan}; +use super::planning::{PlannedBlock, ReadPlan, check_block_limits}; #[cfg(feature = "crypt4gh")] use super::reader::AccessKeys; use super::reader::{ArchiveEntry, ContentAvailability, OpenLimits, archive_entry}; @@ -77,14 +77,15 @@ impl ArchiveView { /// Decodes one planned block from its stored bytes, `BLCK` followed by the payload. /// - /// Checks the marker, the sizes, decryption, decompression and the block identity, and - /// returns the whole verified block. This is CPU work without I/O, so callers may run it - /// on a blocking pool. + /// Checks this view's block limits before anything is copied, then the marker, the sizes, + /// decryption, decompression and the block identity, and returns the whole verified block. + /// This is CPU work without I/O, so callers may run it on a blocking pool. pub fn decode_block( &self, block: &PlannedBlock, stored: &[u8], ) -> Result>, PithosError> { + check_block_limits(&block.descriptor, self.block_limits())?; let expected = block.framed_len(); if stored.len() as u64 != expected { return Err(PithosError::BlockSizeMismatch { diff --git a/crates/pithos_lib/tests/opener.rs b/crates/pithos_lib/tests/opener.rs index 1fe65e4..adf37b0 100644 --- a/crates/pithos_lib/tests/opener.rs +++ b/crates/pithos_lib/tests/opener.rs @@ -314,3 +314,62 @@ fn a_view_decodes_blocks_from_bytes_the_caller_fetched() { let error = view.decode_block(&block, &stored).unwrap_err(); assert!(error.to_string().contains("block marker"), "{error}"); } + +#[test] +fn a_view_checks_its_own_block_limits_for_blocks_planned_by_another_view() { + let bytes = many_blocks(4); + let keys = || AccessKeys::new().with_key(private_key("sender")); + let open = |limits: OpenLimits| { + drive( + &bytes, + OpenOptions::default() + .with_access_keys(keys()) + .with_limits(limits), + ) + .0 + .unwrap() + }; + let permissive = open(OpenLimits::default()); + let block = permissive + .plan_range("data", 0..1) + .unwrap() + .next() + .unwrap() + .unwrap(); + let BlockRequest::Local { offset, len } = block.request() else { + panic!("unexpected external block"); + }; + let stored = &bytes[offset as usize..(offset + len) as usize]; + assert_eq!(permissive.decode_block(&block, stored).unwrap().len(), 16); + + for (limits, field) in [ + ( + OpenLimits { + max_stored_block_bytes: 15, + ..OpenLimits::default() + }, + "stored block", + ), + ( + OpenLimits { + max_decoded_block_bytes: 15, + ..OpenLimits::default() + }, + "decoded block", + ), + ] { + let restrictive = open(limits); + // A short response proves that the limits are checked before the stored bytes. + for response in [stored, &stored[1..]] { + let error = restrictive.decode_block(&block, response).unwrap_err(); + assert!( + matches!( + error, + PithosError::LimitExceeded { field: name, limit: 15, actual: 16 } + if name == field + ), + "{error:?}" + ); + } + } +} From e3b4ada3020794f9a81c38e281fb038879eb872f Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 00:50:11 +0200 Subject: [PATCH 105/166] fix: move recovered block lists into the access state instead of copying keys --- crates/pithos_lib/src/archive/access.rs | 30 ++++++++++-------- crates/pithos_lib/src/archive/opener.rs | 3 +- crates/pithos_lib/src/archive/planning.rs | 21 ++++++++----- crates/pithos_lib/src/archive/reader.rs | 37 ++++++++++++----------- crates/pithos_lib/src/archive/view.rs | 4 +-- 5 files changed, 55 insertions(+), 40 deletions(-) diff --git a/crates/pithos_lib/src/archive/access.rs b/crates/pithos_lib/src/archive/access.rs index e56c270..a0c8feb 100644 --- a/crates/pithos_lib/src/archive/access.rs +++ b/crates/pithos_lib/src/archive/access.rs @@ -1,13 +1,16 @@ -use crate::archive::types::FileId; +use crate::archive::types::{BlockHash, FileId}; use crate::crypto::{BlockKey, FileKey}; use crate::error::PithosError; +use crate::format::file_entry::BlockDataEntry; use std::collections::BTreeMap; +use zeroize::Zeroizing; /// Recovered file secrets have one crate-private owner and never enter an index. pub(crate) struct ResolvedAccess { /// Boxed keys stay in place when the map rearranges its nodes, so no stale copy remains. keys: BTreeMap>, - block_keys: BTreeMap>, + /// The decoded block list of each file, in file order. It is moved here, not copied. + block_keys: BTreeMap>>, provenance: BTreeMap, /// Piece key ids of each file whose block list was sealed in pieces. pieces: BTreeMap>, @@ -80,24 +83,25 @@ impl ResolvedAccess { } } - pub(crate) fn insert_block_keys<'a>( + pub(crate) fn insert_block_keys( &mut self, id: FileId, - entries: impl IntoIterator, + entries: Zeroizing>, ) { - // Inserting one by one avoids the sorted copy that collecting into a map would make. - let mut keys = BTreeMap::new(); - for (hash, key) in entries { - keys.insert(hash, BlockKey::from_protocol(key)); - } - self.block_keys.insert(id, keys); + self.block_keys.insert(id, entries); } + /// The key of the block at `position` in the block list of `id`, if it has `hash`. pub(crate) fn block_key( &self, id: FileId, - hash: crate::archive::types::BlockHash, - ) -> Option<&BlockKey> { - self.block_keys.get(&id).and_then(|keys| keys.get(&hash)) + position: usize, + hash: BlockHash, + ) -> Option { + self.block_keys + .get(&id) + .and_then(|entries| entries.get(position)) + .filter(|(entry_hash, _)| *entry_hash == hash.0) + .map(|(_, key)| BlockKey::from_protocol(key)) } } diff --git a/crates/pithos_lib/src/archive/opener.rs b/crates/pithos_lib/src/archive/opener.rs index 8738ce4..d6d5586 100644 --- a/crates/pithos_lib/src/archive/opener.rs +++ b/crates/pithos_lib/src/archive/opener.rs @@ -2,7 +2,7 @@ use super::access::ResolvedAccess; use super::index::build_effective_index; use super::reader::{ AccessKeys, DecodedDirectoryCounts, OpenLimits, OpenOptions, classify_content_availability, - remaining_deserialization_limits, resolve_block_lists, resolve_recipients, + move_block_keys, remaining_deserialization_limits, resolve_block_lists, resolve_recipients, validate_directory_len, validate_piece_keys, }; use super::types::{FileId, Span, ValidatedSegment}; @@ -389,6 +389,7 @@ fn complete_open( segments.push(validated_segment_from_directory( version, &directory, span, parent, )?); + move_block_keys(&mut directory, &mut access)?; } let index_limits = IndexLimits { max_entries: limits.max_entries, diff --git a/crates/pithos_lib/src/archive/planning.rs b/crates/pithos_lib/src/archive/planning.rs index 25a40bc..9a636af 100644 --- a/crates/pithos_lib/src/archive/planning.rs +++ b/crates/pithos_lib/src/archive/planning.rs @@ -14,6 +14,8 @@ use std::ops::Range; #[derive(Clone, Debug, Eq, PartialEq)] pub struct PlannedBlock { pub(crate) file: FileId, + /// The position of the block in the file's block list. + pub(crate) position: usize, pub(crate) hash: BlockHash, pub(crate) descriptor: BlockDescriptor, pub(crate) output: Range, @@ -73,7 +75,7 @@ impl PlannedBlock { pub struct ReadPlan<'a> { index: &'a ArchiveIndex, file: FileId, - references: std::slice::Iter<'a, BlockHash>, + references: std::iter::Enumerate>, cursor: u64, range: ReadRange, limits: block::Limits, @@ -106,7 +108,7 @@ impl<'a> ReadPlan<'a> { Ok(Self { index, file, - references: references.iter(), + references: references.iter().enumerate(), cursor: 0, range, limits, @@ -114,7 +116,11 @@ impl<'a> ReadPlan<'a> { } /// Plans the block at the cursor, or returns `None` when it lies before the range. - fn step(&mut self, hash: BlockHash) -> Result, PithosError> { + fn step( + &mut self, + position: usize, + hash: BlockHash, + ) -> Result, PithosError> { let descriptor = self .index .descriptor(hash) @@ -127,7 +133,7 @@ impl<'a> ReadPlan<'a> { )?; self.cursor = end; if start >= self.range.end() { - self.references = [].iter(); + self.references = [].iter().enumerate(); return Ok(None); } if end <= self.range.start() { @@ -143,6 +149,7 @@ impl<'a> ReadPlan<'a> { let output_end = convert(self.range.end().min(end) - start)?; Ok(Some(PlannedBlock { file: self.file, + position, hash, descriptor: descriptor.clone(), output: output_start..output_end, @@ -176,12 +183,12 @@ impl Iterator for ReadPlan<'_> { type Item = Result; fn next(&mut self) -> Option { - while let Some(&hash) = self.references.next() { - match self.step(hash) { + while let Some((position, &hash)) = self.references.next() { + match self.step(position, hash) { Ok(Some(block)) => return Some(Ok(block)), Ok(None) => {} Err(error) => { - self.references = [].iter(); + self.references = [].iter().enumerate(); return Some(Err(error)); } } diff --git a/crates/pithos_lib/src/archive/reader.rs b/crates/pithos_lib/src/archive/reader.rs index 4c7d301..96b0e93 100644 --- a/crates/pithos_lib/src/archive/reader.rs +++ b/crates/pithos_lib/src/archive/reader.rs @@ -2,16 +2,14 @@ use super::opener::{ArchiveOpener, OpenSettings}; use super::planning::{BlockRequest, PlannedBlock, ReadPlan}; use super::{AccessProvenance, AppendSnapshot, ArchiveView, FileId, ResolvedAccess, Span}; use crate::archive::index::ArchiveIndex; -use crate::archive::types::{ - BlockHash, BlockLocation, ContentState, Entry, ExternalLocation, ReadRange, -}; +use crate::archive::types::{BlockLocation, ContentState, Entry, ExternalLocation, ReadRange}; #[cfg(feature = "crypt4gh")] use crate::crypto::FileKey; use crate::crypto::{self, PrivateKey}; use crate::error::PithosError; use crate::format::directory::Directory; use crate::format::encryption::RecipientData; -use crate::format::file_entry::{BlockDataEntry, BlockDataState}; +use crate::format::file_entry::BlockDataState; use crate::format::header::FormatVersion; use crate::format::limits::{DeserializationError, DeserializationLimits}; use crate::source::ArchiveSource; @@ -664,6 +662,7 @@ pub(super) fn resolve_recipients( candidates.into_iter().enumerate() { let shared = crypto::derive_shared(secret.as_bytes(), peer)?; + let decrypted; let entries = match data { RecipientData::Encrypted(bytes) => { let nonce = crypto::sealed_nonce(bytes)?; @@ -675,12 +674,14 @@ pub(super) fn resolve_recipients( &nonce, ); let plaintext = crypto::unwrap_recipient_list(&shared, bytes)?; - crate::format::encryption::decode_decrypted_recipient_list( + decrypted = crate::format::encryption::decode_decrypted_recipient_list( &plaintext, &decoded_limits, - )? + )?; + &decrypted } - RecipientData::Decrypted(entries) => entries.clone(), + // A plaintext list is read in place, so its keys are not copied. + RecipientData::Decrypted(entries) => entries, }; for (file_id, file_key) in entries.iter() { access.insert( @@ -711,9 +712,7 @@ pub(super) fn resolve_block_lists( directory.files.try_for_each_mut(|id, file| { let file_id = FileId(id); match &mut file.block_data { - BlockDataState::Decrypted(entries) => { - record_block_keys(access, file_id, entries); - } + BlockDataState::Decrypted(_) => {} BlockDataState::Pieces(pieces) => { if pieces .iter() @@ -763,7 +762,6 @@ pub(super) fn resolve_block_lists( } crate::format::file_entry::validate_unique_block_references(&entries)?; access.insert_pieces(file_id, key_ids); - record_block_keys(access, file_id, &entries); file.block_data = BlockDataState::Decrypted(entries); } BlockDataState::Encrypted(bytes) => { @@ -778,7 +776,6 @@ pub(super) fn resolve_block_lists( &decoded_limits, remaining_block_references, )?; - record_block_keys(access, file_id, &entries); file.block_data = BlockDataState::Decrypted(entries); } } @@ -815,9 +812,15 @@ pub(super) fn validate_piece_keys( Ok(piece_keys.into_iter().max()) } -fn record_block_keys(access: &mut ResolvedAccess, file_id: FileId, entries: &[BlockDataEntry]) { - access.insert_block_keys( - file_id, - entries.iter().map(|(hash, key)| (BlockHash(*hash), key)), - ); +/// Moves every decoded block list into `access`, so each recovered key is kept once. +pub(super) fn move_block_keys( + directory: &mut Directory, + access: &mut ResolvedAccess, +) -> Result<(), PithosError> { + directory.files.try_for_each_mut(|id, file| { + if let BlockDataState::Decrypted(entries) = &mut file.block_data { + access.insert_block_keys(FileId(id), std::mem::take(entries)); + } + Ok(()) + }) } diff --git a/crates/pithos_lib/src/archive/view.rs b/crates/pithos_lib/src/archive/view.rs index 0cc3a5d..19a15c9 100644 --- a/crates/pithos_lib/src/archive/view.rs +++ b/crates/pithos_lib/src/archive/view.rs @@ -97,7 +97,7 @@ impl ArchiveView { crate::format::block::decode_block_marker(&mut marker)?; let key = self .access - .block_key(block.file, block.hash) + .block_key(block.file, block.position, block.hash) .ok_or(PithosError::ContentUnavailable)?; let meta = BlockIndexEntry { offset: 0, @@ -108,7 +108,7 @@ impl ArchiveView { }; block::verify( Zeroizing::new(payload.to_vec()), - key, + &key, block.hash.0, &meta, self.block_limits(), From 838b07831e594bb150dbbdf1dc04ecf58636141b Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 00:50:11 +0200 Subject: [PATCH 106/166] fix: convert block descriptors once while a directory is decoded --- crates/pithos_lib/src/archive/mod.rs | 2 + crates/pithos_lib/src/archive/opener.rs | 43 ++++-- crates/pithos_lib/src/archive/reader.rs | 12 +- crates/pithos_lib/src/archive/validation.rs | 141 +++++++++++++++----- crates/pithos_lib/src/format/directory.rs | 58 ++++++-- crates/pithos_lib/src/fuzzing.rs | 10 +- 6 files changed, 199 insertions(+), 67 deletions(-) diff --git a/crates/pithos_lib/src/archive/mod.rs b/crates/pithos_lib/src/archive/mod.rs index 41e4833..3143c05 100644 --- a/crates/pithos_lib/src/archive/mod.rs +++ b/crates/pithos_lib/src/archive/mod.rs @@ -54,11 +54,13 @@ pub(crate) fn decode_validated_directory( bytes: &[u8], limits: &crate::format::limits::DeserializationLimits, remaining_block_references: &mut u64, + blocks: &mut impl crate::format::directory::BlockSink, ) -> Result { crate::format::directory::decode_complete_directory_with_validation_and_budget( bytes, limits, remaining_block_references, + blocks, |directory| validate_directory_entries(&directory.files), ) } diff --git a/crates/pithos_lib/src/archive/opener.rs b/crates/pithos_lib/src/archive/opener.rs index d6d5586..154d593 100644 --- a/crates/pithos_lib/src/archive/opener.rs +++ b/crates/pithos_lib/src/archive/opener.rs @@ -1,14 +1,14 @@ use super::access::ResolvedAccess; +use super::decode_validated_directory; use super::index::build_effective_index; use super::reader::{ AccessKeys, DecodedDirectoryCounts, OpenLimits, OpenOptions, classify_content_availability, move_block_keys, remaining_deserialization_limits, resolve_block_lists, resolve_recipients, validate_directory_len, validate_piece_keys, }; -use super::types::{FileId, Span, ValidatedSegment}; -use super::validation::IndexLimits; +use super::types::{BlockDescriptor, BlockHash, FileId, Span, ValidatedSegment}; +use super::validation::{DescriptorList, IndexLimits, segment_from_parts}; use super::view::ArchiveView; -use super::{decode_validated_directory, validated_segment_from_directory}; use crate::error::PithosError; use crate::format::directory::Directory; use crate::format::header::{FileHeader, FormatVersion}; @@ -90,11 +90,19 @@ enum Phase { Failed, } +/// One decoded directory. Its block descriptors were converted while it was decoded, so the +/// directory itself holds none. +pub(super) struct DecodedDirectory { + pub(super) directory: Directory, + pub(super) descriptors: Vec<(BlockHash, BlockDescriptor)>, + pub(super) span: Span, +} + /// Directories decoded so far, from the terminal one back toward the base. struct DecodedChain { version: FormatVersion, terminal: Span, - raw: Vec<(Directory, Span)>, + raw: Vec, remaining_block_references: u64, directory_hashes: Vec<[u8; 32]>, } @@ -250,16 +258,23 @@ impl ArchiveOpener { let decoded_chain = &mut chain.decoded_chain; let hash = blake3::hash(&response); decoded_chain.directory_hashes.push(*hash.as_bytes()); + let mut descriptors = DescriptorList::new(decoded_chain.version, response.len()); let directory = decode_validated_directory( &response, &limits, &mut decoded_chain.remaining_block_references, + &mut descriptors, )?; drop(response); + let descriptors = descriptors.into_descriptors(); let next = directory.parent_directory_offset; let span = chain.pending; - chain.decoded.record(&directory); - chain.decoded_chain.raw.push((directory, span)); + chain.decoded.record(&directory, descriptors.len()); + chain.decoded_chain.raw.push(DecodedDirectory { + directory, + descriptors, + span, + }); chain.child_start = span.start(); match next { Some((start, len)) => { @@ -344,7 +359,7 @@ fn complete_open( let maximum_piece_key = validate_piece_keys(version, &raw)?; let mut first_grants = HashMap::new(); - for (directory, _) in &raw { + for DecodedDirectory { directory, .. } in &raw { for (sender, section) in &directory.encryption { for (recipient, recipient_section) in §ion.recipients { let pair = (*sender, *recipient); @@ -362,7 +377,7 @@ fn complete_open( let mut access = ResolvedAccess::new(); let mut recovery_order = 0usize; - for (segment, (directory, _)) in raw.iter().enumerate() { + for (segment, DecodedDirectory { directory, .. }) in raw.iter().enumerate() { resolve_recipients( version, directory, @@ -375,8 +390,12 @@ fn complete_open( } let mut segments: Vec = Vec::new(); - for (segment_index, (directory, span)) in raw.into_iter().enumerate() { - let mut directory = directory; + for (segment_index, decoded) in raw.into_iter().enumerate() { + let DecodedDirectory { + mut directory, + descriptors, + span, + } = decoded; resolve_block_lists( &mut directory, &mut access, @@ -386,9 +405,7 @@ fn complete_open( let parent = segment_index .checked_sub(1) .map(|index| segments[index].span); - segments.push(validated_segment_from_directory( - version, &directory, span, parent, - )?); + segments.push(segment_from_parts(&directory, descriptors, span, parent)?); move_block_keys(&mut directory, &mut access)?; } let index_limits = IndexLimits { diff --git a/crates/pithos_lib/src/archive/reader.rs b/crates/pithos_lib/src/archive/reader.rs index 96b0e93..3b10b6c 100644 --- a/crates/pithos_lib/src/archive/reader.rs +++ b/crates/pithos_lib/src/archive/reader.rs @@ -1,6 +1,6 @@ -use super::opener::{ArchiveOpener, OpenSettings}; +use super::opener::{ArchiveOpener, DecodedDirectory, OpenSettings}; use super::planning::{BlockRequest, PlannedBlock, ReadPlan}; -use super::{AccessProvenance, AppendSnapshot, ArchiveView, FileId, ResolvedAccess, Span}; +use super::{AccessProvenance, AppendSnapshot, ArchiveView, FileId, ResolvedAccess}; use crate::archive::index::ArchiveIndex; use crate::archive::types::{BlockLocation, ContentState, Entry, ExternalLocation, ReadRange}; #[cfg(feature = "crypt4gh")] @@ -38,9 +38,9 @@ pub(super) struct DecodedDirectoryCounts { } impl DecodedDirectoryCounts { - pub(super) fn record(&mut self, directory: &Directory) { + pub(super) fn record(&mut self, directory: &Directory, descriptors: usize) { self.entries += directory.files.len() as u64; - self.descriptors += directory.blocks.len() as u64; + self.descriptors += descriptors as u64; self.references += directory .files .iter() @@ -787,11 +787,11 @@ pub(super) fn resolve_block_lists( /// key id, which new file ids must stay above. pub(super) fn validate_piece_keys( version: FormatVersion, - raw: &[(Directory, Span)], + raw: &[DecodedDirectory], ) -> Result, PithosError> { let mut file_ids = HashSet::new(); let mut piece_keys = HashSet::new(); - for (directory, _) in raw { + for DecodedDirectory { directory, .. } in raw { for (id, _, file) in directory.files.iter() { file_ids.insert(id); if let BlockDataState::Pieces(pieces) = &file.block_data { diff --git a/crates/pithos_lib/src/archive/validation.rs b/crates/pithos_lib/src/archive/validation.rs index 1587669..d921d35 100644 --- a/crates/pithos_lib/src/archive/validation.rs +++ b/crates/pithos_lib/src/archive/validation.rs @@ -4,8 +4,8 @@ use crate::archive::types::{ SegmentEntry, Span, ValidatedSegment, }; use crate::error::PithosError; -use crate::format::block::BlockLocation as FormatBlockLocation; -use crate::format::directory::{Directory, STANDARD_RELATIONSHIPS}; +use crate::format::block::{BlockIndexEntry, BlockLocation as FormatBlockLocation}; +use crate::format::directory::{BlockSink, Directory, STANDARD_RELATIONSHIPS}; use crate::format::file_entry::{BlockDataState, FileEntry, FileType}; use crate::format::header::FormatVersion; use std::collections::{BTreeMap, HashSet}; @@ -38,6 +38,108 @@ pub(crate) fn validated_segment_from_directory( span: Span, parent: Option, ) -> Result { + let descriptors = directory + .blocks + .iter() + .map(|(hash, entry)| block_descriptor(version, *hash, entry)) + .collect::, _>>()?; + segment_from_parts(directory, descriptors, span, parent) +} + +/// Converts the block descriptors of a directory while it is decoded, so the decoded form is +/// the only copy. +pub(crate) struct DescriptorList { + version: FormatVersion, + max_count: usize, + descriptors: Vec<(BlockHash, BlockDescriptor)>, +} + +impl DescriptorList { + pub(crate) fn new(version: FormatVersion, directory_len: usize) -> Self { + Self { + version, + // A descriptor takes at least 37 bytes: the hash and five one-byte fields. + max_count: directory_len / 37, + descriptors: Vec::new(), + } + } + + pub(crate) fn into_descriptors(self) -> Vec<(BlockHash, BlockDescriptor)> { + self.descriptors + } +} + +impl BlockSink for DescriptorList { + fn reserve(&mut self, count: usize) -> Result<(), PithosError> { + let count = count.min(self.max_count); + self.descriptors + .try_reserve_exact(count) + .map_err(|_| PithosError::AllocationFailed { + field: "block descriptors", + size: count as u64, + }) + } + + fn insert(&mut self, hash: [u8; 32], entry: BlockIndexEntry) -> Result<(), PithosError> { + self.descriptors + .push(block_descriptor(self.version, hash, &entry)?); + Ok(()) + } +} + +fn block_descriptor( + version: FormatVersion, + hash: [u8; 32], + block_index_entry: &BlockIndexEntry, +) -> Result<(BlockHash, BlockDescriptor), PithosError> { + let processing = Processing::from_byte(block_index_entry.flags.0, version)?; + if matches!(block_index_entry.location, FormatBlockLocation::Local) + && processing.to_byte() & 0x08 != 0 + && block_index_entry.stored_size < 28 + { + return Err(PithosError::InvalidBlockDescriptor( + "encrypted local block payload is shorter than 28 bytes", + )); + } + let stored_size_with_marker = + block_index_entry + .stored_size + .checked_add(4) + .ok_or(PithosError::InvalidDirectoryRange { + operation: "validate block range", + })?; + let location = match &block_index_entry.location { + FormatBlockLocation::Local => BlockLocation::Local(Span::new( + block_index_entry.offset, + stored_size_with_marker, + )?), + FormatBlockLocation::External { url } => { + BlockLocation::External(ExternalLocation::new(url)) + } + }; + Ok(( + BlockHash(hash), + BlockDescriptor { + stored_size: block_index_entry.stored_size, + original_size: block_index_entry.original_size, + processing, + location, + }, + )) +} + +/// Builds a segment from a decoded directory and its already converted block descriptors. +/// The descriptors are sorted by hash and must be unique. +pub(crate) fn segment_from_parts( + directory: &Directory, + mut descriptors: Vec<(BlockHash, BlockDescriptor)>, + span: Span, + parent: Option, +) -> Result { + descriptors.sort_unstable_by_key(|(hash, _)| *hash); + if descriptors.windows(2).any(|pair| pair[0].0 == pair[1].0) { + return Err(PithosError::DuplicateBlockHash); + } let mut entries = Vec::new(); let mut entry_ids = HashSet::new(); for (id, path, file_entry) in directory.files.iter() { @@ -55,41 +157,6 @@ pub(crate) fn validated_segment_from_directory( }); } validate_relationships(directory)?; - let mut descriptors = Vec::new(); - for (hash, block_index_entry) in &directory.blocks { - let processing = Processing::from_byte(block_index_entry.flags.0, version)?; - if matches!(block_index_entry.location, FormatBlockLocation::Local) - && processing.to_byte() & 0x08 != 0 - && block_index_entry.stored_size < 28 - { - return Err(PithosError::InvalidBlockDescriptor( - "encrypted local block payload is shorter than 28 bytes", - )); - } - let stored_size_with_marker = block_index_entry.stored_size.checked_add(4).ok_or( - PithosError::InvalidDirectoryRange { - operation: "validate block range", - }, - )?; - let location = match &block_index_entry.location { - FormatBlockLocation::Local => BlockLocation::Local(Span::new( - block_index_entry.offset, - stored_size_with_marker, - )?), - FormatBlockLocation::External { url } => { - BlockLocation::External(ExternalLocation::new(url)) - } - }; - descriptors.push(( - BlockHash(*hash), - BlockDescriptor { - stored_size: block_index_entry.stored_size, - original_size: block_index_entry.original_size, - processing, - location, - }, - )); - } let mut relationships = Vec::new(); let mut relation_names: BTreeMap = BTreeMap::new(); for (id, name) in &directory.relations { diff --git a/crates/pithos_lib/src/format/directory.rs b/crates/pithos_lib/src/format/directory.rs index 0c49f1b..a33724d 100644 --- a/crates/pithos_lib/src/format/directory.rs +++ b/crates/pithos_lib/src/format/directory.rs @@ -368,10 +368,46 @@ pub(crate) fn decode_directory( decode_directory_with_budget(reader, limits, &mut remaining_block_references) } +/// Receives the block descriptors of a directory while it is decoded. +pub(crate) trait BlockSink { + /// Prepares for the declared number of descriptors before the first one is decoded. + fn reserve(&mut self, count: usize) -> Result<(), PithosError>; + fn insert(&mut self, hash: [u8; 32], entry: BlockIndexEntry) -> Result<(), PithosError>; +} + +impl BlockSink for IndexMap<[u8; 32], BlockIndexEntry> { + fn reserve(&mut self, _count: usize) -> Result<(), PithosError> { + Ok(()) + } + + fn insert(&mut self, hash: [u8; 32], entry: BlockIndexEntry) -> Result<(), PithosError> { + if self.contains_key(&hash) { + return Err(PithosError::DuplicateBlockHash); + } + IndexMap::insert(self, hash, entry); + Ok(()) + } +} + pub(crate) fn decode_directory_with_budget( reader: &mut R, limits: &DeserializationLimits, remaining_block_references: &mut u64, +) -> Result { + let mut blocks = IndexMap::new(); + let mut directory = + decode_directory_with_sink(reader, limits, remaining_block_references, &mut blocks)?; + directory.blocks = blocks; + Ok(directory) +} + +/// Decodes a directory but hands its block descriptors to `blocks`, so the returned +/// directory has none. +fn decode_directory_with_sink( + reader: &mut R, + limits: &DeserializationLimits, + remaining_block_references: &mut u64, + blocks: &mut impl BlockSink, ) -> Result { let mut identifier = [0; 8]; reader.read_exact(&mut identifier)?; @@ -427,14 +463,11 @@ pub(crate) fn decode_directory_with_budget( limits.max_block_descriptors, "blocks", )?; - let mut blocks = IndexMap::new(); + blocks.reserve(block_count)?; for _ in 0..block_count { let mut hash = [0; 32]; reader.read_exact(&mut hash)?; - if blocks.contains_key(&hash) { - return Err(PithosError::DuplicateBlockHash); - } - blocks.insert(hash, decode_block_index_entry(reader, limits)?); + blocks.insert(hash, decode_block_index_entry(reader, limits)?)?; } let relation_count = bounded_len( reader @@ -477,7 +510,7 @@ pub(crate) fn decode_directory_with_budget( identifier, parent_directory_offset, files, - blocks, + blocks: IndexMap::new(), relations, encryption, dir_len: reader.read_u64::()?, @@ -499,18 +532,24 @@ pub(crate) fn decode_complete_directory_with_validation( validate: impl Fn(&Directory) -> Result<(), PithosError>, ) -> Result { let mut remaining_block_references = limits.max_block_references; - decode_complete_directory_with_validation_and_budget( + let mut blocks = IndexMap::new(); + let mut directory = decode_complete_directory_with_validation_and_budget( bytes, limits, &mut remaining_block_references, + &mut blocks, validate, - ) + )?; + directory.blocks = blocks; + Ok(directory) } +/// Decodes and checks a complete directory. Its block descriptors go to `blocks`. pub(crate) fn decode_complete_directory_with_validation_and_budget( bytes: &[u8], limits: &DeserializationLimits, remaining_block_references: &mut u64, + blocks: &mut impl BlockSink, validate: impl Fn(&Directory) -> Result<(), PithosError>, ) -> Result { if bytes.len() < MIN_DIRECTORY_LEN { @@ -552,7 +591,8 @@ pub(crate) fn decode_complete_directory_with_validation_and_budget( }); } let mut reader = Cursor::new(bytes); - let directory = decode_directory_with_budget(&mut reader, limits, remaining_block_references)?; + let directory = + decode_directory_with_sink(&mut reader, limits, remaining_block_references, blocks)?; validate(&directory)?; if reader.position() != bytes.len() as u64 { return Err(PithosError::DirectoryConsumptionMismatch { diff --git a/crates/pithos_lib/src/fuzzing.rs b/crates/pithos_lib/src/fuzzing.rs index 9d5e875..caf039c 100644 --- a/crates/pithos_lib/src/fuzzing.rs +++ b/crates/pithos_lib/src/fuzzing.rs @@ -35,8 +35,14 @@ pub fn decode_header(data: &[u8]) -> FuzzOutcome { pub fn decode_directory(data: &[u8]) -> FuzzOutcome { let limits = DeserializationLimits::default(); let mut remaining_block_references = limits.max_block_references; - match crate::archive::decode_validated_directory(data, &limits, &mut remaining_block_references) - { + let mut blocks: indexmap::IndexMap<[u8; 32], crate::format::block::BlockIndexEntry> = + indexmap::IndexMap::new(); + match crate::archive::decode_validated_directory( + data, + &limits, + &mut remaining_block_references, + &mut blocks, + ) { Ok(_) => FuzzOutcome::Accepted, Err(error) => FuzzOutcome::Rejected(classify_archive_error(&error)), } From 5d99a0c7dc65e817075288d775e5cac53ae063f6 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 00:50:12 +0200 Subject: [PATCH 107/166] fix: move segment contents into the index instead of keeping copies --- crates/pithos_lib/src/archive/index.rs | 248 ++++++++++++-------- crates/pithos_lib/src/archive/mod.rs | 51 ++-- crates/pithos_lib/src/archive/opener.rs | 8 +- crates/pithos_lib/src/archive/snapshot.rs | 34 +-- crates/pithos_lib/src/archive/validation.rs | 108 +++++---- crates/pithos_lib/src/archive/view.rs | 10 +- 6 files changed, 264 insertions(+), 195 deletions(-) diff --git a/crates/pithos_lib/src/archive/index.rs b/crates/pithos_lib/src/archive/index.rs index b18cc86..6d148ef 100644 --- a/crates/pithos_lib/src/archive/index.rs +++ b/crates/pithos_lib/src/archive/index.rs @@ -2,10 +2,9 @@ use crate::archive::types::{ ArchivePath, BlockDescriptor, BlockHash, ContentState, Entry, FileId, RelationId, SegmentEntry, Span, ValidatedSegment, }; -use crate::archive::validation::{IndexLimits, validate_aggregate, validate_entry}; +use crate::archive::validation::{IndexLimits, SegmentCounts, validate_aggregate, validate_entry}; use crate::error::PithosError; use crate::format::header::FileHeader; -use indexmap::IndexMap; use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::sync::Arc; @@ -23,9 +22,11 @@ pub(crate) struct ArchiveIndex { by_id: BTreeMap, by_path: HashMap, usize>, hierarchy: BTreeMap, - descriptors: IndexMap, + /// Sorted by hash, so lookups need no hash table next to the descriptors. + descriptors: Vec<(BlockHash, BlockDescriptor)>, relationships: BTreeMap>, - segment_spans: Vec, + segment_spans: BTreeSet, + counts: SegmentCounts, maximum_id: Option, } @@ -54,7 +55,7 @@ impl ArchiveIndex { } pub(crate) fn descriptor(&self, hash: BlockHash) -> Option<&BlockDescriptor> { - self.descriptors.get(&hash) + find_descriptor(&self.descriptors, hash) } pub(crate) fn relationships(&self) -> impl Iterator { @@ -67,7 +68,6 @@ impl ArchiveIndex { self.relationships.get(&id).map(Arc::as_ref) } - #[cfg(test)] pub(crate) fn maximum_id(&self) -> Option { self.maximum_id } @@ -77,72 +77,62 @@ impl ArchiveIndex { self.maximum_id = self.maximum_id.max(maximum_piece_key); } - /// Transfers only the effective state needed to form an append snapshot. - pub(crate) fn into_append_snapshot_parts( - self, - ) -> ( - Vec, - IndexMap, - Option, - ) { - (self.entries, self.descriptors, self.maximum_id) + /// Runs the normal merge of `child` as the next segment on a copy of this index. + pub(crate) fn validate_child( + &self, + child: ValidatedSegment, + limits: IndexLimits, + ) -> Result<(), PithosError> { + let mut counts = self.counts; + counts.add(&child); + counts.check(limits)?; + let archive_len = child.span.end(); + let mut index = self.clone(); + index.counts = counts; + index.absorb(child, archive_len)?; + index.finish() } -} -/// Builds a new effective index. Inputs are only borrowed and are unchanged on every error. -pub(crate) fn build_effective_index( - segments: &[ValidatedSegment], - archive_len: u64, - limits: IndexLimits, -) -> Result { - validate_aggregate(segments, limits)?; - let mut entries: Vec = Vec::new(); - let mut by_id = BTreeMap::new(); - let mut by_path: HashMap, usize> = HashMap::new(); - let mut hierarchy = BTreeMap::new(); - let mut descriptors: IndexMap = IndexMap::new(); - let mut relationships: BTreeMap> = BTreeMap::new(); - let mut segment_spans = BTreeSet::new(); + fn new(counts: SegmentCounts) -> Self { + Self { + entries: Vec::new(), + by_id: BTreeMap::new(), + by_path: HashMap::new(), + hierarchy: BTreeMap::new(), + descriptors: Vec::new(), + relationships: BTreeMap::new(), + segment_spans: BTreeSet::new(), + counts, + maximum_id: None, + } + } - for segment in segments { - validate_segment_chain(segment, &segment_spans)?; - let block_region = block_data_region(segment)?; - segment_spans.insert(segment.span); - for (id, name) in &segment.relationships { - match relationships.get(id) { + /// Checks one more segment against the earlier ones and moves its contents in. + fn absorb(&mut self, segment: ValidatedSegment, archive_len: u64) -> Result<(), PithosError> { + validate_segment_chain(&segment, &self.segment_spans)?; + let block_region = block_data_region(&segment)?; + self.segment_spans.insert(segment.span); + for (id, name) in segment.relationships { + match self.relationships.get(&id) { Some(existing) if existing.as_ref() != name.as_ref() => { return Err(PithosError::ConflictingRelationshipDefinition(id.0)); } Some(_) => {} None => { - relationships.insert(*id, Arc::clone(name)); + self.relationships.insert(id, name); } } } - for (hash, descriptor) in &segment.descriptors { - if let Some(existing) = descriptors.get(hash) { - // The current format defines compatibility by identity and plaintext size only. - if existing.original_size != descriptor.original_size { - return Err(PithosError::BlockIndexConflict { - hash: hash.0, - existing_original_size: existing.original_size, - new_original_size: descriptor.original_size, - }); - } - } else { - validate_descriptor(descriptor, archive_len, block_region)?; - descriptors.insert(*hash, descriptor.clone()); - } - } - for SegmentEntry { id, path, entry } in &segment.entries { - validate_entry(path, entry)?; - if by_id.contains_key(id) { + self.absorb_descriptors(segment.descriptors, archive_len, block_region)?; + for SegmentEntry { id, path, entry } in segment.entries { + validate_entry(&path, &entry)?; + if self.by_id.contains_key(&id) { return Err(PithosError::DuplicateFileId(format!( "File id already occupied: {}", id.0 ))); } - if by_path.contains_key(path.as_str()) { + if self.by_path.contains_key(path.as_str()) { return Err(PithosError::PathOccupied(format!( "File path already occupied: {}", path.as_str() @@ -150,60 +140,123 @@ pub(crate) fn build_effective_index( } for (offset, _) in path.as_str().match_indices('/') { let ancestor = &path.as_str()[..offset]; - let Some(ancestor_index) = by_path.get(ancestor) else { + let Some(ancestor_index) = self.by_path.get(ancestor) else { return Err(PithosError::InvalidArchivePath { path: path.as_str().into(), reason: format!("missing directory ancestor {ancestor}"), }); }; - if !entries[*ancestor_index].entry.is_directory() { + if !self.entries[*ancestor_index].entry.is_directory() { return Err(PithosError::InvalidArchivePath { path: path.as_str().into(), reason: format!("file entry {ancestor} is an ancestor"), }); } } - let index = entries.len(); - entries.push(IndexedEntry { - id: *id, - path: path.clone(), - entry: entry.clone(), + let index = self.entries.len(); + self.by_id.insert(id, index); + self.by_path.insert(Arc::from(path.as_str()), index); + self.hierarchy.insert(path.clone(), index); + self.entries.push(IndexedEntry { id, path, entry }); + } + Ok(()) + } + + /// Keeps the earliest descriptor of each block. A later one must agree on the plaintext + /// size. The first segment's list is moved in without a copy. + fn absorb_descriptors( + &mut self, + mut descriptors: Vec<(BlockHash, BlockDescriptor)>, + archive_len: u64, + block_region: Span, + ) -> Result<(), PithosError> { + descriptors.sort_unstable_by_key(|(hash, _)| *hash); + for (hash, descriptor) in &descriptors { + match self.descriptor(*hash) { + // The current format defines compatibility by identity and plaintext size only. + Some(existing) if existing.original_size != descriptor.original_size => { + return Err(PithosError::BlockIndexConflict { + hash: hash.0, + existing_original_size: existing.original_size, + new_original_size: descriptor.original_size, + }); + } + Some(_) => {} + None => validate_descriptor(descriptor, archive_len, block_region)?, + } + } + if self.descriptors.is_empty() { + self.descriptors = descriptors; + return Ok(()); + } + descriptors.retain(|(hash, _)| find_descriptor(&self.descriptors, *hash).is_none()); + self.descriptors + .try_reserve_exact(descriptors.len()) + .map_err(|_| PithosError::AllocationFailed { + field: "block descriptors", + size: descriptors.len() as u64, + })?; + self.descriptors.append(&mut descriptors); + self.descriptors.sort_unstable_by_key(|(hash, _)| *hash); + Ok(()) + } + + /// Runs the checks that need every segment. + fn finish(&mut self) -> Result<(), PithosError> { + let mut local_spans = self + .descriptors + .iter() + .filter_map(|(_, descriptor)| match descriptor.location { + crate::archive::types::BlockLocation::Local(span) => Some(span), + crate::archive::types::BlockLocation::External(_) => None, + }) + .collect::>(); + local_spans.sort_unstable_by_key(|span| span.start()); + if local_spans + .windows(2) + .any(|spans| spans[0].overlaps(spans[1])) + { + return Err(PithosError::InvalidDirectoryRange { + operation: "validate block overlap", }); - by_id.insert(*id, index); - by_path.insert(Arc::from(path.as_str()), index); - hierarchy.insert(path.clone(), index); } + drop(local_spans); + validate_references_and_content( + &self.entries, + &self.by_id, + &self.relationships, + &self.descriptors, + )?; + self.maximum_id = self + .maximum_id + .max(self.entries.iter().map(|entry| entry.id).max()); + Ok(()) } +} - let segment_spans = segment_spans.into_iter().collect::>(); - let mut local_spans = descriptors - .values() - .filter_map(|descriptor| match descriptor.location { - crate::archive::types::BlockLocation::Local(span) => Some(span), - crate::archive::types::BlockLocation::External(_) => None, - }) - .collect::>(); - local_spans.sort_unstable_by_key(|span| span.start()); - if local_spans - .windows(2) - .any(|spans| spans[0].overlaps(spans[1])) - { - return Err(PithosError::InvalidDirectoryRange { - operation: "validate block overlap", - }); +fn find_descriptor( + descriptors: &[(BlockHash, BlockDescriptor)], + hash: BlockHash, +) -> Option<&BlockDescriptor> { + descriptors + .binary_search_by_key(&hash, |(hash, _)| *hash) + .ok() + .map(|index| &descriptors[index].1) +} + +/// Builds a new effective index, moving the contents of `segments` into it. +pub(crate) fn build_effective_index( + segments: Vec, + archive_len: u64, + limits: IndexLimits, +) -> Result { + let counts = validate_aggregate(&segments, limits)?; + let mut index = ArchiveIndex::new(counts); + for segment in segments { + index.absorb(segment, archive_len)?; } - validate_references_and_content(&entries, &by_id, &relationships, &descriptors)?; - let maximum_id = entries.iter().map(|entry| entry.id).max(); - Ok(ArchiveIndex { - entries, - by_id, - by_path, - hierarchy, - descriptors, - relationships, - segment_spans, - maximum_id, - }) + index.finish()?; + Ok(index) } fn validate_segment_chain( @@ -271,7 +324,7 @@ fn validate_references_and_content( entries: &[IndexedEntry], by_id: &BTreeMap, relationships: &BTreeMap>, - descriptors: &IndexMap, + descriptors: &[(BlockHash, BlockDescriptor)], ) -> Result<(), PithosError> { for indexed in entries { for reference in &indexed.entry.metadata().references { @@ -289,9 +342,8 @@ fn validate_references_and_content( continue; }; let actual = blocks.iter().try_fold(0u64, |total, hash| { - let descriptor = descriptors - .get(&hash) - .ok_or(PithosError::MissingBlockDescriptor)?; + let descriptor = + find_descriptor(descriptors, hash).ok_or(PithosError::MissingBlockDescriptor)?; total.checked_add(descriptor.original_size).ok_or( PithosError::AccessibleFileSizeMismatch { expected: content.size, diff --git a/crates/pithos_lib/src/archive/mod.rs b/crates/pithos_lib/src/archive/mod.rs index 3143c05..c1f53de 100644 --- a/crates/pithos_lib/src/archive/mod.rs +++ b/crates/pithos_lib/src/archive/mod.rs @@ -194,7 +194,7 @@ mod tests { }, ]; let index = - build_effective_index(&[segment(entries)], 1_000, IndexLimits::default()).unwrap(); + build_effective_index(vec![segment(entries)], 1_000, IndexLimits::default()).unwrap(); assert_eq!( index.entries().map(|entry| entry.id.0).collect::>(), [7, 3, 4] @@ -235,7 +235,7 @@ mod tests { let older_before = older.clone(); let newer_before = newer.clone(); let index = build_effective_index( - &[older.clone(), newer.clone()], + vec![older.clone(), newer.clone()], 1_000, IndexLimits::default(), ) @@ -249,7 +249,7 @@ mod tests { let conflicting_before = conflicting.clone(); assert!( build_effective_index( - &[older.clone(), conflicting.clone()], + vec![older.clone(), conflicting.clone()], 1_000, IndexLimits::default() ) @@ -283,7 +283,7 @@ mod tests { }; let index = - build_effective_index(&[older.clone(), newer], 1_000, IndexLimits::default()) + build_effective_index(vec![older.clone(), newer], 1_000, IndexLimits::default()) .unwrap(); assert_eq!(index.descriptor(hash), Some(&older.descriptors[0].1)); } @@ -358,7 +358,7 @@ mod tests { for (case, segments) in cases { assert!( - build_effective_index(&segments, 100, IndexLimits::default()).is_err(), + build_effective_index(segments, 100, IndexLimits::default()).is_err(), "accepted {case}" ); } @@ -378,7 +378,7 @@ mod tests { ), ]; - assert!(build_effective_index(&[value], 40, IndexLimits::default()).is_err()); + assert!(build_effective_index(vec![value], 40, IndexLimits::default()).is_err()); } #[test] @@ -391,7 +391,7 @@ mod tests { (BlockHash([2; 32]), local_descriptor(second)), ]; - let index = build_effective_index(&[value], 40, IndexLimits::default()).unwrap(); + let index = build_effective_index(vec![value], 40, IndexLimits::default()).unwrap(); assert_eq!( index.descriptor(BlockHash([1; 32])).unwrap().location, BlockLocation::Local(first) @@ -414,7 +414,8 @@ mod tests { }), }; let index = - build_effective_index(&[segment(vec![entry])], 1_000, IndexLimits::default()).unwrap(); + build_effective_index(vec![segment(vec![entry])], 1_000, IndexLimits::default()) + .unwrap(); assert!(matches!( index.entry(FileId(1)).unwrap().entry, Entry::File(_) @@ -444,7 +445,7 @@ mod tests { }), }]); value.descriptors = vec![(first, descriptor(4)), (second, descriptor(6))]; - let index = build_effective_index(&[value], 1_000, IndexLimits::default()).unwrap(); + let index = build_effective_index(vec![value], 1_000, IndexLimits::default()).unwrap(); let range = ReadRange::new(3..7, 10).unwrap(); let plan = ReadPlan::new(&index, FileId(1), range, test_limits()) .unwrap() @@ -484,7 +485,7 @@ mod tests { (hash(block), local_descriptor(span)) }) .collect(); - build_effective_index(&[value], start + 10, IndexLimits::default()).unwrap() + build_effective_index(vec![value], start + 10, IndexLimits::default()).unwrap() } fn local_offsets(blocks: &[PlannedBlock]) -> Vec { @@ -619,7 +620,7 @@ mod tests { }), }]); value.descriptors = vec![(first, descriptor(4)), (second, descriptor(6))]; - let index = build_effective_index(&[value], 1_000, IndexLimits::default()).unwrap(); + let index = build_effective_index(vec![value], 1_000, IndexLimits::default()).unwrap(); let range = ReadRange::new(0..10, 10).unwrap(); let plan = ReadPlan::new(&index, FileId(1), range, test_limits()).unwrap(); let batches = plan @@ -694,7 +695,8 @@ mod tests { }, }; assert!( - build_effective_index(&[segment(vec![link])], 1_000, IndexLimits::default()).is_err() + build_effective_index(vec![segment(vec![link])], 1_000, IndexLimits::default()) + .is_err() ); } @@ -724,7 +726,7 @@ mod tests { conflicting.reverse(); } assert!( - build_effective_index(&[segment(conflicting)], 1_000, IndexLimits::default()) + build_effective_index(vec![segment(conflicting)], 1_000, IndexLimits::default()) .is_err() ); } @@ -740,7 +742,9 @@ mod tests { entry: file(), }, ]; - assert!(build_effective_index(&[segment(adjacent)], 1_000, IndexLimits::default()).is_ok()); + assert!( + build_effective_index(vec![segment(adjacent)], 1_000, IndexLimits::default()).is_ok() + ); } #[test] @@ -767,13 +771,14 @@ mod tests { vec![directory(2, "parent/child"), directory(1, "parent")], ] { assert!( - build_effective_index(&[segment(entries)], 1_000, IndexLimits::default()).is_err() + build_effective_index(vec![segment(entries)], 1_000, IndexLimits::default()) + .is_err() ); } assert!( build_effective_index( - &[segment(vec![ + vec![segment(vec![ directory(1, "parent"), file(2, "parent/child"), ])], @@ -785,13 +790,17 @@ mod tests { let base = segment_at(100, None, vec![directory(1, "parent")]); let child = segment_at(200, Some(base.span), vec![file(2, "parent/child")]); - assert!(build_effective_index(&[base, child], 1_000, IndexLimits::default()).is_ok()); + assert!(build_effective_index(vec![base, child], 1_000, IndexLimits::default()).is_ok()); let early_child = segment_at(100, None, vec![file(2, "parent/child")]); let late_parent = segment_at(200, Some(early_child.span), vec![directory(1, "parent")]); assert!( - build_effective_index(&[early_child, late_parent], 1_000, IndexLimits::default(),) - .is_err() + build_effective_index( + vec![early_child, late_parent], + 1_000, + IndexLimits::default(), + ) + .is_err() ); } @@ -814,12 +823,12 @@ mod tests { if reverse { conflicting.reverse(); } - prop_assert!(build_effective_index(&[segment(conflicting)], 1_000, IndexLimits::default()).is_err()); + prop_assert!(build_effective_index(vec![segment(conflicting)], 1_000, IndexLimits::default()).is_err()); let adjacent = vec![ SegmentEntry { id: FileId(1), path: ArchivePath::new(&prefix).unwrap(), entry: file() }, SegmentEntry { id: FileId(2), path: ArchivePath::new(format!("{prefix}!")).unwrap(), entry: file() }, ]; - prop_assert!(build_effective_index(&[segment(adjacent)], 1_000, IndexLimits::default()).is_ok()); + prop_assert!(build_effective_index(vec![segment(adjacent)], 1_000, IndexLimits::default()).is_ok()); } } } diff --git a/crates/pithos_lib/src/archive/opener.rs b/crates/pithos_lib/src/archive/opener.rs index 154d593..1069f0a 100644 --- a/crates/pithos_lib/src/archive/opener.rs +++ b/crates/pithos_lib/src/archive/opener.rs @@ -408,6 +408,10 @@ fn complete_open( segments.push(segment_from_parts(&directory, descriptors, span, parent)?); move_block_keys(&mut directory, &mut access)?; } + let recipient_pairs = segments + .iter() + .flat_map(|segment| segment.recipient_pairs.iter().copied()) + .collect(); let index_limits = IndexLimits { max_entries: limits.max_entries, max_descriptors: limits.max_descriptors, @@ -415,7 +419,7 @@ fn complete_open( max_relationships: limits.max_relationships, max_segments: limits.max_parent_directories.saturating_add(1), }; - let mut index = build_effective_index(&segments, archive_len, index_limits)?; + let mut index = build_effective_index(segments, archive_len, index_limits)?; index.cover_piece_keys(maximum_piece_key.map(FileId)); let content_availability = classify_content_availability(&index, settings.external_enabled)?; Ok(ArchiveView { @@ -424,7 +428,7 @@ fn complete_open( metadata_digest, terminal_directory: terminal, index, - segments, + recipient_pairs, index_limits, access, #[cfg(feature = "crypt4gh")] diff --git a/crates/pithos_lib/src/archive/snapshot.rs b/crates/pithos_lib/src/archive/snapshot.rs index d5c9266..5def134 100644 --- a/crates/pithos_lib/src/archive/snapshot.rs +++ b/crates/pithos_lib/src/archive/snapshot.rs @@ -1,5 +1,5 @@ use crate::archive::access::ResolvedAccess; -use crate::archive::index::{ArchiveIndex, build_effective_index}; +use crate::archive::index::ArchiveIndex; use crate::archive::types::{ ArchivePath, BlockDescriptor, BlockHash, Entry, FileId, RecipientPair, Span, ValidatedSegment, }; @@ -14,19 +14,17 @@ use std::sync::Arc; /// The secret-contained state needed to safely plan a direct append or grant. /// -/// This is intentionally derived only from a fully opened archive. It retains validated -/// segments for pure prospective merge, but not the source, decoded directories, or reader facade. +/// This is intentionally derived only from a fully opened archive. It retains the effective +/// index for pure prospective merge, but not the source, decoded directories, or reader facade. pub(crate) struct AppendSnapshot { archive_len: u64, version: FormatVersion, terminal_directory: Span, - maximum_id: Option, - descriptors: IndexMap, + index: ArchiveIndex, entries: BTreeMap, paths: HashMap, FileId>, hierarchy: BTreeMap, relationships: BTreeMap>, - segments: Vec, index_limits: IndexLimits, access: ResolvedAccess, occupied_recipient_pairs: HashSet, @@ -54,7 +52,7 @@ impl AppendSnapshot { version: FormatVersion, terminal_directory: Span, index: ArchiveIndex, - segments: Vec, + occupied_recipient_pairs: HashSet, index_limits: IndexLimits, access: ResolvedAccess, ) -> Self { @@ -62,15 +60,10 @@ impl AppendSnapshot { .relationships() .map(|(id, name)| (id.0, Arc::from(name))) .collect(); - let (index_entries, descriptors, maximum_id) = index.into_append_snapshot_parts(); - let occupied_recipient_pairs = segments - .iter() - .flat_map(|segment| segment.recipient_pairs.iter().copied()) - .collect(); let mut entries = BTreeMap::new(); let mut paths = HashMap::new(); let mut hierarchy = BTreeMap::new(); - for entry in index_entries { + for entry in index.entries() { let kind = snapshot_entry_kind(&entry.entry); let id = entry.id; let path_key = Arc::from(entry.path.as_str()); @@ -81,7 +74,7 @@ impl AppendSnapshot { #[cfg(test)] id, #[cfg(test)] - path: entry.path, + path: entry.path.clone(), kind, }, ); @@ -91,13 +84,11 @@ impl AppendSnapshot { archive_len, version, terminal_directory, - maximum_id, - descriptors, + index, entries, paths, hierarchy, relationships, - segments, index_limits, access, occupied_recipient_pairs, @@ -117,11 +108,11 @@ impl AppendSnapshot { } pub(crate) fn maximum_id(&self) -> Option { - self.maximum_id + self.index.maximum_id() } pub(crate) fn descriptor(&self, hash: BlockHash) -> Option<&BlockDescriptor> { - self.descriptors.get(&hash) + self.index.descriptor(hash) } pub(crate) fn entry(&self, id: FileId) -> Option<&SnapshotEntry> { @@ -210,10 +201,7 @@ impl AppendSnapshot { &self, child: ValidatedSegment, ) -> Result<(), PithosError> { - let archive_len = child.span.end(); - let mut segments = self.segments.clone(); - segments.push(child); - build_effective_index(&segments, archive_len, self.index_limits).map(|_| ()) + self.index.validate_child(child, self.index_limits) } pub(crate) fn sender_pair_is_occupied( diff --git a/crates/pithos_lib/src/archive/validation.rs b/crates/pithos_lib/src/archive/validation.rs index d921d35..ca0d612 100644 --- a/crates/pithos_lib/src/archive/validation.rs +++ b/crates/pithos_lib/src/archive/validation.rs @@ -325,10 +325,66 @@ fn invalid_target(path: &ArchivePath, target: &str, reason: &str) -> PithosError } } +/// Counts summed over the segments of an index, checked against [`IndexLimits`]. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub(crate) struct SegmentCounts { + segments: u64, + entries: u64, + descriptors: u64, + references: u64, + relationships: u64, +} + +impl SegmentCounts { + /// Adds one segment. Sums saturate, so an overflow fails the limit check. + pub(crate) fn add(&mut self, segment: &ValidatedSegment) { + let count = |len: usize| u64::try_from(len).unwrap_or(u64::MAX); + let references = segment.entries.iter().fold(0u64, |total, entry| { + total.saturating_add(count(entry.entry.metadata().references.len())) + }); + self.segments = self.segments.saturating_add(1); + self.entries = self.entries.saturating_add(count(segment.entries.len())); + self.descriptors = self + .descriptors + .saturating_add(count(segment.descriptors.len())); + self.references = self.references.saturating_add(references); + self.relationships = self + .relationships + .saturating_add(count(segment.relationships.len())); + } + + pub(crate) fn check(&self, limits: IndexLimits) -> Result<(), PithosError> { + for (field, actual, limit) in [ + ("directory segments", self.segments, limits.max_segments), + ("entries", self.entries, limits.max_entries), + ( + "block descriptors", + self.descriptors, + limits.max_descriptors, + ), + ("references", self.references, limits.max_references), + ( + "relationship definitions", + self.relationships, + limits.max_relationships, + ), + ] { + if actual > limit { + return Err(PithosError::LimitExceeded { + field, + limit, + actual, + }); + } + } + Ok(()) + } +} + pub(crate) fn validate_aggregate( segments: &[ValidatedSegment], limits: IndexLimits, -) -> Result<(), PithosError> { +) -> Result { if segments.len() as u64 > limits.max_segments { return Err(PithosError::LimitExceeded { field: "directory segments", @@ -336,50 +392,10 @@ pub(crate) fn validate_aggregate( actual: segments.len() as u64, }); } - let (entries, descriptors, references, relationships) = segments.iter().try_fold( - (0u64, 0u64, 0u64, 0u64), - |(entries, descriptors, references, relationships), segment| { - let entry_count = u64::try_from(segment.entries.len()).unwrap_or(u64::MAX); - let descriptor_count = u64::try_from(segment.descriptors.len()).unwrap_or(u64::MAX); - let reference_count = segment - .entries - .iter() - .try_fold(0u64, |total, entry| { - total.checked_add( - u64::try_from(entry.entry.metadata().references.len()).unwrap_or(u64::MAX), - ) - }) - .ok_or(PithosError::LimitExceeded { - field: "references", - limit: limits.max_references, - actual: u64::MAX, - })?; - Ok::<_, PithosError>(( - entries.saturating_add(entry_count), - descriptors.saturating_add(descriptor_count), - references.saturating_add(reference_count), - relationships - .saturating_add(u64::try_from(segment.relationships.len()).unwrap_or(u64::MAX)), - )) - }, - )?; - for (field, actual, limit) in [ - ("entries", entries, limits.max_entries), - ("block descriptors", descriptors, limits.max_descriptors), - ("references", references, limits.max_references), - ( - "relationship definitions", - relationships, - limits.max_relationships, - ), - ] { - if actual > limit { - return Err(PithosError::LimitExceeded { - field, - limit, - actual, - }); - } + let mut counts = SegmentCounts::default(); + for segment in segments { + counts.add(segment); } - Ok(()) + counts.check(limits)?; + Ok(counts) } diff --git a/crates/pithos_lib/src/archive/view.rs b/crates/pithos_lib/src/archive/view.rs index 19a15c9..5ad0dfa 100644 --- a/crates/pithos_lib/src/archive/view.rs +++ b/crates/pithos_lib/src/archive/view.rs @@ -5,13 +5,13 @@ use super::planning::{PlannedBlock, ReadPlan, check_block_limits}; use super::reader::AccessKeys; use super::reader::{ArchiveEntry, ContentAvailability, OpenLimits, archive_entry}; use super::snapshot::AppendSnapshot; -use super::types::{ArchivePath, FileId, ReadRange, Span, ValidatedSegment}; +use super::types::{ArchivePath, FileId, ReadRange, RecipientPair, Span}; use super::validation::IndexLimits; use crate::block; use crate::error::PithosError; use crate::format::block::{BlockIndexEntry, BlockLocation, ProcessingFlags}; use crate::format::header::FormatVersion; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, HashSet}; use std::ops::Range; use zeroize::Zeroizing; @@ -25,7 +25,7 @@ pub struct ArchiveView { pub(super) metadata_digest: [u8; 32], pub(super) terminal_directory: Span, pub(super) index: ArchiveIndex, - pub(super) segments: Vec, + pub(super) recipient_pairs: HashSet, pub(super) index_limits: IndexLimits, pub(super) access: ResolvedAccess, #[cfg(feature = "crypt4gh")] @@ -129,7 +129,7 @@ impl ArchiveView { version, terminal_directory, index, - segments, + recipient_pairs, index_limits, access, .. @@ -139,7 +139,7 @@ impl ArchiveView { version, terminal_directory, index, - segments, + recipient_pairs, index_limits, access, ) From 715445fc9c4cd50f69d99cdfcbafec956122b354 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 00:50:12 +0200 Subject: [PATCH 108/166] test: bound open memory for distinct blocks in several pieces --- crates/pithos_lib/tests/open_memory.rs | 64 +++++++++++++++++--------- 1 file changed, 43 insertions(+), 21 deletions(-) diff --git a/crates/pithos_lib/tests/open_memory.rs b/crates/pithos_lib/tests/open_memory.rs index 1b2cb25..ca125e3 100644 --- a/crates/pithos_lib/tests/open_memory.rs +++ b/crates/pithos_lib/tests/open_memory.rs @@ -58,46 +58,68 @@ unsafe impl GlobalAlloc for Counting { #[global_allocator] static ALLOCATOR: Counting = Counting; -/// One composed file in a single piece that references one 4-byte block `references` times. -/// Its block list is far larger than everything else in the directory. -fn one_piece(references: u32) -> Vec { +/// One composed file of `pieces` pieces with `blocks` 4-byte blocks each. The blocks are +/// distinct, or one block repeated. Returns the archive, its content and its directory size. +fn composed(pieces: u32, blocks: u32, distinct: bool) -> (Vec, Vec, usize) { let processing = ProcessingOptions::new(true, 0).unwrap(); - let mut encoder = PieceEncoder::new(1, vec![public_key("recipient1")], processing).unwrap(); let mut stored = Vec::new(); - for _ in 0..references { - stored.extend(encoder.push(b"same").unwrap()); + let mut content = Vec::new(); + let mut sealed = Vec::new(); + for key_id in 1..=pieces { + let mut encoder = + PieceEncoder::new(key_id.into(), vec![public_key("recipient1")], processing).unwrap(); + for block in 0..blocks { + let plaintext = if distinct { + (key_id * blocks + block).to_be_bytes() + } else { + *b"same" + }; + stored.extend(encoder.push(&plaintext).unwrap()); + content.extend_from_slice(&plaintext); + } + sealed.push(encoder.finish().unwrap()); } - let piece = encoder.finish().unwrap(); let composition = compose( ArchivePath::new("object").unwrap(), EntryMetadata::new(0, 0, 0o644), - &[piece], + &sealed, ) .unwrap(); let mut archive = composition.header().to_vec(); archive.extend_from_slice(&stored); archive.extend_from_slice(composition.directory()); - archive + (archive, content, composition.directory().len()) } -/// Opening holds at most two list-sized buffers at once: the directory response and the -/// decoded sealed list, then the sealed list and its plaintext, then the plaintext and the -/// decoded list. Keeping the response, copying the decoded list and collecting the keys into a -/// map through a sorted copy needed more than five. -#[test] -fn opening_one_large_piece_holds_no_extra_copies_of_its_block_list() { - let references = 100_000; - let source = MemorySource::new(one_piece(references)); +/// Opens `archive` and returns the peak bytes allocated while opening. +fn open_peak(archive: Vec, content: &[u8]) -> usize { + let source = MemorySource::new(archive); let options = OpenOptions::default() .with_access_keys(AccessKeys::new().with_key(private_key("recipient1"))); let baseline = CURRENT.load(Ordering::Relaxed); PEAK.store(baseline, Ordering::Relaxed); let archive = Archive::open(source, options).unwrap(); let peak = PEAK.load(Ordering::Relaxed) - baseline; - let list = references as usize * 64; - assert_eq!(archive.entries().len(), 1); + let mut output = Vec::new(); + archive.copy_to("object", &mut output).unwrap(); assert!( - 2 * peak < 5 * list, - "peak {peak} bytes for a {list}-byte list" + output == content, + "the opened archive returned other content" ); + peak +} + +/// Opening holds about two copies of the directory at most: the encoded directory with its +/// decoded form, then the decoded form with the index and the recovered keys. Converting the +/// directory, copying it into the index and copying the keys into a map needed 4.6 copies. +#[test] +fn opening_pieces_holds_about_two_copies_of_the_directory() { + for (pieces, blocks, distinct) in [(4, 25_000, true), (1, 100_000, false)] { + let (archive, content, directory) = composed(pieces, blocks, distinct); + let peak = open_peak(archive, &content); + assert!( + 2 * peak <= 5 * directory, + "peak {peak} bytes for a {directory}-byte directory" + ); + } } From b18a37ff87a22171011fc37c7cec0a795ee9c4e2 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 00:50:12 +0200 Subject: [PATCH 109/166] feat: start range plans at offset checkpoints instead of the first block --- crates/pithos_lib/src/archive/index.rs | 48 ++++++++++++------- crates/pithos_lib/src/archive/mod.rs | 13 +++++ crates/pithos_lib/src/archive/planning.rs | 13 +++-- crates/pithos_lib/src/archive/types.rs | 34 +++++++++++-- crates/pithos_lib/tests/reader.rs | 58 ++++++++++++++++++++++- 5 files changed, 140 insertions(+), 26 deletions(-) diff --git a/crates/pithos_lib/src/archive/index.rs b/crates/pithos_lib/src/archive/index.rs index 6d148ef..a8f6d64 100644 --- a/crates/pithos_lib/src/archive/index.rs +++ b/crates/pithos_lib/src/archive/index.rs @@ -1,6 +1,6 @@ use crate::archive::types::{ - ArchivePath, BlockDescriptor, BlockHash, ContentState, Entry, FileId, RelationId, SegmentEntry, - Span, ValidatedSegment, + ArchivePath, BlockDescriptor, BlockHash, CHECKPOINT_STRIDE, ContentState, Entry, FileId, + RelationId, SegmentEntry, Span, ValidatedSegment, }; use crate::archive::validation::{IndexLimits, SegmentCounts, validate_aggregate, validate_entry}; use crate::error::PithosError; @@ -222,7 +222,7 @@ impl ArchiveIndex { } drop(local_spans); validate_references_and_content( - &self.entries, + &mut self.entries, &self.by_id, &self.relationships, &self.descriptors, @@ -320,8 +320,9 @@ fn validate_descriptor( Ok(()) } +/// Also records the offset checkpoints of every available content entry. fn validate_references_and_content( - entries: &[IndexedEntry], + entries: &mut [IndexedEntry], by_id: &BTreeMap, relationships: &BTreeMap>, descriptors: &[(BlockHash, BlockDescriptor)], @@ -335,22 +336,37 @@ fn validate_references_and_content( return Err(PithosError::MissingReferenceTarget(reference.target.0)); } } - let Some(content) = indexed.entry.content() else { + let (Entry::File(content) | Entry::Metadata(content)) = &mut indexed.entry else { continue; }; - let ContentState::Available(blocks) = &content.content else { + let ContentState::Available(blocks) = &mut content.content else { continue; }; - let actual = blocks.iter().try_fold(0u64, |total, hash| { - let descriptor = - find_descriptor(descriptors, hash).ok_or(PithosError::MissingBlockDescriptor)?; - total.checked_add(descriptor.original_size).ok_or( - PithosError::AccessibleFileSizeMismatch { - expected: content.size, - actual: u64::MAX, - }, - ) - })?; + let count = blocks.as_slice().len().div_ceil(CHECKPOINT_STRIDE); + let mut checkpoints = Vec::new(); + checkpoints + .try_reserve_exact(count) + .map_err(|_| PithosError::AllocationFailed { + field: "offset checkpoints", + size: count as u64, + })?; + let actual = blocks + .iter() + .enumerate() + .try_fold(0u64, |total, (position, hash)| { + if position % CHECKPOINT_STRIDE == 0 { + checkpoints.push(total); + } + let descriptor = find_descriptor(descriptors, hash) + .ok_or(PithosError::MissingBlockDescriptor)?; + total.checked_add(descriptor.original_size).ok_or( + PithosError::AccessibleFileSizeMismatch { + expected: content.size, + actual: u64::MAX, + }, + ) + })?; + blocks.set_checkpoints(checkpoints); if actual != content.size { return Err(PithosError::AccessibleFileSizeMismatch { expected: content.size, diff --git a/crates/pithos_lib/src/archive/mod.rs b/crates/pithos_lib/src/archive/mod.rs index c1f53de..7c2efb1 100644 --- a/crates/pithos_lib/src/archive/mod.rs +++ b/crates/pithos_lib/src/archive/mod.rs @@ -488,6 +488,19 @@ mod tests { build_effective_index(vec![value], start + 10, IndexLimits::default()).unwrap() } + #[test] + fn plans_start_at_the_offset_checkpoint_before_the_range() { + let index = local_file(1, &[0; 3_000]); + for (start, first_position) in [(0, 0), (1_023, 0), (1_024, 1_024), (2_999, 2_048)] { + let range = ReadRange::new(start..start + 1, 3_000).unwrap(); + let plan = ReadPlan::new(&index, FileId(1), range, test_limits()).unwrap(); + assert_eq!(plan.first_position, first_position); + let blocks = plan.collect::, _>>().unwrap(); + assert_eq!(blocks.len(), 1); + assert_eq!(blocks[0].position, start as usize); + } + } + fn local_offsets(blocks: &[PlannedBlock]) -> Vec { blocks .iter() diff --git a/crates/pithos_lib/src/archive/planning.rs b/crates/pithos_lib/src/archive/planning.rs index 9a636af..b044031 100644 --- a/crates/pithos_lib/src/archive/planning.rs +++ b/crates/pithos_lib/src/archive/planning.rs @@ -76,6 +76,8 @@ pub struct ReadPlan<'a> { index: &'a ArchiveIndex, file: FileId, references: std::iter::Enumerate>, + /// The position of the first reference left in `references`. + pub(super) first_position: usize, cursor: u64, range: ReadRange, limits: block::Limits, @@ -100,8 +102,10 @@ impl<'a> ReadPlan<'a> { let ContentState::Available(references) = &content.content else { return Err(PithosError::ContentUnavailable); }; + // The plan starts at the last offset checkpoint before the range. + let (first_position, cursor) = references.start_near(range.start()); let references = if range.start() < range.end() { - references.as_slice() + &references.as_slice()[first_position..] } else { &[] }; @@ -109,7 +113,8 @@ impl<'a> ReadPlan<'a> { index, file, references: references.iter().enumerate(), - cursor: 0, + first_position, + cursor, range, limits, }) @@ -183,8 +188,8 @@ impl Iterator for ReadPlan<'_> { type Item = Result; fn next(&mut self) -> Option { - while let Some((position, &hash)) = self.references.next() { - match self.step(position, hash) { + while let Some((offset, &hash)) = self.references.next() { + match self.step(self.first_position + offset, hash) { Ok(Some(block)) => return Some(Ok(block)), Ok(None) => {} Err(error) => { diff --git a/crates/pithos_lib/src/archive/types.rs b/crates/pithos_lib/src/archive/types.rs index aad6d9e..c7bc24b 100644 --- a/crates/pithos_lib/src/archive/types.rs +++ b/crates/pithos_lib/src/archive/types.rs @@ -252,20 +252,46 @@ pub(crate) struct EntryMetadata { pub(crate) references: Vec, } +/// The index keeps the logical offset of every this many block references. +pub(crate) const CHECKPOINT_STRIDE: usize = 1024; + #[derive(Clone, Debug, Eq, PartialEq)] -pub(crate) struct BlockReferences(Vec); +pub(crate) struct BlockReferences { + blocks: Vec, + /// The logical offset of reference `k * CHECKPOINT_STRIDE` at position `k`. + checkpoints: Vec, +} impl BlockReferences { pub(crate) fn new(blocks: Vec) -> Self { - Self(blocks) + Self { + blocks, + checkpoints: Vec::new(), + } } pub(crate) fn iter(&self) -> impl ExactSizeIterator + '_ { - self.0.iter().copied() + self.blocks.iter().copied() } pub(crate) fn as_slice(&self) -> &[BlockHash] { - &self.0 + &self.blocks + } + + pub(crate) fn set_checkpoints(&mut self, checkpoints: Vec) { + self.checkpoints = checkpoints; + } + + /// The position and logical offset of the last checkpoint at or before `offset`. + pub(crate) fn start_near(&self, offset: u64) -> (usize, u64) { + let index = self + .checkpoints + .partition_point(|checkpoint| *checkpoint <= offset) + .saturating_sub(1); + match self.checkpoints.get(index) { + Some(checkpoint) => (index * CHECKPOINT_STRIDE, *checkpoint), + None => (0, 0), + } } } diff --git a/crates/pithos_lib/tests/reader.rs b/crates/pithos_lib/tests/reader.rs index 0e87296..65f0058 100644 --- a/crates/pithos_lib/tests/reader.rs +++ b/crates/pithos_lib/tests/reader.rs @@ -3,11 +3,12 @@ mod common; use common::util::{fixture, open, private_key, public_key}; use pithos_lib::adapters::crypt4gh::{self, Crypt4GHError}; use pithos_lib::archive::{ - AccessKeys, ArchivePath, ArchiveWriter, EntryKind, EntryMetadata, ProcessingOptions, - WriteOptions, + AccessKeys, Archive, ArchivePath, ArchiveWriter, Chunking, EntryKind, EntryMetadata, + OpenOptions, ProcessingOptions, WriteOptions, }; use pithos_lib::error::PithosError; use pithos_lib::fs::{ExtractionOptions, extract, extract_all, extract_all_with_options}; +use pithos_lib::source::MemorySource; #[test] fn public_reader_lists_copies_ranges_extracts_and_exports() { @@ -190,3 +191,56 @@ fn extraction_is_no_clobber_no_follow_and_staged() { ); assert!(!outside.join("data").exists()); } + +#[test] +fn ranges_near_offset_checkpoints_and_at_the_tail_match_the_full_read() { + // 3,000 encrypted 16-byte blocks and a short last block span two checkpoints of 1,024. + let content = (0..3_000u64 * 16 + 5) + .map(|byte| (byte % 251) as u8) + .collect::>(); + let sender = private_key("sender"); + let options = WriteOptions::new(sender.duplicate(), vec![sender.public_key()]) + .with_chunking(Chunking::Fixed(16)); + let mut writer = ArchiveWriter::create(Vec::new(), options).unwrap(); + writer + .add_file( + ArchivePath::new("data").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(true, 0).unwrap(), + Some(content.len() as u64), + std::io::Cursor::new(content.clone()), + ) + .unwrap(); + let archive = Archive::open( + MemorySource::new(writer.finish().unwrap()), + OpenOptions::default().with_access_keys(AccessKeys::new().with_key(sender)), + ) + .unwrap(); + let mut full = Vec::new(); + archive.copy_to("data", &mut full).unwrap(); + assert!(full == content); + + let edge = 1_024 * 16; + let len = content.len() as u64; + for range in [ + edge - 1..edge, + edge..edge + 1, + edge - 3..edge + 3, + 2 * edge - 16..2 * edge + 16, + 2 * edge..len, + len - 5..len, + len - 21..len, + len - 1..len, + len..len, + edge..edge, + ] { + let mut output = Vec::new(); + archive + .copy_range_to("data", range.clone(), &mut output) + .unwrap(); + assert!( + output == content[range.start as usize..range.end as usize], + "range {range:?}" + ); + } +} From 353f94216efc56765c4a048f243eec2ccab6653e Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 00:50:13 +0200 Subject: [PATCH 110/166] feat: fetch a repeated adjacent block once --- crates/pithos_lib/src/archive/mod.rs | 2 +- crates/pithos_lib/src/archive/planning.rs | 42 +++++++++++++-- crates/pithos_lib/src/archive/reader.rs | 16 +++++- crates/pithos_lib/tests/opener.rs | 63 +++++++++++++++++++++++ 4 files changed, 116 insertions(+), 7 deletions(-) diff --git a/crates/pithos_lib/src/archive/mod.rs b/crates/pithos_lib/src/archive/mod.rs index 7c2efb1..98cd4f1 100644 --- a/crates/pithos_lib/src/archive/mod.rs +++ b/crates/pithos_lib/src/archive/mod.rs @@ -609,12 +609,12 @@ mod tests { .batches(u64::MAX) .map(|batch| batch.unwrap().request()) .collect::>(); + // The last block repeats the span before it, so it joins that batch without a request. assert_eq!( batches, [ BlockRequest::Local { offset: 16, len: 5 }, BlockRequest::Local { offset: 6, len: 15 }, - BlockRequest::Local { offset: 16, len: 5 }, ] ); } diff --git a/crates/pithos_lib/src/archive/planning.rs b/crates/pithos_lib/src/archive/planning.rs index b044031..1bafe22 100644 --- a/crates/pithos_lib/src/archive/planning.rs +++ b/crates/pithos_lib/src/archive/planning.rs @@ -65,6 +65,11 @@ impl PlannedBlock { pub(crate) fn framed_len(&self) -> u64 { self.descriptor.stored_size.saturating_add(4) } + + /// Whether `other` decodes to the same plaintext: the same file, identity and descriptor. + pub(crate) fn same_block(&self, other: &Self) -> bool { + self.file == other.file && self.hash == other.hash && self.descriptor == other.descriptor + } } /// A lazy read plan: the blocks that intersect a range, in file order. @@ -204,7 +209,9 @@ impl Iterator for ReadPlan<'_> { impl<'a> ReadPlan<'a> { /// Groups consecutive local blocks whose spans are contiguous in storage into one request - /// of at most `max_bytes`. A larger block and every external block form their own batch. + /// of at most `max_bytes`. A block that repeats the span of the block before it joins the + /// batch without adding bytes, so the span is fetched once. A larger block and every + /// external block form their own batch. pub fn batches(self, max_bytes: u64) -> BlockBatches<'a> { BlockBatches { plan: self, @@ -225,21 +232,37 @@ impl BlockBatch { pub fn request(&self) -> BlockRequest { let mut request = self.blocks[0].request(); if let BlockRequest::Local { len, .. } = &mut request { - *len = self.blocks.iter().map(PlannedBlock::framed_len).sum(); + *len = self.stored_len(); } request } + /// The stored bytes of the batch. A block that repeats the span before it adds none. + fn stored_len(&self) -> u64 { + let mut previous = None; + self.blocks + .iter() + .filter(|block| { + let span = block.local_span(); + let repeat = span.is_some() && span == previous; + previous = span; + !repeat + }) + .map(PlannedBlock::framed_len) + .sum() + } + pub fn blocks(&self) -> &[PlannedBlock] { &self.blocks } /// Pairs each block with its stored bytes from a response to [`BlockBatch::request`]. + /// A repeated block gets the same bytes as the block before it. pub fn split<'b>( &'b self, response: &'b [u8], ) -> Result, PithosError> { - let expected = self.blocks.iter().map(PlannedBlock::framed_len).sum(); + let expected = self.stored_len(); if response.len() as u64 != expected { return Err(PithosError::BlockSizeMismatch { expected, @@ -247,9 +270,16 @@ impl BlockBatch { }); } let mut rest = response; + let mut last = None; Ok(self.blocks.iter().map(move |block| { + if let Some((span, stored)) = last + && block.local_span() == Some(span) + { + return (block, stored); + } let (stored, tail) = rest.split_at(block.framed_len() as usize); rest = tail; + last = block.local_span().map(|span| (span, stored)); (block, stored) })) } @@ -275,11 +305,15 @@ impl Iterator for BlockBatches<'_> { blocks: vec![first], })); }; - let (mut end, mut total) = (offset + len, len); + let (mut last, mut end, mut total) = (offset, offset + len, len); let mut blocks = vec![first]; for next in self.plan.by_ref() { match next.as_ref().map(PlannedBlock::local_span) { + Ok(Some((offset, len))) if offset == last && offset + len == end => { + blocks.extend(next.ok()); + } Ok(Some((offset, len))) if offset == end && total + len <= self.max_bytes => { + last = offset; end += len; total += len; blocks.extend(next.ok()); diff --git a/crates/pithos_lib/src/archive/reader.rs b/crates/pithos_lib/src/archive/reader.rs index 3b10b6c..2296464 100644 --- a/crates/pithos_lib/src/archive/reader.rs +++ b/crates/pithos_lib/src/archive/reader.rs @@ -435,10 +435,22 @@ where plan: ReadPlan<'_>, sink: &mut W, ) -> Result<(), PithosError> { + // A block that repeats the block before it reuses its verified plaintext. + let mut last: Option<(PlannedBlock, Zeroizing>)> = None; for block in plan { let block = block?; - let plaintext = self.verified_block(&block)?; - sink.write_all(&plaintext[block.output()])?; + let output = block.output(); + if !last + .as_ref() + .is_some_and(|(previous, _)| previous.same_block(&block)) + { + // The previous block is freed before the next one is decoded. + drop(last.take()); + let plaintext = self.verified_block(&block)?; + last = Some((block, plaintext)); + } + let (_, plaintext) = last.as_ref().expect("the block was verified above"); + sink.write_all(&plaintext[output])?; } Ok(()) } diff --git a/crates/pithos_lib/tests/opener.rs b/crates/pithos_lib/tests/opener.rs index adf37b0..d9133d3 100644 --- a/crates/pithos_lib/tests/opener.rs +++ b/crates/pithos_lib/tests/opener.rs @@ -373,3 +373,66 @@ fn a_view_checks_its_own_block_limits_for_blocks_planned_by_another_view() { } } } + +#[test] +fn repeated_adjacent_blocks_are_fetched_once() { + // Blocks A B B B C: the writer stores B once, right after A, and C after B. + let content = [[0u8; 16], [1; 16], [1; 16], [1; 16], [2; 16]].concat(); + let sender = private_key("sender"); + let options = WriteOptions::new(sender.duplicate(), vec![sender.public_key()]) + .with_chunking(Chunking::Fixed(16)); + let mut writer = ArchiveWriter::create(Vec::new(), options).unwrap(); + writer + .add_file( + ArchivePath::new("data").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(true, 0).unwrap(), + Some(content.len() as u64), + std::io::Cursor::new(content.clone()), + ) + .unwrap(); + let bytes = Arc::<[u8]>::from(writer.finish().unwrap()); + let keys = + || OpenOptions::default().with_access_keys(AccessKeys::new().with_key(sender.duplicate())); + + let reads = Arc::new(Mutex::new(Vec::new())); + let archive = Archive::open( + RecordingSource { + bytes: Arc::clone(&bytes), + reads: Arc::clone(&reads), + }, + keys(), + ) + .unwrap(); + let mut output = Vec::new(); + archive.copy_to("data", &mut output).unwrap(); + assert_eq!(output, content); + assert_eq!(reads.lock().unwrap().len(), 3 + 3); + output.clear(); + archive.copy_range_to("data", 20..60, &mut output).unwrap(); + assert_eq!(output, &content[20..60]); + assert_eq!(reads.lock().unwrap().len(), 6 + 1); + + let view = drive(&bytes, keys()).0.unwrap(); + let batches = view + .plan_range("data", 0..80) + .unwrap() + .batches(1024) + .collect::, _>>() + .unwrap(); + assert_eq!(batches.len(), 1); + let BlockRequest::Local { offset, len } = batches[0].request() else { + panic!("unexpected external block"); + }; + let framed = batches[0].blocks()[0].request(); + let BlockRequest::Local { len: block_len, .. } = framed else { + panic!("unexpected external block"); + }; + assert_eq!(len, 3 * block_len); + let response = &bytes[offset as usize..(offset + len) as usize]; + let mut decoded = Vec::new(); + for (block, stored) in batches[0].split(response).unwrap() { + decoded.extend_from_slice(&view.decode_block(block, stored).unwrap()[block.output()]); + } + assert_eq!(decoded, content); +} From 88da8c4be6abb6f9b49b0560970eb03756c56355 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 01:08:37 +0200 Subject: [PATCH 111/166] fix: encode the piece key grant without a plain copy of the key --- crates/pithos_lib/src/archive/pieces.rs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/crates/pithos_lib/src/archive/pieces.rs b/crates/pithos_lib/src/archive/pieces.rs index 334dfc1..faa44bf 100644 --- a/crates/pithos_lib/src/archive/pieces.rs +++ b/crates/pithos_lib/src/archive/pieces.rs @@ -15,9 +15,7 @@ use crate::crypto::{self, FileKey, PublicKey}; use crate::error::PithosError; use crate::format::block::{BlockIndexEntry, BlockLocation, ProcessingFlags}; use crate::format::directory::{Directory, DirectoryEntries, encode_complete_directory}; -use crate::format::encryption::{ - EncryptionSection, RecipientData, RecipientSection, encode_decrypted_recipient_list, -}; +use crate::format::encryption::{EncryptionSection, RecipientData, RecipientSection}; use crate::format::file_entry::{ BlockDataEntry, BlockDataState, BlockListPiece, FileEntry, FileType, encode_decrypted_block_list, @@ -263,11 +261,11 @@ impl PieceEncoder { let sender = StaticSecret::random(); let sender_public = DalekPublicKey::from(&sender).to_bytes(); - let mut records = Zeroizing::new(Vec::new()); - encode_decrypted_recipient_list( - &[(self.key_id, *piece_key.expose_for_protocol())], - &mut *records, - )?; + // Written straight from the borrowed key into reserved room, so no plain copy remains. + let mut records = Zeroizing::new(Vec::with_capacity(1 + 10 + 32)); + records.push(1); + records.write_varint(self.key_id)?; + records.extend_from_slice(piece_key.expose_for_protocol()); let mut grants = Vec::with_capacity(self.recipients.len()); for recipient in &self.recipients { let nonce = crypto::random_nonce(); From 93c6024ba18b99d3f7060cd4d00cdad822a48d33 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 01:08:38 +0200 Subject: [PATCH 112/166] fix: cap the number of planned blocks in one read batch --- crates/pithos_lib/src/archive/mod.rs | 4 +- crates/pithos_lib/src/archive/planning.rs | 8 +++- crates/pithos_lib/tests/opener.rs | 48 ++++++++++++++++++++++- 3 files changed, 57 insertions(+), 3 deletions(-) diff --git a/crates/pithos_lib/src/archive/mod.rs b/crates/pithos_lib/src/archive/mod.rs index 98cd4f1..b560538 100644 --- a/crates/pithos_lib/src/archive/mod.rs +++ b/crates/pithos_lib/src/archive/mod.rs @@ -28,7 +28,9 @@ pub(crate) use path_validation::{ validate_directory_entries, validate_new_candidate, validate_symlink_target, }; pub use pieces::{Composition, Piece, PieceEncoder, compose}; -pub use planning::{BlockBatch, BlockBatches, BlockRequest, PlannedBlock, ReadPlan}; +pub use planning::{ + BlockBatch, BlockBatches, BlockRequest, MAX_BATCH_BLOCKS, PlannedBlock, ReadPlan, +}; #[cfg(feature = "crypt4gh")] pub(crate) use reader::ContentOperationError; pub use reader::{ diff --git a/crates/pithos_lib/src/archive/planning.rs b/crates/pithos_lib/src/archive/planning.rs index 1bafe22..42835da 100644 --- a/crates/pithos_lib/src/archive/planning.rs +++ b/crates/pithos_lib/src/archive/planning.rs @@ -292,6 +292,9 @@ pub struct BlockBatches<'a> { pending: Option>, } +/// Most planned blocks in one batch. Repeated blocks add no bytes, so this bounds their count. +pub const MAX_BATCH_BLOCKS: usize = 1024; + impl Iterator for BlockBatches<'_> { type Item = Result; @@ -307,7 +310,10 @@ impl Iterator for BlockBatches<'_> { }; let (mut last, mut end, mut total) = (offset, offset + len, len); let mut blocks = vec![first]; - for next in self.plan.by_ref() { + while blocks.len() < MAX_BATCH_BLOCKS { + let Some(next) = self.plan.next() else { + break; + }; match next.as_ref().map(PlannedBlock::local_span) { Ok(Some((offset, len))) if offset == last && offset + len == end => { blocks.extend(next.ok()); diff --git a/crates/pithos_lib/tests/opener.rs b/crates/pithos_lib/tests/opener.rs index d9133d3..1e0715d 100644 --- a/crates/pithos_lib/tests/opener.rs +++ b/crates/pithos_lib/tests/opener.rs @@ -4,7 +4,8 @@ use common::append::append_fixture; use common::util::{fixture, private_key}; use pithos_lib::archive::{ AccessKeys, Archive, ArchiveOpener, ArchivePath, ArchiveView, ArchiveWriter, BlockRequest, - Chunking, EntryMetadata, OpenLimits, OpenOptions, ProcessingOptions, WriteOptions, + Chunking, EntryMetadata, MAX_BATCH_BLOCKS, OpenLimits, OpenOptions, ProcessingOptions, + WriteOptions, }; use pithos_lib::error::PithosError; use pithos_lib::source::{ArchiveSource, MemorySource, SourceError}; @@ -436,3 +437,48 @@ fn repeated_adjacent_blocks_are_fetched_once() { } assert_eq!(decoded, content); } + +#[test] +fn long_runs_of_one_repeated_block_are_split_into_bounded_batches() { + let blocks = 2 * MAX_BATCH_BLOCKS + 3; + let content = [7u8; 16].repeat(blocks); + let sender = private_key("sender"); + let options = WriteOptions::new(sender.duplicate(), vec![sender.public_key()]) + .with_chunking(Chunking::Fixed(16)); + let mut writer = ArchiveWriter::create(Vec::new(), options).unwrap(); + writer + .add_file( + ArchivePath::new("data").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(true, 0).unwrap(), + Some(content.len() as u64), + std::io::Cursor::new(content.clone()), + ) + .unwrap(); + let bytes = writer.finish().unwrap(); + let keys = OpenOptions::default().with_access_keys(AccessKeys::new().with_key(sender)); + let view = drive(&bytes, keys).0.unwrap(); + let batches = view + .plan_range("data", 0..content.len() as u64) + .unwrap() + .batches(1024) + .collect::, _>>() + .unwrap(); + assert_eq!(batches.len(), 3); + assert!( + batches + .iter() + .all(|batch| batch.blocks().len() <= MAX_BATCH_BLOCKS) + ); + let mut decoded = Vec::new(); + for batch in &batches { + let BlockRequest::Local { offset, len } = batch.request() else { + panic!("unexpected external block"); + }; + let response = &bytes[offset as usize..(offset + len) as usize]; + for (block, stored) in batch.split(response).unwrap() { + decoded.extend_from_slice(&view.decode_block(block, stored).unwrap()[block.output()]); + } + } + assert_eq!(decoded, content); +} From 487ff1d8f79bf8849c0c8397be237d4e4c996d0e Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 01:08:38 +0200 Subject: [PATCH 113/166] fix: skip the descriptor sort when a directory adds no descriptors --- crates/pithos_lib/src/archive/index.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/pithos_lib/src/archive/index.rs b/crates/pithos_lib/src/archive/index.rs index a8f6d64..2298b18 100644 --- a/crates/pithos_lib/src/archive/index.rs +++ b/crates/pithos_lib/src/archive/index.rs @@ -190,6 +190,9 @@ impl ArchiveIndex { return Ok(()); } descriptors.retain(|(hash, _)| find_descriptor(&self.descriptors, *hash).is_none()); + if descriptors.is_empty() { + return Ok(()); + } self.descriptors .try_reserve_exact(descriptors.len()) .map_err(|_| PithosError::AllocationFailed { From 8ca0b96f570ac61d6c6d22b726b92cb375442f8a Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 01:16:39 +0200 Subject: [PATCH 114/166] feat: add async source, resolver and blocking hook traits --- crates/pithos_lib/Cargo.toml | 2 + crates/pithos_lib/src/archive/async_reader.rs | 66 +++++++++++++++++++ crates/pithos_lib/src/archive/mod.rs | 4 ++ crates/pithos_lib/src/source.rs | 14 ++++ crates/pithos_lib/src/source/async_source.rs | 30 +++++++++ 5 files changed, 116 insertions(+) create mode 100644 crates/pithos_lib/src/archive/async_reader.rs create mode 100644 crates/pithos_lib/src/source/async_source.rs diff --git a/crates/pithos_lib/Cargo.toml b/crates/pithos_lib/Cargo.toml index 3cd02ee..4203832 100644 --- a/crates/pithos_lib/Cargo.toml +++ b/crates/pithos_lib/Cargo.toml @@ -29,6 +29,8 @@ ro-crate = ["dep:ro-crate-rs", "dep:zip"] # Crypt4GH export. crypt4gh = ["dep:blake2"] fuzzing = ["crypt4gh"] +# Async source, reader and range stream without an async runtime. +async = [] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/crates/pithos_lib/src/archive/async_reader.rs b/crates/pithos_lib/src/archive/async_reader.rs new file mode 100644 index 0000000..9de4d62 --- /dev/null +++ b/crates/pithos_lib/src/archive/async_reader.rs @@ -0,0 +1,66 @@ +use super::reader::{ArchiveFeature, ExternalBlockAccessPolicy, NoExternalBlocks}; +use super::types::ExternalLocation; +use crate::error::PithosError; +use std::future::Future; + +/// Resolves an opaque external block location to exactly one framed `BLCK` value without +/// blocking. +/// +/// The contract equals [`ExternalBlockResolver`](super::ExternalBlockResolver): external reads +/// are enabled only when both a resolver and an access policy are supplied, the resolver must +/// call the policy before its initial access and before every redirect, and it must not accept +/// a response larger than `max_response_size`. The reader rejects a response whose length is +/// not `expected_len`. +pub trait AsyncExternalBlockResolver: Send + Sync { + fn resolve( + &self, + policy: &dyn ExternalBlockAccessPolicy, + location: &ExternalLocation, + expected_len: u64, + max_response_size: u64, + ) -> impl Future, PithosError>> + Send; +} + +impl AsyncExternalBlockResolver for NoExternalBlocks { + fn resolve( + &self, + _policy: &dyn ExternalBlockAccessPolicy, + _location: &ExternalLocation, + _expected_len: u64, + _max_response_size: u64, + ) -> impl Future, PithosError>> + Send { + std::future::ready(Err(PithosError::UnsupportedFeature( + ArchiveFeature::ExternalStorage, + ))) + } +} + +/// Runs CPU work, such as decryption, hashing, decompression and index building, away from +/// the async executor. +/// +/// With Tokio, forward to `tokio::task::spawn_blocking` and resume a panic from the join error. +/// Work that was already handed over may still finish after its future is dropped; the reader +/// then discards the result. +pub trait BlockingHook: Send + Sync { + fn spawn_blocking(&self, task: F) -> impl Future + Send + where + F: FnOnce() -> T + Send + 'static, + T: Send + 'static; +} + +/// Runs CPU work inline on the polling task. +/// +/// Suitable only for small archives and small reads: a large directory or block stalls the +/// executor thread while it is decoded. +#[derive(Clone, Copy, Debug, Default)] +pub struct InlineBlocking; + +impl BlockingHook for InlineBlocking { + fn spawn_blocking(&self, task: F) -> impl Future + Send + where + F: FnOnce() -> T + Send + 'static, + T: Send + 'static, + { + std::future::ready(task()) + } +} diff --git a/crates/pithos_lib/src/archive/mod.rs b/crates/pithos_lib/src/archive/mod.rs index b560538..acd606b 100644 --- a/crates/pithos_lib/src/archive/mod.rs +++ b/crates/pithos_lib/src/archive/mod.rs @@ -6,6 +6,8 @@ mod access; mod append; +#[cfg(feature = "async")] +mod async_reader; mod content_tree; mod index; mod opener; @@ -23,6 +25,8 @@ mod writer; pub(crate) use access::{AccessProvenance, ResolvedAccess}; pub use append::{AppendDurability, AppendObservation, AppendOptions}; +#[cfg(feature = "async")] +pub use async_reader::{AsyncExternalBlockResolver, BlockingHook, InlineBlocking}; pub use opener::{ArchiveOpener, ReadRequest}; pub(crate) use path_validation::{ validate_directory_entries, validate_new_candidate, validate_symlink_target, diff --git a/crates/pithos_lib/src/source.rs b/crates/pithos_lib/src/source.rs index 18fd2eb..5a14a58 100644 --- a/crates/pithos_lib/src/source.rs +++ b/crates/pithos_lib/src/source.rs @@ -4,6 +4,11 @@ use std::os::unix::fs::FileExt; use std::path::PathBuf; use std::sync::Arc; +#[cfg(feature = "async")] +mod async_source; +#[cfg(feature = "async")] +pub use async_source::AsyncArchiveSource; + /// Context for acquisition failures at the fixed, offset-based archive boundary. #[derive(Debug, thiserror::Error)] pub enum SourceError { @@ -31,6 +36,15 @@ pub enum SourceError { RangeOverflow { offset: u64, length: usize }, #[error("remote range at offset {offset}: {message}")] Remote { offset: u64, message: String }, + #[error( + "source response at offset {offset} (revision {revision:?}): expected {expected} bytes, received {actual}" + )] + ResponseLength { + offset: u64, + expected: u64, + actual: u64, + revision: Option, + }, } /// An immutable, positioned byte source for an archive. diff --git a/crates/pithos_lib/src/source/async_source.rs b/crates/pithos_lib/src/source/async_source.rs new file mode 100644 index 0000000..78eb17d --- /dev/null +++ b/crates/pithos_lib/src/source/async_source.rs @@ -0,0 +1,30 @@ +use super::SourceError; +use std::future::Future; + +/// An immutable, positioned byte source read without blocking. +/// +/// The source must present one immutable object revision for the whole open and every later +/// read. Archive metadata and blocks are verified, but changed bytes between reads can still +/// turn a valid archive into errors. A store with versioned objects should pin the revision it +/// opened, for example with a conditional range request, and fail once the object changes. +/// [`AsyncArchiveSource::revision`] names that revision in length errors. +/// +/// Every response must be exactly the requested length. The reader rejects shorter and longer +/// responses before it uses them. +#[allow(clippy::len_without_is_empty)] // Matches the synchronous source contract. +pub trait AsyncArchiveSource: Send + Sync { + /// The archive length in bytes. Callers that already know it can skip this call. + fn len(&self) -> impl Future> + Send; + + /// Reads exactly `len` bytes starting at `offset`. + fn read_at( + &self, + offset: u64, + len: u64, + ) -> impl Future, SourceError>> + Send; + + /// An optional caller-chosen name of the object revision, such as an entity tag. + fn revision(&self) -> Option<&str> { + None + } +} From 3bf0c1466ee802712e0d27f1f0f49977de49095e Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 01:16:40 +0200 Subject: [PATCH 115/166] feat: open archives and stream ranges from async sources --- Cargo.lock | 1 + crates/pithos_lib/Cargo.toml | 3 +- crates/pithos_lib/src/archive/async_reader.rs | 175 ++++++++++- crates/pithos_lib/src/archive/async_stream.rs | 293 ++++++++++++++++++ crates/pithos_lib/src/archive/mod.rs | 6 +- 5 files changed, 475 insertions(+), 3 deletions(-) create mode 100644 crates/pithos_lib/src/archive/async_stream.rs diff --git a/Cargo.lock b/Cargo.lock index bd8d4d7..6d4a426 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1424,6 +1424,7 @@ dependencies = [ "criterion", "digest 0.11.3", "fastcdc", + "futures-core", "hkdf", "indexmap", "integer-encoding", diff --git a/crates/pithos_lib/Cargo.toml b/crates/pithos_lib/Cargo.toml index 4203832..b13e23b 100644 --- a/crates/pithos_lib/Cargo.toml +++ b/crates/pithos_lib/Cargo.toml @@ -30,7 +30,7 @@ ro-crate = ["dep:ro-crate-rs", "dep:zip"] crypt4gh = ["dep:blake2"] fuzzing = ["crypt4gh"] # Async source, reader and range stream without an async runtime. -async = [] +async = ["dep:futures-core"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -44,6 +44,7 @@ crc32fast = "1.5.0" cap-std = "4.0.2" digest = "0.11.3" fastcdc = "4.0.1" +futures-core = { version = "0.3.32", default-features = false, optional = true } hkdf = "0.13.0" indexmap = "2.11.0" integer-encoding = "4.1.0" diff --git a/crates/pithos_lib/src/archive/async_reader.rs b/crates/pithos_lib/src/archive/async_reader.rs index 9de4d62..796dbad 100644 --- a/crates/pithos_lib/src/archive/async_reader.rs +++ b/crates/pithos_lib/src/archive/async_reader.rs @@ -1,7 +1,16 @@ -use super::reader::{ArchiveFeature, ExternalBlockAccessPolicy, NoExternalBlocks}; +use super::async_stream::{RangeStream, ReadLimits}; +use super::opener::ArchiveOpener; +use super::planning::BlockRequest; +use super::reader::{ + ArchiveEntry, ArchiveFeature, ExternalBlockAccessPolicy, NoExternalBlocks, OpenOptions, +}; use super::types::ExternalLocation; +use super::view::ArchiveView; use crate::error::PithosError; +use crate::source::{AsyncArchiveSource, SourceError}; use std::future::Future; +use std::ops::Range; +use std::sync::Arc; /// Resolves an opaque external block location to exactly one framed `BLCK` value without /// blocking. @@ -64,3 +73,167 @@ impl BlockingHook for InlineBlocking { std::future::ready(task()) } } + +/// An archive opened from an [`AsyncArchiveSource`]. +/// +/// Opening issues the requests of [`ArchiveOpener`] one after the other: the header, the +/// footer and each directory. Their count does not depend on the number of blocks. Every +/// directory is decoded through the [`BlockingHook`]. +pub struct AsyncArchive { + source: S, + external: E, + external_access_policy: Option>, + pub(super) hook: B, + pub(super) view: Arc, + limits: ReadLimits, +} + +impl AsyncArchive +where + S: AsyncArchiveSource, + E: AsyncExternalBlockResolver, +{ + /// Opens an archive and decodes its metadata inline. See [`AsyncArchive::open_with_hook`]. + pub async fn open( + source: S, + options: OpenOptions, + known_len: Option, + ) -> Result { + Self::open_with_hook(source, options, known_len, InlineBlocking).await + } +} + +impl AsyncArchive +where + S: AsyncArchiveSource, + E: AsyncExternalBlockResolver, + B: BlockingHook, +{ + /// Opens an archive and runs every metadata step through `hook`. + /// + /// `known_len` skips the length request when the caller already knows the object size. + /// It must be the length of the revision the source serves. + pub async fn open_with_hook( + source: S, + options: OpenOptions, + known_len: Option, + hook: B, + ) -> Result { + let archive_len = match known_len { + Some(len) => len, + None => source.len().await?, + }; + let (settings, external, external_access_policy) = options.into_parts(); + let mut opener = ArchiveOpener::with_settings(archive_len, settings)?; + while let Some(request) = opener.request() { + let response = read_exact(&source, request.offset(), request.len()).await?; + let (returned, result) = hook + .spawn_blocking(move || { + let result = opener.feed(request, response); + (opener, result) + }) + .await; + opener = returned; + result?; + } + Ok(Self { + source, + external, + external_access_policy, + hook, + view: Arc::new(opener.finish()?), + limits: ReadLimits::default(), + }) + } + + /// Replaces the bounds used by later range reads. + pub fn with_read_limits(mut self, limits: ReadLimits) -> Self { + self.limits = limits; + self + } + + /// BLAKE3 over the hashes of every directory, from the base to the terminal directory. + pub fn metadata_digest(&self) -> [u8; 32] { + self.view.metadata_digest() + } + + pub fn entries(&self) -> impl ExactSizeIterator + '_ { + self.view.entries() + } + + pub fn entry(&self, path: &str) -> Result, PithosError> { + self.view.entry(path) + } + + /// The validated metadata, for planning reads without this reader. + pub fn view(&self) -> &ArchiveView { + &self.view + } + + /// Streams the verified plaintext of `range` within the content of `path`, in file order. + /// + /// Path, range and availability errors are returned before any request. Each item is the + /// planned part of one completely verified block. The stream ends after its first error. + /// Dropping it cancels every outstanding request. To move the stream into a task that + /// needs `'static`, share the archive in an `Arc` and read inside an async block. + pub fn read_range( + &self, + path: &str, + range: Range, + ) -> Result, PithosError> { + let plan = self.view.plan_range(path, range)?; + Ok(RangeStream::new( + self, + plan.batches(self.limits.max_request_bytes), + self.limits, + )) + } + + /// Fetches the stored bytes of one batch request. + pub(super) async fn fetch(&self, request: BlockRequest) -> Result, PithosError> { + match request { + BlockRequest::Local { offset, len } => Ok(read_exact(&self.source, offset, len).await?), + BlockRequest::External { location, len } => { + let policy = self.external_access_policy.as_deref().ok_or( + PithosError::UnsupportedFeature(ArchiveFeature::ExternalStorage), + )?; + let max_response_size = self + .view + .limits + .max_stored_block_bytes + .checked_add(4) + .ok_or_else(|| { + PithosError::ExternalBlockFraming("response policy overflow".into()) + })?; + let response = self + .external + .resolve(policy, &location, len, max_response_size) + .await?; + if response.len() as u64 != len { + return Err(PithosError::ExternalBlockFraming( + "response does not match expected size".into(), + )); + } + Ok(response) + } + } + } +} + +/// Reads one exact range and rejects responses of any other length. +async fn read_exact( + source: &S, + offset: u64, + len: u64, +) -> Result, SourceError> { + let response = source.read_at(offset, len).await?; + if response.len() as u64 != len { + return Err(SourceError::ResponseLength { + offset, + expected: len, + actual: response.len() as u64, + revision: source.revision().map(str::to_owned), + }); + } + Ok(response) +} diff --git a/crates/pithos_lib/src/archive/async_stream.rs b/crates/pithos_lib/src/archive/async_stream.rs new file mode 100644 index 0000000..f71aed8 --- /dev/null +++ b/crates/pithos_lib/src/archive/async_stream.rs @@ -0,0 +1,293 @@ +use super::async_reader::{AsyncArchive, AsyncExternalBlockResolver, BlockingHook}; +use super::planning::{BlockBatch, BlockBatches}; +use super::view::ArchiveView; +use crate::error::PithosError; +use crate::source::AsyncArchiveSource; +use futures_core::Stream; +use std::collections::VecDeque; +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; +use std::task::{Context, Poll}; +use zeroize::Zeroizing; + +/// Bounds for one range stream. +/// +/// A batch reserves its response length, one more copy of its largest stored block (the payload +/// copy made while decoding), the decoded size of its blocks, and the planned output of blocks +/// that are only partly in the range. After decoding, the batch keeps only its planned output, +/// which is released as the chunks are delivered. A batch that does not fit next to the work +/// already buffered waits; when nothing else is buffered it starts alone, even if it is larger. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub struct ReadLimits { + /// Source or external requests that may be outstanding at once. Zero counts as one. + pub max_in_flight: usize, + /// The total reservation of all batches that are fetched, decoded or waiting for delivery. + pub max_buffered_bytes: u64, + /// Adjacent local blocks are coalesced into one request of at most this many bytes. + pub max_request_bytes: u64, +} + +impl Default for ReadLimits { + fn default() -> Self { + Self { + max_in_flight: 8, + max_buffered_bytes: 128 * 1024 * 1024, + max_request_bytes: 8 * 1024 * 1024, + } + } +} + +type Pending<'a, T> = Pin> + Send + 'a>>; + +enum State<'a> { + Fetching(Pending<'a, Vec>, BlockBatch), + Decoding(Pending<'a, Vec>>), + Ready(VecDeque>), + Failed(PithosError), + Taken, +} + +struct Slot<'a> { + reserved: u64, + state: State<'a>, +} + +/// Verified plaintext chunks of a range, in file order. See [`AsyncArchive::read_range`]. +pub struct RangeStream<'a, S, E, B> { + archive: &'a AsyncArchive, + batches: Option>, + waiting: Option<(BlockBatch, u64)>, + slots: VecDeque>, + limits: ReadLimits, + buffered: u64, + fetching: usize, +} + +impl<'a, S, E, B> RangeStream<'a, S, E, B> +where + S: AsyncArchiveSource, + E: AsyncExternalBlockResolver, + B: BlockingHook, +{ + pub(super) fn new( + archive: &'a AsyncArchive, + batches: BlockBatches<'a>, + limits: ReadLimits, + ) -> Self { + Self { + archive, + batches: Some(batches), + waiting: None, + slots: VecDeque::new(), + limits, + buffered: 0, + fetching: 0, + } + } + + /// Starts requests while the in-flight and byte bounds allow. Returns whether any started. + fn admit(&mut self) -> bool { + let mut admitted = false; + while self.fetching < self.limits.max_in_flight.max(1) { + let (batch, reserved) = match self.waiting.take() { + Some(next) => next, + None => match self.batches.as_mut().and_then(Iterator::next) { + Some(Ok(batch)) => { + let reserved = reservation(&batch); + (batch, reserved) + } + Some(Err(error)) => { + self.batches = None; + self.slots.push_back(Slot { + reserved: 0, + state: State::Failed(error), + }); + return true; + } + None => { + self.batches = None; + break; + } + }, + }; + if !self.slots.is_empty() + && self.buffered.saturating_add(reserved) > self.limits.max_buffered_bytes + { + self.waiting = Some((batch, reserved)); + break; + } + self.buffered = self.buffered.saturating_add(reserved); + self.fetching += 1; + let fetch = Box::pin(self.archive.fetch(batch.request())); + self.slots.push_back(Slot { + reserved, + state: State::Fetching(fetch, batch), + }); + admitted = true; + } + admitted + } + + /// Polls every outstanding fetch and decode. Returns whether any of them completed. + fn poll_slots(&mut self, cx: &mut Context<'_>) -> bool { + let mut progressed = false; + let mut failed = false; + for slot in &mut self.slots { + let next = + match &mut slot.state { + State::Fetching(fetch, _) => match fetch.as_mut().poll(cx) { + Poll::Ready(result) => { + self.fetching -= 1; + let State::Fetching(_, batch) = + std::mem::replace(&mut slot.state, State::Taken) + else { + unreachable!("the slot was fetching"); + }; + match result { + Ok(stored) => { + let view = Arc::clone(&self.archive.view); + let stored = Zeroizing::new(stored); + State::Decoding(Box::pin(self.archive.hook.spawn_blocking( + move || decode_batch(&view, &batch, &stored), + ))) + } + Err(error) => State::Failed(error), + } + } + Poll::Pending => continue, + }, + State::Decoding(decode) => match decode.as_mut().poll(cx) { + Poll::Ready(Ok(chunks)) => { + let kept = chunks.iter().map(|chunk| chunk.len() as u64).sum::(); + self.buffered -= slot.reserved - kept; + slot.reserved = kept; + State::Ready(chunks.into()) + } + Poll::Ready(Err(error)) => State::Failed(error), + Poll::Pending => continue, + }, + State::Ready(_) | State::Failed(_) | State::Taken => continue, + }; + if matches!(next, State::Failed(_)) { + self.buffered -= slot.reserved; + slot.reserved = 0; + failed = true; + } + slot.state = next; + progressed = true; + } + if failed { + self.batches = None; + self.waiting = None; + } + progressed + } + + /// Delivers the next chunk in file order, if the first batch has one ready. + fn deliver(&mut self) -> Option, PithosError>>> { + let Some(slot) = self.slots.front_mut() else { + let finished = self.batches.is_none() && self.waiting.is_none(); + return finished.then_some(None); + }; + let item = match &mut slot.state { + State::Ready(chunks) => match chunks.pop_front() { + Some(chunk) => { + slot.reserved -= chunk.len() as u64; + self.buffered -= chunk.len() as u64; + Ok(chunk) + } + None => { + self.buffered -= slot.reserved; + self.slots.pop_front(); + return None; + } + }, + State::Failed(_) => { + let State::Failed(error) = std::mem::replace(&mut slot.state, State::Taken) else { + unreachable!("the slot failed"); + }; + // Dropping the remaining slots cancels their requests. + self.slots.clear(); + self.batches = None; + self.waiting = None; + Err(error) + } + State::Fetching(..) | State::Decoding(_) | State::Taken => return None, + }; + if let Some(slot) = self.slots.front() + && matches!(&slot.state, State::Ready(chunks) if chunks.is_empty()) + { + self.buffered -= slot.reserved; + self.slots.pop_front(); + } + Some(Some(item)) + } +} + +impl Stream for RangeStream<'_, S, E, B> +where + S: AsyncArchiveSource, + E: AsyncExternalBlockResolver, + B: BlockingHook, +{ + type Item = Result, PithosError>; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.get_mut(); + loop { + let started = this.admit(); + let progressed = this.poll_slots(cx); + let before = this.slots.len(); + if let Some(item) = this.deliver() { + return Poll::Ready(item); + } + if !started && !progressed && this.slots.len() == before { + return Poll::Pending; + } + } + } +} + +/// The bytes a batch may hold at once. See [`ReadLimits`]. +fn reservation(batch: &BlockBatch) -> u64 { + batch.blocks().iter().fold( + batch + .blocks() + .iter() + .map(|block| block.framed_len()) + .max() + .unwrap_or(0), + |total, block| { + let output = block.output.len() as u64; + let partial = if output == block.descriptor.original_size { + 0 + } else { + output + }; + total + .saturating_add(block.framed_len()) + .saturating_add(block.descriptor.original_size) + .saturating_add(partial) + }, + ) +} + +/// Decodes every block of a batch and keeps only the planned output of each. +fn decode_batch( + view: &ArchiveView, + batch: &BlockBatch, + stored: &[u8], +) -> Result>, PithosError> { + let mut chunks = Vec::with_capacity(batch.blocks().len()); + for (block, bytes) in batch.split(stored)? { + let mut plaintext = view.decode_block(block, bytes)?; + let output = block.output(); + chunks.push(if output.start == 0 && output.end == plaintext.len() { + std::mem::take(&mut *plaintext) + } else { + plaintext[output].to_vec() + }); + } + Ok(chunks) +} diff --git a/crates/pithos_lib/src/archive/mod.rs b/crates/pithos_lib/src/archive/mod.rs index acd606b..46c5059 100644 --- a/crates/pithos_lib/src/archive/mod.rs +++ b/crates/pithos_lib/src/archive/mod.rs @@ -8,6 +8,8 @@ mod access; mod append; #[cfg(feature = "async")] mod async_reader; +#[cfg(feature = "async")] +mod async_stream; mod content_tree; mod index; mod opener; @@ -26,7 +28,9 @@ mod writer; pub(crate) use access::{AccessProvenance, ResolvedAccess}; pub use append::{AppendDurability, AppendObservation, AppendOptions}; #[cfg(feature = "async")] -pub use async_reader::{AsyncExternalBlockResolver, BlockingHook, InlineBlocking}; +pub use async_reader::{AsyncArchive, AsyncExternalBlockResolver, BlockingHook, InlineBlocking}; +#[cfg(feature = "async")] +pub use async_stream::{RangeStream, ReadLimits}; pub use opener::{ArchiveOpener, ReadRequest}; pub(crate) use path_validation::{ validate_directory_entries, validate_new_candidate, validate_symlink_target, From 891228ca7c80150e99e12e7d1eeb99817879d7c2 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 01:16:41 +0200 Subject: [PATCH 116/166] test: cover async open, range bounds, cancellation and the blocking hook --- Cargo.lock | 1 + crates/pithos_lib/Cargo.toml | 5 + crates/pithos_lib/tests/async_reader.rs | 470 ++++++++++++++++++++++++ 3 files changed, 476 insertions(+) create mode 100644 crates/pithos_lib/tests/async_reader.rs diff --git a/Cargo.lock b/Cargo.lock index 6d4a426..7803cd0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1438,6 +1438,7 @@ dependencies = [ "shake", "tempfile", "thiserror", + "tokio", "tracing", "trybuild", "x25519-dalek", diff --git a/crates/pithos_lib/Cargo.toml b/crates/pithos_lib/Cargo.toml index b13e23b..8e55db9 100644 --- a/crates/pithos_lib/Cargo.toml +++ b/crates/pithos_lib/Cargo.toml @@ -66,6 +66,7 @@ criterion = "0.8.2" peak_alloc = "0.3.0" proptest = "1.6.0" tempfile = "3.21" +tokio = { version = "1.52.3", default-features = false, features = ["rt"] } trybuild = "1.0.104" [[bench]] @@ -121,6 +122,10 @@ required-features = ["crypt4gh"] name = "ro_crate" required-features = ["ro-crate"] +[[test]] +name = "async_reader" +required-features = ["async"] + [[test]] name = "compile_fail" required-features = ["crypt4gh"] diff --git a/crates/pithos_lib/tests/async_reader.rs b/crates/pithos_lib/tests/async_reader.rs new file mode 100644 index 0000000..53d6ed4 --- /dev/null +++ b/crates/pithos_lib/tests/async_reader.rs @@ -0,0 +1,470 @@ +mod common; + +use common::keys::private_key; +use futures_core::Stream; +use pithos_lib::archive::{ + AccessKeys, Archive, ArchivePath, ArchiveWriter, AsyncArchive, BlockingHook, Chunking, + EntryMetadata, OpenOptions, ProcessingOptions, ReadLimits, WriteOptions, +}; +use pithos_lib::error::PithosError; +use pithos_lib::source::{AsyncArchiveSource, MemorySource, SourceError}; +use std::future::Future; +use std::pin::{Pin, pin}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::task::{Context, Poll, Waker}; + +/// How the test source answers reads. +#[derive(Clone, Copy, Debug)] +enum Mode { + Ready, + /// Each read returns `Pending` once, so several reads overlap. + YieldOnce, + /// Reads complete only while `Probe::completions` allows it. + Gated, + /// Responses are one byte longer or shorter than requested. + Resize(isize), +} + +#[derive(Default)] +struct Probe { + reads: Mutex>, + len_calls: AtomicUsize, + outstanding: AtomicUsize, + max_outstanding: AtomicUsize, + cancelled: AtomicUsize, + completions: AtomicUsize, + delivered: AtomicUsize, + /// Reads issued while opening, which `max_ahead` ignores. + baseline: AtomicUsize, + max_ahead: AtomicUsize, +} + +struct TestSource { + bytes: Arc<[u8]>, + probe: Arc, + mode: Arc>, +} + +impl TestSource { + fn new(bytes: &[u8]) -> (Self, Arc, Arc>) { + let probe = Arc::new(Probe::default()); + let mode = Arc::new(Mutex::new(Mode::Ready)); + let source = Self { + bytes: Arc::from(bytes), + probe: Arc::clone(&probe), + mode: Arc::clone(&mode), + }; + (source, probe, mode) + } +} + +impl AsyncArchiveSource for TestSource { + async fn len(&self) -> Result { + self.probe.len_calls.fetch_add(1, Ordering::SeqCst); + Ok(self.bytes.len() as u64) + } + + fn read_at( + &self, + offset: u64, + len: u64, + ) -> impl Future, SourceError>> + Send { + let mode = *self.mode.lock().unwrap(); + let mut reads = self.probe.reads.lock().unwrap(); + reads.push((offset, len)); + let ahead = reads.len().saturating_sub( + self.probe.baseline.load(Ordering::SeqCst) + + self.probe.delivered.load(Ordering::SeqCst), + ); + self.probe.max_ahead.fetch_max(ahead, Ordering::SeqCst); + let outstanding = self.probe.outstanding.fetch_add(1, Ordering::SeqCst) + 1; + self.probe + .max_outstanding + .fetch_max(outstanding, Ordering::SeqCst); + let end = match mode { + Mode::Resize(delta) => (offset + len).saturating_add_signed(delta as i64), + _ => offset + len, + }; + Read { + probe: Arc::clone(&self.probe), + response: Some(self.bytes[offset as usize..end as usize].to_vec()), + mode, + yielded: false, + } + } + + fn revision(&self) -> Option<&str> { + Some("etag-1") + } +} + +/// A read that records completion and cancellation. +struct Read { + probe: Arc, + response: Option>, + mode: Mode, + yielded: bool, +} + +impl Future for Read { + type Output = Result, SourceError>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + match this.mode { + Mode::YieldOnce if !this.yielded => { + this.yielded = true; + cx.waker().wake_by_ref(); + return Poll::Pending; + } + Mode::Gated => { + let allowed = this.probe.completions.fetch_update( + Ordering::SeqCst, + Ordering::SeqCst, + |count| count.checked_sub(1), + ); + if allowed.is_err() { + return Poll::Pending; + } + } + _ => {} + } + this.probe.outstanding.fetch_sub(1, Ordering::SeqCst); + Poll::Ready(Ok(this.response.take().unwrap())) + } +} + +impl Drop for Read { + fn drop(&mut self) { + if self.response.is_some() { + self.probe.outstanding.fetch_sub(1, Ordering::SeqCst); + self.probe.cancelled.fetch_add(1, Ordering::SeqCst); + } + } +} + +/// Runs inline and counts its calls. +#[derive(Clone, Default)] +struct CountingHook(Arc); + +impl BlockingHook for CountingHook { + fn spawn_blocking(&self, task: F) -> impl Future + Send + where + F: FnOnce() -> T + Send + 'static, + T: Send + 'static, + { + self.0.fetch_add(1, Ordering::SeqCst); + std::future::ready(task()) + } +} + +/// Drives a future whose wakeups all happen during its own polls. +fn block_on(future: F) -> F::Output { + let mut future = pin!(future); + let mut cx = Context::from_waker(Waker::noop()); + for _ in 0..1_000_000 { + if let Poll::Ready(output) = future.as_mut().poll(&mut cx) { + return output; + } + } + panic!("the future made no progress"); +} + +fn next(stream: &mut S) -> Poll> { + Pin::new(stream).poll_next(&mut Context::from_waker(Waker::noop())) +} + +/// Collects a stream, counting delivered chunks in `probe`. +fn drain(mut stream: S, probe: &Probe) -> (Vec, Option) +where + S: Stream, PithosError>> + Unpin, +{ + let mut output = Vec::new(); + for _ in 0..1_000_000 { + match next(&mut stream) { + Poll::Ready(Some(Ok(chunk))) => { + probe.delivered.fetch_add(1, Ordering::SeqCst); + output.extend_from_slice(&chunk); + } + Poll::Ready(Some(Err(error))) => { + assert!(matches!(next(&mut stream), Poll::Ready(None))); + return (output, Some(error)); + } + Poll::Ready(None) => return (output, None), + Poll::Pending => {} + } + } + panic!("the stream made no progress"); +} + +fn content(count: u64) -> Vec { + (0..count) + .flat_map(|block| [block.to_be_bytes(), (!block).to_be_bytes()]) + .flatten() + .collect() +} + +/// An archive with `count` 16-byte blocks in one file "data". +fn blocks(count: u64, encrypted: bool) -> Vec { + let options = if encrypted { + let sender = private_key("sender"); + WriteOptions::new(sender.duplicate(), vec![sender.public_key()]) + } else { + WriteOptions::base() + }; + let mut writer = + ArchiveWriter::create(Vec::new(), options.with_chunking(Chunking::Fixed(16))).unwrap(); + let content = content(count); + let processing = if encrypted { + ProcessingOptions::new(true, 3) + } else { + ProcessingOptions::new(false, 0) + }; + writer + .add_file( + ArchivePath::new("data").unwrap(), + EntryMetadata::new(0, 0, 0o644), + processing.unwrap(), + Some(content.len() as u64), + std::io::Cursor::new(content), + ) + .unwrap(); + writer.finish().unwrap() +} + +fn sender() -> OpenOptions { + OpenOptions::default().with_access_keys(AccessKeys::new().with_key(private_key("sender"))) +} + +fn open( + bytes: &[u8], + limits: ReadLimits, +) -> (AsyncArchive, Arc, Arc>) { + let (source, probe, mode) = TestSource::new(bytes); + let archive = block_on(AsyncArchive::open(source, OpenOptions::default(), None)).unwrap(); + let opened = probe.reads.lock().unwrap().len(); + probe.baseline.store(opened, Ordering::SeqCst); + probe.max_ahead.store(0, Ordering::SeqCst); + (archive.with_read_limits(limits), probe, mode) +} + +/// Single-block requests that reserve 20 + 20 + 16 bytes each. +fn single_blocks(max_in_flight: usize, max_buffered_bytes: u64) -> ReadLimits { + ReadLimits { + max_in_flight, + max_buffered_bytes, + max_request_bytes: 20, + } +} + +#[test] +fn async_reads_equal_the_sync_archive() { + let bytes = blocks(64, true); + let sync = Archive::open(MemorySource::new(bytes.clone()), sender()).unwrap(); + let (source, probe, mode) = TestSource::new(&bytes); + *mode.lock().unwrap() = Mode::YieldOnce; + let archive = block_on(AsyncArchive::open(source, sender(), None)) + .unwrap() + .with_read_limits(ReadLimits { + max_in_flight: 3, + max_buffered_bytes: 4096, + max_request_bytes: 200, + }); + assert_eq!(archive.metadata_digest(), sync.metadata_digest()); + assert_eq!( + archive.entries().collect::>(), + sync.entries().collect::>() + ); + for range in [0..1024, 0..0, 5..37, 16..32, 1000..1024, 17..18] { + let mut expected = Vec::new(); + sync.copy_range_to("data", range.clone(), &mut expected) + .unwrap(); + let (output, error) = drain(archive.read_range("data", range.clone()).unwrap(), &probe); + assert!(error.is_none(), "{range:?}: {error:?}"); + assert_eq!(output, expected, "{range:?}"); + } + assert_eq!(probe.outstanding.load(Ordering::SeqCst), 0); + assert!(archive.read_range("data", 0..1025).is_err()); + assert!(archive.read_range("missing", 0..1).is_err()); +} + +#[test] +fn open_requests_do_not_depend_on_the_block_count() { + for count in [4, 1_000] { + let bytes = blocks(count, false); + let (source, probe, _) = TestSource::new(&bytes); + let len = bytes.len() as u64; + let archive = block_on(AsyncArchive::open( + source, + OpenOptions::default(), + Some(len), + )) + .unwrap(); + assert_eq!(probe.reads.lock().unwrap().len(), 3, "{count} blocks"); + assert_eq!(probe.len_calls.load(Ordering::SeqCst), 0); + assert_eq!(archive.view().version(), 0x0101); + } + let (_, probe, _) = open(&blocks(4, false), ReadLimits::default()); + assert_eq!(probe.len_calls.load(Ordering::SeqCst), 1); +} + +#[test] +fn reads_respect_the_in_flight_bound() { + let bytes = blocks(32, false); + let (archive, probe, mode) = open(&bytes, single_blocks(2, 1 << 20)); + *mode.lock().unwrap() = Mode::YieldOnce; + let (output, error) = drain(archive.read_range("data", 0..512).unwrap(), &probe); + assert!(error.is_none()); + assert_eq!(output, content(32)); + assert_eq!(probe.max_outstanding.load(Ordering::SeqCst), 2); + assert_eq!(probe.reads.lock().unwrap().len(), 3 + 32); +} + +#[test] +fn reads_respect_the_buffered_byte_bound() { + let bytes = blocks(32, false); + // One reservation, then less than one: each request waits for the previous chunk. + for max_buffered_bytes in [56, 1] { + let (archive, probe, mode) = open(&bytes, single_blocks(8, max_buffered_bytes)); + *mode.lock().unwrap() = Mode::YieldOnce; + let (output, error) = drain(archive.read_range("data", 0..512).unwrap(), &probe); + assert!(error.is_none()); + assert_eq!(output, content(32)); + assert_eq!( + probe.max_ahead.load(Ordering::SeqCst), + 1, + "{max_buffered_bytes}" + ); + } + // Each undelivered block holds at least its 16 output bytes and a new request 56 bytes. + let (archive, probe, _) = open(&bytes, single_blocks(100, 560)); + let (output, _) = drain(archive.read_range("data", 0..512).unwrap(), &probe); + assert_eq!(output, content(32)); + let ahead = probe.max_ahead.load(Ordering::SeqCst); + assert!((10..=(560 - 56) / 16 + 1).contains(&ahead), "{ahead}"); +} + +#[test] +fn dropping_a_stream_cancels_outstanding_requests() { + let bytes = blocks(16, false); + let (archive, probe, mode) = open(&bytes, single_blocks(4, 1 << 20)); + *mode.lock().unwrap() = Mode::Gated; + let mut stream = archive.read_range("data", 0..256).unwrap(); + assert!(next(&mut stream).is_pending()); + assert_eq!(probe.outstanding.load(Ordering::SeqCst), 4); + probe.completions.store(1, Ordering::SeqCst); + let Poll::Ready(Some(Ok(first))) = next(&mut stream) else { + panic!("the first block was released"); + }; + assert_eq!(first, content(1)); + // The completed request was replaced before the chunk was released. + assert_eq!(probe.outstanding.load(Ordering::SeqCst), 4); + drop(stream); + assert_eq!(probe.outstanding.load(Ordering::SeqCst), 0); + assert_eq!(probe.cancelled.load(Ordering::SeqCst), 4); + assert_eq!(probe.reads.lock().unwrap().len(), 3 + 5); +} + +#[test] +fn wrong_response_lengths_fail_without_output() { + let bytes = blocks(8, false); + for delta in [1, -1] { + let (archive, probe, mode) = open(&bytes, ReadLimits::default()); + *mode.lock().unwrap() = Mode::Resize(delta); + let (output, error) = drain(archive.read_range("data", 0..128).unwrap(), &probe); + assert!(output.is_empty()); + let Some(PithosError::Source(SourceError::ResponseLength { + expected, + actual, + revision, + .. + })) = error + else { + panic!("unexpected result {error:?}"); + }; + assert_eq!(actual as i64 - expected as i64, delta as i64); + assert_eq!(revision.as_deref(), Some("etag-1")); + + let (source, _, mode) = TestSource::new(&bytes); + *mode.lock().unwrap() = Mode::Resize(delta); + let result = block_on(AsyncArchive::open(source, OpenOptions::default(), None)); + assert!(matches!( + result, + Err(PithosError::Source(SourceError::ResponseLength { .. })) + )); + } +} + +#[test] +fn the_blocking_hook_runs_open_and_decode() { + let bytes = blocks(8, true); + let hook = CountingHook::default(); + let (source, probe, _) = TestSource::new(&bytes); + let archive = block_on(AsyncArchive::open_with_hook( + source, + sender(), + None, + hook.clone(), + )) + .unwrap() + .with_read_limits(single_blocks(2, 1 << 20)); + // The header, the footer and the one directory. + assert_eq!(hook.0.load(Ordering::SeqCst), 3); + let (output, error) = drain(archive.read_range("data", 0..128).unwrap(), &probe); + assert!(error.is_none()); + assert_eq!(output, content(8)); + assert_eq!(hook.0.load(Ordering::SeqCst), 3 + 8); +} + +struct TokioBlocking; + +impl BlockingHook for TokioBlocking { + fn spawn_blocking(&self, task: F) -> impl Future + Send + where + F: FnOnce() -> T + Send + 'static, + T: Send + 'static, + { + let handle = tokio::task::spawn_blocking(task); + async move { + match handle.await { + Ok(output) => output, + Err(error) => std::panic::resume_unwind(error.into_panic()), + } + } + } +} + +fn assert_send(value: T) -> T { + value +} + +#[test] +fn a_tokio_blocking_pool_reads_like_the_sync_archive() { + let bytes = blocks(64, true); + let sync = Archive::open(MemorySource::new(bytes.clone()), sender()).unwrap(); + let mut expected = Vec::new(); + sync.copy_to("data", &mut expected).unwrap(); + let runtime = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + let output = runtime.block_on(async { + let (source, _, mode) = TestSource::new(&bytes); + *mode.lock().unwrap() = Mode::YieldOnce; + let open = assert_send(AsyncArchive::open_with_hook( + source, + sender(), + None, + TokioBlocking, + )); + let archive = open.await.unwrap(); + let mut stream = assert_send(archive.read_range("data", 0..1024).unwrap()); + let mut output = Vec::new(); + while let Some(chunk) = std::future::poll_fn(|cx| Pin::new(&mut stream).poll_next(cx)).await + { + output.extend_from_slice(&chunk.unwrap()); + } + output + }); + assert_eq!(output, expected); +} From ff50d753801eb57b70efa49dde7925e6a5206e4a Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 01:16:42 +0200 Subject: [PATCH 117/166] test: read external blocks through the async resolver --- crates/pithos_lib/tests/async_reader.rs | 171 +++++++++++++++++++++++- 1 file changed, 169 insertions(+), 2 deletions(-) diff --git a/crates/pithos_lib/tests/async_reader.rs b/crates/pithos_lib/tests/async_reader.rs index 53d6ed4..ccda62f 100644 --- a/crates/pithos_lib/tests/async_reader.rs +++ b/crates/pithos_lib/tests/async_reader.rs @@ -3,8 +3,10 @@ mod common; use common::keys::private_key; use futures_core::Stream; use pithos_lib::archive::{ - AccessKeys, Archive, ArchivePath, ArchiveWriter, AsyncArchive, BlockingHook, Chunking, - EntryMetadata, OpenOptions, ProcessingOptions, ReadLimits, WriteOptions, + AccessKeys, Archive, ArchiveFeature, ArchivePath, ArchiveWriter, AsyncArchive, + AsyncExternalBlockResolver, BlockingHook, Chunking, EntryKind, EntryMetadata, + ExternalBlockAccessPolicy, ExternalLocation, OpenLimits, OpenOptions, ProcessingOptions, + ReadLimits, WriteOptions, }; use pithos_lib::error::PithosError; use pithos_lib::source::{AsyncArchiveSource, MemorySource, SourceError}; @@ -468,3 +470,168 @@ fn a_tokio_blocking_pool_reads_like_the_sync_archive() { }); assert_eq!(output, expected); } + +const SPEC: &str = include_str!("../../../spec/PITHOS_1.0.0_draft.md"); +const INITIAL_TARGET: &str = "https://storage.test/initial"; +const REDIRECT_TARGET: &str = "https://storage.test/redirect"; + +/// CV-LOCAL-HELLO-279 with its block moved to the opaque external location "x". +fn external_hello() -> Vec { + let start = SPEC.find("#### CV-LOCAL-HELLO-279").unwrap(); + let block = SPEC[start..].split("```text\n").nth(1).unwrap(); + let mut bytes = block + .split("```") + .next() + .unwrap() + .lines() + .flat_map(|line| line.split_once(": ").unwrap().1.split_whitespace()) + .map(|hex| u8::from_str_radix(hex, 16).unwrap()) + .collect::>(); + bytes.splice(143..=143, [1, 1, b'x']); + let footer = bytes.len() - 12; + let directory_len = (bytes.len() - 15) as u64; + bytes[footer..footer + 8].copy_from_slice(&directory_len.to_be_bytes()); + let checksum = crc32fast::hash(&bytes[15..bytes.len() - 4]); + let crc = bytes.len() - 4; + bytes[crc..].copy_from_slice(&checksum.to_be_bytes()); + bytes +} + +struct RecordingPolicy { + checks: Mutex>, + denied: Option<&'static str>, +} + +impl ExternalBlockAccessPolicy for RecordingPolicy { + fn allows(&self, target: &str) -> bool { + self.checks.lock().unwrap().push(target.to_owned()); + self.denied != Some(target) + } +} + +/// Follows one redirect and answers with a fixed response. +struct RedirectingResolver { + response: Vec, + calls: Arc>>, +} + +impl AsyncExternalBlockResolver for RedirectingResolver { + async fn resolve( + &self, + policy: &dyn ExternalBlockAccessPolicy, + location: &ExternalLocation, + expected_len: u64, + max_response_size: u64, + ) -> Result, PithosError> { + assert_eq!(location.as_str(), "x"); + for target in [INITIAL_TARGET, REDIRECT_TARGET] { + if !policy.allows(target) { + return Err(PithosError::ExternalBlockAccessDenied); + } + } + self.calls + .lock() + .unwrap() + .push((expected_len, max_response_size)); + Ok(self.response.clone()) + } +} + +/// The result of one external read, the policy checks and the resolver calls. +struct ExternalRead { + result: Result, PithosError>, + checks: Vec, + calls: Vec<(u64, u64)>, +} + +fn external_read( + response: &[u8], + denied: Option<&'static str>, + range: std::ops::Range, +) -> ExternalRead { + let calls = Arc::new(Mutex::new(Vec::new())); + let policy = Arc::new(RecordingPolicy { + checks: Mutex::new(Vec::new()), + denied, + }); + let resolver = RedirectingResolver { + response: response.to_vec(), + calls: Arc::clone(&calls), + }; + let options = OpenOptions::default() + .with_external_resolver(resolver) + .with_external_access_policy(policy.clone()); + let (source, probe, _) = TestSource::new(&external_hello()); + let archive = block_on(AsyncArchive::open(source, options, None)).unwrap(); + let (output, error) = drain(archive.read_range("hello", range).unwrap(), &probe); + assert!(error.is_none() || output.is_empty()); + let result = error.map_or(Ok(output), Err); + let checks = policy.checks.lock().unwrap().clone(); + let calls = calls.lock().unwrap().clone(); + ExternalRead { + result, + checks, + calls, + } +} + +#[test] +fn external_blocks_need_a_resolver_and_a_policy() { + let bytes = external_hello(); + let (source, _, _) = TestSource::new(&bytes); + let disabled = block_on(AsyncArchive::open(source, OpenOptions::default(), None)).unwrap(); + assert!(matches!( + disabled.entry("hello").unwrap().unwrap().kind, + EntryKind::File { + available: false, + .. + } + )); + assert!(matches!( + disabled.read_range("hello", 0..5), + Err(PithosError::UnsupportedFeature( + ArchiveFeature::ExternalStorage + )) + )); + + let (source, _, _) = TestSource::new(&bytes); + let resolver = RedirectingResolver { + response: b"BLCKhello".to_vec(), + calls: Arc::default(), + }; + let options = OpenOptions::default().with_external_resolver(resolver); + let without_policy = block_on(AsyncArchive::open(source, options, None)).unwrap(); + assert!(without_policy.read_range("hello", 0..5).is_err()); +} + +#[test] +fn external_blocks_are_read_through_the_async_resolver() { + let max_response_size = OpenLimits::default().max_stored_block_bytes + 4; + let ExternalRead { + result, + checks, + calls, + } = external_read(b"BLCKhello", None, 1..4); + assert_eq!(result.unwrap(), b"ell"); + assert_eq!(checks, [INITIAL_TARGET, REDIRECT_TARGET]); + assert_eq!(calls, [(9, max_response_size)]); + + let ExternalRead { + result, + checks, + calls, + } = external_read(b"BLCKhello", Some(REDIRECT_TARGET), 0..5); + assert!(matches!( + result, + Err(PithosError::ExternalBlockAccessDenied) + )); + assert_eq!(checks, [INITIAL_TARGET, REDIRECT_TARGET]); + assert!(calls.is_empty()); + + for response in [&b"BLCKhello!"[..], b"BLCKhell"] { + let ExternalRead { result, .. } = external_read(response, None, 0..5); + assert!(matches!(result, Err(PithosError::ExternalBlockFraming(_)))); + } + let ExternalRead { result, .. } = external_read(b"BLCKjello", None, 0..5); + assert!(matches!(result, Err(PithosError::BlockHashMismatch { .. }))); +} From f3b72cfca850822bdc32bc18d37d77fb22b910da Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 01:16:44 +0200 Subject: [PATCH 118/166] chore: describe the async feature in the library readme --- crates/pithos_lib/README.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/crates/pithos_lib/README.md b/crates/pithos_lib/README.md index 19577db..da53cac 100644 --- a/crates/pithos_lib/README.md +++ b/crates/pithos_lib/README.md @@ -87,6 +87,27 @@ Opening validates archive structure and accessible metadata. Payload integrity i `fs::extract` is Linux-only. It walks destination components without following symlinks, stages regular-file output anonymously with Linux `O_TMPFILE`, and publishes without replacing an existing entry. The destination filesystem must support `O_TMPFILE` and `linkat(AT_EMPTY_PATH)`; extraction can therefore fail on a collision rather than overwrite a destination. +## Async reading + +The optional `async` feature adds `source::AsyncArchiveSource`, `archive::AsyncArchive` and +`archive::AsyncExternalBlockResolver`. It depends only on `futures-core` and needs no specific +async runtime. + +- `AsyncArchive::open(source, options, known_len)` issues the header, footer and directory + requests one after the other. Their count does not depend on the number of blocks. Pass the + object length as `known_len` to skip the length request. +- `read_range(path, range)` returns a `Stream` of verified plaintext chunks in file order. + `ReadLimits` bounds the outstanding requests and the bytes held for responses, decoding and + ordered delivery. Dropping the stream cancels its outstanding requests. +- The source must serve one immutable object revision for the open and every later read, and + each response must have exactly the requested length. +- Decoding directories and blocks is CPU work. `open` runs it inline, which suits only small + archives. Pass a `BlockingHook` to `open_with_hook` to move it to a blocking pool, for example + one that forwards to `tokio::task::spawn_blocking`. +- External blocks follow the rules of the synchronous reader: they are read only when both a + resolver and an access policy are supplied, and the resolver checks the policy before its + initial access and before every redirect. + ## RO-Crate ingestion Load a directory or ZIP RO-Crate, then convert its retained source through a configured writer: From b732166c52c99c9b66b41d4b9d3a183f074ddc0b Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 01:16:45 +0200 Subject: [PATCH 119/166] fix: split async read requests by their full decode reservation --- crates/pithos_lib/src/archive/async_stream.rs | 168 ++++++++++++------ 1 file changed, 118 insertions(+), 50 deletions(-) diff --git a/crates/pithos_lib/src/archive/async_stream.rs b/crates/pithos_lib/src/archive/async_stream.rs index f71aed8..08439fe 100644 --- a/crates/pithos_lib/src/archive/async_stream.rs +++ b/crates/pithos_lib/src/archive/async_stream.rs @@ -1,7 +1,8 @@ use super::async_reader::{AsyncArchive, AsyncExternalBlockResolver, BlockingHook}; -use super::planning::{BlockBatch, BlockBatches}; +use super::planning::{BlockBatch, BlockBatches, BlockRequest, PlannedBlock}; use super::view::ArchiveView; use crate::error::PithosError; +use crate::format::block::ProcessingFlags; use crate::source::AsyncArchiveSource; use futures_core::Stream; use std::collections::VecDeque; @@ -13,11 +14,14 @@ use zeroize::Zeroizing; /// Bounds for one range stream. /// -/// A batch reserves its response length, one more copy of its largest stored block (the payload -/// copy made while decoding), the decoded size of its blocks, and the planned output of blocks -/// that are only partly in the range. After decoding, the batch keeps only its planned output, -/// which is released as the chunks are delivered. A batch that does not fit next to the work -/// already buffered waits; when nothing else is buffered it starts alone, even if it is larger. +/// Each request reserves its response, the planned output of its blocks, and the largest +/// working set of decoding one of them: the payload copy, the decrypted bytes of an encrypted +/// compressed block, the decoded block and a copy of a partial output. After decoding, the +/// request keeps only its planned output, which is released as the chunks are delivered. +/// +/// Adjacent blocks share a request only while the whole reservation fits `max_buffered_bytes`. +/// A request that does not fit next to the work already buffered waits. The only request that +/// may exceed the bound is a single block whose own reservation is larger; it starts alone. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub struct ReadLimits { /// Source or external requests that may be outstanding at once. Zero counts as one. @@ -41,7 +45,7 @@ impl Default for ReadLimits { type Pending<'a, T> = Pin> + Send + 'a>>; enum State<'a> { - Fetching(Pending<'a, Vec>, BlockBatch), + Fetching(Pending<'a, Vec>, Vec), Decoding(Pending<'a, Vec>>), Ready(VecDeque>), Failed(PithosError), @@ -57,7 +61,7 @@ struct Slot<'a> { pub struct RangeStream<'a, S, E, B> { archive: &'a AsyncArchive, batches: Option>, - waiting: Option<(BlockBatch, u64)>, + waiting: VecDeque, slots: VecDeque>, limits: ReadLimits, buffered: u64, @@ -78,7 +82,7 @@ where Self { archive, batches: Some(batches), - waiting: None, + waiting: VecDeque::new(), slots: VecDeque::new(), limits, buffered: 0, @@ -86,16 +90,20 @@ where } } + /// The bytes currently reserved by requests that are fetched, decoded or waiting for + /// delivery, as described in [`ReadLimits`]. + pub fn buffered_bytes(&self) -> u64 { + self.buffered + } + /// Starts requests while the in-flight and byte bounds allow. Returns whether any started. fn admit(&mut self) -> bool { let mut admitted = false; while self.fetching < self.limits.max_in_flight.max(1) { - let (batch, reserved) = match self.waiting.take() { - Some(next) => next, - None => match self.batches.as_mut().and_then(Iterator::next) { + if self.waiting.is_empty() { + match self.batches.as_mut().and_then(Iterator::next) { Some(Ok(batch)) => { - let reserved = reservation(&batch); - (batch, reserved) + self.waiting = split(&batch, self.limits.max_buffered_bytes); } Some(Err(error)) => { self.batches = None; @@ -109,20 +117,24 @@ where self.batches = None; break; } - }, + } + } + let Some(group) = self.waiting.pop_front() else { + continue; }; + let reserved = group.reserved(); if !self.slots.is_empty() && self.buffered.saturating_add(reserved) > self.limits.max_buffered_bytes { - self.waiting = Some((batch, reserved)); + self.waiting.push_front(group); break; } self.buffered = self.buffered.saturating_add(reserved); self.fetching += 1; - let fetch = Box::pin(self.archive.fetch(batch.request())); + let fetch = Box::pin(self.archive.fetch(group.request())); self.slots.push_back(Slot { reserved, - state: State::Fetching(fetch, batch), + state: State::Fetching(fetch, group.blocks), }); admitted = true; } @@ -139,7 +151,7 @@ where State::Fetching(fetch, _) => match fetch.as_mut().poll(cx) { Poll::Ready(result) => { self.fetching -= 1; - let State::Fetching(_, batch) = + let State::Fetching(_, blocks) = std::mem::replace(&mut slot.state, State::Taken) else { unreachable!("the slot was fetching"); @@ -149,7 +161,7 @@ where let view = Arc::clone(&self.archive.view); let stored = Zeroizing::new(stored); State::Decoding(Box::pin(self.archive.hook.spawn_blocking( - move || decode_batch(&view, &batch, &stored), + move || decode_group(&view, &blocks, &stored), ))) } Err(error) => State::Failed(error), @@ -179,7 +191,7 @@ where } if failed { self.batches = None; - self.waiting = None; + self.waiting.clear(); } progressed } @@ -187,7 +199,7 @@ where /// Delivers the next chunk in file order, if the first batch has one ready. fn deliver(&mut self) -> Option, PithosError>>> { let Some(slot) = self.slots.front_mut() else { - let finished = self.batches.is_none() && self.waiting.is_none(); + let finished = self.batches.is_none() && self.waiting.is_empty(); return finished.then_some(None); }; let item = match &mut slot.state { @@ -210,7 +222,7 @@ where // Dropping the remaining slots cancels their requests. self.slots.clear(); self.batches = None; - self.waiting = None; + self.waiting.clear(); Err(error) } State::Fetching(..) | State::Decoding(_) | State::Taken => return None, @@ -249,38 +261,94 @@ where } } -/// The bytes a batch may hold at once. See [`ReadLimits`]. -fn reservation(batch: &BlockBatch) -> u64 { - batch.blocks().iter().fold( - batch - .blocks() - .iter() - .map(|block| block.framed_len()) - .max() - .unwrap_or(0), - |total, block| { - let output = block.output.len() as u64; - let partial = if output == block.descriptor.original_size { - 0 - } else { - output - }; - total - .saturating_add(block.framed_len()) - .saturating_add(block.descriptor.original_size) - .saturating_add(partial) - }, - ) +/// Adjacent planned blocks fetched by one request, with the parts of their reservation. +#[derive(Default)] +struct Group { + blocks: Vec, + response: u64, + output: u64, + working: u64, +} + +impl Group { + /// The bytes this request may hold at once. See [`ReadLimits`]. + fn reserved(&self) -> u64 { + self.response + .saturating_add(self.output) + .saturating_add(self.working) + } + + fn push(&mut self, block: PlannedBlock) { + let framed = block.framed_len(); + let original = block.descriptor.original_size; + let output = block.output.len() as u64; + let flags = ProcessingFlags::from_byte(block.descriptor.processing.to_byte()); + // Decrypting before decompressing holds the payload copy and the decrypted bytes. + let decrypted = if flags.is_encrypted() && flags.get_compression_level() > 0 { + framed + } else { + 0 + }; + let partial = if output == original { 0 } else { output }; + let working = framed + .saturating_add(decrypted) + .saturating_add(original) + .saturating_add(partial); + self.response = self.response.saturating_add(framed); + self.output = self.output.saturating_add(output); + self.working = self.working.max(working); + self.blocks.push(block); + } + + fn request(&self) -> BlockRequest { + let mut request = self.blocks[0].request(); + if let BlockRequest::Local { len, .. } = &mut request { + *len = self.response; + } + request + } +} + +/// Splits a batch into requests whose reservation fits `max_bytes`, keeping file order. +/// A block that does not fit even alone becomes a request of its own. +fn split(batch: &BlockBatch, max_bytes: u64) -> VecDeque { + let mut groups = VecDeque::new(); + let mut current = Group::default(); + for block in batch.blocks() { + let mut candidate = Group { + blocks: Vec::new(), + ..current + }; + candidate.push(block.clone()); + if !current.blocks.is_empty() && candidate.reserved() > max_bytes { + groups.push_back(std::mem::take(&mut current)); + } + current.push(block.clone()); + } + if !current.blocks.is_empty() { + groups.push_back(current); + } + groups } -/// Decodes every block of a batch and keeps only the planned output of each. -fn decode_batch( +/// Decodes every block of a request and keeps only the planned output of each. +fn decode_group( view: &ArchiveView, - batch: &BlockBatch, + blocks: &[PlannedBlock], stored: &[u8], ) -> Result>, PithosError> { - let mut chunks = Vec::with_capacity(batch.blocks().len()); - for (block, bytes) in batch.split(stored)? { + let expected = blocks.iter().map(PlannedBlock::framed_len).sum::(); + if stored.len() as u64 != expected { + return Err(PithosError::BlockSizeMismatch { + expected, + actual: stored.len() as u64, + }); + } + let mut chunks = Vec::with_capacity(blocks.len()); + let mut rest = stored; + for block in blocks { + let (bytes, tail) = rest.split_at(block.framed_len() as usize); + rest = tail; let mut plaintext = view.decode_block(block, bytes)?; let output = block.output(); chunks.push(if output.start == 0 && output.end == plaintext.len() { From 8ded1b0b66f15efcd4204a2c6c68851f220d382d Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 01:16:45 +0200 Subject: [PATCH 120/166] test: gate async reads and decodes and check compressed read budgets --- crates/pithos_lib/tests/async_reader.rs | 383 ++++++++++++++++++++++-- 1 file changed, 361 insertions(+), 22 deletions(-) diff --git a/crates/pithos_lib/tests/async_reader.rs b/crates/pithos_lib/tests/async_reader.rs index ccda62f..ac7878d 100644 --- a/crates/pithos_lib/tests/async_reader.rs +++ b/crates/pithos_lib/tests/async_reader.rs @@ -4,13 +4,15 @@ use common::keys::private_key; use futures_core::Stream; use pithos_lib::archive::{ AccessKeys, Archive, ArchiveFeature, ArchivePath, ArchiveWriter, AsyncArchive, - AsyncExternalBlockResolver, BlockingHook, Chunking, EntryKind, EntryMetadata, - ExternalBlockAccessPolicy, ExternalLocation, OpenLimits, OpenOptions, ProcessingOptions, - ReadLimits, WriteOptions, + AsyncExternalBlockResolver, BlockRequest, BlockingHook, Chunking, EntryKind, EntryMetadata, + ExternalBlockAccessPolicy, ExternalLocation, NoExternalBlocks, OpenLimits, OpenOptions, + ProcessingOptions, ReadLimits, WriteOptions, }; use pithos_lib::error::PithosError; use pithos_lib::source::{AsyncArchiveSource, MemorySource, SourceError}; +use std::collections::HashSet; use std::future::Future; +use std::ops::Range; use std::pin::{Pin, pin}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; @@ -22,7 +24,8 @@ enum Mode { Ready, /// Each read returns `Pending` once, so several reads overlap. YieldOnce, - /// Reads complete only while `Probe::completions` allows it. + /// A read completes once its offset is in `Probe::released`, and fails if it is also in + /// `Probe::failing`. Gated, /// Responses are one byte longer or shorter than requested. Resize(isize), @@ -35,7 +38,8 @@ struct Probe { outstanding: AtomicUsize, max_outstanding: AtomicUsize, cancelled: AtomicUsize, - completions: AtomicUsize, + released: Mutex>, + failing: Mutex>, delivered: AtomicUsize, /// Reads issued while opening, which `max_ahead` ignores. baseline: AtomicUsize, @@ -89,6 +93,7 @@ impl AsyncArchiveSource for TestSource { _ => offset + len, }; Read { + offset, probe: Arc::clone(&self.probe), response: Some(self.bytes[offset as usize..end as usize].to_vec()), mode, @@ -103,6 +108,7 @@ impl AsyncArchiveSource for TestSource { /// A read that records completion and cancellation. struct Read { + offset: u64, probe: Arc, response: Option>, mode: Mode, @@ -120,20 +126,20 @@ impl Future for Read { cx.waker().wake_by_ref(); return Poll::Pending; } - Mode::Gated => { - let allowed = this.probe.completions.fetch_update( - Ordering::SeqCst, - Ordering::SeqCst, - |count| count.checked_sub(1), - ); - if allowed.is_err() { - return Poll::Pending; - } + Mode::Gated if !this.probe.released.lock().unwrap().contains(&this.offset) => { + return Poll::Pending; } _ => {} } this.probe.outstanding.fetch_sub(1, Ordering::SeqCst); - Poll::Ready(Ok(this.response.take().unwrap())) + let response = this.response.take().unwrap(); + if this.probe.failing.lock().unwrap().contains(&this.offset) { + return Poll::Ready(Err(SourceError::Remote { + offset: this.offset, + message: "injected failure".into(), + })); + } + Poll::Ready(Ok(response)) } } @@ -235,6 +241,164 @@ fn blocks(count: u64, encrypted: bool) -> Vec { writer.finish().unwrap() } +/// One planned block: its stored offset, framed length, decoded size and planned output. +#[derive(Clone, Copy, Debug)] +struct Planned { + offset: u64, + framed: u64, + output: u64, +} + +fn planned(archive: &AsyncArchive, range: Range) -> Vec +where + E: AsyncExternalBlockResolver, + B: BlockingHook, +{ + archive + .view() + .plan_range("data", range) + .unwrap() + .map(|block| { + let block = block.unwrap(); + let BlockRequest::Local { offset, len } = block.request() else { + panic!("unexpected external block"); + }; + Planned { + offset, + framed: len, + output: block.output().len() as u64, + } + }) + .collect() +} + +fn release(probe: &Probe, offset: u64) { + probe.released.lock().unwrap().insert(offset); +} + +/// Hook tasks run only once their call index is released, or all of them while `open` is set. +#[derive(Default)] +struct HookGate { + open: std::sync::atomic::AtomicBool, + started: AtomicUsize, + finished: AtomicUsize, + dropped: AtomicUsize, + released: Mutex>, +} + +#[derive(Clone, Default)] +struct GatedHook(Arc); + +impl BlockingHook for GatedHook { + fn spawn_blocking(&self, task: F) -> impl Future + Send + where + F: FnOnce() -> T + Send + 'static, + T: Send + 'static, + { + GatedTask { + index: self.0.started.fetch_add(1, Ordering::SeqCst), + gate: Arc::clone(&self.0), + task: Some(Box::new(task)), + } + } +} + +struct GatedTask { + index: usize, + gate: Arc, + task: Option>, +} + +impl T, T> Future for GatedTask { + type Output = T; + + fn poll(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + let released = this.gate.open.load(Ordering::SeqCst) + || this.gate.released.lock().unwrap().contains(&this.index); + if !released { + return Poll::Pending; + } + let output = (this.task.take().unwrap())(); + this.gate.finished.fetch_add(1, Ordering::SeqCst); + Poll::Ready(output) + } +} + +impl Drop for GatedTask { + fn drop(&mut self) { + if self.task.is_some() { + self.gate.dropped.fetch_add(1, Ordering::SeqCst); + } + } +} + +/// Opens with a hook that runs the open steps and then holds every decode. +fn open_gated( + bytes: &[u8], + limits: ReadLimits, +) -> ( + AsyncArchive, + Arc, + GatedHook, +) { + let (source, probe, mode) = TestSource::new(bytes); + let hook = GatedHook::default(); + hook.0.open.store(true, Ordering::SeqCst); + let archive = block_on(AsyncArchive::open_with_hook( + source, + OpenOptions::default(), + None, + hook.clone(), + )) + .unwrap() + .with_read_limits(limits); + hook.0.open.store(false, Ordering::SeqCst); + *mode.lock().unwrap() = Mode::Gated; + (archive, probe, hook) +} + +/// Blocks of 4096 bytes that are zero except for their index, so they compress well. +fn compressible(count: u64) -> Vec { + (0..count) + .flat_map(|block| { + let mut bytes = vec![0; 4096]; + bytes[..8].copy_from_slice(&block.to_be_bytes()); + bytes + }) + .collect() +} + +fn compressed_blocks(count: u64, encrypted: bool) -> Vec { + let sender = private_key("sender"); + let options = WriteOptions::new(sender.duplicate(), vec![sender.public_key()]) + .with_chunking(Chunking::Fixed(4096)); + let mut writer = ArchiveWriter::create(Vec::new(), options).unwrap(); + let content = compressible(count); + writer + .add_file( + ArchivePath::new("data").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(encrypted, 3).unwrap(), + Some(content.len() as u64), + std::io::Cursor::new(content), + ) + .unwrap(); + writer.finish().unwrap() +} + +/// The documented reservation of one request over full, encrypted and compressed blocks. +fn reservation(blocks: &[Planned]) -> u64 { + let response = blocks.iter().map(|block| block.framed).sum::(); + let output = blocks.iter().map(|block| block.output).sum::(); + let working = blocks + .iter() + .map(|block| 2 * block.framed + block.output) + .max() + .unwrap(); + response + output + working +} + fn sender() -> OpenOptions { OpenOptions::default().with_access_keys(AccessKeys::new().with_key(private_key("sender"))) } @@ -242,16 +406,24 @@ fn sender() -> OpenOptions { fn open( bytes: &[u8], limits: ReadLimits, +) -> (AsyncArchive, Arc, Arc>) { + open_as(bytes, OpenOptions::default(), limits) +} + +fn open_as( + bytes: &[u8], + options: OpenOptions, + limits: ReadLimits, ) -> (AsyncArchive, Arc, Arc>) { let (source, probe, mode) = TestSource::new(bytes); - let archive = block_on(AsyncArchive::open(source, OpenOptions::default(), None)).unwrap(); + let archive = block_on(AsyncArchive::open(source, options, None)).unwrap(); let opened = probe.reads.lock().unwrap().len(); probe.baseline.store(opened, Ordering::SeqCst); probe.max_ahead.store(0, Ordering::SeqCst); (archive.with_read_limits(limits), probe, mode) } -/// Single-block requests that reserve 20 + 20 + 16 bytes each. +/// Single-block requests that reserve 20 + 16 + (20 + 16) bytes each. fn single_blocks(max_in_flight: usize, max_buffered_bytes: u64) -> ReadLimits { ReadLimits { max_in_flight, @@ -327,7 +499,7 @@ fn reads_respect_the_in_flight_bound() { fn reads_respect_the_buffered_byte_bound() { let bytes = blocks(32, false); // One reservation, then less than one: each request waits for the previous chunk. - for max_buffered_bytes in [56, 1] { + for max_buffered_bytes in [72, 1] { let (archive, probe, mode) = open(&bytes, single_blocks(8, max_buffered_bytes)); *mode.lock().unwrap() = Mode::YieldOnce; let (output, error) = drain(archive.read_range("data", 0..512).unwrap(), &probe); @@ -339,12 +511,12 @@ fn reads_respect_the_buffered_byte_bound() { "{max_buffered_bytes}" ); } - // Each undelivered block holds at least its 16 output bytes and a new request 56 bytes. - let (archive, probe, _) = open(&bytes, single_blocks(100, 560)); + // Each undelivered block holds at least its 16 output bytes and a new request 72 bytes. + let (archive, probe, _) = open(&bytes, single_blocks(100, 720)); let (output, _) = drain(archive.read_range("data", 0..512).unwrap(), &probe); assert_eq!(output, content(32)); let ahead = probe.max_ahead.load(Ordering::SeqCst); - assert!((10..=(560 - 56) / 16 + 1).contains(&ahead), "{ahead}"); + assert!((10..=(720 - 72) / 16 + 1).contains(&ahead), "{ahead}"); } #[test] @@ -355,7 +527,7 @@ fn dropping_a_stream_cancels_outstanding_requests() { let mut stream = archive.read_range("data", 0..256).unwrap(); assert!(next(&mut stream).is_pending()); assert_eq!(probe.outstanding.load(Ordering::SeqCst), 4); - probe.completions.store(1, Ordering::SeqCst); + release(&probe, planned(&archive, 0..16)[0].offset); let Poll::Ready(Some(Ok(first))) = next(&mut stream) else { panic!("the first block was released"); }; @@ -635,3 +807,170 @@ fn external_blocks_are_read_through_the_async_resolver() { let ExternalRead { result, .. } = external_read(b"BLCKjello", None, 0..5); assert!(matches!(result, Err(PithosError::BlockHashMismatch { .. }))); } + +#[test] +fn reads_completed_in_reverse_are_delivered_in_file_order() { + let bytes = blocks(8, false); + let (archive, probe, mode) = open(&bytes, single_blocks(4, 1 << 20)); + *mode.lock().unwrap() = Mode::Gated; + let blocks = planned(&archive, 0..64); + let mut stream = archive.read_range("data", 0..64).unwrap(); + assert!(next(&mut stream).is_pending()); + for block in blocks[1..].iter().rev() { + release(&probe, block.offset); + assert!(next(&mut stream).is_pending()); + } + assert_eq!(probe.outstanding.load(Ordering::SeqCst), 1); + release(&probe, blocks[0].offset); + for block in 0..4 { + let Poll::Ready(Some(Ok(chunk))) = next(&mut stream) else { + panic!("block {block} was released"); + }; + assert_eq!(chunk, content(block + 1)[block as usize * 16..]); + } + assert!(matches!(next(&mut stream), Poll::Ready(None))); +} + +#[test] +fn decodes_completed_in_reverse_are_delivered_in_file_order() { + let bytes = blocks(4, false); + let (archive, probe, hook) = open_gated(&bytes, single_blocks(4, 1 << 20)); + for block in planned(&archive, 0..64) { + release(&probe, block.offset); + } + let mut stream = archive.read_range("data", 0..64).unwrap(); + assert!(next(&mut stream).is_pending()); + assert_eq!(hook.0.started.load(Ordering::SeqCst), 3 + 4); + for index in (4..7).rev() { + hook.0.released.lock().unwrap().insert(index); + assert!(next(&mut stream).is_pending()); + } + assert_eq!(hook.0.finished.load(Ordering::SeqCst), 3 + 3); + hook.0.released.lock().unwrap().insert(3); + let (output, error) = drain(stream, &probe); + assert!(error.is_none()); + assert_eq!(output, content(4)); +} + +#[test] +fn a_later_failure_ends_the_stream_after_the_earlier_output() { + let bytes = blocks(8, false); + let (archive, probe, mode) = open(&bytes, single_blocks(4, 1 << 20)); + *mode.lock().unwrap() = Mode::Gated; + let blocks = planned(&archive, 0..128); + probe.failing.lock().unwrap().insert(blocks[2].offset); + let mut stream = archive.read_range("data", 0..128).unwrap(); + assert!(next(&mut stream).is_pending()); + release(&probe, blocks[2].offset); + assert!(next(&mut stream).is_pending()); + release(&probe, blocks[1].offset); + assert!(next(&mut stream).is_pending()); + release(&probe, blocks[0].offset); + let (output, error) = drain(stream, &probe); + assert_eq!(output, content(2)); + assert!(matches!( + error, + Some(PithosError::Source(SourceError::Remote { offset, .. })) if offset == blocks[2].offset + )); + // The pending sibling was cancelled and no request started after the failure. + assert_eq!(probe.cancelled.load(Ordering::SeqCst), 1); + assert_eq!(probe.outstanding.load(Ordering::SeqCst), 0); + assert_eq!(probe.reads.lock().unwrap().len(), 3 + 4); +} + +#[test] +fn dropping_a_stream_during_a_decode_drops_the_hook_task() { + let bytes = blocks(8, false); + let (archive, probe, hook) = open_gated(&bytes, single_blocks(4, 1 << 20)); + let first = planned(&archive, 0..16)[0]; + release(&probe, first.offset); + let mut stream = archive.read_range("data", 0..128).unwrap(); + assert!(next(&mut stream).is_pending()); + assert_eq!(hook.0.started.load(Ordering::SeqCst), 3 + 1); + assert_eq!(probe.outstanding.load(Ordering::SeqCst), 4); + drop(stream); + assert_eq!(hook.0.dropped.load(Ordering::SeqCst), 1); + assert_eq!(hook.0.finished.load(Ordering::SeqCst), 3); + assert_eq!(probe.cancelled.load(Ordering::SeqCst), 4); + assert_eq!(probe.outstanding.load(Ordering::SeqCst), 0); +} + +#[test] +fn compressed_blocks_share_a_request_only_within_the_byte_bound() { + let bytes = compressed_blocks(32, true); + let limits = ReadLimits { + max_in_flight: 8, + max_buffered_bytes: 40_000, + max_request_bytes: 1 << 20, + }; + let (archive, probe, _) = open_as(&bytes, sender(), limits); + let blocks = planned(&archive, 0..32 * 4096); + // Compression is kept: every block stores far fewer bytes than it decodes to. + assert!(blocks.iter().all(|block| block.framed * 10 < block.output)); + let mut stream = archive.read_range("data", 0..32 * 4096).unwrap(); + let mut output = Vec::new(); + let mut peak = 0; + while let Poll::Ready(Some(chunk)) = next(&mut stream) { + peak = peak.max(stream.buffered_bytes()); + output.extend_from_slice(&chunk.unwrap()); + } + assert!(matches!(next(&mut stream), Poll::Ready(None))); + assert_eq!(output, compressible(32)); + assert!(peak > 0 && peak <= limits.max_buffered_bytes, "{peak}"); + let reads = probe.reads.lock().unwrap()[3..].to_vec(); + assert!(reads.len() > 1 && reads.len() < blocks.len(), "{reads:?}"); + let mut rest = &blocks[..]; + for (offset, len) in reads { + let count = rest + .iter() + .scan(0, |total, block| { + *total += block.framed; + Some(*total) + }) + .position(|total| total == len) + .unwrap() + + 1; + assert_eq!(rest[0].offset, offset); + assert!(reservation(&rest[..count]) <= limits.max_buffered_bytes); + rest = &rest[count..]; + } + assert!(rest.is_empty()); + + // A bound below one block still reads, one block per request. + let limits = ReadLimits { + max_buffered_bytes: 100, + ..limits + }; + let (archive, probe, _) = open_as(&bytes, sender(), limits); + let (output, error) = drain(archive.read_range("data", 0..32 * 4096).unwrap(), &probe); + assert!(error.is_none()); + assert_eq!(output, compressible(32)); + let reads = probe.reads.lock().unwrap()[3..].to_vec(); + assert_eq!(reads.len(), blocks.len()); + assert!( + reads + .iter() + .zip(&blocks) + .all(|(read, block)| read.1 == block.framed) + ); +} + +#[test] +fn encrypted_compressed_blocks_reserve_the_decrypted_bytes() { + for encrypted in [true, false] { + let bytes = compressed_blocks(4, encrypted); + let (archive, probe, mode) = open_as(&bytes, sender(), single_blocks(1, 1 << 20)); + *mode.lock().unwrap() = Mode::Gated; + let block = planned(&archive, 0..4096)[0]; + assert!(block.framed * 10 < block.output); + let mut stream = archive.read_range("data", 0..4096).unwrap(); + assert!(next(&mut stream).is_pending()); + let decrypted = if encrypted { block.framed } else { 0 }; + let expected = block.framed + block.output + (block.framed + decrypted + block.output); + assert_eq!(stream.buffered_bytes(), expected, "encrypted {encrypted}"); + release(&probe, block.offset); + let (output, error) = drain(stream, &probe); + assert!(error.is_none()); + assert_eq!(output, compressible(1)); + } +} From 5f5ed43b77c05393d67da14e6f171a297c3365d4 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 01:16:46 +0200 Subject: [PATCH 121/166] fix: fetch and reserve a repeated adjacent block once in async reads --- crates/pithos_lib/src/archive/async_stream.rs | 40 +++++++++++++++++-- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/crates/pithos_lib/src/archive/async_stream.rs b/crates/pithos_lib/src/archive/async_stream.rs index 08439fe..443ffe5 100644 --- a/crates/pithos_lib/src/archive/async_stream.rs +++ b/crates/pithos_lib/src/archive/async_stream.rs @@ -268,6 +268,7 @@ struct Group { response: u64, output: u64, working: u64, + last_span: Option<(u64, u64)>, } impl Group { @@ -294,7 +295,12 @@ impl Group { .saturating_add(decrypted) .saturating_add(original) .saturating_add(partial); - self.response = self.response.saturating_add(framed); + // A block that repeats the span before it reuses the bytes already fetched. + let span = local_span(&block); + if span.is_none() || span != self.last_span { + self.response = self.response.saturating_add(framed); + } + self.last_span = span; self.output = self.output.saturating_add(output); self.working = self.working.max(working); self.blocks.push(block); @@ -309,6 +315,13 @@ impl Group { } } +fn local_span(block: &PlannedBlock) -> Option<(u64, u64)> { + match block.request() { + BlockRequest::Local { offset, len } => Some((offset, len)), + BlockRequest::External { .. } => None, + } +} + /// Splits a batch into requests whose reservation fits `max_bytes`, keeping file order. /// A block that does not fit even alone becomes a request of its own. fn split(batch: &BlockBatch, max_bytes: u64) -> VecDeque { @@ -337,7 +350,17 @@ fn decode_group( blocks: &[PlannedBlock], stored: &[u8], ) -> Result>, PithosError> { - let expected = blocks.iter().map(PlannedBlock::framed_len).sum::(); + let mut previous = None; + let expected = blocks + .iter() + .filter(|block| { + let span = local_span(block); + let repeat = span.is_some() && span == previous; + previous = span; + !repeat + }) + .map(PlannedBlock::framed_len) + .sum::(); if stored.len() as u64 != expected { return Err(PithosError::BlockSizeMismatch { expected, @@ -346,9 +369,18 @@ fn decode_group( } let mut chunks = Vec::with_capacity(blocks.len()); let mut rest = stored; + let mut last: Option<((u64, u64), &[u8])> = None; for block in blocks { - let (bytes, tail) = rest.split_at(block.framed_len() as usize); - rest = tail; + let span = local_span(block); + let bytes = match last { + Some((previous, bytes)) if span == Some(previous) => bytes, + _ => { + let (bytes, tail) = rest.split_at(block.framed_len() as usize); + rest = tail; + bytes + } + }; + last = span.map(|span| (span, bytes)); let mut plaintext = view.decode_block(block, bytes)?; let output = block.output(); chunks.push(if output.start == 0 && output.end == plaintext.len() { From a8265e832524890ac5e04b55b567ed8965cc4e51 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 01:16:47 +0200 Subject: [PATCH 122/166] test: read repeated adjacent blocks through the async stream --- crates/pithos_lib/tests/async_reader.rs | 46 +++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/crates/pithos_lib/tests/async_reader.rs b/crates/pithos_lib/tests/async_reader.rs index ac7878d..20226aa 100644 --- a/crates/pithos_lib/tests/async_reader.rs +++ b/crates/pithos_lib/tests/async_reader.rs @@ -974,3 +974,49 @@ fn encrypted_compressed_blocks_reserve_the_decrypted_bytes() { assert_eq!(output, compressible(1)); } } + +#[test] +fn repeated_adjacent_blocks_are_read_in_file_order() { + // Blocks A B B B C: the writer stores B once, right after A, and C after B. + let content = [[0u8; 16], [1; 16], [1; 16], [1; 16], [2; 16]].concat(); + let key = private_key("sender"); + let options = WriteOptions::new(key.duplicate(), vec![key.public_key()]) + .with_chunking(Chunking::Fixed(16)); + let mut writer = ArchiveWriter::create(Vec::new(), options).unwrap(); + writer + .add_file( + ArchivePath::new("data").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(true, 0).unwrap(), + Some(content.len() as u64), + std::io::Cursor::new(content.clone()), + ) + .unwrap(); + let bytes = writer.finish().unwrap(); + for (max_buffered_bytes, range) in [(1 << 20, 0..80), (1 << 20, 20..60), (1, 0..80)] { + let limits = ReadLimits { + max_in_flight: 4, + max_buffered_bytes, + max_request_bytes: 1024, + }; + let (archive, probe, _) = open_as(&bytes, sender(), limits); + let (output, error) = drain(archive.read_range("data", range.clone()).unwrap(), &probe); + assert!(error.is_none(), "{error:?}"); + assert_eq!(output, content[range.start as usize..range.end as usize]); + // Each request covers distinct stored bytes of the archive. + let reads = probe.reads.lock().unwrap()[3..].to_vec(); + let stored = planned(&archive, range.clone()) + .iter() + .map(|block| (block.offset, block.framed)) + .collect::>(); + assert!(reads.len() <= planned(&archive, range).len()); + for (offset, len) in reads { + let covered = stored + .iter() + .filter(|(start, framed)| *start >= offset && start + framed <= offset + len) + .map(|(_, framed)| framed) + .sum::(); + assert_eq!(covered, len); + } + } +} From 30bfeea9a3b3a006d895402edcbf296a3b21b3dc Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 02:18:07 +0200 Subject: [PATCH 123/166] chore: rename the draft spec to version 1.1.0 --- README.md | 2 +- crates/pithos_lib/tests/async_reader.rs | 2 +- crates/pithos_lib/tests/draft_spec_vectors.rs | 2 +- spec/{PITHOS_1.0.0_draft.md => PITHOS_1.1.0_draft.md} | 0 4 files changed, 3 insertions(+), 3 deletions(-) rename spec/{PITHOS_1.0.0_draft.md => PITHOS_1.1.0_draft.md} (100%) diff --git a/README.md b/README.md index 75a4141..a10f5b5 100644 --- a/README.md +++ b/README.md @@ -56,4 +56,4 @@ The filesystem operations are Linux-only and require destination filesystem supp Version 0.8 is a source break from 0.7. Use the selected public API in `pithos_lib::archive`, `crypto`, `source`, `fs`, and `adapters`; old model, helper, and wire-record paths are not compatibility APIs. -This branch implements the wire rules in the [Pithos 1.0 draft](spec/PITHOS_1.0.0_draft.md). Cargo package versioning is maintained separately from the wire-format version. +This branch implements the wire rules in the [Pithos 1.1 draft](spec/PITHOS_1.1.0_draft.md). Cargo package versioning is maintained separately from the wire-format version. diff --git a/crates/pithos_lib/tests/async_reader.rs b/crates/pithos_lib/tests/async_reader.rs index 20226aa..d59dd46 100644 --- a/crates/pithos_lib/tests/async_reader.rs +++ b/crates/pithos_lib/tests/async_reader.rs @@ -643,7 +643,7 @@ fn a_tokio_blocking_pool_reads_like_the_sync_archive() { assert_eq!(output, expected); } -const SPEC: &str = include_str!("../../../spec/PITHOS_1.0.0_draft.md"); +const SPEC: &str = include_str!("../../../spec/PITHOS_1.1.0_draft.md"); const INITIAL_TARGET: &str = "https://storage.test/initial"; const REDIRECT_TARGET: &str = "https://storage.test/redirect"; diff --git a/crates/pithos_lib/tests/draft_spec_vectors.rs b/crates/pithos_lib/tests/draft_spec_vectors.rs index bc0e29f..324c784 100644 --- a/crates/pithos_lib/tests/draft_spec_vectors.rs +++ b/crates/pithos_lib/tests/draft_spec_vectors.rs @@ -14,7 +14,7 @@ use pithos_lib::source::MemorySource; use std::io::Cursor; use x25519_dalek::{PublicKey, StaticSecret}; -const SPEC: &str = include_str!("../../../spec/PITHOS_1.0.0_draft.md"); +const SPEC: &str = include_str!("../../../spec/PITHOS_1.1.0_draft.md"); const RELATIONS: [(u8, &str); 10] = [ (0, "DESCRIBES"), (1, "ANNOTATES"), diff --git a/spec/PITHOS_1.0.0_draft.md b/spec/PITHOS_1.1.0_draft.md similarity index 100% rename from spec/PITHOS_1.0.0_draft.md rename to spec/PITHOS_1.1.0_draft.md From fd77e5f79bf0a5cd9abfe9c6b737dd35cd0983e0 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 02:18:07 +0200 Subject: [PATCH 124/166] test: check that secret buffers are wiped before they are freed --- crates/pithos_lib/src/format/mod.rs | 2 + crates/pithos_lib/src/format/zeroing_tests.rs | 128 ++++++++++++++++++ 2 files changed, 130 insertions(+) create mode 100644 crates/pithos_lib/src/format/zeroing_tests.rs diff --git a/crates/pithos_lib/src/format/mod.rs b/crates/pithos_lib/src/format/mod.rs index cf55c6f..e52a864 100644 --- a/crates/pithos_lib/src/format/mod.rs +++ b/crates/pithos_lib/src/format/mod.rs @@ -11,3 +11,5 @@ pub(crate) mod file_entry; pub(crate) mod header; pub(crate) mod limits; pub(crate) mod primitives; +#[cfg(test)] +mod zeroing_tests; diff --git a/crates/pithos_lib/src/format/zeroing_tests.rs b/crates/pithos_lib/src/format/zeroing_tests.rs new file mode 100644 index 0000000..26d422b --- /dev/null +++ b/crates/pithos_lib/src/format/zeroing_tests.rs @@ -0,0 +1,128 @@ +//! Checks that secret buffers are wiped before their allocations are freed. +//! +//! The unit-test binary uses an allocator that inspects freed blocks only while the current +//! thread watches, so other tests running in parallel are not affected. + +use crate::format::encryption::decode_decrypted_recipient_list; +use crate::format::limits::DeserializationLimits; +use crate::format::primitives::reserve_secret; +use std::alloc::{GlobalAlloc, Layout, System}; +use std::cell::Cell; +use zeroize::Zeroizing; + +const SECRET: [u8; 32] = { + let mut bytes = [0; 32]; + let mut index = 0; + while index < 32 { + bytes[index] = (index as u8).wrapping_mul(37) ^ 0xA5; + index += 1; + } + bytes +}; + +thread_local! { + static WATCHED_SIZE: Cell> = const { Cell::new(None) }; + static ZEROED_FREES: Cell = const { Cell::new(0) }; + static SECRET_FREES: Cell = const { Cell::new(0) }; +} + +struct Inspecting; + +impl Inspecting { + fn watching() -> bool { + WATCHED_SIZE.try_with(Cell::get).ok().flatten().is_some() + } + + /// Records whether a block about to be freed is wiped or still holds the secret. + unsafe fn inspect(pointer: *mut u8, layout: Layout) { + let Ok(Some(size)) = WATCHED_SIZE.try_with(Cell::get) else { + return; + }; + let bytes = unsafe { std::slice::from_raw_parts(pointer, layout.size()) }; + if layout.size() == size && bytes.iter().all(|byte| *byte == 0) { + ZEROED_FREES.set(ZEROED_FREES.get() + 1); + } + if bytes.windows(SECRET.len()).any(|window| window == SECRET) { + SECRET_FREES.set(SECRET_FREES.get() + 1); + } + } +} + +unsafe impl GlobalAlloc for Inspecting { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + unsafe { System.alloc(layout) } + } + + unsafe fn dealloc(&self, pointer: *mut u8, layout: Layout) { + unsafe { + Self::inspect(pointer, layout); + System.dealloc(pointer, layout); + } + } + + unsafe fn realloc(&self, pointer: *mut u8, layout: Layout, new_size: usize) -> *mut u8 { + if !Self::watching() { + return unsafe { System.realloc(pointer, layout, new_size) }; + } + // While watching, a reallocation frees the old block through `dealloc`. + let new_layout = unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) }; + let moved = unsafe { self.alloc(new_layout) }; + if !moved.is_null() { + unsafe { + std::ptr::copy_nonoverlapping(pointer, moved, layout.size().min(new_size)); + self.dealloc(pointer, layout); + } + } + moved + } +} + +#[global_allocator] +static ALLOCATOR: Inspecting = Inspecting; + +/// Runs `action` while watching freed blocks of `size` bytes. Returns the number of wiped +/// blocks of that size and the number of freed blocks of any size that held the secret. +fn watch(size: usize, action: impl FnOnce() -> T) -> (T, usize, usize) { + ZEROED_FREES.set(0); + SECRET_FREES.set(0); + WATCHED_SIZE.set(Some(size)); + let result = action(); + WATCHED_SIZE.set(None); + (result, ZEROED_FREES.get(), SECRET_FREES.get()) +} + +#[test] +fn secret_growth_and_drop_wipe_freed_buffers() { + let mut secrets = Zeroizing::new(vec![SECRET]); + let (grown, zeroed, leaked) = watch(32, || reserve_secret(&mut secrets, 3, "secrets")); + grown.unwrap(); + assert_eq!((zeroed, leaked), (1, 0)); + assert_eq!(secrets.as_slice(), &[SECRET]); + let capacity = secrets.capacity(); + assert!(capacity >= 4); + + let ((), zeroed, leaked) = watch(capacity * 32, || drop(secrets)); + assert_eq!((zeroed, leaked), (1, 0)); +} + +#[test] +fn ordinary_growth_is_detected() { + let mut plain = vec![SECRET]; + let ((), _, leaked) = watch(32, || plain.reserve_exact(7)); + assert_eq!(leaked, 1); +} + +#[test] +fn decode_error_wipes_partial_secrets() { + let mut bytes = vec![2]; + bytes.push(1); + bytes.extend_from_slice(&SECRET); + bytes.push(1); + bytes.extend_from_slice(&[0; 32]); + let limits = DeserializationLimits::default(); + let (decoded, zeroed, leaked) = watch(2 * size_of::<(u64, [u8; 32])>(), || { + decode_decrypted_recipient_list(&bytes, &limits).map(|_| ()) + }); + assert!(decoded.is_err()); + assert_eq!((zeroed, leaked), (1, 0)); +} From 6939619145bd4616daf9fea1cbfadd68729354da Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 02:18:08 +0200 Subject: [PATCH 125/166] fix: reuse a repeated adjacent block in Crypt4GH export --- crates/pithos_lib/src/archive/reader.rs | 31 +++++++++----- .../src/archive/reader_private_tests.rs | 40 +++++++++++++++++++ 2 files changed, 62 insertions(+), 9 deletions(-) diff --git a/crates/pithos_lib/src/archive/reader.rs b/crates/pithos_lib/src/archive/reader.rs index 2296464..82939b9 100644 --- a/crates/pithos_lib/src/archive/reader.rs +++ b/crates/pithos_lib/src/archive/reader.rs @@ -24,6 +24,14 @@ pub(crate) enum ContentOperationError { Core(PithosError), Callback(E), } + +#[cfg(feature = "crypt4gh")] +impl From for ContentOperationError { + fn from(error: PithosError) -> Self { + Self::Core(error) + } +} + use std::ops::Range; use x25519_dalek::PublicKey as DalekPublicKey; use zeroize::Zeroizing; @@ -406,7 +414,7 @@ where pub(crate) fn for_each_verified_file_block( &self, id: FileId, - mut operation: impl FnMut(Zeroizing>) -> Result<(), CallbackError>, + mut operation: impl FnMut(&[u8]) -> Result<(), CallbackError>, ) -> Result<(), ContentOperationError> { self.require_content_available(id) .map_err(ContentOperationError::Core)?; @@ -417,13 +425,9 @@ where let plan = ReadRange::new(0..size, size) .and_then(|range| self.view.plan(id, range)) .map_err(ContentOperationError::Core)?; - for block in plan { - let plaintext = block - .and_then(|block| self.verified_block(&block)) - .map_err(ContentOperationError::Core)?; - operation(plaintext).map_err(ContentOperationError::Callback)?; - } - Ok(()) + self.for_each_output(plan, |plaintext| { + operation(plaintext).map_err(ContentOperationError::Callback) + }) } pub(crate) fn require_content_available(&self, id: FileId) -> Result<(), PithosError> { @@ -435,6 +439,15 @@ where plan: ReadPlan<'_>, sink: &mut W, ) -> Result<(), PithosError> { + self.for_each_output(plan, |bytes| Ok(sink.write_all(bytes)?)) + } + + /// Passes the planned part of each verified block to `operation`, in file order. + fn for_each_output>( + &self, + plan: ReadPlan<'_>, + mut operation: impl FnMut(&[u8]) -> Result<(), Failure>, + ) -> Result<(), Failure> { // A block that repeats the block before it reuses its verified plaintext. let mut last: Option<(PlannedBlock, Zeroizing>)> = None; for block in plan { @@ -450,7 +463,7 @@ where last = Some((block, plaintext)); } let (_, plaintext) = last.as_ref().expect("the block was verified above"); - sink.write_all(&plaintext[output])?; + operation(&plaintext[output])?; } Ok(()) } diff --git a/crates/pithos_lib/src/archive/reader_private_tests.rs b/crates/pithos_lib/src/archive/reader_private_tests.rs index 4f604a9..a376b7b 100644 --- a/crates/pithos_lib/src/archive/reader_private_tests.rs +++ b/crates/pithos_lib/src/archive/reader_private_tests.rs @@ -1565,6 +1565,46 @@ fn unsupported_external_content_preflights_the_whole_file_and_adapters() { assert!(exported.is_empty()); } } + +#[cfg(feature = "crypt4gh")] +#[test] +fn crypt4gh_blocks_read_a_repeated_adjacent_block_once() { + let blocks = [[0u8; 16], [1; 16], [1; 16], [1; 16], [2; 16]]; + let sender = private("sender"); + let options = WriteOptions::new(sender.duplicate(), vec![sender.public_key()]) + .with_chunking(Chunking::Fixed(16)); + let mut writer = ArchiveWriter::create(Vec::new(), options).unwrap(); + writer + .add_file( + ArchivePath::new("data").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(true, 0).unwrap(), + Some(80), + Cursor::new(blocks.concat()), + ) + .unwrap(); + let reads = Arc::new(Mutex::new(Vec::new())); + let archive = Archive::open( + RecordingSource { + bytes: Arc::from(writer.finish().unwrap()), + reads: Arc::clone(&reads), + }, + OpenOptions::default().with_access_keys(AccessKeys::new().with_key(sender)), + ) + .unwrap(); + let opened = reads.lock().unwrap().len(); + let (id, _) = archive.view().content_id("data").unwrap(); + let mut output = Vec::new(); + let result = archive.for_each_verified_file_block(id, |plaintext| { + output.push(plaintext.to_vec()); + Ok::<_, ()>(()) + }); + assert!(result.is_ok()); + assert_eq!(output, blocks); + // A, B and C are each read once. + assert_eq!(reads.lock().unwrap().len(), opened + 3); +} + #[cfg(feature = "crypt4gh")] use crate::adapters::crypt4gh; use crate::fs::extract; From 136275554e8ea5acf36e335fa152c7c0ad7e74d2 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 02:18:08 +0200 Subject: [PATCH 126/166] chore: explain why repeated external blocks keep separate batches --- crates/pithos_lib/src/archive/planning.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/pithos_lib/src/archive/planning.rs b/crates/pithos_lib/src/archive/planning.rs index 42835da..5270d9d 100644 --- a/crates/pithos_lib/src/archive/planning.rs +++ b/crates/pithos_lib/src/archive/planning.rs @@ -304,6 +304,8 @@ impl Iterator for BlockBatches<'_> { Err(error) => return Some(Err(error)), }; let Some((offset, len)) = first.local_span() else { + // A repeated external block could reuse the response, but the async stream detects + // repeats by local span only, so joining external repeats needs a change there too. return Some(Ok(BlockBatch { blocks: vec![first], })); From b3c801d9bb088a1494f9a434bd87c183ca6ddfa9 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 02:18:08 +0200 Subject: [PATCH 127/166] chore: list the format 1.1 and 0.8 API changes in the readme --- README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/README.md b/README.md index a10f5b5..722ee4f 100644 --- a/README.md +++ b/README.md @@ -57,3 +57,21 @@ The filesystem operations are Linux-only and require destination filesystem supp Version 0.8 is a source break from 0.7. Use the selected public API in `pithos_lib::archive`, `crypto`, `source`, `fs`, and `adapters`; old model, helper, and wire-record paths are not compatibility APIs. This branch implements the wire rules in the [Pithos 1.1 draft](spec/PITHOS_1.1.0_draft.md). Cargo package versioning is maintained separately from the wire-format version. + +## Format 1.1 + +Format 1.1 keeps every 1.0 structure and encoding. This version reads 1.0 and 1.1 archives, writes new archives as 1.1, and appends to an archive with the rules of its own version. The changes are: + +- The header version is `0x0101`. +- A recipient grant key is derived with HKDF-SHA256 instead of using the raw X25519 shared secret. +- A file's block list can be sealed in independent pieces, each with its own key. Pieces can be joined later without opening a key. +- An encrypted block can use a random key instead of its content-derived key. Such blocks are never deduplicated. +- An encrypted block payload can use AES-256-GCM instead of ChaCha20-Poly1305. + +Breaking changes in the 0.8 API: + +- New writers use fixed 4 MiB blocks by default. Use `Chunking::ContentDefined` with `WriteOptions::with_chunking` for FastCDC blocks. +- The RO-Crate and Crypt4GH adapters are behind the default features `ro-crate` and `crypt4gh`. +- `Archive::open` no longer reads block markers. A missing or changed marker fails when the block is read. +- The default `OpenLimits` admit objects up to 5 TiB. +- The minimum supported Rust version is 1.89. From 21eaf2b90b0f8e1d4076a5379842859b574988a9 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 02:45:30 +0200 Subject: [PATCH 128/166] fix: keep only the planned bytes of encrypted async outputs --- crates/pithos_lib/src/archive/async_stream.rs | 20 +++++--- crates/pithos_lib/src/crypto.rs | 34 ++++++++----- crates/pithos_lib/tests/async_reader.rs | 50 ++++++++++++++++++- 3 files changed, 83 insertions(+), 21 deletions(-) diff --git a/crates/pithos_lib/src/archive/async_stream.rs b/crates/pithos_lib/src/archive/async_stream.rs index 443ffe5..735dd1b 100644 --- a/crates/pithos_lib/src/archive/async_stream.rs +++ b/crates/pithos_lib/src/archive/async_stream.rs @@ -17,7 +17,8 @@ use zeroize::Zeroizing; /// Each request reserves its response, the planned output of its blocks, and the largest /// working set of decoding one of them: the payload copy, the decrypted bytes of an encrypted /// compressed block, the decoded block and a copy of a partial output. After decoding, the -/// request keeps only its planned output, which is released as the chunks are delivered. +/// request keeps only its planned output in buffers of exactly that size, which are released +/// as the chunks are delivered. /// /// Adjacent blocks share a request only while the whole reservation fits `max_buffered_bytes`. /// A request that does not fit next to the work already buffered waits. The only request that @@ -171,8 +172,8 @@ where }, State::Decoding(decode) => match decode.as_mut().poll(cx) { Poll::Ready(Ok(chunks)) => { - let kept = chunks.iter().map(|chunk| chunk.len() as u64).sum::(); - self.buffered -= slot.reserved - kept; + let kept = chunks.iter().map(retained).sum::(); + self.buffered = self.buffered - slot.reserved + kept; slot.reserved = kept; State::Ready(chunks.into()) } @@ -205,8 +206,8 @@ where let item = match &mut slot.state { State::Ready(chunks) => match chunks.pop_front() { Some(chunk) => { - slot.reserved -= chunk.len() as u64; - self.buffered -= chunk.len() as u64; + slot.reserved -= retained(&chunk); + self.buffered -= retained(&chunk); Ok(chunk) } None => { @@ -344,6 +345,11 @@ fn split(batch: &BlockBatch, max_bytes: u64) -> VecDeque { groups } +/// The bytes a delivered chunk holds: its capacity, not only its length. +fn retained(chunk: &Vec) -> u64 { + chunk.capacity() as u64 +} + /// Decodes every block of a request and keeps only the planned output of each. fn decode_group( view: &ArchiveView, @@ -383,7 +389,9 @@ fn decode_group( last = span.map(|span| (span, bytes)); let mut plaintext = view.decode_block(block, bytes)?; let output = block.output(); - chunks.push(if output.start == 0 && output.end == plaintext.len() { + let whole = output.start == 0 && output.end == plaintext.len(); + // A buffer with spare capacity is copied, so a chunk keeps only its planned bytes. + chunks.push(if whole && plaintext.capacity() == plaintext.len() { std::mem::take(&mut *plaintext) } else { plaintext[output].to_vec() diff --git a/crates/pithos_lib/src/crypto.rs b/crates/pithos_lib/src/crypto.rs index d3b75b2..3f627ba 100644 --- a/crates/pithos_lib/src/crypto.rs +++ b/crates/pithos_lib/src/crypto.rs @@ -8,7 +8,10 @@ use crate::format::header::FormatVersion; use aes_gcm::Aes256Gcm; use chacha20poly1305::{ ChaCha20Poly1305, Key, KeyInit, Nonce, - aead::{self, Aead, AeadCore, AeadInOut, Generate, Payload, consts::U12}, + aead::{ + self, Aead, AeadCore, AeadInOut, Generate, Payload, + consts::{U12, U16}, + }, }; use digest::{ExtendableOutput, Update, XofReader}; use hkdf::Hkdf; @@ -460,28 +463,33 @@ fn seal_with>( Ok(payload) } -fn open_with>( +/// Decrypts into a buffer of exactly the plaintext size, so it keeps no capacity for the tag. +fn open_with>( key: &[u8; 32], payload: &[u8], ) -> Result>, CryptoError> { if payload.len() < 15 { return Err(CryptoError::EncryptedPayloadTooShort); } - let (nonce, ciphertext) = payload.split_at(12); + let len = payload + .len() + .checked_sub(28) + .ok_or(CryptoError::AuthenticationFailed)?; + let (nonce, sealed) = payload.split_at(12); + let (message, tag) = sealed.split_at(len); let cipher = C::new_from_slice(key).map_err(|_| CryptoError::CipherInitialization)?; - let nonce: [u8; 12] = nonce - .try_into() - .map_err(|_| CryptoError::EncryptedPayloadTooShort)?; + let nonce: [u8; 12] = nonce.try_into().expect("12-byte nonce"); + let tag = aead::Tag::::try_from(tag).expect("16-byte tag"); + let mut plaintext = Zeroizing::new(message.to_vec()); cipher - .decrypt( + .decrypt_inout_detached( &aead::Nonce::::from(nonce), - Payload { - msg: ciphertext, - aad: b"", - }, + b"", + plaintext.as_mut_slice().into(), + &tag, ) - .map(Zeroizing::new) - .map_err(|_| CryptoError::AuthenticationFailed) + .map_err(|_| CryptoError::AuthenticationFailed)?; + Ok(plaintext) } #[cfg(test)] diff --git a/crates/pithos_lib/tests/async_reader.rs b/crates/pithos_lib/tests/async_reader.rs index d59dd46..6fba001 100644 --- a/crates/pithos_lib/tests/async_reader.rs +++ b/crates/pithos_lib/tests/async_reader.rs @@ -5,8 +5,8 @@ use futures_core::Stream; use pithos_lib::archive::{ AccessKeys, Archive, ArchiveFeature, ArchivePath, ArchiveWriter, AsyncArchive, AsyncExternalBlockResolver, BlockRequest, BlockingHook, Chunking, EntryKind, EntryMetadata, - ExternalBlockAccessPolicy, ExternalLocation, NoExternalBlocks, OpenLimits, OpenOptions, - ProcessingOptions, ReadLimits, WriteOptions, + ExternalBlockAccessPolicy, ExternalLocation, MAX_BATCH_BLOCKS, NoExternalBlocks, OpenLimits, + OpenOptions, ProcessingOptions, ReadLimits, WriteOptions, }; use pithos_lib::error::PithosError; use pithos_lib::source::{AsyncArchiveSource, MemorySource, SourceError}; @@ -1020,3 +1020,49 @@ fn repeated_adjacent_blocks_are_read_in_file_order() { } } } + +#[test] +fn encrypted_repeated_blocks_retain_only_their_output() { + // One stored 16-byte block repeated past the batch cap. + let count = MAX_BATCH_BLOCKS + 100; + let content = [7u8; 16].repeat(count); + let key = private_key("sender"); + let options = WriteOptions::new(key.duplicate(), vec![key.public_key()]) + .with_chunking(Chunking::Fixed(16)); + let mut writer = ArchiveWriter::create(Vec::new(), options).unwrap(); + writer + .add_file( + ArchivePath::new("data").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(true, 0).unwrap(), + Some(content.len() as u64), + std::io::Cursor::new(content.clone()), + ) + .unwrap(); + let bytes = writer.finish().unwrap(); + let limits = ReadLimits { + max_in_flight: 4, + max_buffered_bytes: 20_000, + max_request_bytes: 1024, + }; + let (archive, probe, mode) = open_as(&bytes, sender(), limits); + *mode.lock().unwrap() = Mode::YieldOnce; + let mut stream = archive.read_range("data", 0..content.len() as u64).unwrap(); + let mut output = Vec::new(); + let mut peak = 0; + while let Poll::Ready(Some(chunk)) = next(&mut stream) { + let chunk = chunk.unwrap(); + assert_eq!(chunk.capacity(), chunk.len()); + peak = peak.max(stream.buffered_bytes()); + output.extend_from_slice(&chunk); + } + assert!(matches!(next(&mut stream), Poll::Ready(None))); + assert_eq!(output, content); + assert!( + peak > 16 * 1024 && peak <= limits.max_buffered_bytes, + "{peak}" + ); + // Both batches were outstanding at once, and each fetched the stored block once. + assert_eq!(probe.reads.lock().unwrap()[3..].len(), 2); + assert_eq!(probe.max_outstanding.load(Ordering::SeqCst), 2); +} From 74635130a603e6d7b1b4e38467dbc7023498a8d7 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 02:45:30 +0200 Subject: [PATCH 129/166] fix: release async read reservations after a terminal failure --- crates/pithos_lib/src/archive/async_stream.rs | 3 ++- crates/pithos_lib/tests/async_reader.rs | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/crates/pithos_lib/src/archive/async_stream.rs b/crates/pithos_lib/src/archive/async_stream.rs index 735dd1b..e992b33 100644 --- a/crates/pithos_lib/src/archive/async_stream.rs +++ b/crates/pithos_lib/src/archive/async_stream.rs @@ -220,8 +220,9 @@ where let State::Failed(error) = std::mem::replace(&mut slot.state, State::Taken) else { unreachable!("the slot failed"); }; - // Dropping the remaining slots cancels their requests. + // Dropping the remaining slots cancels their requests and frees their bytes. self.slots.clear(); + self.buffered = 0; self.batches = None; self.waiting.clear(); Err(error) diff --git a/crates/pithos_lib/tests/async_reader.rs b/crates/pithos_lib/tests/async_reader.rs index 6fba001..411c850 100644 --- a/crates/pithos_lib/tests/async_reader.rs +++ b/crates/pithos_lib/tests/async_reader.rs @@ -866,12 +866,15 @@ fn a_later_failure_ends_the_stream_after_the_earlier_output() { release(&probe, blocks[1].offset); assert!(next(&mut stream).is_pending()); release(&probe, blocks[0].offset); - let (output, error) = drain(stream, &probe); + let (output, error) = drain(&mut stream, &probe); assert_eq!(output, content(2)); assert!(matches!( error, Some(PithosError::Source(SourceError::Remote { offset, .. })) if offset == blocks[2].offset )); + assert_eq!(stream.buffered_bytes(), 0); + assert!(matches!(next(&mut stream), Poll::Ready(None))); + assert_eq!(stream.buffered_bytes(), 0); // The pending sibling was cancelled and no request started after the failure. assert_eq!(probe.cancelled.load(Ordering::SeqCst), 1); assert_eq!(probe.outstanding.load(Ordering::SeqCst), 0); From b539e25f7c67be57cc17857bea54470e56db9ce6 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 02:45:31 +0200 Subject: [PATCH 130/166] test: drop an async stream while a worker thread decodes --- crates/pithos_lib/tests/async_reader.rs | 102 +++++++++++++++++++++++- 1 file changed, 100 insertions(+), 2 deletions(-) diff --git a/crates/pithos_lib/tests/async_reader.rs b/crates/pithos_lib/tests/async_reader.rs index 411c850..5db10d4 100644 --- a/crates/pithos_lib/tests/async_reader.rs +++ b/crates/pithos_lib/tests/async_reader.rs @@ -14,9 +14,10 @@ use std::collections::HashSet; use std::future::Future; use std::ops::Range; use std::pin::{Pin, pin}; -use std::sync::atomic::{AtomicUsize, Ordering}; -use std::sync::{Arc, Mutex}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex, mpsc}; use std::task::{Context, Poll, Waker}; +use std::time::Duration; /// How the test source answers reads. #[derive(Clone, Copy, Debug)] @@ -1069,3 +1070,100 @@ fn encrypted_repeated_blocks_retain_only_their_output() { assert_eq!(probe.reads.lock().unwrap()[3..].len(), 2); assert_eq!(probe.max_outstanding.load(Ordering::SeqCst), 2); } + +/// A generous cap that turns a lost signal into a failure instead of a hang. +const HANG_CAP: Duration = Duration::from_secs(60); + +/// What a worker thread reports and waits for. +struct WorkerSignals { + started: mpsc::Sender<()>, + release: Mutex>, + /// Whether the reader still received the result. + finished: mpsc::Sender, +} + +/// Runs tasks inline until `threaded` is set, then each on its own thread once released. +#[derive(Clone)] +struct WorkerHook { + threaded: Arc, + signals: Arc, + workers: Arc>>>, +} + +impl BlockingHook for WorkerHook { + fn spawn_blocking(&self, task: F) -> impl Future + Send + where + F: FnOnce() -> T + Send + 'static, + T: Send + 'static, + { + let (sender, receiver) = mpsc::channel(); + if self.threaded.load(Ordering::SeqCst) { + let signals = Arc::clone(&self.signals); + let worker = std::thread::spawn(move || { + signals.started.send(()).unwrap(); + signals + .release + .lock() + .unwrap() + .recv_timeout(HANG_CAP) + .unwrap(); + let delivered = sender.send(task()).is_ok(); + signals.finished.send(delivered).unwrap(); + }); + self.workers.lock().unwrap().push(worker); + } else { + sender.send(task()).unwrap(); + } + std::future::poll_fn(move |_| match receiver.try_recv() { + Ok(output) => Poll::Ready(output), + Err(mpsc::TryRecvError::Empty) => Poll::Pending, + Err(mpsc::TryRecvError::Disconnected) => panic!("the worker stopped"), + }) + } +} + +#[test] +fn dropping_a_stream_while_a_worker_decodes_discards_the_result() { + let bytes = blocks(8, true); + let (started, started_signal) = mpsc::channel(); + let (release, release_signal) = mpsc::channel(); + let (finished, finished_signal) = mpsc::channel(); + let hook = WorkerHook { + threaded: Arc::default(), + signals: Arc::new(WorkerSignals { + started, + release: Mutex::new(release_signal), + finished, + }), + workers: Arc::default(), + }; + let (source, probe, _) = TestSource::new(&bytes); + let archive = block_on(AsyncArchive::open_with_hook( + source, + sender(), + None, + hook.clone(), + )) + .unwrap() + .with_read_limits(single_blocks(1, 1)); + hook.threaded.store(true, Ordering::SeqCst); + let mut stream = archive.read_range("data", 0..128).unwrap(); + assert!(next(&mut stream).is_pending()); + started_signal.recv_timeout(HANG_CAP).unwrap(); + drop(stream); + release.send(()).unwrap(); + // The worker completed, but nobody received its result. + assert!(!finished_signal.recv_timeout(HANG_CAP).unwrap()); + let workers = std::mem::take(&mut *hook.workers.lock().unwrap()); + assert_eq!(workers.len(), 1); + for worker in workers { + worker.join().unwrap(); + } + // Only the test and the archive still hold the hook state, and the archive reads again. + assert_eq!(Arc::strong_count(&hook.signals), 2); + assert_eq!(probe.outstanding.load(Ordering::SeqCst), 0); + hook.threaded.store(false, Ordering::SeqCst); + let (output, error) = drain(archive.read_range("data", 0..128).unwrap(), &probe); + assert!(error.is_none()); + assert_eq!(output, content(8)); +} From f3e043c9c56870a1367d212afa6b35b422d193d5 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 02:45:31 +0200 Subject: [PATCH 131/166] feat: add an owned async range stream --- crates/pithos_lib/src/archive/async_reader.rs | 28 +- crates/pithos_lib/src/archive/async_stream.rs | 259 +++++++++++++----- crates/pithos_lib/src/archive/mod.rs | 2 +- crates/pithos_lib/src/archive/planning.rs | 135 ++++++++- 4 files changed, 345 insertions(+), 79 deletions(-) diff --git a/crates/pithos_lib/src/archive/async_reader.rs b/crates/pithos_lib/src/archive/async_reader.rs index 796dbad..e0f9c04 100644 --- a/crates/pithos_lib/src/archive/async_reader.rs +++ b/crates/pithos_lib/src/archive/async_reader.rs @@ -1,4 +1,4 @@ -use super::async_stream::{RangeStream, ReadLimits}; +use super::async_stream::{OwnedRangeStream, RangeStream, ReadLimits}; use super::opener::ArchiveOpener; use super::planning::BlockRequest; use super::reader::{ @@ -174,8 +174,8 @@ where /// /// Path, range and availability errors are returned before any request. Each item is the /// planned part of one completely verified block. The stream ends after its first error. - /// Dropping it cancels every outstanding request. To move the stream into a task that - /// needs `'static`, share the archive in an `Arc` and read inside an async block. + /// Dropping it cancels every outstanding request. For a `'static` stream, use + /// [`AsyncArchive::read_range_owned`]. pub fn read_range( &self, path: &str, @@ -189,6 +189,28 @@ where )) } + /// Streams like [`AsyncArchive::read_range`], but the stream holds the shared archive + /// instead of a borrow. It is `Send + 'static`, so a server handler can return it as a + /// response body or move it into a task. Dropping it cancels every outstanding request. + pub fn read_range_owned( + self: Arc, + path: &str, + range: Range, + ) -> Result, PithosError> + where + S: 'static, + E: 'static, + B: 'static, + { + let cursor = self + .view + .plan_range(path, range)? + .batches(self.limits.max_request_bytes) + .detach(); + let limits = self.limits; + Ok(OwnedRangeStream::new(self, cursor, limits)) + } + /// Fetches the stored bytes of one batch request. pub(super) async fn fetch(&self, request: BlockRequest) -> Result, PithosError> { match request { diff --git a/crates/pithos_lib/src/archive/async_stream.rs b/crates/pithos_lib/src/archive/async_stream.rs index e992b33..619cfb4 100644 --- a/crates/pithos_lib/src/archive/async_stream.rs +++ b/crates/pithos_lib/src/archive/async_stream.rs @@ -1,5 +1,5 @@ use super::async_reader::{AsyncArchive, AsyncExternalBlockResolver, BlockingHook}; -use super::planning::{BlockBatch, BlockBatches, BlockRequest, PlannedBlock}; +use super::planning::{BatchCursor, BlockBatch, BlockBatches, BlockRequest, PlannedBlock}; use super::view::ArchiveView; use crate::error::PithosError; use crate::format::block::ProcessingFlags; @@ -60,13 +60,7 @@ struct Slot<'a> { /// Verified plaintext chunks of a range, in file order. See [`AsyncArchive::read_range`]. pub struct RangeStream<'a, S, E, B> { - archive: &'a AsyncArchive, - batches: Option>, - waiting: VecDeque, - slots: VecDeque>, - limits: ReadLimits, - buffered: u64, - fetching: usize, + engine: Engine<'a, &'a AsyncArchive, BlockBatches<'a>>, } impl<'a, S, E, B> RangeStream<'a, S, E, B> @@ -81,7 +75,145 @@ where limits: ReadLimits, ) -> Self { Self { - archive, + engine: Engine::new(archive, batches, limits), + } + } + + /// The bytes currently reserved by requests that are fetched, decoded or waiting for + /// delivery, as described in [`ReadLimits`]. + pub fn buffered_bytes(&self) -> u64 { + self.engine.buffered + } +} + +/// A [`RangeStream`] that shares the archive instead of borrowing it. +/// See [`AsyncArchive::read_range_owned`]. +pub struct OwnedRangeStream { + engine: Engine<'static, Arc>, SharedBatches>, +} + +impl OwnedRangeStream +where + S: AsyncArchiveSource + 'static, + E: AsyncExternalBlockResolver + 'static, + B: BlockingHook + 'static, +{ + pub(super) fn new( + archive: Arc>, + cursor: BatchCursor, + limits: ReadLimits, + ) -> Self { + let batches = SharedBatches { + view: Arc::clone(&archive.view), + cursor, + }; + Self { + engine: Engine::new(archive, batches, limits), + } + } + + /// The bytes currently reserved, as in [`RangeStream::buffered_bytes`]. + pub fn buffered_bytes(&self) -> u64 { + self.engine.buffered + } +} + +/// The batches of a plan over a shared view, planned one at a time. +struct SharedBatches { + view: Arc, + cursor: BatchCursor, +} + +impl Iterator for SharedBatches { + type Item = Result; + + fn next(&mut self) -> Option { + self.cursor.next_batch(&self.view.index) + } +} + +/// How a stream reaches its archive: borrowed, or shared by an owned stream. +trait Reader<'a> { + fn fetch(&self, request: BlockRequest) -> Pending<'a, Vec>; + + fn decode( + &self, + blocks: Vec, + stored: Zeroizing>, + ) -> Pending<'a, Vec>>; +} + +impl<'a, S, E, B> Reader<'a> for &'a AsyncArchive +where + S: AsyncArchiveSource, + E: AsyncExternalBlockResolver, + B: BlockingHook, +{ + fn fetch(&self, request: BlockRequest) -> Pending<'a, Vec> { + Box::pin(AsyncArchive::fetch(*self, request)) + } + + fn decode( + &self, + blocks: Vec, + stored: Zeroizing>, + ) -> Pending<'a, Vec>> { + let archive: &'a AsyncArchive = self; + let view = Arc::clone(&archive.view); + Box::pin( + archive + .hook + .spawn_blocking(move || decode_group(&view, &blocks, &stored)), + ) + } +} + +impl Reader<'static> for Arc> +where + S: AsyncArchiveSource + 'static, + E: AsyncExternalBlockResolver + 'static, + B: BlockingHook + 'static, +{ + fn fetch(&self, request: BlockRequest) -> Pending<'static, Vec> { + let archive = Arc::clone(self); + Box::pin(async move { AsyncArchive::fetch(&archive, request).await }) + } + + fn decode( + &self, + blocks: Vec, + stored: Zeroizing>, + ) -> Pending<'static, Vec>> { + let archive = Arc::clone(self); + Box::pin(async move { + let view = Arc::clone(&archive.view); + archive + .hook + .spawn_blocking(move || decode_group(&view, &blocks, &stored)) + .await + }) + } +} + +/// The ordering, bounds and cancellation shared by both range streams. +struct Engine<'a, R, P> { + reader: R, + batches: Option

    , + waiting: VecDeque, + slots: VecDeque>, + limits: ReadLimits, + buffered: u64, + fetching: usize, +} + +impl<'a, R, P> Engine<'a, R, P> +where + R: Reader<'a>, + P: Iterator>, +{ + fn new(reader: R, batches: P, limits: ReadLimits) -> Self { + Self { + reader, batches: Some(batches), waiting: VecDeque::new(), slots: VecDeque::new(), @@ -91,12 +223,6 @@ where } } - /// The bytes currently reserved by requests that are fetched, decoded or waiting for - /// delivery, as described in [`ReadLimits`]. - pub fn buffered_bytes(&self) -> u64 { - self.buffered - } - /// Starts requests while the in-flight and byte bounds allow. Returns whether any started. fn admit(&mut self) -> bool { let mut admitted = false; @@ -132,7 +258,7 @@ where } self.buffered = self.buffered.saturating_add(reserved); self.fetching += 1; - let fetch = Box::pin(self.archive.fetch(group.request())); + let fetch = self.reader.fetch(group.request()); self.slots.push_back(Slot { reserved, state: State::Fetching(fetch, group.blocks), @@ -147,41 +273,36 @@ where let mut progressed = false; let mut failed = false; for slot in &mut self.slots { - let next = - match &mut slot.state { - State::Fetching(fetch, _) => match fetch.as_mut().poll(cx) { - Poll::Ready(result) => { - self.fetching -= 1; - let State::Fetching(_, blocks) = - std::mem::replace(&mut slot.state, State::Taken) - else { - unreachable!("the slot was fetching"); - }; - match result { - Ok(stored) => { - let view = Arc::clone(&self.archive.view); - let stored = Zeroizing::new(stored); - State::Decoding(Box::pin(self.archive.hook.spawn_blocking( - move || decode_group(&view, &blocks, &stored), - ))) - } - Err(error) => State::Failed(error), + let next = match &mut slot.state { + State::Fetching(fetch, _) => match fetch.as_mut().poll(cx) { + Poll::Ready(result) => { + self.fetching -= 1; + let State::Fetching(_, blocks) = + std::mem::replace(&mut slot.state, State::Taken) + else { + unreachable!("the slot was fetching"); + }; + match result { + Ok(stored) => { + State::Decoding(self.reader.decode(blocks, Zeroizing::new(stored))) } + Err(error) => State::Failed(error), } - Poll::Pending => continue, - }, - State::Decoding(decode) => match decode.as_mut().poll(cx) { - Poll::Ready(Ok(chunks)) => { - let kept = chunks.iter().map(retained).sum::(); - self.buffered = self.buffered - slot.reserved + kept; - slot.reserved = kept; - State::Ready(chunks.into()) - } - Poll::Ready(Err(error)) => State::Failed(error), - Poll::Pending => continue, - }, - State::Ready(_) | State::Failed(_) | State::Taken => continue, - }; + } + Poll::Pending => continue, + }, + State::Decoding(decode) => match decode.as_mut().poll(cx) { + Poll::Ready(Ok(chunks)) => { + let kept = chunks.iter().map(retained).sum::(); + self.buffered = self.buffered - slot.reserved + kept; + slot.reserved = kept; + State::Ready(chunks.into()) + } + Poll::Ready(Err(error)) => State::Failed(error), + Poll::Pending => continue, + }, + State::Ready(_) | State::Failed(_) | State::Taken => continue, + }; if matches!(next, State::Failed(_)) { self.buffered -= slot.reserved; slot.reserved = 0; @@ -237,6 +358,20 @@ where } Some(Some(item)) } + + fn poll_next(&mut self, cx: &mut Context<'_>) -> Poll, PithosError>>> { + loop { + let started = self.admit(); + let progressed = self.poll_slots(cx); + let before = self.slots.len(); + if let Some(item) = self.deliver() { + return Poll::Ready(item); + } + if !started && !progressed && self.slots.len() == before { + return Poll::Pending; + } + } + } } impl Stream for RangeStream<'_, S, E, B> @@ -248,18 +383,20 @@ where type Item = Result, PithosError>; fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - let this = self.get_mut(); - loop { - let started = this.admit(); - let progressed = this.poll_slots(cx); - let before = this.slots.len(); - if let Some(item) = this.deliver() { - return Poll::Ready(item); - } - if !started && !progressed && this.slots.len() == before { - return Poll::Pending; - } - } + self.get_mut().engine.poll_next(cx) + } +} + +impl Stream for OwnedRangeStream +where + S: AsyncArchiveSource + 'static, + E: AsyncExternalBlockResolver + 'static, + B: BlockingHook + 'static, +{ + type Item = Result, PithosError>; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + self.get_mut().engine.poll_next(cx) } } diff --git a/crates/pithos_lib/src/archive/mod.rs b/crates/pithos_lib/src/archive/mod.rs index 46c5059..81b3918 100644 --- a/crates/pithos_lib/src/archive/mod.rs +++ b/crates/pithos_lib/src/archive/mod.rs @@ -30,7 +30,7 @@ pub use append::{AppendDurability, AppendObservation, AppendOptions}; #[cfg(feature = "async")] pub use async_reader::{AsyncArchive, AsyncExternalBlockResolver, BlockingHook, InlineBlocking}; #[cfg(feature = "async")] -pub use async_stream::{RangeStream, ReadLimits}; +pub use async_stream::{OwnedRangeStream, RangeStream, ReadLimits}; pub use opener::{ArchiveOpener, ReadRequest}; pub(crate) use path_validation::{ validate_directory_entries, validate_new_candidate, validate_symlink_target, diff --git a/crates/pithos_lib/src/archive/planning.rs b/crates/pithos_lib/src/archive/planning.rs index 5270d9d..f37a0ba 100644 --- a/crates/pithos_lib/src/archive/planning.rs +++ b/crates/pithos_lib/src/archive/planning.rs @@ -1,7 +1,7 @@ use crate::archive::index::ArchiveIndex; use crate::archive::types::{ - BlockDescriptor, BlockHash, BlockLocation, ContentState, Entry, ExternalLocation, FileId, - ReadRange, + BlockDescriptor, BlockHash, BlockLocation, BlockReferences, ContentState, Entry, + ExternalLocation, FileId, ReadRange, }; use crate::block; use crate::error::PithosError; @@ -95,18 +95,7 @@ impl<'a> ReadPlan<'a> { range: ReadRange, limits: block::Limits, ) -> Result { - let content = index - .entry(file) - .and_then(|entry| match &entry.entry { - Entry::File(content) | Entry::Metadata(content) => Some(content), - Entry::Directory(_) | Entry::Symlink { .. } => None, - }) - .ok_or_else(|| { - PithosError::InvalidBlockDataState("only data/metadata entries have content".into()) - })?; - let ContentState::Available(references) = &content.content else { - return Err(PithosError::ContentUnavailable); - }; + let references = available_references(index, file)?; // The plan starts at the last offset checkpoint before the range. let (first_position, cursor) = references.start_near(range.start()); let references = if range.start() < range.end() { @@ -125,6 +114,42 @@ impl<'a> ReadPlan<'a> { }) } + /// Continues a plan at the reference `position`, whose block starts at `cursor`. + /// `None` continues a plan that has ended. + #[cfg(feature = "async")] + fn resume( + index: &'a ArchiveIndex, + file: FileId, + range: ReadRange, + limits: block::Limits, + position: Option, + cursor: u64, + ) -> Result { + let references = match position { + Some(position) => available_references(index, file)? + .as_slice() + .get(position..) + .unwrap_or_default(), + None => &[], + }; + Ok(Self { + index, + file, + references: references.iter().enumerate(), + first_position: position.unwrap_or_default(), + cursor, + range, + limits, + }) + } + + /// The position of the next reference, or `None` when the plan has ended. + #[cfg(feature = "async")] + fn position(&self) -> Option { + let (offset, _) = self.references.clone().next()?; + Some(self.first_position + offset) + } + /// Plans the block at the cursor, or returns `None` when it lies before the range. fn step( &mut self, @@ -167,6 +192,26 @@ impl<'a> ReadPlan<'a> { } } +/// The block references of an available data or metadata entry. +fn available_references( + index: &ArchiveIndex, + file: FileId, +) -> Result<&BlockReferences, PithosError> { + let content = index + .entry(file) + .and_then(|entry| match &entry.entry { + Entry::File(content) | Entry::Metadata(content) => Some(content), + Entry::Directory(_) | Entry::Symlink { .. } => None, + }) + .ok_or_else(|| { + PithosError::InvalidBlockDataState("only data/metadata entries have content".into()) + })?; + match &content.content { + ContentState::Available(references) => Ok(references), + ContentState::Unavailable => Err(PithosError::ContentUnavailable), + } +} + /// Checks a block against the stored and decoded size limits. pub(crate) fn check_block_limits( descriptor: &BlockDescriptor, @@ -335,3 +380,65 @@ impl Iterator for BlockBatches<'_> { Some(Ok(BlockBatch { blocks })) } } + +#[cfg(feature = "async")] +impl BlockBatches<'_> { + /// Detaches the remaining batches from the index. See [`BatchCursor::next_batch`]. + pub(crate) fn detach(self) -> BatchCursor { + BatchCursor { + file: self.plan.file, + range: self.plan.range, + limits: self.plan.limits, + position: self.plan.position(), + cursor: self.plan.cursor, + max_bytes: self.max_bytes, + pending: self.pending, + } + } +} + +/// The remaining batches of a plan without a borrow of the index, for owned streams. +#[cfg(feature = "async")] +pub(crate) struct BatchCursor { + file: FileId, + range: ReadRange, + limits: block::Limits, + position: Option, + cursor: u64, + max_bytes: u64, + pending: Option>, +} + +#[cfg(feature = "async")] +impl BatchCursor { + /// Plans the next batch. `index` must be the index the cursor was detached from. + pub(crate) fn next_batch( + &mut self, + index: &ArchiveIndex, + ) -> Option> { + let plan = ReadPlan::resume( + index, + self.file, + self.range, + self.limits, + self.position, + self.cursor, + ); + let plan = match plan { + Ok(plan) => plan, + Err(error) => { + self.position = None; + self.pending = None; + return Some(Err(error)); + } + }; + let mut batches = BlockBatches { + plan, + max_bytes: self.max_bytes, + pending: self.pending.take(), + }; + let batch = batches.next(); + *self = batches.detach(); + batch + } +} From 46f42adeceaba20eacdd601567d4b6102b137d0d Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 02:45:31 +0200 Subject: [PATCH 132/166] test: compare owned and borrowed async range streams --- crates/pithos_lib/tests/async_reader.rs | 97 +++++++++++++++++++++++++ 1 file changed, 97 insertions(+) diff --git a/crates/pithos_lib/tests/async_reader.rs b/crates/pithos_lib/tests/async_reader.rs index 5db10d4..8f4a3c4 100644 --- a/crates/pithos_lib/tests/async_reader.rs +++ b/crates/pithos_lib/tests/async_reader.rs @@ -1167,3 +1167,100 @@ fn dropping_a_stream_while_a_worker_decodes_discards_the_result() { assert!(error.is_none()); assert_eq!(output, content(8)); } + +fn assert_owned(value: T) -> T { + value +} + +#[test] +fn owned_streams_equal_borrowed_streams() { + let bytes = blocks(64, true); + let (source, probe, mode) = TestSource::new(&bytes); + *mode.lock().unwrap() = Mode::YieldOnce; + let archive = block_on(AsyncArchive::open(source, sender(), None)) + .unwrap() + .with_read_limits(ReadLimits { + max_in_flight: 3, + max_buffered_bytes: 4096, + max_request_bytes: 200, + }); + let archive = Arc::new(archive); + for range in [0..1024, 0..0, 5..37, 16..32, 1000..1024, 17..18] { + let start = probe.reads.lock().unwrap().len(); + let borrowed = drain(archive.read_range("data", range.clone()).unwrap(), &probe); + let middle = probe.reads.lock().unwrap().len(); + let stream = Arc::clone(&archive) + .read_range_owned("data", range.clone()) + .unwrap(); + let owned = drain(assert_owned(stream), &probe); + assert!(borrowed.1.is_none() && owned.1.is_none(), "{range:?}"); + assert_eq!(owned.0, borrowed.0, "{range:?}"); + assert_eq!( + owned.0, + content(64)[range.start as usize..range.end as usize] + ); + // Both streams issue the same requests in the same order. + let reads = probe.reads.lock().unwrap(); + assert_eq!(reads[start..middle], reads[middle..], "{range:?}"); + } + assert_eq!(probe.outstanding.load(Ordering::SeqCst), 0); + assert!( + Arc::clone(&archive) + .read_range_owned("data", 0..1025) + .is_err() + ); + assert!( + Arc::clone(&archive) + .read_range_owned("missing", 0..1) + .is_err() + ); + assert_eq!(Arc::strong_count(&archive), 1); +} + +#[test] +fn owned_streams_run_in_spawned_tokio_tasks() { + let bytes = blocks(64, true); + let runtime = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + let output = runtime.block_on(async { + let (source, _, mode) = TestSource::new(&bytes); + *mode.lock().unwrap() = Mode::YieldOnce; + let archive = AsyncArchive::open_with_hook(source, sender(), None, TokioBlocking) + .await + .unwrap(); + let mut stream = Arc::new(archive) + .read_range_owned("data", 16..1000) + .unwrap(); + let task = tokio::spawn(async move { + let mut output = Vec::new(); + while let Some(chunk) = + std::future::poll_fn(|cx| Pin::new(&mut stream).poll_next(cx)).await + { + output.extend_from_slice(&chunk.unwrap()); + } + output + }); + task.await.unwrap() + }); + assert_eq!(output, content(64)[16..1000]); +} + +#[test] +fn dropping_an_owned_stream_cancels_outstanding_requests() { + let bytes = blocks(16, false); + let (archive, probe, mode) = open(&bytes, single_blocks(4, 1 << 20)); + *mode.lock().unwrap() = Mode::Gated; + let archive = Arc::new(archive); + let mut stream = Arc::clone(&archive) + .read_range_owned("data", 0..256) + .unwrap(); + assert!(next(&mut stream).is_pending()); + assert_eq!(probe.outstanding.load(Ordering::SeqCst), 4); + // The stream and each of its four requests hold the archive. + assert_eq!(Arc::strong_count(&archive), 1 + 1 + 4); + drop(stream); + assert_eq!(probe.outstanding.load(Ordering::SeqCst), 0); + assert_eq!(probe.cancelled.load(Ordering::SeqCst), 4); + assert_eq!(Arc::strong_count(&archive), 1); +} From 7688dbe9087d7b3061486b2865b7ec599aa075ce Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 02:45:32 +0200 Subject: [PATCH 133/166] chore: describe the owned async range stream in the readme --- crates/pithos_lib/README.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/crates/pithos_lib/README.md b/crates/pithos_lib/README.md index da53cac..c5e1dd7 100644 --- a/crates/pithos_lib/README.md +++ b/crates/pithos_lib/README.md @@ -99,6 +99,10 @@ async runtime. - `read_range(path, range)` returns a `Stream` of verified plaintext chunks in file order. `ReadLimits` bounds the outstanding requests and the bytes held for responses, decoding and ordered delivery. Dropping the stream cancels its outstanding requests. +- `read_range_owned(path, range)` takes the archive in an `Arc` and returns an + `OwnedRangeStream` that holds it. It reads like `read_range`, with the same order, bounds and + cancellation, but it is `Send + 'static`. A server handler can return it as a streaming + response body, for example after mapping its error type. - The source must serve one immutable object revision for the open and every later read, and each response must have exactly the requested length. - Decoding directories and blocks is CPU work. `open` runs it inline, which suits only small @@ -108,6 +112,25 @@ async runtime. resolver and an access policy are supplied, and the resolver checks the policy before its initial access and before every redirect. +A handler that shares one opened archive can return a body stream like this: + +```rust,no_run +use futures_core::Stream; +use pithos_lib::archive::AsyncArchive; +use pithos_lib::error::PithosError; +use pithos_lib::source::AsyncArchiveSource; +use std::ops::Range; +use std::sync::Arc; + +fn body( + archive: &Arc>, + path: &str, + range: Range, +) -> Result, PithosError>> + Send + 'static, PithosError> { + Arc::clone(archive).read_range_owned(path, range) +} +``` + ## RO-Crate ingestion Load a directory or ZIP RO-Crate, then convert its retained source through a configured writer: From 214d3b9113c2b026ceb4424ae2629a3c2362cf14 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 02:45:42 +0200 Subject: [PATCH 134/166] fix: fetch a repeated adjacent external block once --- crates/pithos_lib/src/archive/async_stream.rs | 31 +++----- crates/pithos_lib/src/archive/mod.rs | 43 +++++++----- crates/pithos_lib/src/archive/planning.rs | 48 ++++++++----- crates/pithos_lib/tests/async_reader.rs | 70 ++++++++++++++++++- 4 files changed, 134 insertions(+), 58 deletions(-) diff --git a/crates/pithos_lib/src/archive/async_stream.rs b/crates/pithos_lib/src/archive/async_stream.rs index 619cfb4..09baeed 100644 --- a/crates/pithos_lib/src/archive/async_stream.rs +++ b/crates/pithos_lib/src/archive/async_stream.rs @@ -407,7 +407,7 @@ struct Group { response: u64, output: u64, working: u64, - last_span: Option<(u64, u64)>, + last: Option, } impl Group { @@ -434,14 +434,13 @@ impl Group { .saturating_add(decrypted) .saturating_add(original) .saturating_add(partial); - // A block that repeats the span before it reuses the bytes already fetched. - let span = local_span(&block); - if span.is_none() || span != self.last_span { + // A block that repeats the request before it reuses the bytes already fetched. + if !self.last.as_ref().is_some_and(|last| block.repeats(last)) { self.response = self.response.saturating_add(framed); } - self.last_span = span; self.output = self.output.saturating_add(output); self.working = self.working.max(working); + self.last = Some(block.clone()); self.blocks.push(block); } @@ -454,13 +453,6 @@ impl Group { } } -fn local_span(block: &PlannedBlock) -> Option<(u64, u64)> { - match block.request() { - BlockRequest::Local { offset, len } => Some((offset, len)), - BlockRequest::External { .. } => None, - } -} - /// Splits a batch into requests whose reservation fits `max_bytes`, keeping file order. /// A block that does not fit even alone becomes a request of its own. fn split(batch: &BlockBatch, max_bytes: u64) -> VecDeque { @@ -469,6 +461,7 @@ fn split(batch: &BlockBatch, max_bytes: u64) -> VecDeque { for block in batch.blocks() { let mut candidate = Group { blocks: Vec::new(), + last: current.last.clone(), ..current }; candidate.push(block.clone()); @@ -494,13 +487,12 @@ fn decode_group( blocks: &[PlannedBlock], stored: &[u8], ) -> Result>, PithosError> { - let mut previous = None; + let mut previous: Option<&PlannedBlock> = None; let expected = blocks .iter() .filter(|block| { - let span = local_span(block); - let repeat = span.is_some() && span == previous; - previous = span; + let repeat = previous.is_some_and(|previous| block.repeats(previous)); + previous = Some(block); !repeat }) .map(PlannedBlock::framed_len) @@ -513,18 +505,17 @@ fn decode_group( } let mut chunks = Vec::with_capacity(blocks.len()); let mut rest = stored; - let mut last: Option<((u64, u64), &[u8])> = None; + let mut last: Option<(&PlannedBlock, &[u8])> = None; for block in blocks { - let span = local_span(block); let bytes = match last { - Some((previous, bytes)) if span == Some(previous) => bytes, + Some((previous, bytes)) if block.repeats(previous) => bytes, _ => { let (bytes, tail) = rest.split_at(block.framed_len() as usize); rest = tail; bytes } }; - last = span.map(|span| (span, bytes)); + last = Some((block, bytes)); let mut plaintext = view.decode_block(block, bytes)?; let output = block.output(); let whole = output.start == 0 && output.end == plaintext.len(); diff --git a/crates/pithos_lib/src/archive/mod.rs b/crates/pithos_lib/src/archive/mod.rs index 81b3918..33e3c2d 100644 --- a/crates/pithos_lib/src/archive/mod.rs +++ b/crates/pithos_lib/src/archive/mod.rs @@ -450,8 +450,10 @@ mod tests { path: ArchivePath::new("data").unwrap(), entry: Entry::File(ContentEntry { metadata: metadata(), - size: 10, - content: ContentState::Available(BlockReferences::new(vec![first, second])), + size: 20, + content: ContentState::Available(BlockReferences::new(vec![ + first, second, second, first, + ])), }), }]); value.descriptors = vec![(first, descriptor(4)), (second, descriptor(6))]; @@ -630,7 +632,7 @@ mod tests { } #[test] - fn external_blocks_are_a_distinct_request_kind_and_never_join_a_batch() { + fn external_blocks_join_a_batch_only_when_repeated() { let first = BlockHash([1; 32]); let second = BlockHash([2; 32]); let mut value = segment(vec![SegmentEntry { @@ -638,31 +640,34 @@ mod tests { path: ArchivePath::new("data").unwrap(), entry: Entry::File(ContentEntry { metadata: metadata(), - size: 10, - content: ContentState::Available(BlockReferences::new(vec![first, second])), + size: 20, + content: ContentState::Available(BlockReferences::new(vec![ + first, second, second, first, + ])), }), }]); value.descriptors = vec![(first, descriptor(4)), (second, descriptor(6))]; let index = build_effective_index(vec![value], 1_000, IndexLimits::default()).unwrap(); - let range = ReadRange::new(0..10, 10).unwrap(); + let range = ReadRange::new(0..20, 20).unwrap(); let plan = ReadPlan::new(&index, FileId(1), range, test_limits()).unwrap(); let batches = plan .batches(u64::MAX) - .map(|batch| batch.unwrap().request()) - .collect::>(); + .collect::, _>>() + .unwrap(); + let external = |len| BlockRequest::External { + location: ExternalLocation::new("opaque"), + len, + }; + // Only the repeated block joins the batch before it; equal locations alone do not. assert_eq!( - batches, - [ - BlockRequest::External { - location: ExternalLocation::new("opaque"), - len: 8 - }, - BlockRequest::External { - location: ExternalLocation::new("opaque"), - len: 10 - }, - ] + batches.iter().map(BlockBatch::request).collect::>(), + [external(8), external(10), external(8)] ); + assert_eq!(batches[1].blocks().len(), 2); + let response = (0..10).collect::>(); + let parts = batches[1].split(&response).unwrap().collect::>(); + assert!(parts.iter().all(|(_, stored)| *stored == &response[..])); + assert!(batches[1].split(&[0; 20]).is_err()); } #[test] diff --git a/crates/pithos_lib/src/archive/planning.rs b/crates/pithos_lib/src/archive/planning.rs index f37a0ba..25330d6 100644 --- a/crates/pithos_lib/src/archive/planning.rs +++ b/crates/pithos_lib/src/archive/planning.rs @@ -66,6 +66,13 @@ impl PlannedBlock { self.descriptor.stored_size.saturating_add(4) } + /// Whether this block is read by the same request as `previous`: the same location and + /// length, local or external. Such a block reuses the stored bytes of `previous`. + pub(crate) fn repeats(&self, previous: &Self) -> bool { + self.descriptor.location == previous.descriptor.location + && self.framed_len() == previous.framed_len() + } + /// Whether `other` decodes to the same plaintext: the same file, identity and descriptor. pub(crate) fn same_block(&self, other: &Self) -> bool { self.file == other.file && self.hash == other.hash && self.descriptor == other.descriptor @@ -254,9 +261,9 @@ impl Iterator for ReadPlan<'_> { impl<'a> ReadPlan<'a> { /// Groups consecutive local blocks whose spans are contiguous in storage into one request - /// of at most `max_bytes`. A block that repeats the span of the block before it joins the - /// batch without adding bytes, so the span is fetched once. A larger block and every - /// external block form their own batch. + /// of at most `max_bytes`. A block with the same request as the block before it, local or + /// external, joins the batch without adding bytes, so its bytes are fetched once. A larger + /// block forms its own batch, and an external block is joined only by its repeats. pub fn batches(self, max_bytes: u64) -> BlockBatches<'a> { BlockBatches { plan: self, @@ -282,15 +289,14 @@ impl BlockBatch { request } - /// The stored bytes of the batch. A block that repeats the span before it adds none. + /// The stored bytes of the batch. A block that repeats the block before it adds none. fn stored_len(&self) -> u64 { - let mut previous = None; + let mut previous: Option<&PlannedBlock> = None; self.blocks .iter() .filter(|block| { - let span = block.local_span(); - let repeat = span.is_some() && span == previous; - previous = span; + let repeat = previous.is_some_and(|previous| block.repeats(previous)); + previous = Some(block); !repeat }) .map(PlannedBlock::framed_len) @@ -315,16 +321,16 @@ impl BlockBatch { }); } let mut rest = response; - let mut last = None; + let mut last: Option<(&PlannedBlock, &[u8])> = None; Ok(self.blocks.iter().map(move |block| { - if let Some((span, stored)) = last - && block.local_span() == Some(span) + if let Some((previous, stored)) = last + && block.repeats(previous) { return (block, stored); } let (stored, tail) = rest.split_at(block.framed_len() as usize); rest = tail; - last = block.local_span().map(|span| (span, stored)); + last = Some((block, stored)); (block, stored) })) } @@ -349,11 +355,19 @@ impl Iterator for BlockBatches<'_> { Err(error) => return Some(Err(error)), }; let Some((offset, len)) = first.local_span() else { - // A repeated external block could reuse the response, but the async stream detects - // repeats by local span only, so joining external repeats needs a change there too. - return Some(Ok(BlockBatch { - blocks: vec![first], - })); + // An external block is joined only by repeats of itself, which reuse its response. + let mut blocks = vec![first]; + while blocks.len() < MAX_BATCH_BLOCKS { + match self.plan.next() { + Some(Ok(next)) if next.repeats(&blocks[0]) => blocks.push(next), + Some(next) => { + self.pending = Some(next); + break; + } + None => break, + } + } + return Some(Ok(BlockBatch { blocks })); }; let (mut last, mut end, mut total) = (offset, offset + len, len); let mut blocks = vec![first]; diff --git a/crates/pithos_lib/tests/async_reader.rs b/crates/pithos_lib/tests/async_reader.rs index 8f4a3c4..183dc39 100644 --- a/crates/pithos_lib/tests/async_reader.rs +++ b/crates/pithos_lib/tests/async_reader.rs @@ -5,8 +5,8 @@ use futures_core::Stream; use pithos_lib::archive::{ AccessKeys, Archive, ArchiveFeature, ArchivePath, ArchiveWriter, AsyncArchive, AsyncExternalBlockResolver, BlockRequest, BlockingHook, Chunking, EntryKind, EntryMetadata, - ExternalBlockAccessPolicy, ExternalLocation, MAX_BATCH_BLOCKS, NoExternalBlocks, OpenLimits, - OpenOptions, ProcessingOptions, ReadLimits, WriteOptions, + ExternalBlockAccessPolicy, ExternalBlockResolver, ExternalLocation, MAX_BATCH_BLOCKS, + NoExternalBlocks, OpenLimits, OpenOptions, ProcessingOptions, ReadLimits, WriteOptions, }; use pithos_lib::error::PithosError; use pithos_lib::source::{AsyncArchiveSource, MemorySource, SourceError}; @@ -650,6 +650,11 @@ const REDIRECT_TARGET: &str = "https://storage.test/redirect"; /// CV-LOCAL-HELLO-279 with its block moved to the opaque external location "x". fn external_hello() -> Vec { + repeated_external_hello(1) +} + +/// [`external_hello`] whose file lists its one block `count` times. +fn repeated_external_hello(count: u8) -> Vec { let start = SPEC.find("#### CV-LOCAL-HELLO-279").unwrap(); let block = SPEC[start..].split("```text\n").nth(1).unwrap(); let mut bytes = block @@ -660,7 +665,12 @@ fn external_hello() -> Vec { .flat_map(|line| line.split_once(": ").unwrap().1.split_whitespace()) .map(|hex| u8::from_str_radix(hex, 16).unwrap()) .collect::>(); + // The block list count, then the file size, before the list grows by whole pairs. + bytes[0x22] = count; + bytes[0x65] = 5 * count; bytes.splice(143..=143, [1, 1, b'x']); + let pair = bytes[0x23..0x63].to_vec(); + bytes.splice(0x63..0x63, pair.repeat(usize::from(count) - 1)); let footer = bytes.len() - 12; let directory_len = (bytes.len() - 15) as u64; bytes[footer..footer + 8].copy_from_slice(&directory_len.to_be_bytes()); @@ -710,6 +720,25 @@ impl AsyncExternalBlockResolver for RedirectingResolver { } } +impl ExternalBlockResolver for RedirectingResolver { + fn resolve( + &self, + policy: &dyn ExternalBlockAccessPolicy, + location: &ExternalLocation, + expected_len: u64, + max_response_size: u64, + ) -> Result, PithosError> { + let resolve = AsyncExternalBlockResolver::resolve( + self, + policy, + location, + expected_len, + max_response_size, + ); + block_on(resolve) + } +} + /// The result of one external read, the policy checks and the resolver calls. struct ExternalRead { result: Result, PithosError>, @@ -1264,3 +1293,40 @@ fn dropping_an_owned_stream_cancels_outstanding_requests() { assert_eq!(probe.cancelled.load(Ordering::SeqCst), 4); assert_eq!(Arc::strong_count(&archive), 1); } + +#[test] +fn repeated_external_blocks_are_resolved_once() { + let bytes = repeated_external_hello(3); + let expected = (9, OpenLimits::default().max_stored_block_bytes + 4); + let options = |calls: &Arc>>| { + let policy = Arc::new(RecordingPolicy { + checks: Mutex::new(Vec::new()), + denied: None, + }); + let resolver = RedirectingResolver { + response: b"BLCKhello".to_vec(), + calls: Arc::clone(calls), + }; + OpenOptions::default() + .with_external_resolver(resolver) + .with_external_access_policy(policy) + }; + for range in [0..15, 3..12] { + let wanted = &b"hellohellohello"[range.start as usize..range.end as usize]; + let calls = Arc::default(); + let sync = Archive::open(MemorySource::new(bytes.clone()), options(&calls)).unwrap(); + let mut output = Vec::new(); + sync.copy_range_to("hello", range.clone(), &mut output) + .unwrap(); + assert_eq!(output, wanted); + assert_eq!(*calls.lock().unwrap(), [expected], "sync {range:?}"); + + let calls = Arc::default(); + let (source, probe, _) = TestSource::new(&bytes); + let archive = block_on(AsyncArchive::open(source, options(&calls), None)).unwrap(); + let (output, error) = drain(archive.read_range("hello", range.clone()).unwrap(), &probe); + assert!(error.is_none(), "{error:?}"); + assert_eq!(output, wanted); + assert_eq!(*calls.lock().unwrap(), [expected], "async {range:?}"); + } +} From a171fdd6cd1ca967c023ef864c70a25b439afde0 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 03:23:43 +0200 Subject: [PATCH 135/166] fix: inspect only initialized blocks in the zeroization test allocator --- crates/pithos_lib/src/format/zeroing_tests.rs | 76 ++++++++++++++++--- 1 file changed, 66 insertions(+), 10 deletions(-) diff --git a/crates/pithos_lib/src/format/zeroing_tests.rs b/crates/pithos_lib/src/format/zeroing_tests.rs index 26d422b..acb60f4 100644 --- a/crates/pithos_lib/src/format/zeroing_tests.rs +++ b/crates/pithos_lib/src/format/zeroing_tests.rs @@ -1,7 +1,8 @@ //! Checks that secret buffers are wiped before their allocations are freed. //! //! The unit-test binary uses an allocator that inspects freed blocks only while the current -//! thread watches, so other tests running in parallel are not affected. +//! thread watches, so other tests running in parallel are not affected. It reads only blocks +//! whose bytes are known to be initialized: blocks it filled itself and buffers the test names. use crate::format::encryption::decode_decrypted_recipient_list; use crate::format::limits::DeserializationLimits; @@ -20,10 +21,16 @@ const SECRET: [u8; 32] = { bytes }; +/// Most blocks one watch can track; the tests allocate far fewer. +const TRACKED_CAPACITY: usize = 64; +/// Fill byte for blocks allocated while watching, so every inspected byte is initialized. +const FILL: u8 = 0xEE; + thread_local! { static WATCHED_SIZE: Cell> = const { Cell::new(None) }; static ZEROED_FREES: Cell = const { Cell::new(0) }; static SECRET_FREES: Cell = const { Cell::new(0) }; + static TRACKED: Cell<[usize; TRACKED_CAPACITY]> = const { Cell::new([0; TRACKED_CAPACITY]) }; } struct Inspecting; @@ -33,11 +40,38 @@ impl Inspecting { WATCHED_SIZE.try_with(Cell::get).ok().flatten().is_some() } - /// Records whether a block about to be freed is wiped or still holds the secret. + fn track(pointer: *mut u8) { + let _ = TRACKED.try_with(|tracked| { + let mut slots = tracked.get(); + if let Some(slot) = slots.iter_mut().find(|slot| **slot == 0) { + *slot = pointer as usize; + tracked.set(slots); + } + }); + } + + /// Removes `pointer` from the tracked blocks and reports whether it was tracked. + fn untrack(pointer: *mut u8) -> bool { + TRACKED + .try_with(|tracked| { + let mut slots = tracked.get(); + let found = slots.iter_mut().find(|slot| **slot == pointer as usize); + let was_tracked = found.map(|slot| *slot = 0).is_some(); + tracked.set(slots); + was_tracked + }) + .unwrap_or(false) + } + + /// Records whether a tracked block about to be freed is wiped or still holds the secret. unsafe fn inspect(pointer: *mut u8, layout: Layout) { let Ok(Some(size)) = WATCHED_SIZE.try_with(Cell::get) else { return; }; + if !Self::untrack(pointer) { + return; + } + // Tracked blocks were filled at allocation or named by the test as fully initialized. let bytes = unsafe { std::slice::from_raw_parts(pointer, layout.size()) }; if layout.size() == size && bytes.iter().all(|byte| *byte == 0) { ZEROED_FREES.set(ZEROED_FREES.get() + 1); @@ -50,7 +84,12 @@ impl Inspecting { unsafe impl GlobalAlloc for Inspecting { unsafe fn alloc(&self, layout: Layout) -> *mut u8 { - unsafe { System.alloc(layout) } + let pointer = unsafe { System.alloc(layout) }; + if Self::watching() && !pointer.is_null() { + unsafe { std::ptr::write_bytes(pointer, FILL, layout.size()) }; + Self::track(pointer); + } + pointer } unsafe fn dealloc(&self, pointer: *mut u8, layout: Layout) { @@ -80,35 +119,52 @@ unsafe impl GlobalAlloc for Inspecting { #[global_allocator] static ALLOCATOR: Inspecting = Inspecting; -/// Runs `action` while watching freed blocks of `size` bytes. Returns the number of wiped -/// blocks of that size and the number of freed blocks of any size that held the secret. -fn watch(size: usize, action: impl FnOnce() -> T) -> (T, usize, usize) { +/// Runs `action` while watching freed blocks of `size` bytes. `initialized` names existing +/// buffers whose whole allocation is initialized; blocks allocated during the watch count too. +/// Returns the number of wiped blocks of that size and of freed blocks that held the secret. +fn watch( + size: usize, + initialized: &[*const u8], + action: impl FnOnce() -> T, +) -> (T, usize, usize) { ZEROED_FREES.set(0); SECRET_FREES.set(0); + TRACKED.set([0; TRACKED_CAPACITY]); + for pointer in initialized { + Inspecting::track(pointer.cast_mut()); + } WATCHED_SIZE.set(Some(size)); let result = action(); WATCHED_SIZE.set(None); + TRACKED.set([0; TRACKED_CAPACITY]); (result, ZEROED_FREES.get(), SECRET_FREES.get()) } #[test] fn secret_growth_and_drop_wipe_freed_buffers() { + // `vec![SECRET]` has no spare capacity, so its whole allocation is initialized. let mut secrets = Zeroizing::new(vec![SECRET]); - let (grown, zeroed, leaked) = watch(32, || reserve_secret(&mut secrets, 3, "secrets")); + let initialized = [secrets.as_ptr().cast::()]; + let (grown, zeroed, leaked) = watch(32, &initialized, || { + reserve_secret(&mut secrets, 3, "secrets") + }); grown.unwrap(); assert_eq!((zeroed, leaked), (1, 0)); assert_eq!(secrets.as_slice(), &[SECRET]); let capacity = secrets.capacity(); assert!(capacity >= 4); - let ((), zeroed, leaked) = watch(capacity * 32, || drop(secrets)); + // The grown buffer was allocated, and so filled, while the first watch ran. + let initialized = [secrets.as_ptr().cast::()]; + let ((), zeroed, leaked) = watch(capacity * 32, &initialized, || drop(secrets)); assert_eq!((zeroed, leaked), (1, 0)); } #[test] fn ordinary_growth_is_detected() { let mut plain = vec![SECRET]; - let ((), _, leaked) = watch(32, || plain.reserve_exact(7)); + let initialized = [plain.as_ptr().cast::()]; + let ((), _, leaked) = watch(32, &initialized, || plain.reserve_exact(7)); assert_eq!(leaked, 1); } @@ -120,7 +176,7 @@ fn decode_error_wipes_partial_secrets() { bytes.push(1); bytes.extend_from_slice(&[0; 32]); let limits = DeserializationLimits::default(); - let (decoded, zeroed, leaked) = watch(2 * size_of::<(u64, [u8; 32])>(), || { + let (decoded, zeroed, leaked) = watch(2 * size_of::<(u64, [u8; 32])>(), &[], || { decode_decrypted_recipient_list(&bytes, &limits).map(|_| ()) }); assert!(decoded.is_err()); From 114909b4a83a0dea266389e6986ae449ac8b4ba6 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 03:23:44 +0200 Subject: [PATCH 136/166] feat: report the format version and block-list form when listing --- crates/pithos/src/main.rs | 17 ++++++++- crates/pithos/tests/create.rs | 9 +++-- crates/pithos_lib/src/archive/mod.rs | 2 +- crates/pithos_lib/src/archive/opener.rs | 22 +++++++++--- crates/pithos_lib/src/archive/reader.rs | 15 ++++++++ crates/pithos_lib/src/archive/view.rs | 26 +++++++++----- crates/pithos_lib/tests/pieces.rs | 46 +++++++++++++++++++++++-- 7 files changed, 116 insertions(+), 21 deletions(-) diff --git a/crates/pithos/src/main.rs b/crates/pithos/src/main.rs index 185e486..290d6e1 100644 --- a/crates/pithos/src/main.rs +++ b/crates/pithos/src/main.rs @@ -375,8 +375,23 @@ fn run() -> Result<(), PithosCliError> { } ReadCommands::List { file } => { let archive = open_archive(&file, optional_private_key(&cli.secret_key)?)?; + let version = archive.view().version(); + write_stdout(format_args!( + "format version {}.{}", + version >> 8, + version & 0xff + ))?; for entry in archive.entries() { - write_stdout(format_args!("{} {:?} {}", entry.id, entry.kind, entry.path))?; + match entry.block_list { + Some(form) => write_stdout(format_args!( + "{} {:?} {:?} {}", + entry.id, entry.kind, form, entry.path + ))?, + None => write_stdout(format_args!( + "{} {:?} {}", + entry.id, entry.kind, entry.path + ))?, + } } } ReadCommands::All { file } => { diff --git a/crates/pithos/tests/create.rs b/crates/pithos/tests/create.rs index 0beea67..0beb9dc 100644 --- a/crates/pithos/tests/create.rs +++ b/crates/pithos/tests/create.rs @@ -191,11 +191,10 @@ fn encrypted_archives_list_but_do_not_read_without_a_key() { .output() .unwrap(); assert!(listing.status.success()); - assert!( - String::from_utf8(listing.stdout) - .unwrap() - .contains("available: false") - ); + let listing = String::from_utf8(listing.stdout).unwrap(); + assert!(listing.contains("available: false")); + assert!(listing.starts_with("format version 1.1\n"), "{listing}"); + assert!(listing.contains(" Sealed "), "{listing}"); let read = command() .args(["read", "data", archive.to_str().unwrap(), "encrypted.txt"]) diff --git a/crates/pithos_lib/src/archive/mod.rs b/crates/pithos_lib/src/archive/mod.rs index 33e3c2d..65197e1 100644 --- a/crates/pithos_lib/src/archive/mod.rs +++ b/crates/pithos_lib/src/archive/mod.rs @@ -42,7 +42,7 @@ pub use planning::{ #[cfg(feature = "crypt4gh")] pub(crate) use reader::ContentOperationError; pub use reader::{ - AccessKeys, Archive, ArchiveEntry, ArchiveFeature, ArchiveReference, EntryKind, + AccessKeys, Archive, ArchiveEntry, ArchiveFeature, ArchiveReference, BlockListForm, EntryKind, ExternalBlockAccessPolicy, ExternalBlockResolver, NoExternalBlocks, OpenLimits, OpenOptions, }; pub(crate) use snapshot::AppendSnapshot; diff --git a/crates/pithos_lib/src/archive/opener.rs b/crates/pithos_lib/src/archive/opener.rs index 1069f0a..99164c2 100644 --- a/crates/pithos_lib/src/archive/opener.rs +++ b/crates/pithos_lib/src/archive/opener.rs @@ -2,17 +2,18 @@ use super::access::ResolvedAccess; use super::decode_validated_directory; use super::index::build_effective_index; use super::reader::{ - AccessKeys, DecodedDirectoryCounts, OpenLimits, OpenOptions, classify_content_availability, - move_block_keys, remaining_deserialization_limits, resolve_block_lists, resolve_recipients, - validate_directory_len, validate_piece_keys, + AccessKeys, BlockListForm, DecodedDirectoryCounts, OpenLimits, OpenOptions, + classify_content_availability, move_block_keys, remaining_deserialization_limits, + resolve_block_lists, resolve_recipients, validate_directory_len, validate_piece_keys, }; use super::types::{BlockDescriptor, BlockHash, FileId, Span, ValidatedSegment}; use super::validation::{DescriptorList, IndexLimits, segment_from_parts}; use super::view::ArchiveView; use crate::error::PithosError; use crate::format::directory::Directory; +use crate::format::file_entry::{BlockDataState, FileType}; use crate::format::header::{FileHeader, FormatVersion}; -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::sync::atomic::{AtomicU64, Ordering}; use zeroize::Zeroizing; @@ -390,12 +391,24 @@ fn complete_open( } let mut segments: Vec = Vec::new(); + let mut block_lists = BTreeMap::new(); for (segment_index, decoded) in raw.into_iter().enumerate() { let DecodedDirectory { mut directory, descriptors, span, } = decoded; + // Record each stored form before key resolution replaces sealed lists. + for (id, _, file) in directory.files.iter() { + if matches!(file.file_type, FileType::Data | FileType::Metadata) { + let form = match &file.block_data { + BlockDataState::Decrypted(_) => BlockListForm::Plain, + BlockDataState::Encrypted(_) => BlockListForm::Sealed, + BlockDataState::Pieces(_) => BlockListForm::Pieces, + }; + block_lists.insert(FileId(id), form); + } + } resolve_block_lists( &mut directory, &mut access, @@ -435,5 +448,6 @@ fn complete_open( access_keys: std::mem::take(&mut settings.keys), limits, content_availability, + block_lists, }) } diff --git a/crates/pithos_lib/src/archive/reader.rs b/crates/pithos_lib/src/archive/reader.rs index 82939b9..87b9057 100644 --- a/crates/pithos_lib/src/archive/reader.rs +++ b/crates/pithos_lib/src/archive/reader.rs @@ -273,11 +273,24 @@ pub enum EntryKind { Symlink { target: String }, } +/// How a content entry stores its block list in the archive. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum BlockListForm { + /// The block list is stored in the clear. + Plain, + /// The block list is sealed with one file key. + Sealed, + /// The block list is sealed in independent pieces (format 1.1). + Pieces, +} + #[derive(Clone, Debug, Eq, PartialEq)] pub struct ArchiveEntry { pub id: u64, pub path: String, pub kind: EntryKind, + /// The stored block-list form of a file or metadata entry; `None` for other entries. + pub block_list: Option, pub created: u64, pub modified: u64, pub permissions: u32, @@ -530,12 +543,14 @@ pub(super) fn archive_entry( index: &ArchiveIndex, entry: &super::index::IndexedEntry, content_availability: &BTreeMap, + block_lists: &BTreeMap, ) -> ArchiveEntry { let metadata = entry.entry.metadata(); ArchiveEntry { id: entry.id.0, path: entry.path.as_str().to_owned(), kind: entry_kind(&entry.entry, entry.id, content_availability), + block_list: block_lists.get(&entry.id).copied(), created: metadata.created, modified: metadata.modified, permissions: metadata.permissions, diff --git a/crates/pithos_lib/src/archive/view.rs b/crates/pithos_lib/src/archive/view.rs index 5ad0dfa..767cdd7 100644 --- a/crates/pithos_lib/src/archive/view.rs +++ b/crates/pithos_lib/src/archive/view.rs @@ -3,7 +3,7 @@ use super::index::ArchiveIndex; use super::planning::{PlannedBlock, ReadPlan, check_block_limits}; #[cfg(feature = "crypt4gh")] use super::reader::AccessKeys; -use super::reader::{ArchiveEntry, ContentAvailability, OpenLimits, archive_entry}; +use super::reader::{ArchiveEntry, BlockListForm, ContentAvailability, OpenLimits, archive_entry}; use super::snapshot::AppendSnapshot; use super::types::{ArchivePath, FileId, ReadRange, RecipientPair, Span}; use super::validation::IndexLimits; @@ -32,6 +32,7 @@ pub struct ArchiveView { pub(super) access_keys: AccessKeys, pub(super) limits: OpenLimits, pub(super) content_availability: BTreeMap, + pub(super) block_lists: BTreeMap, } impl ArchiveView { @@ -47,17 +48,26 @@ impl ArchiveView { } pub fn entries(&self) -> impl ExactSizeIterator + '_ { - self.index - .entries() - .map(|entry| archive_entry(&self.index, entry, &self.content_availability)) + self.index.entries().map(|entry| { + archive_entry( + &self.index, + entry, + &self.content_availability, + &self.block_lists, + ) + }) } pub fn entry(&self, path: &str) -> Result, PithosError> { let path = ArchivePath::new(path)?; - Ok(self - .index - .entry_at_path(&path) - .map(|entry| archive_entry(&self.index, entry, &self.content_availability))) + Ok(self.index.entry_at_path(&path).map(|entry| { + archive_entry( + &self.index, + entry, + &self.content_availability, + &self.block_lists, + ) + })) } /// Plans a read of `range` within the content of `path`. diff --git a/crates/pithos_lib/tests/pieces.rs b/crates/pithos_lib/tests/pieces.rs index 17dc2a8..8271a84 100644 --- a/crates/pithos_lib/tests/pieces.rs +++ b/crates/pithos_lib/tests/pieces.rs @@ -2,8 +2,9 @@ mod common; use common::util::{private_key, public_key}; use pithos_lib::archive::{ - AccessKeys, AppendOptions, Archive, ArchivePath, BlockKeyMode, EntryKind, EntryMetadata, - OpenOptions, PayloadCipher, Piece, PieceEncoder, ProcessingOptions, compose, + AccessKeys, AppendOptions, Archive, ArchivePath, ArchiveWriter, BlockKeyMode, BlockListForm, + EntryKind, EntryMetadata, OpenOptions, PayloadCipher, Piece, PieceEncoder, ProcessingOptions, + WriteOptions, compose, }; use pithos_lib::error::{DeserializationError, PithosError}; use pithos_lib::source::MemorySource; @@ -464,3 +465,44 @@ fn unique_key_pieces_record_no_content_hash_unless_asked() { Some(*blake3::hash(&file).as_bytes()) ); } + +#[test] +fn entries_report_how_their_block_list_is_stored() { + let composed = open(assemble(&[encode(1, b"pieces")]), recipient()); + assert_eq!( + composed.entry("object").unwrap().unwrap().block_list, + Some(BlockListForm::Pieces) + ); + + for (options, form) in [ + (WriteOptions::base(), BlockListForm::Plain), + ( + WriteOptions::new(private_key("recipient1"), vec![public_key("recipient1")]), + BlockListForm::Sealed, + ), + ] { + let plain = form == BlockListForm::Plain; + let mut writer = ArchiveWriter::create(Vec::new(), options).unwrap(); + writer + .add_directory( + ArchivePath::new("folder").unwrap(), + EntryMetadata::new(0, 0, 0o755), + ) + .unwrap(); + writer + .add_file( + ArchivePath::new("folder/data").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(!plain, 0).unwrap(), + None, + std::io::Cursor::new(b"content"), + ) + .unwrap(); + let archive = open(writer.finish().unwrap(), recipient()); + assert_eq!( + archive.entry("folder/data").unwrap().unwrap().block_list, + Some(form) + ); + assert_eq!(archive.entry("folder").unwrap().unwrap().block_list, None); + } +} From 3d8b19073195fa692d11eb2e56d6a60d5048a6e7 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 03:40:00 +0200 Subject: [PATCH 137/166] fix: track a grown test allocation only when its source was initialized --- crates/pithos_lib/src/format/zeroing_tests.rs | 43 ++++++++++++++++--- 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/crates/pithos_lib/src/format/zeroing_tests.rs b/crates/pithos_lib/src/format/zeroing_tests.rs index acb60f4..7af9d3a 100644 --- a/crates/pithos_lib/src/format/zeroing_tests.rs +++ b/crates/pithos_lib/src/format/zeroing_tests.rs @@ -31,6 +31,7 @@ thread_local! { static ZEROED_FREES: Cell = const { Cell::new(0) }; static SECRET_FREES: Cell = const { Cell::new(0) }; static TRACKED: Cell<[usize; TRACKED_CAPACITY]> = const { Cell::new([0; TRACKED_CAPACITY]) }; + static TRACKING_FULL: Cell = const { Cell::new(false) }; } struct Inspecting; @@ -43,9 +44,13 @@ impl Inspecting { fn track(pointer: *mut u8) { let _ = TRACKED.try_with(|tracked| { let mut slots = tracked.get(); - if let Some(slot) = slots.iter_mut().find(|slot| **slot == 0) { - *slot = pointer as usize; - tracked.set(slots); + match slots.iter_mut().find(|slot| **slot == 0) { + Some(slot) => { + *slot = pointer as usize; + tracked.set(slots); + } + // A skipped block could hide a leak, so `watch` fails the test instead. + None => TRACKING_FULL.set(true), } }); } @@ -103,14 +108,23 @@ unsafe impl GlobalAlloc for Inspecting { if !Self::watching() { return unsafe { System.realloc(pointer, layout, new_size) }; } - // While watching, a reallocation frees the old block through `dealloc`. + // While watching, a reallocation frees the old block through `dealloc`. The copy keeps + // any uninitialized bytes of the old block, so the new block is tracked only when the + // old block was known to be fully initialized. + let old_tracked = TRACKED + .try_with(|tracked| tracked.get().contains(&(pointer as usize))) + .unwrap_or(false); let new_layout = unsafe { Layout::from_size_align_unchecked(new_size, layout.align()) }; - let moved = unsafe { self.alloc(new_layout) }; + let moved = unsafe { System.alloc(new_layout) }; if !moved.is_null() { unsafe { + std::ptr::write_bytes(moved, FILL, new_size); std::ptr::copy_nonoverlapping(pointer, moved, layout.size().min(new_size)); - self.dealloc(pointer, layout); } + if old_tracked { + Self::track(moved); + } + unsafe { self.dealloc(pointer, layout) }; } moved } @@ -130,6 +144,7 @@ fn watch( ZEROED_FREES.set(0); SECRET_FREES.set(0); TRACKED.set([0; TRACKED_CAPACITY]); + TRACKING_FULL.set(false); for pointer in initialized { Inspecting::track(pointer.cast_mut()); } @@ -137,6 +152,10 @@ fn watch( let result = action(); WATCHED_SIZE.set(None); TRACKED.set([0; TRACKED_CAPACITY]); + assert!( + !TRACKING_FULL.get(), + "the watch tracked more blocks than it can hold" + ); (result, ZEROED_FREES.get(), SECRET_FREES.get()) } @@ -182,3 +201,15 @@ fn decode_error_wipes_partial_secrets() { assert!(decoded.is_err()); assert_eq!((zeroed, leaked), (1, 0)); } + +#[test] +fn untracked_buffers_with_spare_capacity_are_never_read() { + // Spare capacity is uninitialized, so neither this block nor its grown copy may be read. + let mut partial = Vec::with_capacity(8); + partial.push(SECRET); + let ((), zeroed, leaked) = watch(32, &[], || { + partial.reserve_exact(64); + drop(std::mem::take(&mut partial)); + }); + assert_eq!((zeroed, leaked), (0, 0)); +} From 20c5d3ede3ef3b1554b330bbb19e0b823e3be4eb Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 09:59:53 +0200 Subject: [PATCH 138/166] test: run compile-fail checks only on the MSRV compiler --- Cargo.lock | 1 + crates/pithos_lib/Cargo.toml | 1 + crates/pithos_lib/tests/compile_fail.rs | 2 ++ 3 files changed, 4 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 7803cd0..6d2403a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1434,6 +1434,7 @@ dependencies = [ "proptest", "ro-crate-rs", "rustix", + "rustversion", "sha2", "shake", "tempfile", diff --git a/crates/pithos_lib/Cargo.toml b/crates/pithos_lib/Cargo.toml index 8e55db9..457ec39 100644 --- a/crates/pithos_lib/Cargo.toml +++ b/crates/pithos_lib/Cargo.toml @@ -65,6 +65,7 @@ zeroize = "1.8.2" criterion = "0.8.2" peak_alloc = "0.3.0" proptest = "1.6.0" +rustversion = "1.0.23" tempfile = "3.21" tokio = { version = "1.52.3", default-features = false, features = ["rt"] } trybuild = "1.0.104" diff --git a/crates/pithos_lib/tests/compile_fail.rs b/crates/pithos_lib/tests/compile_fail.rs index b7c29da..a69a3b5 100644 --- a/crates/pithos_lib/tests/compile_fail.rs +++ b/crates/pithos_lib/tests/compile_fail.rs @@ -1,3 +1,5 @@ +// Compiler messages change between releases, so the expected output pins the MSRV compiler. +#[rustversion::attr(not(stable(1.89)), ignore = "expected output is from Rust 1.89")] #[test] fn secret_ownership_and_writer_typestate_are_compile_time_enforced() { let cases = trybuild::TestCases::new(); From f620458bfe1cee7ad7fa837db2a86d2f1dc0d12f Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 10:01:28 +0200 Subject: [PATCH 139/166] feat: split piece content with FastCDC as an option --- crates/pithos_lib/src/archive/pieces.rs | 130 +++++++++++++++++++----- crates/pithos_lib/tests/pieces.rs | 34 ++++++- 2 files changed, 138 insertions(+), 26 deletions(-) diff --git a/crates/pithos_lib/src/archive/pieces.rs b/crates/pithos_lib/src/archive/pieces.rs index faa44bf..cb16184 100644 --- a/crates/pithos_lib/src/archive/pieces.rs +++ b/crates/pithos_lib/src/archive/pieces.rs @@ -7,9 +7,7 @@ use crate::archive::content_tree::{self, ContentTree, Subtree, TreeHasher}; use crate::archive::path_validation::validate_entry; use crate::archive::types::{ArchivePath, Processing}; -use crate::archive::writer::{ - BlockKeyMode, Chunking, EntryMetadata, ProcessingOptions, validate_block_size, -}; +use crate::archive::writer::{BlockKeyMode, CdcConfig, Chunking, EntryMetadata, ProcessingOptions}; use crate::block; use crate::crypto::{self, FileKey, PublicKey}; use crate::error::PithosError; @@ -22,6 +20,7 @@ use crate::format::file_entry::{ }; use crate::format::header::{FileHeader, FormatVersion, encode_header}; use crate::format::limits::DeserializationError; +use fastcdc::v2020::{FastCDC, Normalization}; use indexmap::IndexMap; use integer_encoding::{VarIntReader, VarIntWriter}; use std::collections::HashSet; @@ -62,8 +61,8 @@ pub struct Piece { /// Encodes the blocks of one piece. The caller appends every returned byte string, in order, /// to the piece's stored bytes. /// -/// [`PieceEncoder::write`] and [`PieceEncoder::flush`] split the content into fixed-size blocks, -/// which is the recommended use. [`PieceEncoder::push`] instead encodes caller-chosen blocks. +/// [`PieceEncoder::write`] and [`PieceEncoder::flush`] split the content with the configured +/// [`Chunking`], fixed 4 MiB blocks by default. [`PieceEncoder::push`] encodes caller-chosen blocks. /// /// The encoder also records BLAKE3 subtree chaining values of the piece plaintext, so that /// [`Composition::content_hash`] can report the whole-file BLAKE3. They are plaintext @@ -78,7 +77,7 @@ pub struct PieceEncoder { seen: HashSet<[u8; 32]>, stored_len: u64, original_size: u64, - block_size: usize, + chunking: Chunking, pending: Zeroizing>, content_offset: u64, tree: Option, @@ -119,7 +118,7 @@ impl PieceEncoder { seen: HashSet::new(), stored_len: 0, original_size: 0, - block_size: Chunking::DEFAULT_BLOCK_SIZE, + chunking: Chunking::default(), pending: Zeroizing::new(Vec::new()), content_offset: 0, tree: (processing.key_mode() != BlockKeyMode::Unique).then(|| TreeHasher::new(0)), @@ -155,49 +154,84 @@ impl PieceEncoder { Ok(()) } - /// Sets the block size used by [`PieceEncoder::write`]. The default is 4 MiB. + /// Sets fixed blocks of `size` bytes for [`PieceEncoder::write`]. The default is 4 MiB. /// Fails once content was added, because buffered bytes assume the old size. - pub fn with_block_size(mut self, size: usize) -> Result { + pub fn with_block_size(self, size: usize) -> Result { + self.with_chunking(Chunking::Fixed(size)) + } + + /// Sets how [`PieceEncoder::write`] splits content into blocks. The default is fixed 4 MiB. + /// FastCDC gives the same blocks as [`ArchiveWriter`](crate::archive::ArchiveWriter) for the + /// same content. Fails once content was added. + pub fn with_chunking(mut self, chunking: Chunking) -> Result { self.ensure_no_content()?; - validate_block_size(size)?; - self.block_size = size; + chunking.validate()?; + self.chunking = chunking; Ok(self) } /// Adds content and returns the stored bytes of every block it completes. - /// Blocks have the configured size no matter how the content is split across calls. - /// After the last write, call [`PieceEncoder::flush`] for the short final block. + /// Block boundaries do not depend on how the content is split across calls. + /// After the last write, call [`PieceEncoder::flush`] for the final blocks. pub fn write(&mut self, mut content: &[u8]) -> Result, PithosError> { let mut stored = Vec::new(); + let (buffer_size, cdc) = match self.chunking { + Chunking::Fixed(size) => (size, None), + Chunking::ContentDefined(cdc) => (cdc.max_size(), Some(cdc)), + }; while !content.is_empty() { - if self.pending.is_empty() && content.len() >= self.block_size { - let (block, rest) = content.split_at(self.block_size); + if cdc.is_none() && self.pending.is_empty() && content.len() >= buffer_size { + let (block, rest) = content.split_at(buffer_size); stored.extend(self.encode(block)?); content = rest; continue; } - let missing = self.block_size - self.pending.len(); - // Reserving the rest of the block once keeps plaintext from being copied on growth. + let missing = buffer_size - self.pending.len(); + // Reserving the rest of the buffer once keeps plaintext from being copied on growth. self.pending.reserve_exact(missing); let take = missing.min(content.len()); self.pending.extend_from_slice(&content[..take]); content = &content[take..]; - if self.pending.len() == self.block_size { - stored.extend(self.flush()?); + if self.pending.len() == buffer_size { + stored.extend(match cdc { + Some(cdc) => self.cut_block(cdc)?, + None => self.flush()?, + }); } } Ok(stored) } - /// Encodes the bytes buffered by [`PieceEncoder::write`] as one block, if there are any. + /// Encodes the bytes buffered by [`PieceEncoder::write`] as the final blocks, if there are any. pub fn flush(&mut self) -> Result, PithosError> { - if self.pending.is_empty() { - return Ok(Vec::new()); + let Chunking::ContentDefined(cdc) = self.chunking else { + if self.pending.is_empty() { + return Ok(Vec::new()); + } + let block = std::mem::take(&mut self.pending); + let stored = self.encode(&block); + self.pending = block; + self.pending.clear(); + return stored; + }; + let mut stored = Vec::new(); + while !self.pending.is_empty() { + stored.extend(self.cut_block(cdc)?); } + Ok(stored) + } + + /// Encodes the first FastCDC block of the buffered bytes and removes it from the buffer. + /// Like `StreamCDC`, it cuts only with a full buffer or at the end of the content. + fn cut_block(&mut self, cdc: CdcConfig) -> Result, PithosError> { + let (min, avg, max) = (cdc.min_size(), cdc.avg_size(), cdc.max_size()); + let len = self.pending.len(); + let (_, end) = + FastCDC::with_level(&self.pending, min, avg, max, Normalization::Level1).cut(0, len); let block = std::mem::take(&mut self.pending); - let stored = self.encode(&block); + let stored = self.encode(&block[..end]); self.pending = block; - self.pending.clear(); + self.pending.drain(..end); stored } @@ -735,6 +769,54 @@ mod tests { } } + #[test] + fn written_fragments_give_the_same_fastcdc_blocks_as_the_archive_writer() { + let mut state = 0x9e37_79b9_7f4a_7c15u64; + let content = (0..20_000) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state as u8 + }) + .collect::>(); + let cdc = CdcConfig::new(64, 256, 1024).unwrap(); + let stream = fastcdc::v2020::StreamCDC::with_level( + Cursor::new(&content), + cdc.min_size(), + cdc.avg_size(), + cdc.max_size(), + Normalization::Level1, + ); + let expected = stream + .map(|chunk| chunk.unwrap().length as u64) + .collect::>(); + assert!(expected.len() > 20); + let recipient = crate::crypto::PrivateKey::generate().public_key(); + let processing = ProcessingOptions::new(true, 0).unwrap(); + let encoder = || PieceEncoder::new(1, vec![recipient], processing).unwrap(); + for fragment in [1, 7, 1000, 4096, content.len()] { + let chunking = Chunking::ContentDefined(cdc); + let mut written = encoder().with_chunking(chunking).unwrap(); + let mut stored = 0; + for part in content.chunks(fragment) { + stored += written.write(part).unwrap().len(); + } + stored += written.flush().unwrap().len(); + let piece = written.finish().unwrap(); + let sizes = piece.blocks.iter().map(|block| block.original_size); + assert_eq!(sizes.collect::>(), expected, "{fragment}"); + assert_eq!(piece.stored_len, stored as u64); + } + + let mut started = encoder(); + started.write(b"pending").unwrap(); + assert!(matches!( + started.with_chunking(Chunking::ContentDefined(cdc)), + Err(PithosError::PieceContentStarted) + )); + } + #[test] fn written_fragments_give_the_same_fixed_blocks_as_pushed_blocks() { let content = (0..2500u32) diff --git a/crates/pithos_lib/tests/pieces.rs b/crates/pithos_lib/tests/pieces.rs index 8271a84..8fef078 100644 --- a/crates/pithos_lib/tests/pieces.rs +++ b/crates/pithos_lib/tests/pieces.rs @@ -3,8 +3,8 @@ mod common; use common::util::{private_key, public_key}; use pithos_lib::archive::{ AccessKeys, AppendOptions, Archive, ArchivePath, ArchiveWriter, BlockKeyMode, BlockListForm, - EntryKind, EntryMetadata, OpenOptions, PayloadCipher, Piece, PieceEncoder, ProcessingOptions, - WriteOptions, compose, + CdcConfig, Chunking, EntryKind, EntryMetadata, OpenOptions, PayloadCipher, Piece, PieceEncoder, + ProcessingOptions, WriteOptions, compose, }; use pithos_lib::error::{DeserializationError, PithosError}; use pithos_lib::source::MemorySource; @@ -406,6 +406,36 @@ fn content_hash_of_aligned_parts_is_the_blake3_of_the_file() { } } +#[test] +fn content_defined_pieces_compose_with_the_file_content_hash() { + let file = content(7, 300_000); + let chunking = Chunking::ContentDefined(CdcConfig::new(1024, 4096, 16_384).unwrap()); + let processing = ProcessingOptions::new(true, 3).unwrap(); + let mut parts = Vec::new(); + for (key_id, range) in [(1, 0..102_400), (2, 102_400..file.len())] { + let mut encoder = PieceEncoder::new(key_id, vec![public_key("recipient1")], processing) + .unwrap() + .with_chunking(chunking) + .unwrap() + .with_content_offset(range.start as u64) + .unwrap(); + let mut stored = Vec::new(); + for fragment in file[range].chunks(5000) { + stored.extend(encoder.write(fragment).unwrap()); + } + stored.extend(encoder.flush().unwrap()); + parts.push((stored, encoder.finish().unwrap())); + } + let pieces = parts.iter().map(|(_, piece)| piece.clone()); + let expected = *blake3::hash(&file).as_bytes(); + assert_eq!(content_hash(&pieces.collect::>()), Some(expected)); + + let archive = open(assemble(&parts), recipient()); + let mut output = Vec::new(); + archive.copy_to("object", &mut output).unwrap(); + assert_eq!(output, file); +} + #[test] fn content_hash_of_one_piece_and_of_an_empty_file() { let file = content(6, 131_079); From 8374e85b9463468140cb3ec4bbfe56345015528a Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 10:01:39 +0200 Subject: [PATCH 140/166] chore: present content-defined and fixed blocks as equal methods in the spec --- README.md | 2 +- spec/PITHOS_1.1.0_draft.md | 33 +++++++++++++++++++++------------ 2 files changed, 22 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 722ee4f..92066c8 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ Format 1.1 keeps every 1.0 structure and encoding. This version reads 1.0 and 1. Breaking changes in the 0.8 API: -- New writers use fixed 4 MiB blocks by default. Use `Chunking::ContentDefined` with `WriteOptions::with_chunking` for FastCDC blocks. +- New writers use fixed 4 MiB blocks by default. Use `Chunking::ContentDefined` with `WriteOptions::with_chunking` or `PieceEncoder::with_chunking` for FastCDC blocks. - The RO-Crate and Crypt4GH adapters are behind the default features `ro-crate` and `crypt4gh`. - `Archive::open` no longer reads block markers. A missing or changed marker fails when the block is read. - The default `OpenLimits` admit objects up to 5 TiB. diff --git a/spec/PITHOS_1.1.0_draft.md b/spec/PITHOS_1.1.0_draft.md index e92e614..2aebd73 100644 --- a/spec/PITHOS_1.1.0_draft.md +++ b/spec/PITHOS_1.1.0_draft.md @@ -9,7 +9,7 @@ This document specifies the Pithos file format using the key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" as described in [RFC 2119](https://www.rfc-editor.org/rfc/rfc2119). -Pithos is an append-only archive format designed for efficient storage and sharing of scientific data. It combines block-level deduplication, convergent encryption, and flexible metadata support optimized for object storage systems. +Pithos is an append-only archive format designed for efficient storage and sharing of scientific data. It combines block-level deduplication with content-defined or fixed-size blocks, convergent encryption, and flexible metadata support optimized for object storage systems. **Illustrative data model.** Rust declarations in this document illustrate the data model only; implementations may use different declarations. Normative @@ -1232,21 +1232,30 @@ Version 1.1 uses these ASCII strings, without a terminator: This appendix suggests writer choices only. It does not define archive conformance or reader behavior. -**Recommended block size:** fixed 4 MiB (4,194,304 byte) blocks. Every block -of an independently encoded part has this size except its last, which may be -shorter. A file written in one pass therefore has at most one short block, at -its end. A file joined from pieces has one possible short block at the end of -each piece: two 5 MiB pieces give blocks of 4, 1, 4 and 1 MiB. Block boundaries -depend only on byte offsets within a part, not on how the input was read. This -suits object storage and multipart uploads, where parts are written -independently. -Content-defined chunking (for example FastCDC) is optional. It can find more -repeated blocks when content shifts, at the cost of variable block sizes. +**Block boundaries:** two methods suit different writers. One archive, and one +file joined from pieces, may use both. + +- **Content-defined chunking:** FastCDC chooses boundaries with a gear hash of + the content, so content that moves after an insertion or deletion still + gives equal blocks. It suits a file written in one pass. Suggested sizes are + a 1 MiB minimum, a 4 MiB average and a 16 MiB maximum, with level-one + normalization. +- **Fixed-size blocks:** 4 MiB (4,194,304 byte) blocks. Every block of an + independently encoded part has this size except its last, which may be + shorter. A file joined from pieces has one possible short block at the end of + each piece: two 5 MiB pieces give blocks of 4, 1, 4 and 1 MiB. Boundaries + depend only on byte offsets within a part. This suits object storage and + multipart uploads, where parts are written independently. + +Either method gives the same boundaries no matter how the input was read. +Content-defined boundaries in a joined file restart at the start of every +piece. **Directory size:** each block adds one descriptor and one block-list entry, so the directory grows with the block count. A 5 TiB file at 4 MiB blocks has about 1.31 million blocks and roughly 150 MB of directory. Smaller blocks -increase this size in proportion. +increase this size in proportion. With content-defined chunking the count +depends on the content; the minimum block size bounds it. **Recommended compression mapping:** a writer may map ProcessingFlags values `1` through `7` to Zstandard levels `1`, `4`, `8`, `11`, `15`, `18`, and `22`, From c716893940fe69fb760a879062bb55432136427e Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 10:11:23 +0200 Subject: [PATCH 141/166] chore: restore the CI job names that the main ruleset requires --- .github/workflows/checks.yaml | 2 +- .github/workflows/ci.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/checks.yaml b/.github/workflows/checks.yaml index f2b96f8..8a78c52 100644 --- a/.github/workflows/checks.yaml +++ b/.github/workflows/checks.yaml @@ -47,7 +47,7 @@ jobs: - name: Run Clippy run: cargo +stable clippy --locked --workspace --all-targets --all-features -- -D warnings - dependency-policy: + deny: runs-on: ubuntu-latest timeout-minutes: 10 steps: diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index eba3bd7..d4887a9 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -13,7 +13,7 @@ concurrency: cancel-in-progress: true jobs: - stable: + tests: runs-on: ubuntu-latest timeout-minutes: 20 steps: From 62713710c7c7918828463ada993aed0bd48265e6 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 10:24:39 +0200 Subject: [PATCH 142/166] feat: read Pithos 0.7 archives without appending to them --- crates/pithos/src/main.rs | 14 +- crates/pithos_lib/src/archive/mod.rs | 2 + crates/pithos_lib/src/archive/opener.rs | 5 +- crates/pithos_lib/src/archive/reader.rs | 2 +- crates/pithos_lib/src/archive/types.rs | 2 +- crates/pithos_lib/src/archive/view.rs | 3 +- crates/pithos_lib/src/crypto.rs | 2 +- crates/pithos_lib/src/format/directory.rs | 120 +++++++++++++++++- crates/pithos_lib/src/format/header.rs | 4 + crates/pithos_lib/src/fs/append.rs | 15 ++- crates/pithos_lib/src/fuzzing.rs | 1 + crates/pithos_lib/tests/append_workflow.rs | 29 ++++- crates/pithos_lib/tests/data/README.md | 3 + .../pithos_lib/tests/data/pithos-0.7.0.pith | Bin 0 -> 1389 bytes .../pithos_lib/tests/data/pithos-0.7.3.pith | Bin 0 -> 1389 bytes crates/pithos_lib/tests/format_validation.rs | 22 ++-- crates/pithos_lib/tests/reader.rs | 53 ++++++++ 17 files changed, 247 insertions(+), 30 deletions(-) create mode 100644 crates/pithos_lib/tests/data/pithos-0.7.0.pith create mode 100644 crates/pithos_lib/tests/data/pithos-0.7.3.pith diff --git a/crates/pithos/src/main.rs b/crates/pithos/src/main.rs index 290d6e1..4720469 100644 --- a/crates/pithos/src/main.rs +++ b/crates/pithos/src/main.rs @@ -375,12 +375,14 @@ fn run() -> Result<(), PithosCliError> { } ReadCommands::List { file } => { let archive = open_archive(&file, optional_private_key(&cli.secret_key)?)?; - let version = archive.view().version(); - write_stdout(format_args!( - "format version {}.{}", - version >> 8, - version & 0xff - ))?; + match archive.view().version() { + 0x8002 => write_stdout(format_args!("format version 0.7"))?, + version => write_stdout(format_args!( + "format version {}.{}", + version >> 8, + version & 0xff + ))?, + } for entry in archive.entries() { match entry.block_list { Some(form) => write_stdout(format_args!( diff --git a/crates/pithos_lib/src/archive/mod.rs b/crates/pithos_lib/src/archive/mod.rs index 65197e1..01b17d6 100644 --- a/crates/pithos_lib/src/archive/mod.rs +++ b/crates/pithos_lib/src/archive/mod.rs @@ -62,6 +62,7 @@ pub(crate) fn metadata_digest(directory_hashes: &[[u8; 32]]) -> [u8; 32] { pub(crate) fn decode_validated_directory( bytes: &[u8], + version: crate::format::header::FormatVersion, limits: &crate::format::limits::DeserializationLimits, remaining_block_references: &mut u64, blocks: &mut impl crate::format::directory::BlockSink, @@ -71,6 +72,7 @@ pub(crate) fn decode_validated_directory( limits, remaining_block_references, blocks, + version == crate::format::header::FormatVersion::V0_7, |directory| validate_directory_entries(&directory.files), ) } diff --git a/crates/pithos_lib/src/archive/opener.rs b/crates/pithos_lib/src/archive/opener.rs index 99164c2..670c745 100644 --- a/crates/pithos_lib/src/archive/opener.rs +++ b/crates/pithos_lib/src/archive/opener.rs @@ -262,6 +262,7 @@ impl ArchiveOpener { let mut descriptors = DescriptorList::new(decoded_chain.version, response.len()); let directory = decode_validated_directory( &response, + decoded_chain.version, &limits, &mut decoded_chain.remaining_block_references, &mut descriptors, @@ -364,7 +365,9 @@ fn complete_open( for (sender, section) in &directory.encryption { for (recipient, recipient_section) in §ion.recipients { let pair = (*sender, *recipient); - if let Some(first) = first_grants.get(&pair) + // Pithos 0.7 appends reused the writer key, so one pair may hold several grants. + if version != FormatVersion::V0_7 + && let Some(first) = first_grants.get(&pair) && *first != &recipient_section.recipient_data { return Err(PithosError::ConflictingRecipientGrant); diff --git a/crates/pithos_lib/src/archive/reader.rs b/crates/pithos_lib/src/archive/reader.rs index 87b9057..0569e77 100644 --- a/crates/pithos_lib/src/archive/reader.rs +++ b/crates/pithos_lib/src/archive/reader.rs @@ -835,7 +835,7 @@ pub(super) fn validate_piece_keys( for (id, _, file) in directory.files.iter() { file_ids.insert(id); if let BlockDataState::Pieces(pieces) = &file.block_data { - if version == FormatVersion::V1_0 { + if version != FormatVersion::V1_1 { return Err(PithosError::UnsupportedBlockListPieces); } for piece in pieces { diff --git a/crates/pithos_lib/src/archive/types.rs b/crates/pithos_lib/src/archive/types.rs index c7bc24b..376ffb4 100644 --- a/crates/pithos_lib/src/archive/types.rs +++ b/crates/pithos_lib/src/archive/types.rs @@ -181,7 +181,7 @@ impl Processing { } let encrypted = value & 0x08 != 0; if value & ProcessingFlags::VERSION_1_1_MASK != 0 { - if version == FormatVersion::V1_0 { + if version != FormatVersion::V1_1 { return Err(PithosError::UnsupportedProcessingFlags(value)); } if !encrypted { diff --git a/crates/pithos_lib/src/archive/view.rs b/crates/pithos_lib/src/archive/view.rs index 767cdd7..733d007 100644 --- a/crates/pithos_lib/src/archive/view.rs +++ b/crates/pithos_lib/src/archive/view.rs @@ -42,7 +42,8 @@ impl ArchiveView { self.metadata_digest } - /// The format version stored in the file header: `0x0100` for 1.0, `0x0101` for 1.1. + /// The format version stored in the file header: `0x0100` for 1.0, `0x0101` for 1.1, and + /// `0x8002` for a read-only Pithos 0.7 archive. pub fn version(&self) -> u16 { self.version.wire() } diff --git a/crates/pithos_lib/src/crypto.rs b/crates/pithos_lib/src/crypto.rs index 3f627ba..4d45078 100644 --- a/crates/pithos_lib/src/crypto.rs +++ b/crates/pithos_lib/src/crypto.rs @@ -270,7 +270,7 @@ pub(crate) fn grant_wrapping_key( nonce: &[u8; 12], ) -> SharedSecret { match version { - FormatVersion::V1_0 => shared, + FormatVersion::V0_7 | FormatVersion::V1_0 => shared, FormatVersion::V1_1 => { let mut info = Zeroizing::new([0u8; GRANT_KEY_INFO.len() + 64]); info[..GRANT_KEY_INFO.len()].copy_from_slice(GRANT_KEY_INFO); diff --git a/crates/pithos_lib/src/format/directory.rs b/crates/pithos_lib/src/format/directory.rs index a33724d..cc8f32b 100644 --- a/crates/pithos_lib/src/format/directory.rs +++ b/crates/pithos_lib/src/format/directory.rs @@ -6,8 +6,8 @@ use crate::format::encryption::{ }; use crate::format::error::SerializationError; use crate::format::file_entry::{ - BlockDataState, FileEntry, FileType, decode_file_entry, encode_file_entry, - validate_unique_block_references, + BlockDataState, FileEntry, FileType, VALID_PERMISSION_BITS, decode_file_entry, + encode_file_entry, validate_unique_block_references, }; use crate::format::limits::{DeserializationError, DeserializationLimits}; use crate::format::primitives::{ @@ -232,6 +232,106 @@ pub(crate) const STANDARD_RELATIONSHIPS: [(u64, &str); 10] = [ (9, "OUTPUT_FROM"), ]; +/// The names Pithos 0.7 stored for the standard relationships, in ID order. +const RELATIONSHIP_NAMES_0_7: [&str; 10] = [ + "Describes", + "Annotates", + "Derived_From", + "Source_Of", + "Previous_Version", + "Next_Version", + "Part_of", + "Contains", + "Input_To", + "Output_From", +]; + +/// Rewrites a decoded Pithos 0.7 directory into its version 1.0 form: standard relationship +/// names, permissions without file type bits, and size 0 for directories and symlinks. +fn upgrade_0_7_directory(directory: &mut Directory) { + for (id, name) in &mut directory.relations { + let index = usize::try_from(*id).unwrap_or(usize::MAX); + if RELATIONSHIP_NAMES_0_7.get(index) == Some(&name.as_str()) { + *name = STANDARD_RELATIONSHIPS[index].1.to_owned(); + } + } + let Ok(()) = directory + .files + .try_for_each_mut::(|_, entry| { + entry.permissions &= VALID_PERMISSION_BITS; + if matches!(entry.file_type, FileType::Directory | FileType::Symlink) { + entry.file_size = 0; + } + Ok(()) + }); +} + +/// The CRC-32 that Pithos 0.7.0 stored. It covers the parent option only when present, the file, +/// block and encryption items without their counts, the relationships with their count, and the +/// directory length as ULEB128. Returns `None` when the bytes do not decode. +fn checksum_0_7_0(bytes: &[u8], limits: &DeserializationLimits) -> Option { + let body = &bytes[..bytes.len() - 12]; + let mut reader = Cursor::new(body); + reader.set_position(DIRECTORY_MARKER.len() as u64); + let mut hasher = crc32fast::Hasher::new(); + let cover = |hasher: &mut crc32fast::Hasher, start: u64, reader: &Cursor<&[u8]>| { + hasher.update(&body[start as usize..reader.position() as usize]); + }; + let start = reader.position(); + if reader.read_u8().ok()? == 1 { + reader.read_varint::().ok()?; + reader.read_varint::().ok()?; + cover(&mut hasher, start, &reader); + } + let mut references = limits.max_references; + let mut block_references = limits.max_block_references; + let files = bounded_len(reader.read_varint().ok()?, limits.max_file_entries, "files").ok()?; + let start = reader.position(); + for _ in 0..files { + reader.read_varint::().ok()?; + decode_string(&mut reader, limits).ok()?; + decode_file_entry(&mut reader, limits, &mut references, &mut block_references).ok()?; + } + cover(&mut hasher, start, &reader); + let blocks = bounded_len( + reader.read_varint().ok()?, + limits.max_block_descriptors, + "blocks", + ); + let start = reader.position(); + for _ in 0..blocks.ok()? { + reader.read_exact(&mut [0; 32]).ok()?; + decode_block_index_entry(&mut reader, limits).ok()?; + } + cover(&mut hasher, start, &reader); + let start = reader.position(); + let relations = bounded_len( + reader.read_varint().ok()?, + limits.max_relationships, + "relations", + ); + for _ in 0..relations.ok()? { + reader.read_varint::().ok()?; + decode_string(&mut reader, limits).ok()?; + } + cover(&mut hasher, start, &reader); + let limit = limits.max_collection_entries; + let sections = bounded_len(reader.read_varint().ok()?, limit, "encryption").ok()?; + let start = reader.position(); + for _ in 0..sections { + reader.read_exact(&mut [0; 32]).ok()?; + decode_encryption_section(&mut reader, limits).ok()?; + } + cover(&mut hasher, start, &reader); + if reader.position() != body.len() as u64 { + return None; + } + let mut length = Vec::new(); + length.write_varint(bytes.len() as u64).ok()?; + hasher.update(&length); + Some(hasher.finalize()) +} + impl Directory { pub(crate) fn new( parent_directory_offset: Option<(u64, u64)>, @@ -523,12 +623,13 @@ pub(crate) fn decode_complete_directory( bytes: &[u8], limits: &DeserializationLimits, ) -> Result { - decode_complete_directory_with_validation(bytes, limits, |_| Ok(())) + decode_complete_directory_with_validation(bytes, limits, false, |_| Ok(())) } pub(crate) fn decode_complete_directory_with_validation( bytes: &[u8], limits: &DeserializationLimits, + pithos_0_7: bool, validate: impl Fn(&Directory) -> Result<(), PithosError>, ) -> Result { let mut remaining_block_references = limits.max_block_references; @@ -538,6 +639,7 @@ pub(crate) fn decode_complete_directory_with_validation( limits, &mut remaining_block_references, &mut blocks, + pithos_0_7, validate, )?; directory.blocks = blocks; @@ -545,11 +647,13 @@ pub(crate) fn decode_complete_directory_with_validation( } /// Decodes and checks a complete directory. Its block descriptors go to `blocks`. +/// With `pithos_0_7`, it also accepts the Pithos 0.7.0 checksum and returns the version 1.0 form. pub(crate) fn decode_complete_directory_with_validation_and_budget( bytes: &[u8], limits: &DeserializationLimits, remaining_block_references: &mut u64, blocks: &mut impl BlockSink, + pithos_0_7: bool, validate: impl Fn(&Directory) -> Result<(), PithosError>, ) -> Result { if bytes.len() < MIN_DIRECTORY_LEN { @@ -584,15 +688,20 @@ pub(crate) fn decode_complete_directory_with_validation_and_budget( .expect("checked directory footer"), ); let computed_crc = crc32fast::hash(&bytes[..bytes.len() - 4]); - if encoded_crc != computed_crc { + if encoded_crc != computed_crc + && !(pithos_0_7 && checksum_0_7_0(bytes, limits) == Some(encoded_crc)) + { return Err(PithosError::DirectoryChecksumMismatch { expected: computed_crc, actual: encoded_crc, }); } let mut reader = Cursor::new(bytes); - let directory = + let mut directory = decode_directory_with_sink(&mut reader, limits, remaining_block_references, blocks)?; + if pithos_0_7 { + upgrade_0_7_directory(&mut directory); + } validate(&directory)?; if reader.position() != bytes.len() as u64 { return Err(PithosError::DirectoryConsumptionMismatch { @@ -698,6 +807,7 @@ mod tests { decode_complete_directory_with_validation( &bytes, &DeserializationLimits::default(), + false, |directory| { if directory.files.get_by_path("/invalid").is_some() { Err(PithosError::InvalidArchivePath { diff --git a/crates/pithos_lib/src/format/header.rs b/crates/pithos_lib/src/format/header.rs index e5c3c3d..760c23c 100644 --- a/crates/pithos_lib/src/format/header.rs +++ b/crates/pithos_lib/src/format/header.rs @@ -25,6 +25,8 @@ impl FileHeader { /// The wire rules of one archive. An append keeps the version of the archive it extends. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(crate) enum FormatVersion { + /// Pithos 0.7, read only. It stored version 1.0 as ULEB128 `80 02`, which reads as `0x8002`. + V0_7, V1_0, V1_1, } @@ -35,6 +37,7 @@ impl FormatVersion { pub(crate) fn from_wire(version: u16) -> Option { match version { + 0x8002 => Some(Self::V0_7), 0x0100 => Some(Self::V1_0), 0x0101 => Some(Self::V1_1), _ => None, @@ -43,6 +46,7 @@ impl FormatVersion { pub(crate) fn wire(self) -> u16 { match self { + Self::V0_7 => 0x8002, Self::V1_0 => 0x0100, Self::V1_1 => 0x0101, } diff --git a/crates/pithos_lib/src/fs/append.rs b/crates/pithos_lib/src/fs/append.rs index 93e1acd..10fc982 100644 --- a/crates/pithos_lib/src/fs/append.rs +++ b/crates/pithos_lib/src/fs/append.rs @@ -4,6 +4,7 @@ use crate::archive::{ }; use crate::crypto::PrivateKey; use crate::error::PithosError; +use crate::format::header::FormatVersion; use crate::fs::FsError; use crate::fs::ingest::{InputManifest, build_input_manifest}; use crate::source::{ArchiveSource, FileSource, SourceError}; @@ -333,15 +334,23 @@ fn open_snapshot( reads: Arc, bytes: Arc, ) -> Result { - Archive::open( + let archive = Archive::open( AppendSource { source: FileSource::from_file(locked.try_clone()?)?, reads, bytes, }, OpenOptions::default().with_access_keys(AccessKeys::new().with_key(sender)), - ) - .map(Archive::into_append_snapshot) + )?; + let version = archive.view().version(); + // Pithos 0.7 archives are read only. + if version == FormatVersion::V0_7.wire() { + return Err(PithosError::UnsupportedFileVersion { + supported: FormatVersion::CURRENT.wire(), + actual: version, + }); + } + Ok(archive.into_append_snapshot()) } fn fresh_wrapping_sender(snapshot: &AppendSnapshot, options: &AppendOptions) -> PrivateKey { diff --git a/crates/pithos_lib/src/fuzzing.rs b/crates/pithos_lib/src/fuzzing.rs index caf039c..fcbd5c9 100644 --- a/crates/pithos_lib/src/fuzzing.rs +++ b/crates/pithos_lib/src/fuzzing.rs @@ -39,6 +39,7 @@ pub fn decode_directory(data: &[u8]) -> FuzzOutcome { indexmap::IndexMap::new(); match crate::archive::decode_validated_directory( data, + crate::format::header::FormatVersion::CURRENT, &limits, &mut remaining_block_references, &mut blocks, diff --git a/crates/pithos_lib/tests/append_workflow.rs b/crates/pithos_lib/tests/append_workflow.rs index 6b2fb01..02bb842 100644 --- a/crates/pithos_lib/tests/append_workflow.rs +++ b/crates/pithos_lib/tests/append_workflow.rs @@ -9,7 +9,7 @@ use pithos_lib::archive::{ EntryMetadata, PayloadCipher, ProcessingOptions, WriteOptions, }; use pithos_lib::error::PithosError; -use pithos_lib::fs::{FsError, append_files}; +use pithos_lib::fs::{FsError, append_files, grant_readers}; use std::io::Cursor; use std::path::PathBuf; @@ -32,6 +32,33 @@ fn version_1_0_base_archive(temporary: &tempfile::TempDir) -> PathBuf { path } +#[test] +fn appends_and_grants_to_pithos_0_7_archives_are_refused_before_writing() { + let temporary = tempfile::tempdir().unwrap(); + let archive = temporary.path().join("pithos-0.7.pith"); + std::fs::copy("tests/data/pithos-0.7.3.pith", &archive).unwrap(); + let original = std::fs::read(&archive).unwrap(); + let source = temporary.path().join("appended.txt"); + std::fs::write(&source, b"appended payload").unwrap(); + let options = || AppendOptions::new(private_key("recipient1"), vec![public_key("recipient2")]); + let refused = |result| { + matches!( + result, + Err(FsError::Core { + source: PithosError::UnsupportedFileVersion { actual: 0x8002, .. }, + .. + }) + ) + }; + assert!(refused( + append_files(&archive, options(), &[source]).map(|_| ()) + )); + assert!(refused( + grant_readers(&archive, options(), &[2]).map(|_| ()) + )); + assert_eq!(std::fs::read(&archive).unwrap(), original); +} + #[test] fn appends_to_version_1_0_archives_reject_version_1_1_processing_before_writing() { let temporary = tempfile::tempdir().unwrap(); diff --git a/crates/pithos_lib/tests/data/README.md b/crates/pithos_lib/tests/data/README.md index 4947dc3..5536caa 100644 --- a/crates/pithos_lib/tests/data/README.md +++ b/crates/pithos_lib/tests/data/README.md @@ -6,6 +6,9 @@ This directory is for repository-owned integration-test inputs. The fixtures are `ro_crate_directory.rs`. It covers upstream graph parsing and Pithos directory conversion. Its `literature/pithos-placeholder.pdf` is a minimal repository-owned PDF retained solely to cover a non-text `application/pdf` member and `encodingFormat` handling. +- `pithos-0.7.0.pith` and `pithos-0.7.3.pith` were written by the Pithos 0.7.0 release binary and + by 0.7.3 with the `sender` and `recipient1` test keys: a small directory tree, then one appended + file. They cover reading Pithos 0.7 archives (spec Appendix C). - `keys/` contains test-only PEM key pairs consumed through `tests/common/keys.rs` by archive, filesystem, adapter, and CLI integration tests. They are public repository test material, not operational credentials. diff --git a/crates/pithos_lib/tests/data/pithos-0.7.0.pith b/crates/pithos_lib/tests/data/pithos-0.7.0.pith new file mode 100644 index 0000000000000000000000000000000000000000..7ffbbc78bf7a4cad284779ca5a4561163edd9cf4 GIT binary patch literal 1389 zcmWIW4Do1Sa`JKZzPsn-Vng+LOE%mQWYBIYRrlAp)Do_CxJ-M^t`Lot->Ut$fK;V? zE>8boQ_pkkrq_<08lT&uTYfG5z$kyB-0}K({(A@AA_IU1`UkrNF|aYPq$HLkGB7f1 znc8uU4NRoGXl7(!VB`Qv=oclHqe$Ak1xYe-Cgv1mBH{$#8zc5OJYfn8?i zR1W|9sm5OwFFlfE+QXLm_g)r5V)8%n$uFYLey~2Xtndw=iNubrRRt^_3b`#gyi1&I zVxIglaNFdVoY3c6zEHv8a-i}&cF}uZEwn#(|2?ox@EpQjYz@s!3=GVCV0WeF7L-&% z?3XR%D4rdm!QsAjR#?oFyNe^Gr!X$Py_+X6tBbFJ6QQ4B2}nPSAk1I-NvS0;FN9S* zYg2Ibwb^=QZdUn2=Oasu)0?!XW|gyFSd&wBsyQ**Q}YkxT{op@V_@2h#DMnAd(#TnY)k$rcRKL5&!&S2C$NH@z{&}B zLQZC0HZyX-^ZY-|#K6b{405>BnAW8CPTG9kvfNhU>{OjUuUzbxEM3|EXKKi{^r^d* zt_JI}DY0=dc=QO~E%Uk3HoL^W=3DSveTE7&u*0 zi<65olTwQrIUV!z@=Fp+Qj3{*TvCfN%TiO~-HP&anK^^=ON)|I64i3+}g3^-skbF*V|I!i= z6J#i3jgQdza`)LS%Vd7qi0#p=Og`Z1ytKQ>${>5fF}KM3HH^2|nDRBs}Jj)E^09JG}v*BFH~-J-s9sT4`O9S z4hyfDE*^FG5l`UtyO(q8RbNCu8PS+DJTJ2?KYU(K=VB%Ai@TYobi52Hv z7gy8B#kFrD{I1M;@j&xwLppox|K2;FiqG_K;|Y_l?dnKZROj0F{OVSw)2H^`i7Bg? z!2kx#_S3vhD}wS^Wa*oGZ$4XJJ}4TWx7J(ngt&~8;Dc9I^At}zhQ@zUxqmYdmdO|c z_p>pwai%Bc=E9PjUYI+cTGJ0ta66TabCv7 ziHY*6HJcF0jSZaK7!UGP`0kK+wClWrgVW5!Di7ah|EpS09Df|}ZjIfiSI^QWu!9oO zkWL?V1rs-0NcFaV;E~&QGwP0T?A+MbWz$c^r!;&{4`ay{kjV=1b;=Lg>aTU~;k|di ezPdb5T^XHmfg$ok$>-0ar=JV!)*X2YD#nf literal 0 HcmV?d00001 diff --git a/crates/pithos_lib/tests/data/pithos-0.7.3.pith b/crates/pithos_lib/tests/data/pithos-0.7.3.pith new file mode 100644 index 0000000000000000000000000000000000000000..169ef2aff36c9e3dd250a96476bfe52e573b768b GIT binary patch literal 1389 zcmWIW4Do1Sa`JKZ*3XW+cxrA&iR|seb8Ov0uKr%c{Qkfx%@1GfTORM`*i|0L0#da) zr<(gv*H5|6;i`&ue!5CmqpVN<`}$)S_aZ;lhkuu=`vw3F^bd9kVqjxnNl7e8WME|2 zGPUCx8< z{J6Npv*5U)O#;{BUCOuD&%b=qnJaE)-leZ6&V-2Xn=G+{mA%9v!D#>c6`TFOe4O(% zea)*?J6hwG7%tqGb>owXwW0KV4S&~ZFA}!ye7pX`iqB;y`UEGeKDg%E13rWkSiw$U zVE+FfL2kUim;Tjc#3##?O6`=o!iEiJBm z_H=Rb<1N)c;v3iB^<(%hpKLWH;z=Y!1EWv-#;0K#FPhvr7EiusDIJkp?Wu9))tm6f zeR3yqwjZhA&%mY|w|m9gdGi02#1^J#x6WPiy1i!O1{IyX7euQ3cUPX1y>ywoW6r__ zCYv`dGyC*U&%Js3)E5#96V3NXEPLhmg;k*Hhu^f&zroErF0(FPXrd>>6W zz%m(Q;C?nnHqP|K++0|4JD7XWY>LWXhh;1G`C75REkDkFsb{{@Q;zj}-tzqkG(N?^ z@qCv0wVoYS*)eO*EZuzaGH=kV%8+YU&m8$JKbLV!)4`uJd^=8XPyPMTb<#g;3DrD< zr$xtu!1KzE%`}FEr z+5~n`A{x@^!>;uAi!=Yfzu(HY=Kk+_M_TV3oj+rZUX^=ncYr&~jN45M*-8?*PO+R| l{BdU1=HrL6*}iOZ?$v3i@Qz*cX6mI9NaQkZPs}-%4FIlQUGx9| literal 0 HcmV?d00001 diff --git a/crates/pithos_lib/tests/format_validation.rs b/crates/pithos_lib/tests/format_validation.rs index 917f689..9320686 100644 --- a/crates/pithos_lib/tests/format_validation.rs +++ b/crates/pithos_lib/tests/format_validation.rs @@ -29,17 +29,19 @@ fn public_open_rejects_a_footer_claiming_an_impossible_directory() { } #[test] -fn public_open_rejects_the_legacy_varint_header() { - let (mut bytes, _) = archive_with_entries(Vec::new()); - bytes[..6].copy_from_slice(b"PITH\x80\x02"); +fn public_open_rejects_unknown_header_versions() { + for version in [0x0102u16, 0x8003] { + let (mut bytes, _) = archive_with_entries(Vec::new()); + bytes[4..6].copy_from_slice(&version.to_be_bytes()); - assert!(matches!( - Archive::open(MemorySource::new(bytes), OpenOptions::default()), - Err(PithosError::UnsupportedFileVersion { - supported: 0x0101, - actual: 0x8002, - }) - )); + assert!(matches!( + Archive::open(MemorySource::new(bytes), OpenOptions::default()), + Err(PithosError::UnsupportedFileVersion { + supported: 0x0101, + actual, + }) if actual == version + )); + } } fn archive_with_entries(entries: Vec<(&str, EntryMetadata)>) -> (Vec, PrivateKey) { diff --git a/crates/pithos_lib/tests/reader.rs b/crates/pithos_lib/tests/reader.rs index 65f0058..05d4593 100644 --- a/crates/pithos_lib/tests/reader.rs +++ b/crates/pithos_lib/tests/reader.rs @@ -10,6 +10,59 @@ use pithos_lib::error::PithosError; use pithos_lib::fs::{ExtractionOptions, extract, extract_all, extract_all_with_options}; use pithos_lib::source::MemorySource; +const PITHOS_0_7_FIXTURES: [&str; 2] = [ + "tests/data/pithos-0.7.0.pith", + "tests/data/pithos-0.7.3.pith", +]; + +#[test] +fn pithos_0_7_archives_read_with_their_content_and_metadata() { + for fixture in PITHOS_0_7_FIXTURES { + let path = std::path::Path::new(fixture); + let archive = open(path, AccessKeys::new().with_key(private_key("recipient1"))); + assert_eq!(archive.view().version(), 0x8002, "{fixture}"); + let files: [(&str, &[u8]); 4] = [ + ("alpha.txt", b"alpha\n"), + ("data/empty.txt", b""), + ("data/raw/beta.txt", b"beta\n"), + ("gamma.txt", b"gamma\n"), + ]; + for (path, expected) in files { + let mut content = Vec::new(); + archive.copy_to(path, &mut content).unwrap(); + assert_eq!(content, expected, "{fixture} {path}"); + } + let alpha = archive.entry("alpha.txt").unwrap().unwrap(); + assert_eq!(alpha.permissions, 0o600, "{fixture}"); + let data = archive.entry("data").unwrap().unwrap(); + assert!(matches!(data.kind, EntryKind::Directory)); + assert_eq!(data.permissions, 0o750, "{fixture}"); + let link = archive.entry("data/link").unwrap().unwrap(); + assert!(matches!(link.kind, EntryKind::Symlink { ref target } if target == "raw/beta.txt")); + } +} + +#[test] +fn pithos_0_7_archives_reject_a_changed_directory() { + for fixture in PITHOS_0_7_FIXTURES { + let mut bytes = std::fs::read(fixture).unwrap(); + let path = bytes + .windows(8) + .rposition(|window| window == b"data/raw") + .unwrap(); + bytes[path] ^= 1; + let options = OpenOptions::default() + .with_access_keys(AccessKeys::new().with_key(private_key("recipient1"))); + assert!( + matches!( + Archive::open(MemorySource::new(bytes), options), + Err(PithosError::DirectoryChecksumMismatch { .. }) + ), + "{fixture}" + ); + } +} + #[test] fn public_reader_lists_copies_ranges_extracts_and_exports() { let temporary = tempfile::tempdir().unwrap(); From a3f95d0a56ddd701ae41b7646937fa824a325288 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 10:24:39 +0200 Subject: [PATCH 143/166] chore: describe reading Pithos 0.7 archives in the spec --- README.md | 2 +- spec/PITHOS_1.1.0_draft.md | 32 ++++++++++++++++++++++++++++++-- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 92066c8..8276f2a 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,7 @@ This branch implements the wire rules in the [Pithos 1.1 draft](spec/PITHOS_1.1. ## Format 1.1 -Format 1.1 keeps every 1.0 structure and encoding. This version reads 1.0 and 1.1 archives, writes new archives as 1.1, and appends to an archive with the rules of its own version. The changes are: +Format 1.1 keeps every 1.0 structure and encoding. This version reads 1.0 and 1.1 archives, writes new archives as 1.1, and appends to an archive with the rules of its own version. It also reads archives written by Pithos 0.7, but does not append to them (spec Appendix C). The changes are: - The header version is `0x0101`. - A recipient grant key is derived with HKDF-SHA256 instead of using the raw X25519 shared secret. diff --git a/spec/PITHOS_1.1.0_draft.md b/spec/PITHOS_1.1.0_draft.md index 2aebd73..cb06aea 100644 --- a/spec/PITHOS_1.1.0_draft.md +++ b/spec/PITHOS_1.1.0_draft.md @@ -170,7 +170,8 @@ pub struct FileHeader { Readers MUST reject a header whose magic is not `PITH` or whose version is neither `0x0100` nor `0x0101`. The encoded form is exactly six bytes: `PITH 01 01` -for version 1.1 and `PITH 01 00` for version 1.0. +for version 1.1 and `PITH 01 00` for version 1.0. As the only exception, a reader +MAY open a Pithos 0.7 archive, whose version reads as `0x8002` (Appendix C). ### 4.2 Block Storage @@ -1181,7 +1182,7 @@ Non-POSIX platforms MAY retain them as metadata without an ACL mapping. ## 9. Future Extensions The format reserves space for future extensions: -- Header versions other than `0x0100` and `0x0101` +- Header versions other than `0x0100`, `0x0101` and the Pithos 0.7 value `0x8002` - FileType values 4-255 - ProcessingFlags bits 6-7 - Relationship IDs 10 through 999, for future standard relationships @@ -1211,6 +1212,7 @@ Extensions MUST maintain backwards compatibility for reading. - Version 1.0: `0x0100` - Version 1.1: `0x0101` +- Pithos 0.7, read only (Appendix C): `0x8002` ### 10.3 Key-Derivation Context Strings @@ -1628,4 +1630,30 @@ byte and then refreshing the CRC. | Version 1.1 pieces and grants | 1.3, 4.4.2, 5.3, 8.2 | CV-PIECES-HELLO-766, PV-RECIPIENT-WRAP-11; *case:* tag `02` in a version 1.0 archive; *case:* a reader granted only some of a file's piece keys | Valid archive; content readable only with every piece key; reject tag `02` in version 1.0; a partly granted file is unavailable | | Version 1.0 compatibility | 1.3, 5.3 | CV-BASE-EMPTY-146, CV-LOCAL-HELLO-279, PV-RECIPIENT-WRAP-01; *case:* an append and a reader grant on a version 1.0 archive keep header `0x0100` and the version 1.0 grant construction | Readable under the version 1.0 rules; under the version 1.1 rules the same grants fail authentication and the open fails, and the reverse | | Grant authentication | 5.3, 7, 8.2 | *case:* one changed ciphertext byte in a grant addressed to the reader, while an older grant still gives access, for a recipient key and for a sender key through sender-key recovery; *case:* a reader with no grant | The open fails for an addressed grant that fails authentication; with no grant the open succeeds, the content is unavailable, and reads fail without output | +| Pithos 0.7 archives (optional) | Appendix C | *case:* archives written by Pithos 0.7.0 and 0.7.3, each with one append; *case:* one changed path byte in a Directory; *case:* an append or grant to such an archive | Content, permissions and symlink targets read back; the changed Directory fails its checksum; the append and the grant fail before writing | | Whole-file hash of joined parts (informative) | Appendix A | *case:* aligned parts, a short last part, empty parts, one part, and an empty file; *case:* a part that ends inside a 1024-byte chunk | The hash equals the BLAKE3 of the full read; no hash for the misaligned part | + +## Appendix C. Reading Pithos 0.7 Archives (Optional) + +Pithos 0.7 wrote archives before this specification fixed the header encoding. +A reader MAY open them read only. It then applies the version 1.0 rules with the +differences below. Writers MUST NOT create such archives or append to them. + +1. **Header:** the version is the ULEB128 encoding of `0x0100`, the bytes + `80 02`. Read as the fixed-width field of Section 4.1, it is `0x8002`. +2. **Directory checksum:** Pithos 0.7.3 computes the CRC as in Section 4.3. + Pithos 0.7.0 computes it over other bytes: the parent option tag and values + only when a parent is present, the `files`, `blocks` and `encryption` items + without their counts, the `relations` vector with its count, and `dir_len` as + ULEB128 instead of `u64be`. The stored CRC MUST match one of the two. +3. **Relationship names:** the ten standard relationships are stored as + `Describes`, `Annotates`, `Derived_From`, `Source_Of`, `Previous_Version`, + `Next_Version`, `Part_of`, `Contains`, `Input_To` and `Output_From`. A reader + treats them as the names of Section 4.3.2. +4. **Permissions:** `permissions` stores a full POSIX mode, including the file + type bits. A reader keeps only the low 12 bits. +5. **Sizes:** `Directory` and `Symlink` entries may store a nonzero `file_size`. + A reader treats it as `0`. +6. **Recipient grants:** an append reused the archive's sender key, so one + sender and recipient pair may have different grants in several Directories. + A reader uses each of them instead of rejecting the conflict. From 4562aca3a75e5eafb97ce84e1459a01a127703ce Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 10:54:00 +0200 Subject: [PATCH 144/166] fix: wipe AEAD cipher keys on drop --- Cargo.lock | 6 ++++++ crates/pithos_lib/Cargo.toml | 4 ++-- crates/pithos_lib/src/crypto.rs | 6 ++++++ fuzz/Cargo.lock | 6 ++++++ 4 files changed, 20 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6d2403a..2afc8a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -27,6 +27,7 @@ dependencies = [ "cipher", "cpubits", "cpufeatures", + "zeroize", ] [[package]] @@ -41,6 +42,7 @@ dependencies = [ "ctr", "ctutils", "ghash", + "zeroize", ] [[package]] @@ -312,6 +314,7 @@ dependencies = [ "cipher", "cpufeatures", "rand_core 0.10.1", + "zeroize", ] [[package]] @@ -324,6 +327,7 @@ dependencies = [ "chacha20", "cipher", "poly1305", + "zeroize", ] [[package]] @@ -858,6 +862,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eecf2d5dc9b66b732b97707a0210906b1d30523eb773193ab777c0c84b3e8d5" dependencies = [ "polyval", + "zeroize", ] [[package]] @@ -1518,6 +1523,7 @@ dependencies = [ "cpubits", "cpufeatures", "universal-hash", + "zeroize", ] [[package]] diff --git a/crates/pithos_lib/Cargo.toml b/crates/pithos_lib/Cargo.toml index 457ec39..05e7e3b 100644 --- a/crates/pithos_lib/Cargo.toml +++ b/crates/pithos_lib/Cargo.toml @@ -35,11 +35,11 @@ async = ["dep:futures-core"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -aes-gcm = "0.11.1" +aes-gcm = { version = "0.11.1", features = ["zeroize"] } blake2 = { version = "0.10.6", optional = true } blake3 = "1.8.2" byteorder = "1.5.0" -chacha20poly1305 = "0.11.0" +chacha20poly1305 = { version = "0.11.0", features = ["zeroize"] } crc32fast = "1.5.0" cap-std = "4.0.2" digest = "0.11.3" diff --git a/crates/pithos_lib/src/crypto.rs b/crates/pithos_lib/src/crypto.rs index 4d45078..746c915 100644 --- a/crates/pithos_lib/src/crypto.rs +++ b/crates/pithos_lib/src/crypto.rs @@ -506,4 +506,10 @@ mod zeroization_tests { assert_zeroize_on_drop::(); assert_zeroize_on_drop::>>(); } + + #[test] + fn aead_ciphers_wipe_keys() { + assert_zeroize_on_drop::(); + assert_zeroize_on_drop::(); + } } diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index 08fe60b..a5e708a 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -27,6 +27,7 @@ dependencies = [ "cipher", "cpubits", "cpufeatures", + "zeroize", ] [[package]] @@ -41,6 +42,7 @@ dependencies = [ "ctr", "ctutils", "ghash", + "zeroize", ] [[package]] @@ -223,6 +225,7 @@ dependencies = [ "cipher", "cpufeatures", "rand_core", + "zeroize", ] [[package]] @@ -235,6 +238,7 @@ dependencies = [ "chacha20", "cipher", "poly1305", + "zeroize", ] [[package]] @@ -600,6 +604,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2eecf2d5dc9b66b732b97707a0210906b1d30523eb773193ab777c0c84b3e8d5" dependencies = [ "polyval", + "zeroize", ] [[package]] @@ -1131,6 +1136,7 @@ dependencies = [ "cpubits", "cpufeatures", "universal-hash", + "zeroize", ] [[package]] From ab2302d6c39cb2571c09c5e067913be70357a199 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 10:57:08 +0200 Subject: [PATCH 145/166] fix: wipe undelivered plaintext of async range streams --- crates/pithos_lib/src/archive/async_stream.rs | 25 +++--- .../src/archive/reader_private_tests.rs | 85 +++++++++++++++++++ crates/pithos_lib/src/format/mod.rs | 2 +- crates/pithos_lib/src/format/zeroing_tests.rs | 4 +- 4 files changed, 101 insertions(+), 15 deletions(-) diff --git a/crates/pithos_lib/src/archive/async_stream.rs b/crates/pithos_lib/src/archive/async_stream.rs index 09baeed..17fd4e1 100644 --- a/crates/pithos_lib/src/archive/async_stream.rs +++ b/crates/pithos_lib/src/archive/async_stream.rs @@ -47,8 +47,8 @@ type Pending<'a, T> = Pin> + Send enum State<'a> { Fetching(Pending<'a, Vec>, Vec), - Decoding(Pending<'a, Vec>>), - Ready(VecDeque>), + Decoding(Pending<'a, Vec>>>), + Ready(VecDeque>>), Failed(PithosError), Taken, } @@ -140,7 +140,7 @@ trait Reader<'a> { &self, blocks: Vec, stored: Zeroizing>, - ) -> Pending<'a, Vec>>; + ) -> Pending<'a, Vec>>>; } impl<'a, S, E, B> Reader<'a> for &'a AsyncArchive @@ -157,7 +157,7 @@ where &self, blocks: Vec, stored: Zeroizing>, - ) -> Pending<'a, Vec>> { + ) -> Pending<'a, Vec>>> { let archive: &'a AsyncArchive = self; let view = Arc::clone(&archive.view); Box::pin( @@ -183,7 +183,7 @@ where &self, blocks: Vec, stored: Zeroizing>, - ) -> Pending<'static, Vec>> { + ) -> Pending<'static, Vec>>> { let archive = Arc::clone(self); Box::pin(async move { let view = Arc::clone(&archive.view); @@ -326,10 +326,11 @@ where }; let item = match &mut slot.state { State::Ready(chunks) => match chunks.pop_front() { - Some(chunk) => { + Some(mut chunk) => { slot.reserved -= retained(&chunk); self.buffered -= retained(&chunk); - Ok(chunk) + // Only the delivered chunk leaves its wiping wrapper. + Ok(std::mem::take(&mut *chunk)) } None => { self.buffered -= slot.reserved; @@ -477,7 +478,7 @@ fn split(batch: &BlockBatch, max_bytes: u64) -> VecDeque { } /// The bytes a delivered chunk holds: its capacity, not only its length. -fn retained(chunk: &Vec) -> u64 { +fn retained(chunk: &Zeroizing>) -> u64 { chunk.capacity() as u64 } @@ -486,7 +487,7 @@ fn decode_group( view: &ArchiveView, blocks: &[PlannedBlock], stored: &[u8], -) -> Result>, PithosError> { +) -> Result>>, PithosError> { let mut previous: Option<&PlannedBlock> = None; let expected = blocks .iter() @@ -516,14 +517,14 @@ fn decode_group( } }; last = Some((block, bytes)); - let mut plaintext = view.decode_block(block, bytes)?; + let plaintext = view.decode_block(block, bytes)?; let output = block.output(); let whole = output.start == 0 && output.end == plaintext.len(); // A buffer with spare capacity is copied, so a chunk keeps only its planned bytes. chunks.push(if whole && plaintext.capacity() == plaintext.len() { - std::mem::take(&mut *plaintext) + plaintext } else { - plaintext[output].to_vec() + Zeroizing::new(plaintext[output].to_vec()) }); } Ok(chunks) diff --git a/crates/pithos_lib/src/archive/reader_private_tests.rs b/crates/pithos_lib/src/archive/reader_private_tests.rs index a376b7b..8323ca4 100644 --- a/crates/pithos_lib/src/archive/reader_private_tests.rs +++ b/crates/pithos_lib/src/archive/reader_private_tests.rs @@ -2079,3 +2079,88 @@ fn version_1_1_appends_reuse_ciphers_and_give_unique_blocks_fresh_identities() { assert_eq!(output, content, "{name}"); } } + +#[cfg(feature = "async")] +mod async_wiping { + use super::*; + use crate::archive::AsyncArchive; + use crate::format::zeroing_tests::{SECRET, watch}; + use crate::source::AsyncArchiveSource; + use futures_core::Stream; + use std::future::Future; + use std::pin::{Pin, pin}; + use std::task::{Context, Poll, Waker}; + + struct Bytes(Vec); + + impl AsyncArchiveSource for Bytes { + async fn len(&self) -> Result { + Ok(self.0.len() as u64) + } + + async fn read_at(&self, offset: u64, len: u64) -> Result, SourceError> { + Ok(self.0[offset as usize..(offset + len) as usize].to_vec()) + } + } + + fn ready(future: F) -> F::Output { + match pin!(future).poll(&mut Context::from_waker(Waker::noop())) { + Poll::Ready(output) => output, + Poll::Pending => panic!("an in-memory read is pending"), + } + } + + /// A plain archive whose file "data" has four 64-byte blocks that each hold the secret. + fn secret_blocks() -> Vec { + let content: Vec = (0..4) + .flat_map(|block| [SECRET, [0xB0 + block; 32]]) + .flatten() + .collect(); + let options = WriteOptions::base().with_chunking(Chunking::Fixed(64)); + let mut writer = ArchiveWriter::create(Vec::new(), options).unwrap(); + writer + .add_file( + ArchivePath::new("data").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(false, 0).unwrap(), + Some(content.len() as u64), + Cursor::new(content), + ) + .unwrap(); + writer.finish().unwrap() + } + + /// Takes the first item of a stream over all blocks, then drops the stream. + /// Returns the item length and the number of freed buffers that still held the secret. + fn first_then_drop(bytes: Vec) -> (Result, usize) { + let source = Bytes(bytes); + let archive = ready(AsyncArchive::open(source, OpenOptions::default(), None)).unwrap(); + let (first, _, leaked) = watch(64, &[], || { + let mut stream = archive.read_range("data", 0..256).unwrap(); + let mut cx = Context::from_waker(Waker::noop()); + let Poll::Ready(Some(first)) = Pin::new(&mut stream).poll_next(&mut cx) else { + panic!("an in-memory stream has no first item"); + }; + drop(stream); + first.map(|chunk| Zeroizing::new(chunk).len()) + }); + (first, leaked) + } + + #[test] + fn dropped_stream_wipes_chunks() { + let (first, leaked) = first_then_drop(secret_blocks()); + assert_eq!(first.unwrap(), 64); + assert_eq!(leaked, 0); + } + + #[test] + fn decode_failure_wipes_chunks() { + let mut bytes = secret_blocks(); + let second = bytes.windows(32).position(|window| window == [0xB1; 32]); + bytes[second.unwrap()] ^= 1; + let (first, leaked) = first_then_drop(bytes); + assert!(matches!(first, Err(PithosError::BlockHashMismatch { .. }))); + assert_eq!(leaked, 0); + } +} diff --git a/crates/pithos_lib/src/format/mod.rs b/crates/pithos_lib/src/format/mod.rs index e52a864..d48ce1f 100644 --- a/crates/pithos_lib/src/format/mod.rs +++ b/crates/pithos_lib/src/format/mod.rs @@ -12,4 +12,4 @@ pub(crate) mod header; pub(crate) mod limits; pub(crate) mod primitives; #[cfg(test)] -mod zeroing_tests; +pub(crate) mod zeroing_tests; diff --git a/crates/pithos_lib/src/format/zeroing_tests.rs b/crates/pithos_lib/src/format/zeroing_tests.rs index 7af9d3a..63b854a 100644 --- a/crates/pithos_lib/src/format/zeroing_tests.rs +++ b/crates/pithos_lib/src/format/zeroing_tests.rs @@ -11,7 +11,7 @@ use std::alloc::{GlobalAlloc, Layout, System}; use std::cell::Cell; use zeroize::Zeroizing; -const SECRET: [u8; 32] = { +pub(crate) const SECRET: [u8; 32] = { let mut bytes = [0; 32]; let mut index = 0; while index < 32 { @@ -136,7 +136,7 @@ static ALLOCATOR: Inspecting = Inspecting; /// Runs `action` while watching freed blocks of `size` bytes. `initialized` names existing /// buffers whose whole allocation is initialized; blocks allocated during the watch count too. /// Returns the number of wiped blocks of that size and of freed blocks that held the secret. -fn watch( +pub(crate) fn watch( size: usize, initialized: &[*const u8], action: impl FnOnce() -> T, From db98af59201ae54e81ee2a8ab268796b51b5db01 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 11:01:29 +0200 Subject: [PATCH 146/166] fix: cut content-defined writer blocks from a wiped buffer --- crates/pithos_lib/src/archive/pieces.rs | 8 +- crates/pithos_lib/src/archive/writer.rs | 123 +++++++++++++++++------- 2 files changed, 89 insertions(+), 42 deletions(-) diff --git a/crates/pithos_lib/src/archive/pieces.rs b/crates/pithos_lib/src/archive/pieces.rs index cb16184..31ade99 100644 --- a/crates/pithos_lib/src/archive/pieces.rs +++ b/crates/pithos_lib/src/archive/pieces.rs @@ -20,7 +20,6 @@ use crate::format::file_entry::{ }; use crate::format::header::{FileHeader, FormatVersion, encode_header}; use crate::format::limits::DeserializationError; -use fastcdc::v2020::{FastCDC, Normalization}; use indexmap::IndexMap; use integer_encoding::{VarIntReader, VarIntWriter}; use std::collections::HashSet; @@ -224,10 +223,7 @@ impl PieceEncoder { /// Encodes the first FastCDC block of the buffered bytes and removes it from the buffer. /// Like `StreamCDC`, it cuts only with a full buffer or at the end of the content. fn cut_block(&mut self, cdc: CdcConfig) -> Result, PithosError> { - let (min, avg, max) = (cdc.min_size(), cdc.avg_size(), cdc.max_size()); - let len = self.pending.len(); - let (_, end) = - FastCDC::with_level(&self.pending, min, avg, max, Normalization::Level1).cut(0, len); + let end = cdc.first_cut(&self.pending); let block = std::mem::take(&mut self.pending); let stored = self.encode(&block[..end]); self.pending = block; @@ -786,7 +782,7 @@ mod tests { cdc.min_size(), cdc.avg_size(), cdc.max_size(), - Normalization::Level1, + fastcdc::v2020::Normalization::Level1, ); let expected = stream .map(|chunk| chunk.unwrap().length as u64) diff --git a/crates/pithos_lib/src/archive/writer.rs b/crates/pithos_lib/src/archive/writer.rs index 3c11efa..fa13aba 100644 --- a/crates/pithos_lib/src/archive/writer.rs +++ b/crates/pithos_lib/src/archive/writer.rs @@ -19,7 +19,7 @@ use crate::format::error::SerializationError; use crate::format::file_entry::{BlockDataState, FileEntry, FileType, Reference}; use crate::format::header::FormatVersion; use crate::format::{directory, header}; -use fastcdc::v2020::{Normalization, StreamCDC}; +use fastcdc::v2020::{FastCDC, Normalization}; use indexmap::IndexMap; use std::collections::{HashMap, HashSet}; use std::io::{self, Read, Write}; @@ -72,6 +72,14 @@ impl CdcConfig { pub fn max_size(self) -> usize { self.max_size } + + /// The length of the first FastCDC block of `data`, with level-one normalization. + pub(crate) fn first_cut(self, data: &[u8]) -> usize { + let (min, avg, max) = (self.min_size, self.avg_size, self.max_size); + FastCDC::with_level(data, min, avg, max, Normalization::Level1) + .cut(0, data.len()) + .1 + } } impl Default for CdcConfig { @@ -123,8 +131,10 @@ enum Chunker { block: Zeroizing>, }, ContentDefined { - stream: StreamCDC, - block: Zeroizing>, + content: R, + cdc: CdcConfig, + buffer: Zeroizing>, + cut: usize, }, } @@ -137,14 +147,10 @@ impl Chunker { block: Zeroizing::new(Vec::new()), }, Chunking::ContentDefined(cdc) => Self::ContentDefined { - stream: StreamCDC::with_level( - content, - cdc.min_size, - cdc.avg_size, - cdc.max_size, - Normalization::Level1, - ), - block: Zeroizing::new(Vec::new()), + content, + cdc, + buffer: Zeroizing::new(Vec::new()), + cut: 0, }, } } @@ -163,13 +169,23 @@ impl Chunker { content.by_ref().take(*size as u64).read_to_end(block)?; Ok((!block.is_empty()).then_some(block.as_slice())) } - Self::ContentDefined { stream, block } => match stream.next() { - Some(chunk) => { - *block = Zeroizing::new(chunk?.data); - Ok(Some(block.as_slice())) + Self::ContentDefined { + content, + cdc, + buffer, + cut, + } => { + // Like `StreamCDC`, this cuts only with a full buffer or at the end of the content. + buffer.drain(..std::mem::take(cut)); + let missing = cdc.max_size - buffer.len(); + buffer.reserve_exact(missing); + content.by_ref().take(missing as u64).read_to_end(buffer)?; + if buffer.is_empty() { + return Ok(None); } - None => Ok(None), - }, + *cut = cdc.first_cut(buffer); + Ok(Some(&buffer[..*cut])) + } } } } @@ -2486,25 +2502,28 @@ mod tests { } } + /// The plaintext hash and size of every block written for `content`, in order. + fn written_blocks(chunking: Chunking, content: impl Read) -> Vec<([u8; 32], u64)> { + let options = options().with_chunking(chunking); + let mut writer = ArchiveWriter::create(Vec::new(), options).unwrap(); + writer + .add_file( + ArchivePath::new("data").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(true, 0).unwrap(), + None, + content, + ) + .unwrap(); + let blocks = writer.directory.blocks.iter(); + blocks + .map(|(hash, block)| (*hash, block.original_size)) + .collect() + } + #[test] fn fixed_blocks_ignore_read_sizes_and_end_with_one_short_block() { - fn blocks(content: impl Read) -> Vec<([u8; 32], u64)> { - let options = options().with_chunking(Chunking::Fixed(1000)); - let mut writer = ArchiveWriter::create(Vec::new(), options).unwrap(); - writer - .add_file( - ArchivePath::new("data").unwrap(), - EntryMetadata::new(0, 0, 0o644), - ProcessingOptions::new(true, 0).unwrap(), - None, - content, - ) - .unwrap(); - let blocks = writer.directory.blocks.iter(); - blocks - .map(|(hash, block)| (*hash, block.original_size)) - .collect() - } + let fixed = Chunking::Fixed(1000); let content = (0..2500u32) .map(|index| (index * 7) as u8) .collect::>(); @@ -2513,11 +2532,43 @@ mod tests { .chunks(1000) .map(|chunk| (*blake3::hash(chunk).as_bytes(), chunk.len() as u64)) .collect::>(); - assert_eq!(blocks(Cursor::new(content)), expected); - assert_eq!(blocks(ByteReader(content)), expected); + assert_eq!(written_blocks(fixed, Cursor::new(content)), expected); + assert_eq!(written_blocks(fixed, ByteReader(content)), expected); } } + #[test] + fn content_defined_blocks_match_stream_cdc() { + let mut state = 0x9e37_79b9_7f4a_7c15u64; + let content = (0..20_000) + .map(|_| { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + state as u8 + }) + .collect::>(); + let cdc = CdcConfig::new(64, 256, 1024).unwrap(); + let (min, avg, max) = (cdc.min_size, cdc.avg_size, cdc.max_size); + let stream = fastcdc::v2020::StreamCDC::with_level( + Cursor::new(&content), + min, + avg, + max, + Normalization::Level1, + ); + let expected = stream + .map(|chunk| { + let chunk = chunk.unwrap(); + (*blake3::hash(&chunk.data).as_bytes(), chunk.length as u64) + }) + .collect::>(); + assert!(expected.len() > 20); + let chunking = Chunking::ContentDefined(cdc); + assert_eq!(written_blocks(chunking, Cursor::new(&content)), expected); + assert_eq!(written_blocks(chunking, ByteReader(&content)), expected); + } + #[test] fn fixed_block_sizes_are_validated_before_header_output() { assert_eq!( From 2d6f562729a411c949fd51731b5a1cff97cebcb5 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 11:04:55 +0200 Subject: [PATCH 147/166] fix: export readable plain files to Crypt4GH --- crates/pithos_lib/src/adapters/crypt4gh.rs | 80 +++++++++++++++++++++- crates/pithos_lib/src/archive/reader.rs | 43 +++++++----- 2 files changed, 104 insertions(+), 19 deletions(-) diff --git a/crates/pithos_lib/src/adapters/crypt4gh.rs b/crates/pithos_lib/src/adapters/crypt4gh.rs index a08b2a3..2d88638 100644 --- a/crates/pithos_lib/src/adapters/crypt4gh.rs +++ b/crates/pithos_lib/src/adapters/crypt4gh.rs @@ -43,7 +43,7 @@ fn derive_ga4gh_header_key( /// /// The export validates archive content before encrypting it and reports archive, /// cryptographic, and sink failures with operation context. New headers use the -/// GA4GH KDF. +/// GA4GH KDF. Plain content, which no reader key protects, gets a fresh writer key. pub fn export( archive: &Archive, path: &str, @@ -618,3 +618,81 @@ mod zeroization_tests { ); } } + +#[cfg(test)] +mod export_tests { + use super::*; + use crate::archive::{ + ArchivePath, ArchiveWriter, EntryMetadata, OpenOptions, ProcessingOptions, WriteOptions, + }; + use crate::source::MemorySource; + + /// Decrypts an export to one recipient with that recipient's key. + fn decrypt(exported: &[u8], recipient: &PrivateKey) -> Vec { + let header_len = 16 + u32::from_le_bytes(exported[16..20].try_into().unwrap()) as usize; + let header = Crypt4GHHeader::try_from(&exported[..header_len]).unwrap(); + let [packet] = header.header_packets.as_slice() else { + panic!("the export has one recipient packet"); + }; + let PacketData::Encrypted(sealed) = &packet.packet_data else { + panic!("a parsed packet is encrypted"); + }; + let writer = &packet.writers_pubkey; + let secret = recipient.as_dalek_static_secret().to_bytes(); + let shared = crypto::derive_shared(&secret, writer).unwrap(); + let reader = recipient.public_key().into_dalek_public_key().to_bytes(); + let header_key = derive_ga4gh_header_key(shared.expose_for_protocol(), &reader, writer); + let packet_data = ChaCha20Poly1305::new_from_slice(&*header_key) + .unwrap() + .decrypt( + &Nonce::from(packet.nonce), + [sealed.as_slice(), packet.mac.as_slice()] + .concat() + .as_slice(), + ) + .unwrap(); + let data_key = ChaCha20Poly1305::new_from_slice(&packet_data[8..40]).unwrap(); + exported[header_len..] + .chunks(CRYPT4GH_ENCRYPTED_BLOCK_SIZE) + .flat_map(|segment| { + let nonce: [u8; 12] = segment[..12].try_into().unwrap(); + data_key + .decrypt(&Nonce::from(nonce), &segment[12..]) + .unwrap() + }) + .collect() + } + + #[test] + fn plain_content_round_trips() { + let content = (0..100_000u32) + .map(|index| (index % 251) as u8) + .collect::>(); + let mut writer = ArchiveWriter::create(Vec::new(), WriteOptions::base()).unwrap(); + writer + .add_file( + ArchivePath::new("data").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(false, 0).unwrap(), + Some(content.len() as u64), + std::io::Cursor::new(&content), + ) + .unwrap(); + let source = MemorySource::new(writer.finish().unwrap()); + let archive = Archive::open(source, OpenOptions::default()).unwrap(); + let mut copied = Vec::new(); + archive.copy_to("data", &mut copied).unwrap(); + assert_eq!(copied, content); + + let recipient = PrivateKey::generate(); + let mut exported = Vec::new(); + export( + &archive, + "data", + vec![recipient.public_key()], + &mut exported, + ) + .unwrap(); + assert_eq!(decrypt(&exported, &recipient), content); + } +} diff --git a/crates/pithos_lib/src/archive/reader.rs b/crates/pithos_lib/src/archive/reader.rs index 0569e77..2b8ea45 100644 --- a/crates/pithos_lib/src/archive/reader.rs +++ b/crates/pithos_lib/src/archive/reader.rs @@ -393,33 +393,40 @@ where .map_err(ContentOperationError::Core)?; let fresh_key; let (key, key_owner) = match self.view.access.key(id) { - Some(key) => (key, id), + Some(key) => (key, Some(id)), None => { - // A file sealed in pieces has no file key, so the export gets a fresh one. + // Files in pieces and plain files have no file key, so the export gets a fresh one. let first_piece = self .view .access .grant_keys(id) - .and_then(|keys| keys.first().map(|(key_id, _)| *key_id)) - .ok_or(PithosError::ContentUnavailable) - .map_err(ContentOperationError::Core)?; + .and_then(|keys| keys.first().map(|(key_id, _)| *key_id)); fresh_key = FileKey::from_bytes(x25519_dalek::StaticSecret::random().to_bytes()); (&fresh_key, first_piece) } }; - let provenance = self - .view - .access - .provenance(key_owner) - .ok_or(PithosError::ContentUnavailable) - .map_err(ContentOperationError::Core)?; - let reader = self - .view - .access_keys - .0 - .get(provenance.access_key) - .ok_or(PithosError::ContentUnavailable) - .map_err(ContentOperationError::Core)?; + let fresh_sender; + let reader = match key_owner { + Some(key_owner) => { + let provenance = self + .view + .access + .provenance(key_owner) + .ok_or(PithosError::ContentUnavailable) + .map_err(ContentOperationError::Core)?; + self.view + .access_keys + .0 + .get(provenance.access_key) + .ok_or(PithosError::ContentUnavailable) + .map_err(ContentOperationError::Core)? + } + // Plain content has no reader key, so a fresh key writes the Crypt4GH header. + None => { + fresh_sender = PrivateKey::generate(); + &fresh_sender + } + }; operation(id, reader, key).map_err(ContentOperationError::Callback) } From 729b001c08f712aa5d37d272de1e613e18570d87 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 11:05:55 +0200 Subject: [PATCH 148/166] chore: run the ignored 5 TiB block-count test in CI --- .github/workflows/ci.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d4887a9..e5cc0e1 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -44,3 +44,21 @@ jobs: - name: Run MSRV workspace tests run: cargo +1.89.0 test --locked --workspace --all-features + + block-count: + runs-on: ubuntu-latest + timeout-minutes: 40 + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Install stable Rust + uses: dtolnay/rust-toolchain@stable + + - name: Cache Rust build artifacts + uses: Swatinem/rust-cache@v2 + + - name: Run the ignored 5 TiB block-count test in release mode + run: >- + cargo +stable test --locked --release -p pithos_lib --test pieces + a_composition_with_the_block_count_of_5_tib_opens_with_default_limits -- --ignored --exact From 89b4ff4dff1958f9360e57da93b551547ee631a8 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 11:06:38 +0200 Subject: [PATCH 149/166] chore: document that the library builds only on Linux --- README.md | 2 +- crates/pithos_lib/README.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8276f2a..4455b01 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ For the complete command workflow, including extracting entries, see the [CLI gu - Local filesystem operations that avoid symlink traversal and refuse to overwrite existing extracted files. - RO-Crate directory and ZIP conversion, plus Crypt4GH export for readable entries. -The filesystem operations are Linux-only and require destination filesystem support for `O_TMPFILE` and `linkat(AT_EMPTY_PATH)`. Core archive reading and writing do not require Linux filesystem access. +The filesystem operations are Linux-only and require destination filesystem support for `O_TMPFILE` and `linkat(AT_EMPTY_PATH)`. Core archive reading and writing do not require Linux filesystem access at run time. The crates still build only on Linux, even with default features off. The Linux filesystem modules and the Unix file source are always compiled. ## Crates diff --git a/crates/pithos_lib/README.md b/crates/pithos_lib/README.md index c5e1dd7..694d1e7 100644 --- a/crates/pithos_lib/README.md +++ b/crates/pithos_lib/README.md @@ -150,4 +150,6 @@ The upstream parser accepts RO-Crate 1.1 and 1.2 metadata without upstream valid ## Platform behavior +`pithos_lib` builds only on Linux, even with `default-features = false`. The Linux filesystem modules in `fs` and the Unix file source in `source` are always compiled. + Filesystem ingestion, extraction, append, and grants are Linux-only in 0.8. Append uses a cooperating-writer advisory lock and can request `AppendDurability::SyncAll`; it attempts to truncate a failed child directory, but it cannot promise rollback after an unrecoverable host or power failure. From 3b6be76defe5bfcd39ce223239dee6fb45d4dcb4 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 11:34:52 +0200 Subject: [PATCH 150/166] fix: wipe partial plaintext when block decompression fails --- crates/pithos_lib/src/block.rs | 36 ++++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/crates/pithos_lib/src/block.rs b/crates/pithos_lib/src/block.rs index 995d17f..15cb0fa 100644 --- a/crates/pithos_lib/src/block.rs +++ b/crates/pithos_lib/src/block.rs @@ -115,7 +115,7 @@ pub(crate) fn verify( stored }; if meta.flags.get_compression_level() > 0 { - plaintext = Zeroizing::new(decompress(&plaintext, meta.original_size)?); + plaintext = decompress(&plaintext, meta.original_size)?; } if plaintext.len() as u64 != meta.original_size { return Err(PithosError::BlockSizeMismatch { @@ -171,14 +171,19 @@ fn compress(input: &[u8], level: i32) -> Result, PithosError> { }) } -fn decompress(input: &[u8], expected_size: u64) -> Result, PithosError> { +fn decompress(input: &[u8], expected_size: u64) -> Result>, PithosError> { let size = usize::try_from(expected_size).map_err(|_| PithosError::InvalidDirectoryRange { operation: "convert decoded block size", })?; - bulk::decompress(input, size).map_err(|source| PithosError::Compression { - operation: "decompress block", - source, - }) + // Decompressing into a wiped buffer also wipes the partial output of a failed frame. + let mut output = Zeroizing::new(Vec::with_capacity(size)); + bulk::Decompressor::new() + .and_then(|mut decompressor| decompressor.decompress_to_buffer(input, &mut *output)) + .map_err(|source| PithosError::Compression { + operation: "decompress block", + source, + })?; + Ok(output) } #[cfg(test)] @@ -448,4 +453,23 @@ mod tests { .is_err() ); } + + #[test] + fn failed_decompression_wipes_partial_plaintext() { + use crate::format::zeroing_tests::{SECRET, watch}; + let plain = SECRET.repeat(9_375); + let encoded = encode(&plain, ProcessingFlags::new(true, Some(2)), [7; 12]).unwrap(); + assert!(encoded.flags.get_compression_level() > 0); + let declared = plain.len() - 1; + let meta = descriptor(&encoded, declared); + let limits = Limits { + max_stored_bytes: 1024 * 1024, + max_decoded_bytes: 1024 * 1024, + }; + let (result, zeroed, leaked) = watch(declared, &[], || { + verify(encoded.stored, &encoded.key, encoded.hash, &meta, limits).map(drop) + }); + assert!(matches!(result, Err(PithosError::Compression { .. }))); + assert_eq!((zeroed, leaked), (1, 0)); + } } From 378de450fe99fb2f6f340bb8f2be0b81503f8b37 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 11:37:52 +0200 Subject: [PATCH 151/166] fix: stop reading content-defined input after its end --- crates/pithos_lib/src/archive/writer.rs | 79 +++++++++++++++++++------ 1 file changed, 62 insertions(+), 17 deletions(-) diff --git a/crates/pithos_lib/src/archive/writer.rs b/crates/pithos_lib/src/archive/writer.rs index fa13aba..3e68b85 100644 --- a/crates/pithos_lib/src/archive/writer.rs +++ b/crates/pithos_lib/src/archive/writer.rs @@ -135,6 +135,7 @@ enum Chunker { cdc: CdcConfig, buffer: Zeroizing>, cut: usize, + ended: bool, }, } @@ -151,6 +152,7 @@ impl Chunker { cdc, buffer: Zeroizing::new(Vec::new()), cut: 0, + ended: false, }, } } @@ -174,12 +176,17 @@ impl Chunker { cdc, buffer, cut, + ended, } => { // Like `StreamCDC`, this cuts only with a full buffer or at the end of the content. buffer.drain(..std::mem::take(cut)); - let missing = cdc.max_size - buffer.len(); - buffer.reserve_exact(missing); - content.by_ref().take(missing as u64).read_to_end(buffer)?; + if !*ended { + let missing = cdc.max_size - buffer.len(); + buffer.reserve_exact(missing); + content.by_ref().take(missing as u64).read_to_end(buffer)?; + // A short fill means a read returned 0. Like `StreamCDC`, it is the end. + *ended = buffer.len() < cdc.max_size; + } if buffer.is_empty() { return Ok(None); } @@ -2537,38 +2544,76 @@ mod tests { } } - #[test] - fn content_defined_blocks_match_stream_cdc() { + /// Pseudo-random bytes, so no two content-defined blocks are equal. + fn noise(len: usize) -> Vec { let mut state = 0x9e37_79b9_7f4a_7c15u64; - let content = (0..20_000) + (0..len) .map(|_| { state ^= state << 13; state ^= state >> 7; state ^= state << 17; state as u8 }) - .collect::>(); - let cdc = CdcConfig::new(64, 256, 1024).unwrap(); + .collect() + } + + /// The plaintext hash and size of every `StreamCDC` block of `content`, in order. + fn stream_blocks(cdc: CdcConfig, content: impl Read) -> Vec<([u8; 32], u64)> { let (min, avg, max) = (cdc.min_size, cdc.avg_size, cdc.max_size); - let stream = fastcdc::v2020::StreamCDC::with_level( - Cursor::new(&content), - min, - avg, - max, - Normalization::Level1, - ); - let expected = stream + let stream = + fastcdc::v2020::StreamCDC::with_level(content, min, avg, max, Normalization::Level1); + stream .map(|chunk| { let chunk = chunk.unwrap(); (*blake3::hash(&chunk.data).as_bytes(), chunk.length as u64) }) - .collect::>(); + .collect() + } + + #[test] + fn content_defined_blocks_match_stream_cdc() { + let content = noise(20_000); + let cdc = CdcConfig::new(64, 256, 1024).unwrap(); + let expected = stream_blocks(cdc, Cursor::new(&content)); assert!(expected.len() > 20); let chunking = Chunking::ContentDefined(cdc); assert_eq!(written_blocks(chunking, Cursor::new(&content)), expected); assert_eq!(written_blocks(chunking, ByteReader(&content)), expected); } + /// Answers reads from fixed steps: bytes, an empty step for one end of input, or a failure. + struct Steps(std::collections::VecDeque>>); + + impl Read for Steps { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + match self.0.pop_front() { + None => Ok(0), + Some(Err(error)) => Err(error), + Some(Ok(mut data)) => { + let count = data.len().min(buffer.len()); + buffer[..count].copy_from_slice(&data[..count]); + if count < data.len() { + self.0.push_front(Ok(data.split_off(count))); + } + Ok(count) + } + } + } + } + + #[test] + fn content_defined_reads_stop_at_the_end() { + let first = noise(500); + let steps = |last| Steps([Ok(first.clone()), Ok(Vec::new()), last].into()); + let cdc = CdcConfig::new(64, 256, 1024).unwrap(); + let expected = stream_blocks(cdc, steps(Ok(noise(500)))); + assert_eq!(expected.iter().map(|block| block.1).sum::(), 500); + let chunking = Chunking::ContentDefined(cdc); + assert_eq!(written_blocks(chunking, steps(Ok(noise(500)))), expected); + let failing = steps(Err(io::Error::other("read after the end"))); + assert_eq!(written_blocks(chunking, failing), expected); + } + #[test] fn fixed_block_sizes_are_validated_before_header_output() { assert_eq!( From 59b105cca76e18ed82421703fd83e9ec81b8041b Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 11:39:32 +0200 Subject: [PATCH 152/166] fix: wipe rejected async source and resolver responses --- crates/pithos_lib/src/archive/async_reader.rs | 18 ++++++--- crates/pithos_lib/src/archive/async_stream.rs | 12 +++--- .../src/archive/reader_private_tests.rs | 39 +++++++++++++++---- 3 files changed, 48 insertions(+), 21 deletions(-) diff --git a/crates/pithos_lib/src/archive/async_reader.rs b/crates/pithos_lib/src/archive/async_reader.rs index e0f9c04..f374706 100644 --- a/crates/pithos_lib/src/archive/async_reader.rs +++ b/crates/pithos_lib/src/archive/async_reader.rs @@ -11,6 +11,7 @@ use crate::source::{AsyncArchiveSource, SourceError}; use std::future::Future; use std::ops::Range; use std::sync::Arc; +use zeroize::Zeroizing; /// Resolves an opaque external block location to exactly one framed `BLCK` value without /// blocking. @@ -126,10 +127,10 @@ where let (settings, external, external_access_policy) = options.into_parts(); let mut opener = ArchiveOpener::with_settings(archive_len, settings)?; while let Some(request) = opener.request() { - let response = read_exact(&source, request.offset(), request.len()).await?; + let mut response = read_exact(&source, request.offset(), request.len()).await?; let (returned, result) = hook .spawn_blocking(move || { - let result = opener.feed(request, response); + let result = opener.feed(request, std::mem::take(&mut *response)); (opener, result) }) .await; @@ -212,7 +213,10 @@ where } /// Fetches the stored bytes of one batch request. - pub(super) async fn fetch(&self, request: BlockRequest) -> Result, PithosError> { + pub(super) async fn fetch( + &self, + request: BlockRequest, + ) -> Result>, PithosError> { match request { BlockRequest::Local { offset, len } => Ok(read_exact(&self.source, offset, len).await?), BlockRequest::External { location, len } => { @@ -230,7 +234,8 @@ where let response = self .external .resolve(policy, &location, len, max_response_size) - .await?; + .await + .map(Zeroizing::new)?; if response.len() as u64 != len { return Err(PithosError::ExternalBlockFraming( "response does not match expected size".into(), @@ -247,8 +252,9 @@ async fn read_exact( source: &S, offset: u64, len: u64, -) -> Result, SourceError> { - let response = source.read_at(offset, len).await?; +) -> Result>, SourceError> { + // A rejected response may hold plaintext, so it is wiped like an accepted one. + let response = Zeroizing::new(source.read_at(offset, len).await?); if response.len() as u64 != len { return Err(SourceError::ResponseLength { offset, diff --git a/crates/pithos_lib/src/archive/async_stream.rs b/crates/pithos_lib/src/archive/async_stream.rs index 17fd4e1..8614aab 100644 --- a/crates/pithos_lib/src/archive/async_stream.rs +++ b/crates/pithos_lib/src/archive/async_stream.rs @@ -46,7 +46,7 @@ impl Default for ReadLimits { type Pending<'a, T> = Pin> + Send + 'a>>; enum State<'a> { - Fetching(Pending<'a, Vec>, Vec), + Fetching(Pending<'a, Zeroizing>>, Vec), Decoding(Pending<'a, Vec>>>), Ready(VecDeque>>), Failed(PithosError), @@ -134,7 +134,7 @@ impl Iterator for SharedBatches { /// How a stream reaches its archive: borrowed, or shared by an owned stream. trait Reader<'a> { - fn fetch(&self, request: BlockRequest) -> Pending<'a, Vec>; + fn fetch(&self, request: BlockRequest) -> Pending<'a, Zeroizing>>; fn decode( &self, @@ -149,7 +149,7 @@ where E: AsyncExternalBlockResolver, B: BlockingHook, { - fn fetch(&self, request: BlockRequest) -> Pending<'a, Vec> { + fn fetch(&self, request: BlockRequest) -> Pending<'a, Zeroizing>> { Box::pin(AsyncArchive::fetch(*self, request)) } @@ -174,7 +174,7 @@ where E: AsyncExternalBlockResolver + 'static, B: BlockingHook + 'static, { - fn fetch(&self, request: BlockRequest) -> Pending<'static, Vec> { + fn fetch(&self, request: BlockRequest) -> Pending<'static, Zeroizing>> { let archive = Arc::clone(self); Box::pin(async move { AsyncArchive::fetch(&archive, request).await }) } @@ -283,9 +283,7 @@ where unreachable!("the slot was fetching"); }; match result { - Ok(stored) => { - State::Decoding(self.reader.decode(blocks, Zeroizing::new(stored))) - } + Ok(stored) => State::Decoding(self.reader.decode(blocks, stored)), Err(error) => State::Failed(error), } } diff --git a/crates/pithos_lib/src/archive/reader_private_tests.rs b/crates/pithos_lib/src/archive/reader_private_tests.rs index 8323ca4..3a65ec6 100644 --- a/crates/pithos_lib/src/archive/reader_private_tests.rs +++ b/crates/pithos_lib/src/archive/reader_private_tests.rs @@ -2089,17 +2089,23 @@ mod async_wiping { use futures_core::Stream; use std::future::Future; use std::pin::{Pin, pin}; + use std::sync::atomic::AtomicBool; use std::task::{Context, Poll, Waker}; - struct Bytes(Vec); + /// Serves `bytes`, but one byte short once `short` is set. + struct Bytes { + bytes: Vec, + short: Arc, + } impl AsyncArchiveSource for Bytes { async fn len(&self) -> Result { - Ok(self.0.len() as u64) + Ok(self.bytes.len() as u64) } async fn read_at(&self, offset: u64, len: u64) -> Result, SourceError> { - Ok(self.0[offset as usize..(offset + len) as usize].to_vec()) + let end = offset + len - u64::from(self.short.load(Ordering::Relaxed)); + Ok(self.bytes[offset as usize..end as usize].to_vec()) } } @@ -2130,11 +2136,15 @@ mod async_wiping { writer.finish().unwrap() } + fn open(bytes: Vec, short: &Arc) -> AsyncArchive { + let short = Arc::clone(short); + let source = Bytes { bytes, short }; + ready(AsyncArchive::open(source, OpenOptions::default(), None)).unwrap() + } + /// Takes the first item of a stream over all blocks, then drops the stream. /// Returns the item length and the number of freed buffers that still held the secret. - fn first_then_drop(bytes: Vec) -> (Result, usize) { - let source = Bytes(bytes); - let archive = ready(AsyncArchive::open(source, OpenOptions::default(), None)).unwrap(); + fn first_then_drop(archive: &AsyncArchive) -> (Result, usize) { let (first, _, leaked) = watch(64, &[], || { let mut stream = archive.read_range("data", 0..256).unwrap(); let mut cx = Context::from_waker(Waker::noop()); @@ -2149,7 +2159,7 @@ mod async_wiping { #[test] fn dropped_stream_wipes_chunks() { - let (first, leaked) = first_then_drop(secret_blocks()); + let (first, leaked) = first_then_drop(&open(secret_blocks(), &Arc::default())); assert_eq!(first.unwrap(), 64); assert_eq!(leaked, 0); } @@ -2159,8 +2169,21 @@ mod async_wiping { let mut bytes = secret_blocks(); let second = bytes.windows(32).position(|window| window == [0xB1; 32]); bytes[second.unwrap()] ^= 1; - let (first, leaked) = first_then_drop(bytes); + let (first, leaked) = first_then_drop(&open(bytes, &Arc::default())); assert!(matches!(first, Err(PithosError::BlockHashMismatch { .. }))); assert_eq!(leaked, 0); } + + #[test] + fn short_response_is_wiped() { + let short = Arc::new(AtomicBool::new(false)); + let archive = open(secret_blocks(), &short); + short.store(true, Ordering::Relaxed); + let (first, leaked) = first_then_drop(&archive); + assert!(matches!( + first, + Err(PithosError::Source(SourceError::ResponseLength { .. })) + )); + assert_eq!(leaked, 0); + } } From b0264efb96d7d222e0f5de0b2f92a415b2bf29db Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 11:40:54 +0200 Subject: [PATCH 153/166] feat: import and export raw X25519 keys --- README.md | 4 ++++ crates/pithos_lib/src/crypto.rs | 17 +++++++++++++++++ crates/pithos_lib/tests/crypto.rs | 23 ++++++++++++++++++++++- 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4455b01..7426d2b 100644 --- a/README.md +++ b/README.md @@ -75,3 +75,7 @@ Breaking changes in the 0.8 API: - `Archive::open` no longer reads block markers. A missing or changed marker fails when the block is read. - The default `OpenLimits` admit objects up to 5 TiB. - The minimum supported Rust version is 1.89. + +Additions in the 0.8 API: + +- `PublicKey::from_raw`, `PublicKey::as_bytes` and `PrivateKey::from_raw` import and export raw 32-byte X25519 keys without PEM. `PrivateKey::from_raw` takes a `Zeroizing` container. diff --git a/crates/pithos_lib/src/crypto.rs b/crates/pithos_lib/src/crypto.rs index 746c915..aab900d 100644 --- a/crates/pithos_lib/src/crypto.rs +++ b/crates/pithos_lib/src/crypto.rs @@ -111,6 +111,12 @@ impl PrivateKey { Self(StaticSecret::random().to_bytes()) } + /// Takes 32 raw X25519 private key bytes. The consumed container is wiped when it drops, + /// so neither the caller nor Pithos keeps an unwiped copy. + pub fn from_raw(bytes: Zeroizing<[u8; 32]>) -> Self { + Self(*bytes) + } + /// Duplicate a secret only when a distinct owner is required by an API boundary. pub fn duplicate(&self) -> Self { Self(self.0) @@ -188,6 +194,17 @@ impl fmt::Debug for PrivateKey { } impl PublicKey { + /// Accepts 32 raw X25519 public key bytes that pass the same checks as a PEM public key. + pub fn from_raw(bytes: [u8; 32]) -> Result { + validate_x25519_public_key(&bytes)?; + Ok(Self(bytes)) + } + + /// The 32 raw X25519 public key bytes. + pub fn as_bytes(&self) -> &[u8; 32] { + &self.0 + } + pub fn from_public_pem_bytes(pem: &[u8]) -> Result { let pem = std::str::from_utf8(pem).map_err(|_| CryptoError::InvalidKeyDocument)?; let (label, document) = diff --git a/crates/pithos_lib/tests/crypto.rs b/crates/pithos_lib/tests/crypto.rs index 5a8c5d6..c19d86d 100644 --- a/crates/pithos_lib/tests/crypto.rs +++ b/crates/pithos_lib/tests/crypto.rs @@ -1,7 +1,9 @@ use pithos_lib::crypto::{ - CryptoError, generate_private_key, parse_private_pem, parse_public_pem, serialize_public_pem, + CryptoError, PrivateKey, PublicKey, generate_private_key, parse_private_pem, parse_public_pem, + serialize_public_pem, }; use pkcs8::{Document, LineEnding}; +use zeroize::Zeroizing; #[test] fn curated_pem_api_preserves_fixture_bytes_and_redacts_private_debug() { @@ -61,3 +63,22 @@ fn non_contributory_x25519_public_keys_are_rejected_at_every_public_boundary() { assert!(parse_public_pem(zero_public_pem.as_bytes()).is_err()); } + +#[test] +fn raw_keys_match_their_pem_documents() { + let public = parse_public_pem(&std::fs::read("tests/data/keys/sender_public.pem").unwrap()); + let public = public.unwrap(); + assert_eq!(PublicKey::from_raw(*public.as_bytes()).unwrap(), public); + assert!(matches!( + PublicKey::from_raw([0; 32]), + Err(CryptoError::NonContributoryPublicKey) + )); + + // A PKCS#8 X25519 document ends with the 32 raw private key bytes. + let private_pem = std::fs::read_to_string("tests/data/keys/sender_private.pem").unwrap(); + let (_, document) = Document::from_pem(&private_pem).unwrap(); + let raw = <[u8; 32]>::try_from(&document.as_bytes()[16..]).unwrap(); + let private = PrivateKey::from_raw(Zeroizing::new(raw)); + assert_eq!(private.public_key(), public); + assert_eq!(format!("{private:?}"), "PrivateKey([REDACTED])"); +} From af1076d22723c13a2791f737f9731b2ddcd3276f Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 11:47:06 +0200 Subject: [PATCH 154/166] feat: replace the reader grants of an archive without re-encoding --- README.md | 1 + .../src/archive/grant_replacement.rs | 151 ++++++++++++++++++ crates/pithos_lib/src/archive/index.rs | 5 + crates/pithos_lib/src/archive/mod.rs | 2 + crates/pithos_lib/src/archive/pieces.rs | 41 +++-- crates/pithos_lib/src/archive/reader.rs | 2 +- crates/pithos_lib/src/error.rs | 2 + crates/pithos_lib/tests/pieces.rs | 113 ++++++++++++- 8 files changed, 299 insertions(+), 18 deletions(-) create mode 100644 crates/pithos_lib/src/archive/grant_replacement.rs diff --git a/README.md b/README.md index 7426d2b..8792aee 100644 --- a/README.md +++ b/README.md @@ -79,3 +79,4 @@ Breaking changes in the 0.8 API: Additions in the 0.8 API: - `PublicKey::from_raw`, `PublicKey::as_bytes` and `PrivateKey::from_raw` import and export raw 32-byte X25519 keys without PEM. `PrivateKey::from_raw` takes a `Zeroizing` container. +- `ArchiveView::replace_grants` plans a copy of a version 1.1 archive with one directory that grants its keys only to new recipients. It reuses every block and sealed block list unchanged. diff --git a/crates/pithos_lib/src/archive/grant_replacement.rs b/crates/pithos_lib/src/archive/grant_replacement.rs new file mode 100644 index 0000000..3401261 --- /dev/null +++ b/crates/pithos_lib/src/archive/grant_replacement.rs @@ -0,0 +1,151 @@ +//! Replaces the reader grants of an archive without re-encoding its content. + +use super::pieces::{distinct_keys, v1_1_header}; +use super::reader::deserialization_limits; +use super::types::FileId; +use super::view::ArchiveView; +use crate::crypto::{self, PublicKey}; +use crate::error::PithosError; +use crate::format::directory::{ + Directory, decode_complete_directory_with_validation, encode_complete_directory, +}; +use crate::format::encryption::{ + EncryptionSection, RecipientData, RecipientSection, encode_decrypted_recipient_list, +}; +use crate::format::file_entry::BlockDataState; +use crate::format::header::{FileHeader, FormatVersion}; +use indexmap::IndexMap; +use std::ops::Range; +use x25519_dalek::{PublicKey as DalekPublicKey, StaticSecret}; +use zeroize::Zeroizing; + +/// An archive that grants the keys of an existing archive only to new recipients. +/// +/// Write [`GrantReplacement::header`], then the old archive bytes in +/// [`GrantReplacement::copy_range`] unchanged, then [`GrantReplacement::directory`]. +/// It holds no key material. +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct GrantReplacement { + copy_range: Range, + directory: Vec, + archive_len: u64, +} + +impl GrantReplacement { + pub fn header(&self) -> [u8; FileHeader::ENCODED_LEN] { + v1_1_header() + } + + /// The block region of the old archive. Its bytes keep their offsets in the new archive. + pub fn copy_range(&self) -> Range { + self.copy_range.clone() + } + + pub fn directory(&self) -> &[u8] { + &self.directory + } + + pub fn archive_len(&self) -> u64 { + self.archive_len + } + + /// The digest the new archive reports on open, for + /// [`crate::archive::OpenOptions::with_expected_metadata_digest`]. + pub fn metadata_digest(&self) -> [u8; 32] { + crate::archive::metadata_digest(&[*blake3::hash(&self.directory).as_bytes()]) + } +} + +impl ArchiveView { + /// The byte range of the terminal directory, which [`ArchiveView::replace_grants`] needs. + pub fn directory_range(&self) -> Range { + self.terminal_directory.start()..self.terminal_directory.end() + } + + /// Plans a copy of this archive that grants every file and piece key this view recovered + /// only to `recipients`, under a fresh sender key and version 1.1 grant keys. + /// + /// `directory` must hold the stored bytes at [`ArchiveView::directory_range`]; they must + /// match the metadata digest. Every block payload and sealed block list stays unchanged. + /// Fails with [`PithosError::GrantReplacementUnsupported`] unless the archive is version + /// 1.1 with one directory, and with [`PithosError::ContentUnavailable`] when a key needed + /// by a sealed block list was not recovered. + pub fn replace_grants( + &self, + directory: &[u8], + recipients: Vec, + ) -> Result { + if self.version != FormatVersion::V1_1 || self.index.directory_count() != 1 { + return Err(PithosError::GrantReplacementUnsupported); + } + if recipients.is_empty() { + return Err(PithosError::WriterRequiresRecipient); + } + let recipients = distinct_keys(recipients)?; + let hash = *blake3::hash(directory).as_bytes(); + if crate::archive::metadata_digest(&[hash]) != self.metadata_digest { + return Err(PithosError::MetadataDigestMismatch); + } + let limits = deserialization_limits(self.limits); + let mut directory = + decode_complete_directory_with_validation(directory, &limits, false, |_| Ok(()))?; + let list = self.key_list(&directory)?; + directory.encryption = grant(&list, &recipients)?; + let directory = encode_complete_directory(&directory)?; + let copy_range = FileHeader::ENCODED_LEN as u64..self.terminal_directory.start(); + Ok(GrantReplacement { + archive_len: copy_range.end + directory.len() as u64, + copy_range, + directory, + }) + } + + /// Encodes the recipient list of every key that a sealed block list of `directory` needs. + fn key_list(&self, directory: &Directory) -> Result>, PithosError> { + let mut ids = Vec::new(); + for (id, _, file) in directory.files.iter() { + match &file.block_data { + BlockDataState::Encrypted(_) => ids.push(id), + BlockDataState::Pieces(pieces) => ids.extend(pieces.iter().map(|p| p.key_id)), + BlockDataState::Decrypted(_) => {} + } + } + // Exact capacities keep the keys from moving to new buffers while they are written. + let mut records = Zeroizing::new(Vec::with_capacity(ids.len())); + for id in ids { + let key = self.access.key(FileId(id)); + let key = key.ok_or(PithosError::ContentUnavailable)?; + records.push((id, *key.expose_for_protocol())); + } + let mut list = Zeroizing::new(Vec::with_capacity(10 + records.len() * 42)); + encode_decrypted_recipient_list(&records, &mut *list)?; + Ok(list) + } +} + +/// Seals `list` for every recipient under one fresh sender key. +fn grant( + list: &[u8], + recipients: &[[u8; 32]], +) -> Result, PithosError> { + let sender = StaticSecret::random(); + let sender_public = DalekPublicKey::from(&sender).to_bytes(); + let mut sections = IndexMap::with_capacity(recipients.len()); + for recipient in recipients { + let nonce = crypto::random_nonce(); + let key = crypto::grant_wrapping_key( + FormatVersion::V1_1, + crypto::derive_shared(sender.as_bytes(), recipient)?, + &sender_public, + recipient, + &nonce, + ); + let wrapped = crypto::wrap_recipient_list_with_nonce(&key, list, nonce)?; + let recipient_data = RecipientData::Encrypted(wrapped); + sections.insert(*recipient, RecipientSection { recipient_data }); + } + let section = EncryptionSection { + recipients: sections, + }; + Ok(IndexMap::from([(sender_public, section)])) +} diff --git a/crates/pithos_lib/src/archive/index.rs b/crates/pithos_lib/src/archive/index.rs index 2298b18..7b071e3 100644 --- a/crates/pithos_lib/src/archive/index.rs +++ b/crates/pithos_lib/src/archive/index.rs @@ -41,6 +41,11 @@ impl ArchiveIndex { .and_then(|index| self.entries.get(*index)) } + /// The number of directories, from the base to the terminal directory. + pub(crate) fn directory_count(&self) -> usize { + self.segment_spans.len() + } + pub(crate) fn entry_at_path(&self, path: &ArchivePath) -> Option<&IndexedEntry> { self.by_path .get(path.as_str()) diff --git a/crates/pithos_lib/src/archive/mod.rs b/crates/pithos_lib/src/archive/mod.rs index 01b17d6..077a966 100644 --- a/crates/pithos_lib/src/archive/mod.rs +++ b/crates/pithos_lib/src/archive/mod.rs @@ -11,6 +11,7 @@ mod async_reader; #[cfg(feature = "async")] mod async_stream; mod content_tree; +mod grant_replacement; mod index; mod opener; mod path_validation; @@ -31,6 +32,7 @@ pub use append::{AppendDurability, AppendObservation, AppendOptions}; pub use async_reader::{AsyncArchive, AsyncExternalBlockResolver, BlockingHook, InlineBlocking}; #[cfg(feature = "async")] pub use async_stream::{OwnedRangeStream, RangeStream, ReadLimits}; +pub use grant_replacement::GrantReplacement; pub use opener::{ArchiveOpener, ReadRequest}; pub(crate) use path_validation::{ validate_directory_entries, validate_new_candidate, validate_symlink_target, diff --git a/crates/pithos_lib/src/archive/pieces.rs b/crates/pithos_lib/src/archive/pieces.rs index 31ade99..70e701a 100644 --- a/crates/pithos_lib/src/archive/pieces.rs +++ b/crates/pithos_lib/src/archive/pieces.rs @@ -97,17 +97,7 @@ impl PieceEncoder { return Err(PithosError::WriterRequiresRecipient); } processing.validate_for(FormatVersion::V1_1)?; - let recipients = recipients - .into_iter() - .map(|recipient| recipient.into_dalek_public_key().to_bytes()) - .collect::>(); - let mut unique = HashSet::with_capacity(recipients.len()); - if recipients - .iter() - .any(|recipient| !unique.insert(*recipient)) - { - return Err(PithosError::DuplicateRecipientKey); - } + let recipients = distinct_keys(recipients)?; Ok(Self { key_id, recipients, @@ -533,6 +523,30 @@ fn read_count(reader: &mut Cursor<&[u8]>, item_len: u64) -> Result) -> Result, PithosError> { + let recipients = recipients + .into_iter() + .map(|recipient| recipient.into_dalek_public_key().to_bytes()) + .collect::>(); + let mut unique = HashSet::with_capacity(recipients.len()); + if recipients + .iter() + .any(|recipient| !unique.insert(*recipient)) + { + return Err(PithosError::DuplicateRecipientKey); + } + Ok(recipients) +} + +/// The file header of a version 1.1 archive. +pub(crate) fn v1_1_header() -> [u8; FileHeader::ENCODED_LEN] { + let mut header = [0; FileHeader::ENCODED_LEN]; + encode_header(&FormatVersion::V1_1.header(), &mut header.as_mut_slice()) + .expect("the header fits its fixed length"); + header +} + fn checked_add(left: u64, right: u64) -> Result { left.checked_add(right) .ok_or(PithosError::InvalidDirectoryRange { @@ -552,10 +566,7 @@ pub struct Composition { impl Composition { pub fn header(&self) -> [u8; FileHeader::ENCODED_LEN] { - let mut header = [0; FileHeader::ENCODED_LEN]; - encode_header(&FormatVersion::V1_1.header(), &mut header.as_mut_slice()) - .expect("the header fits its fixed length"); - header + v1_1_header() } /// Archive offset at which each piece's stored bytes start. diff --git a/crates/pithos_lib/src/archive/reader.rs b/crates/pithos_lib/src/archive/reader.rs index 2b8ea45..ef720bc 100644 --- a/crates/pithos_lib/src/archive/reader.rs +++ b/crates/pithos_lib/src/archive/reader.rs @@ -608,7 +608,7 @@ pub(super) fn classify_content_availability( Ok(availability) } -fn deserialization_limits(limits: OpenLimits) -> DeserializationLimits { +pub(super) fn deserialization_limits(limits: OpenLimits) -> DeserializationLimits { DeserializationLimits { max_collection_entries: limits.max_entries, max_file_entries: limits.max_entries, diff --git a/crates/pithos_lib/src/error.rs b/crates/pithos_lib/src/error.rs index 98a00fa..75ec624 100644 --- a/crates/pithos_lib/src/error.rs +++ b/crates/pithos_lib/src/error.rs @@ -97,6 +97,8 @@ pub enum PithosError { PieceContentStarted, #[error("archive metadata does not match the expected digest")] MetadataDigestMismatch, + #[error("grant replacement needs a version 1.1 archive with exactly one directory")] + GrantReplacementUnsupported, #[error("duplicate encoded recipient file id")] DuplicateRecipientFileId, #[error("reserved processing bits are set: {0:#04x}")] diff --git a/crates/pithos_lib/tests/pieces.rs b/crates/pithos_lib/tests/pieces.rs index 8fef078..d1f130f 100644 --- a/crates/pithos_lib/tests/pieces.rs +++ b/crates/pithos_lib/tests/pieces.rs @@ -3,8 +3,8 @@ mod common; use common::util::{private_key, public_key}; use pithos_lib::archive::{ AccessKeys, AppendOptions, Archive, ArchivePath, ArchiveWriter, BlockKeyMode, BlockListForm, - CdcConfig, Chunking, EntryKind, EntryMetadata, OpenOptions, PayloadCipher, Piece, PieceEncoder, - ProcessingOptions, WriteOptions, compose, + CdcConfig, Chunking, EntryKind, EntryMetadata, GrantReplacement, OpenOptions, PayloadCipher, + Piece, PieceEncoder, ProcessingOptions, WriteOptions, compose, }; use pithos_lib::error::{DeserializationError, PithosError}; use pithos_lib::source::MemorySource; @@ -536,3 +536,112 @@ fn entries_report_how_their_block_list_is_stored() { assert_eq!(archive.entry("folder").unwrap().unwrap().block_list, None); } } + +/// Plans grants for recipient2 only, from an archive opened with recipient1's key. +fn replace_grants(bytes: &[u8]) -> Result { + let archive = open(bytes.to_vec(), recipient()); + let directory = &bytes[archive.view().directory_range().start as usize..]; + archive + .view() + .replace_grants(directory, vec![public_key("recipient2")]) +} + +/// Replaces the grants of `bytes` and checks the new archive and both readers. +fn check_replaced_grants(bytes: Vec, path: &str, expected: &[u8]) { + let plan = replace_grants(&bytes).unwrap(); + let range = plan.copy_range(); + let mut replaced = plan.header().to_vec(); + replaced.extend_from_slice(&bytes[range.start as usize..range.end as usize]); + replaced.extend_from_slice(plan.directory()); + assert_eq!(replaced.len() as u64, plan.archive_len()); + // The header and every stored block keep their bytes and offsets. + let old = open(bytes.clone(), recipient()); + assert_eq!(range.end, old.view().directory_range().start); + assert_eq!(replaced[..range.end as usize], bytes[..range.end as usize]); + assert_ne!(plan.metadata_digest(), old.metadata_digest()); + + let reader = AccessKeys::new().with_key(private_key("recipient2")); + let options = OpenOptions::default().with_access_keys(reader); + let source = MemorySource::new(replaced.clone()); + let archive = Archive::open( + source, + options.with_expected_metadata_digest(plan.metadata_digest()), + ) + .unwrap(); + let mut output = Vec::new(); + archive.copy_to(path, &mut output).unwrap(); + assert_eq!(output, expected); + let stale = OpenOptions::default().with_expected_metadata_digest(old.metadata_digest()); + assert!(matches!( + Archive::open(MemorySource::new(replaced.clone()), stale), + Err(PithosError::MetadataDigestMismatch) + )); + let former = open(replaced, recipient()); + assert!(matches!( + former.copy_to(path, &mut Vec::new()), + Err(PithosError::ContentUnavailable) + )); +} + +#[test] +fn replaced_grants_move_piece_keys_to_a_new_reader() { + let first = content(6, 2500); + let second = content(7, 900); + let parts = vec![encode(2, &first), encode(5, &second)]; + check_replaced_grants(assemble(&parts), "object", &[first, second].concat()); +} + +#[test] +fn replaced_grants_move_a_file_key_to_a_new_reader() { + let options = WriteOptions::new(private_key("sender"), vec![public_key("recipient1")]); + let options = options.with_chunking(Chunking::Fixed(BLOCK)); + let mut writer = ArchiveWriter::create(Vec::new(), options).unwrap(); + let data = content(8, 3000); + writer + .add_file( + ArchivePath::new("data").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(true, 3).unwrap(), + None, + std::io::Cursor::new(&data), + ) + .unwrap(); + check_replaced_grants(writer.finish().unwrap(), "data", &data); +} + +#[test] +fn grant_replacement_rejects_unsupported_archives_and_missing_keys() { + let bytes = assemble(&[encode(1, &content(9, 1500))]); + let without_keys = open(bytes.clone(), AccessKeys::new()); + let directory = &bytes[without_keys.view().directory_range().start as usize..]; + assert!(matches!( + without_keys + .view() + .replace_grants(directory, vec![public_key("recipient2")]), + Err(PithosError::ContentUnavailable) + )); + let archive = open(bytes.clone(), recipient()); + assert!(matches!( + archive + .view() + .replace_grants(&bytes, vec![public_key("recipient2")]), + Err(PithosError::MetadataDigestMismatch) + )); + + let temporary = tempfile::tempdir().unwrap(); + let path = temporary.path().join("appended.pith"); + std::fs::write(&path, &bytes).unwrap(); + pithos_lib::fs::grant_readers( + &path, + AppendOptions::new(private_key("recipient1"), vec![public_key("recipient2")]), + &[0], + ) + .unwrap(); + let older = std::fs::read("tests/data/pithos-0.7.0.pith").unwrap(); + for bytes in [std::fs::read(&path).unwrap(), older] { + assert!(matches!( + replace_grants(&bytes), + Err(PithosError::GrantReplacementUnsupported) + )); + } +} From f4b97a3f31c40a6cd73636eae2539b7c2cd7b77b Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 12:22:03 +0200 Subject: [PATCH 155/166] fix: reject a grant replacement whose archive length overflows --- .../src/archive/grant_replacement.rs | 7 ++- crates/pithos_lib/tests/pieces.rs | 47 ++++++++++++++++++- 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/crates/pithos_lib/src/archive/grant_replacement.rs b/crates/pithos_lib/src/archive/grant_replacement.rs index 3401261..93df99a 100644 --- a/crates/pithos_lib/src/archive/grant_replacement.rs +++ b/crates/pithos_lib/src/archive/grant_replacement.rs @@ -93,8 +93,13 @@ impl ArchiveView { directory.encryption = grant(&list, &recipients)?; let directory = encode_complete_directory(&directory)?; let copy_range = FileHeader::ENCODED_LEN as u64..self.terminal_directory.start(); + let archive_len = copy_range.end.checked_add(directory.len() as u64).ok_or( + PithosError::InvalidDirectoryRange { + operation: "add the replacement directory length", + }, + )?; Ok(GrantReplacement { - archive_len: copy_range.end + directory.len() as u64, + archive_len, copy_range, directory, }) diff --git a/crates/pithos_lib/tests/pieces.rs b/crates/pithos_lib/tests/pieces.rs index d1f130f..11367e0 100644 --- a/crates/pithos_lib/tests/pieces.rs +++ b/crates/pithos_lib/tests/pieces.rs @@ -7,7 +7,7 @@ use pithos_lib::archive::{ Piece, PieceEncoder, ProcessingOptions, WriteOptions, compose, }; use pithos_lib::error::{DeserializationError, PithosError}; -use pithos_lib::source::MemorySource; +use pithos_lib::source::{ArchiveSource, MemorySource, SourceError}; const BLOCK: usize = 1000; @@ -645,3 +645,48 @@ fn grant_replacement_rejects_unsupported_archives_and_missing_keys() { )); } } + +/// Serves an archive with its directory moved to the end of a `u64::MAX` byte object. +struct AtEnd { + bytes: Vec, + directory: usize, +} + +impl ArchiveSource for AtEnd { + fn len(&self) -> Result { + Ok(u64::MAX) + } + + fn read_exact_at(&self, offset: u64, buffer: &mut [u8]) -> Result<(), SourceError> { + let tail_start = u64::MAX - (self.bytes.len() - self.directory) as u64; + let start = match offset.checked_sub(tail_start) { + Some(position) => self.directory + position as usize, + None => offset as usize, + }; + buffer.copy_from_slice(&self.bytes[start..start + buffer.len()]); + Ok(()) + } +} + +#[test] +fn grant_replacement_rejects_an_archive_length_overflow() { + let bytes = assemble(&[encode(1, &content(9, 1500))]); + let directory = open(bytes.clone(), recipient()) + .view() + .directory_range() + .start as usize; + let source = AtEnd { + bytes: bytes.clone(), + directory, + }; + let options = OpenOptions::default().with_access_keys(recipient()); + let archive = Archive::open(source, options).unwrap(); + // Two recipients make the new directory larger than the old one. + let recipients = vec![public_key("recipient2"), public_key("sender")]; + assert!(matches!( + archive + .view() + .replace_grants(&bytes[directory..], recipients), + Err(PithosError::InvalidDirectoryRange { .. }) + )); +} From 4505c715fac086fd22350c2743c2a738a3ec0a62 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 12:27:38 +0200 Subject: [PATCH 156/166] fix: wipe failed and rejected synchronous block reads --- crates/pithos_lib/src/archive/reader.rs | 13 +- .../src/archive/reader_private_tests.rs | 114 ++++++++++++++---- 2 files changed, 101 insertions(+), 26 deletions(-) diff --git a/crates/pithos_lib/src/archive/reader.rs b/crates/pithos_lib/src/archive/reader.rs index ef720bc..8fc250b 100644 --- a/crates/pithos_lib/src/archive/reader.rs +++ b/crates/pithos_lib/src/archive/reader.rs @@ -328,8 +328,8 @@ where let (settings, external, external_access_policy) = options.into_parts(); let mut opener = ArchiveOpener::with_settings(archive_len, settings)?; while let Some(request) = opener.request() { - let response = read_source(&source, request.offset(), request.len(), "directory")?; - opener.feed(request, response)?; + let mut response = read_source(&source, request.offset(), request.len(), "directory")?; + opener.feed(request, std::mem::take(&mut *response))?; } let view = opener.finish()?; Ok(Self { @@ -507,6 +507,8 @@ where PithosError::ExternalBlockFraming("response policy overflow".into()) })?, )?; + // A rejected response may hold plaintext, so it is wiped like an accepted one. + let response = Zeroizing::new(response); if response.len() as u64 != len { return Err(PithosError::ExternalBlockFraming( "response does not match expected size".into(), @@ -515,7 +517,7 @@ where response } }; - self.view.decode_block(block, &Zeroizing::new(stored)) + self.view.decode_block(block, &stored) } } @@ -662,10 +664,11 @@ fn read_source( offset: u64, len: u64, field: &'static str, -) -> Result, PithosError> { +) -> Result>, PithosError> { let len = usize::try_from(len) .map_err(|_| PithosError::InvalidDirectoryRange { operation: field })?; - let mut bytes = Vec::new(); + // A failed read may have filled part of the buffer, so it is wiped like a complete one. + let mut bytes = Zeroizing::new(Vec::new()); bytes .try_reserve_exact(len) .map_err(|_| PithosError::AllocationFailed { diff --git a/crates/pithos_lib/src/archive/reader_private_tests.rs b/crates/pithos_lib/src/archive/reader_private_tests.rs index 3a65ec6..4ee964d 100644 --- a/crates/pithos_lib/src/archive/reader_private_tests.rs +++ b/crates/pithos_lib/src/archive/reader_private_tests.rs @@ -2080,11 +2080,103 @@ fn version_1_1_appends_reuse_ciphers_and_give_unique_blocks_fresh_identities() { } } +mod read_wiping { + use super::*; + use crate::format::zeroing_tests::{SECRET, watch}; + use std::sync::atomic::AtomicBool; + + /// A plain archive whose file "data" has four 64-byte blocks that each hold the secret. + pub(super) fn secret_blocks() -> Vec { + let content: Vec = (0..4) + .flat_map(|block| [SECRET, [0xB0 + block; 32]]) + .flatten() + .collect(); + let options = WriteOptions::base().with_chunking(Chunking::Fixed(64)); + let mut writer = ArchiveWriter::create(Vec::new(), options).unwrap(); + writer + .add_file( + ArchivePath::new("data").unwrap(), + EntryMetadata::new(0, 0, 0o644), + ProcessingOptions::new(false, 0).unwrap(), + Some(content.len() as u64), + Cursor::new(content), + ) + .unwrap(); + writer.finish().unwrap() + } + + /// Serves `bytes`. Once `failing` is set, a read fills all but one byte and fails. + struct Failing { + bytes: Vec, + failing: Arc, + } + + impl ArchiveSource for Failing { + fn len(&self) -> Result { + Ok(self.bytes.len() as u64) + } + + fn read_exact_at(&self, offset: u64, output: &mut [u8]) -> Result<(), SourceError> { + let start = offset as usize; + let filled = output.len() - usize::from(self.failing.load(Ordering::Relaxed)); + output[..filled].copy_from_slice(&self.bytes[start..start + filled]); + if filled < output.len() { + return Err(SourceError::UnexpectedEof { + offset, + expected: output.len(), + actual: filled, + }); + } + Ok(()) + } + } + + #[test] + fn failed_block_read_is_wiped() { + let failing = Arc::new(AtomicBool::new(false)); + let source = Failing { + bytes: secret_blocks(), + failing: Arc::clone(&failing), + }; + let archive = Archive::open(source, OpenOptions::default()).unwrap(); + failing.store(true, Ordering::Relaxed); + let (result, _, leaked) = watch(0, &[], || archive.copy_to("data", &mut Vec::new())); + assert!(matches!( + result, + Err(PithosError::Source(SourceError::UnexpectedEof { .. })) + )); + assert_eq!(leaked, 0); + } + + #[test] + fn rejected_resolver_response_is_wiped() { + let temporary = tempfile::tempdir().unwrap(); + let path = temporary.path().join("external.pith"); + std::fs::write(&path, secret_blocks()).unwrap(); + let mut response = as_external(&path); + response.pop(); + let resolver = CountingResolver { + response: Arc::from(response), + calls: Arc::default(), + expected: Arc::default(), + }; + let options = OpenOptions::default() + .with_external_resolver(resolver) + .with_external_access_policy(allowing_policy()); + let source = MemorySource::new(std::fs::read(&path).unwrap()); + let archive = Archive::open(source, options).unwrap(); + let (result, _, leaked) = watch(0, &[], || archive.copy_to("data", &mut Vec::new())); + assert!(matches!(result, Err(PithosError::ExternalBlockFraming(_)))); + assert_eq!(leaked, 0); + } +} + #[cfg(feature = "async")] mod async_wiping { + use super::read_wiping::secret_blocks; use super::*; use crate::archive::AsyncArchive; - use crate::format::zeroing_tests::{SECRET, watch}; + use crate::format::zeroing_tests::watch; use crate::source::AsyncArchiveSource; use futures_core::Stream; use std::future::Future; @@ -2116,26 +2208,6 @@ mod async_wiping { } } - /// A plain archive whose file "data" has four 64-byte blocks that each hold the secret. - fn secret_blocks() -> Vec { - let content: Vec = (0..4) - .flat_map(|block| [SECRET, [0xB0 + block; 32]]) - .flatten() - .collect(); - let options = WriteOptions::base().with_chunking(Chunking::Fixed(64)); - let mut writer = ArchiveWriter::create(Vec::new(), options).unwrap(); - writer - .add_file( - ArchivePath::new("data").unwrap(), - EntryMetadata::new(0, 0, 0o644), - ProcessingOptions::new(false, 0).unwrap(), - Some(content.len() as u64), - Cursor::new(content), - ) - .unwrap(); - writer.finish().unwrap() - } - fn open(bytes: Vec, short: &Arc) -> AsyncArchive { let short = Arc::clone(short); let source = Bytes { bytes, short }; From ebe6ae3b04c55154a08c3d47c37a443d75db3f37 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 12:30:26 +0200 Subject: [PATCH 157/166] fix: stop reading fixed-size input after its end --- crates/pithos_lib/src/archive/writer.rs | 30 +++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/crates/pithos_lib/src/archive/writer.rs b/crates/pithos_lib/src/archive/writer.rs index 3e68b85..4801f56 100644 --- a/crates/pithos_lib/src/archive/writer.rs +++ b/crates/pithos_lib/src/archive/writer.rs @@ -129,6 +129,7 @@ enum Chunker { content: R, size: usize, block: Zeroizing>, + ended: bool, }, ContentDefined { content: R, @@ -146,6 +147,7 @@ impl Chunker { content, size, block: Zeroizing::new(Vec::new()), + ended: false, }, Chunking::ContentDefined(cdc) => Self::ContentDefined { content, @@ -164,11 +166,17 @@ impl Chunker { content, size, block, + ended, } => { - // Reserving the whole block once keeps plaintext from being copied on growth. block.clear(); + if *ended { + return Ok(None); + } + // Reserving the whole block once keeps plaintext from being copied on growth. block.reserve_exact(*size); content.by_ref().take(*size as u64).read_to_end(block)?; + // A short block means a read returned 0, so it is the last one. + *ended = block.len() < *size; Ok((!block.is_empty()).then_some(block.as_slice())) } Self::ContentDefined { @@ -2601,10 +2609,28 @@ mod tests { } } + /// Answers with `first`, then reports the end once, then answers with `last`. + fn ended_after(first: &[u8], last: io::Result>) -> Steps { + Steps([Ok(first.to_vec()), Ok(Vec::new()), last].into()) + } + + #[test] + fn fixed_reads_stop_at_the_end() { + let first = noise(500); + let expected = vec![(*blake3::hash(&first).as_bytes(), 500)]; + let fixed = Chunking::Fixed(1024); + assert_eq!( + written_blocks(fixed, ended_after(&first, Ok(noise(500)))), + expected + ); + let failing = ended_after(&first, Err(io::Error::other("read after the end"))); + assert_eq!(written_blocks(fixed, failing), expected); + } + #[test] fn content_defined_reads_stop_at_the_end() { let first = noise(500); - let steps = |last| Steps([Ok(first.clone()), Ok(Vec::new()), last].into()); + let steps = |last| ended_after(&first, last); let cdc = CdcConfig::new(64, 256, 1024).unwrap(); let expected = stream_blocks(cdc, steps(Ok(noise(500)))); assert_eq!(expected.iter().map(|block| block.1).sum::(), 500); From d89de5945fd9f5f4eef9b3abcc03645d643a7e87 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 12:31:23 +0200 Subject: [PATCH 158/166] fix: reject non-contributory keys in piece records --- crates/pithos_lib/src/archive/pieces.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/crates/pithos_lib/src/archive/pieces.rs b/crates/pithos_lib/src/archive/pieces.rs index 70e701a..bec6633 100644 --- a/crates/pithos_lib/src/archive/pieces.rs +++ b/crates/pithos_lib/src/archive/pieces.rs @@ -416,10 +416,12 @@ impl Piece { let list_len = read_count(&mut reader, 1)?; let sealed_list = read_vec(&mut reader, list_len)?; let sender = read_array(&mut reader)?; + crypto::validate_x25519_public_key(&sender)?; let grant_count = read_count(&mut reader, 32 + 1)?; let mut grants = Vec::with_capacity(grant_count); for _ in 0..grant_count { let recipient = read_array(&mut reader)?; + crypto::validate_x25519_public_key(&recipient)?; let len = read_count(&mut reader, 1)?; grants.push((recipient, read_vec(&mut reader, len)?)); } @@ -776,6 +778,28 @@ mod tests { } } + #[test] + fn piece_records_reject_non_contributory_keys() { + let recipient = crate::crypto::PrivateKey::generate().public_key(); + let processing = ProcessingOptions::new(true, 0).unwrap(); + let mut encoder = PieceEncoder::new(1, vec![recipient], processing).unwrap(); + encoder.push(b"piece").unwrap(); + let piece = encoder.finish().unwrap(); + assert_eq!(Piece::from_bytes(&piece.to_bytes()).unwrap(), piece); + let mut sender = piece.clone(); + sender.sender = [0; 32]; + let mut grant = piece; + grant.grants[0].0 = [0; 32]; + for damaged in [sender, grant] { + assert!(matches!( + Piece::from_bytes(&damaged.to_bytes()), + Err(PithosError::Crypt( + crate::crypto::CryptoError::NonContributoryPublicKey + )) + )); + } + } + #[test] fn written_fragments_give_the_same_fastcdc_blocks_as_the_archive_writer() { let mut state = 0x9e37_79b9_7f4a_7c15u64; From 9c7bce0b2e48482b001df136b0dc1cd5638728d7 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 12:32:14 +0200 Subject: [PATCH 159/166] fix: compress blocks into a wiped buffer --- crates/pithos_lib/src/block.rs | 32 ++++++++++++++++++-------------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/crates/pithos_lib/src/block.rs b/crates/pithos_lib/src/block.rs index 15cb0fa..c1cf65a 100644 --- a/crates/pithos_lib/src/block.rs +++ b/crates/pithos_lib/src/block.rs @@ -58,14 +58,12 @@ fn encode_with_key( let hash = block_hash(requested_flags, &key, plaintext); let mut flags = requested_flags; let compression_level = zstd_level(flags); - let mut stored = Zeroizing::new( - if compression_level > 0 && probe_ratio(plaintext, compression_level)? < 0.85 { - compress(plaintext, compression_level)? - } else { - flags.set_compression_level(0); - plaintext.to_vec() - }, - ); + let mut stored = if compression_level > 0 && probe_ratio(plaintext, compression_level)? < 0.85 { + compress(plaintext, compression_level)? + } else { + flags.set_compression_level(0); + Zeroizing::new(plaintext.to_vec()) + }; if flags.is_aes_256_gcm() { stored = Zeroizing::new(crypto::seal_block_aes_with_nonce(&key, &stored, nonce)?); } else if flags.is_encrypted() { @@ -160,15 +158,21 @@ fn probe_ratio(input: &[u8], level: i32) -> Result { return Ok(1.0); } let sample = &input[..input.len().min(4096)]; - let compressed = Zeroizing::new(compress(sample, level)?); + let compressed = compress(sample, level)?; Ok(compressed.len() as f64 / sample.len() as f64) } -fn compress(input: &[u8], level: i32) -> Result, PithosError> { - bulk::compress(input, level).map_err(|source| PithosError::Compression { - operation: "compress block", - source, - }) +fn compress(input: &[u8], level: i32) -> Result>, PithosError> { + // Compressing into a wiped buffer also wipes the partial output of a failed call. + let bound = zstd::zstd_safe::compress_bound(input.len()); + let mut output = Zeroizing::new(Vec::with_capacity(bound)); + bulk::Compressor::new(level) + .and_then(|mut compressor| compressor.compress_to_buffer(input, &mut *output)) + .map_err(|source| PithosError::Compression { + operation: "compress block", + source, + })?; + Ok(output) } fn decompress(input: &[u8], expected_size: u64) -> Result>, PithosError> { From bf8673c06511966fda5950c3956d9e76096950d4 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 12:33:10 +0200 Subject: [PATCH 160/166] test: wipe a short async resolver response --- .../src/archive/reader_private_tests.rs | 42 ++++++++++++++++++- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/crates/pithos_lib/src/archive/reader_private_tests.rs b/crates/pithos_lib/src/archive/reader_private_tests.rs index 4ee964d..326caae 100644 --- a/crates/pithos_lib/src/archive/reader_private_tests.rs +++ b/crates/pithos_lib/src/archive/reader_private_tests.rs @@ -2175,7 +2175,7 @@ mod read_wiping { mod async_wiping { use super::read_wiping::secret_blocks; use super::*; - use crate::archive::AsyncArchive; + use crate::archive::{AsyncArchive, AsyncExternalBlockResolver, ExternalLocation}; use crate::format::zeroing_tests::watch; use crate::source::AsyncArchiveSource; use futures_core::Stream; @@ -2216,7 +2216,9 @@ mod async_wiping { /// Takes the first item of a stream over all blocks, then drops the stream. /// Returns the item length and the number of freed buffers that still held the secret. - fn first_then_drop(archive: &AsyncArchive) -> (Result, usize) { + fn first_then_drop( + archive: &AsyncArchive, + ) -> (Result, usize) { let (first, _, leaked) = watch(64, &[], || { let mut stream = archive.read_range("data", 0..256).unwrap(); let mut cx = Context::from_waker(Waker::noop()); @@ -2258,4 +2260,40 @@ mod async_wiping { )); assert_eq!(leaked, 0); } + + /// Answers every external block with the same response. + struct Answer(Vec); + + impl AsyncExternalBlockResolver for Answer { + fn resolve( + &self, + _policy: &dyn ExternalBlockAccessPolicy, + _location: &ExternalLocation, + _expected_len: u64, + _max_response_size: u64, + ) -> impl Future, PithosError>> + Send { + std::future::ready(Ok(self.0.clone())) + } + } + + #[test] + fn short_resolver_response_is_wiped() { + let temporary = tempfile::tempdir().unwrap(); + let path = temporary.path().join("external.pith"); + std::fs::write(&path, secret_blocks()).unwrap(); + let mut response = as_external(&path); + response.pop(); + let bytes = std::fs::read(&path).unwrap(); + let source = Bytes { + bytes, + short: Arc::default(), + }; + let options = OpenOptions::default() + .with_external_resolver(Answer(response)) + .with_external_access_policy(allowing_policy()); + let archive = ready(AsyncArchive::open(source, options, None)).unwrap(); + let (first, leaked) = first_then_drop(&archive); + assert!(matches!(first, Err(PithosError::ExternalBlockFraming(_)))); + assert_eq!(leaked, 0); + } } From ac92e3b5338f75ce0d9129a9d4018c601cd31d86 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 12:33:45 +0200 Subject: [PATCH 161/166] chore: document grant replacement limits, raw key wiping and zstd buffers --- crates/pithos_lib/README.md | 4 ++++ crates/pithos_lib/src/archive/grant_replacement.rs | 5 +++++ crates/pithos_lib/src/crypto.rs | 4 ++-- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/crates/pithos_lib/README.md b/crates/pithos_lib/README.md index 694d1e7..d52d605 100644 --- a/crates/pithos_lib/README.md +++ b/crates/pithos_lib/README.md @@ -148,6 +148,10 @@ write_ro_crate(writer, loaded, ProcessingOptions::default())?; The upstream parser accepts RO-Crate 1.1 and 1.2 metadata without upstream validation or warning emission; Pithos then applies its own source, path, limit, metadata, and conversion policy. Conversion stores the inspected `ro-crate-metadata.json` bytes rather than reserializing the graph. ZIP conversion streams retained members and does not extract them first. +## Memory wiping + +Pithos wipes its owned Rust buffers. zstd internal allocations may retain plaintext after being freed. + ## Platform behavior `pithos_lib` builds only on Linux, even with `default-features = false`. The Linux filesystem modules in `fs` and the Unix file source in `source` are always compiled. diff --git a/crates/pithos_lib/src/archive/grant_replacement.rs b/crates/pithos_lib/src/archive/grant_replacement.rs index 93df99a..2405a39 100644 --- a/crates/pithos_lib/src/archive/grant_replacement.rs +++ b/crates/pithos_lib/src/archive/grant_replacement.rs @@ -70,6 +70,11 @@ impl ArchiveView { /// Fails with [`PithosError::GrantReplacementUnsupported`] unless the archive is version /// 1.1 with one directory, and with [`PithosError::ContentUnavailable`] when a key needed /// by a sealed block list was not recovered. + /// + /// This replaces grants. It leaves content keys unchanged. Readers that already recovered + /// those keys can still decrypt retained content. Replacing grants does not restrict + /// entries whose block lists are stored in the clear. The new directory can be larger than + /// the old one, so reopening may need larger [`OpenLimits`](crate::archive::OpenLimits). pub fn replace_grants( &self, directory: &[u8], diff --git a/crates/pithos_lib/src/crypto.rs b/crates/pithos_lib/src/crypto.rs index aab900d..046c802 100644 --- a/crates/pithos_lib/src/crypto.rs +++ b/crates/pithos_lib/src/crypto.rs @@ -111,8 +111,8 @@ impl PrivateKey { Self(StaticSecret::random().to_bytes()) } - /// Takes 32 raw X25519 private key bytes. The consumed container is wiped when it drops, - /// so neither the caller nor Pithos keeps an unwiped copy. + /// Takes 32 raw X25519 private key bytes. Consumes and wipes the supplied container. + /// The returned key wipes its bytes on drop. Callers must wipe other copies. pub fn from_raw(bytes: Zeroizing<[u8; 32]>) -> Self { Self(*bytes) } From c4469b2cc00c865c9fb960ae109dc5b1d63d492d Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 12:58:24 +0200 Subject: [PATCH 162/166] fix: read manifest sources from the start on every ingestion --- crates/pithos_lib/src/fs/ingest.rs | 32 ++++++++++++--- crates/pithos_lib/tests/fs_ingest.rs | 61 ++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 5 deletions(-) diff --git a/crates/pithos_lib/src/fs/ingest.rs b/crates/pithos_lib/src/fs/ingest.rs index 876054d..fb1b4e8 100644 --- a/crates/pithos_lib/src/fs/ingest.rs +++ b/crates/pithos_lib/src/fs/ingest.rs @@ -13,7 +13,8 @@ use cap_std::time::{SystemClock, SystemTime as CapSystemTime}; use rustix::fs::{Mode, OFlags}; use std::collections::BTreeMap; use std::fs::{self, File}; -use std::os::unix::fs::{FileTypeExt, MetadataExt, PermissionsExt}; +use std::io::Read; +use std::os::unix::fs::{FileExt, FileTypeExt, MetadataExt, PermissionsExt}; use std::path::{Component, Path, PathBuf}; use std::time::SystemTime; @@ -121,6 +122,21 @@ fn same_cap_file_identity(left: &cap_std::fs::Metadata, right: &fs::Metadata) -> left.dev() == right.dev() && left.ino() == right.ino() } +/// Reads a retained source from its start with positioned reads. Each ingestion keeps its own +/// offset, so ingesting a manifest again or after a failure reads the whole file again. +struct SourceReader<'a> { + file: &'a File, + offset: u64, +} + +impl Read for SourceReader<'_> { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + let count = self.file.read_at(buffer, self.offset)?; + self.offset += count as u64; + Ok(count) + } +} + enum ManifestKind { File { expected_size: u64 }, Directory, @@ -160,7 +176,10 @@ impl InputManifest { entry.metadata.clone(), processing, Some(*expected_size), - source.try_clone().map_err(PithosError::Io)?, + SourceReader { + file: source, + offset: 0, + }, )?; } ManifestKind::Directory => { @@ -178,8 +197,8 @@ impl InputManifest { Ok(()) } - /// Executes a preassigned append plan. The ID assertion occurs before a source is cloned - /// or read, so a stale plan cannot consume filesystem content. + /// Executes a preassigned append plan. The ID assertion occurs before a source is read, + /// so a stale plan cannot consume filesystem content. pub(crate) fn ingest_planned( &self, ids: &[u64], @@ -200,7 +219,10 @@ impl InputManifest { entry.metadata.clone(), processing, Some(*expected_size), - source.try_clone().map_err(PithosError::Io)?, + SourceReader { + file: source, + offset: 0, + }, )?; } ManifestKind::Directory => { diff --git a/crates/pithos_lib/tests/fs_ingest.rs b/crates/pithos_lib/tests/fs_ingest.rs index c10bf7a..0ba364d 100644 --- a/crates/pithos_lib/tests/fs_ingest.rs +++ b/crates/pithos_lib/tests/fs_ingest.rs @@ -7,6 +7,7 @@ use pithos_lib::fs::FsError; use pithos_lib::fs::ingest::build_input_manifest; use pithos_lib::source::MemorySource; use std::fs; +use std::io::{self, Write}; use std::path::Path; fn write_manifest(manifest: &pithos_lib::fs::ingest::InputManifest) -> Archive { @@ -189,3 +190,63 @@ fn wide_directory_ingestion_and_archive_traversal_complete() { assert!(archive.entry("entry-0000").unwrap().is_some()); assert!(archive.entry("entry-0999").unwrap().is_some()); } + +/// A manifest with one file "file.txt" holding `content`. +fn file_manifest( + temporary: &tempfile::TempDir, + content: &[u8], +) -> pithos_lib::fs::ingest::InputManifest { + let source = temporary.path().join("source"); + fs::create_dir_all(&source).unwrap(); + fs::write(source.join("file.txt"), content).unwrap(); + build_input_manifest(&[source]).unwrap() +} + +fn read_file(archive: &Archive) -> Vec { + let mut output = Vec::new(); + archive.copy_to("file.txt", &mut output).unwrap(); + output +} + +#[test] +fn a_manifest_can_be_ingested_again() { + let temporary = tempfile::tempdir().unwrap(); + let manifest = file_manifest(&temporary, b"reused content"); + for _ in 0..2 { + assert_eq!(read_file(&write_manifest(&manifest)), b"reused content"); + } +} + +/// Accepts the first `remaining` bytes, then fails every write. +struct FailingSink { + remaining: usize, +} + +impl Write for FailingSink { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.remaining = self + .remaining + .checked_sub(bytes.len()) + .ok_or_else(|| io::Error::other("sink failure"))?; + Ok(bytes.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} + +#[test] +fn a_manifest_ingests_again_after_a_sink_failure() { + let temporary = tempfile::tempdir().unwrap(); + let manifest = file_manifest(&temporary, b"retried content"); + let sender = PrivateKey::generate(); + let options = WriteOptions::new(sender.duplicate(), vec![sender.public_key()]); + let mut writer = ArchiveWriter::create(FailingSink { remaining: 16 }, options).unwrap(); + assert!( + manifest + .ingest(&mut writer, ProcessingOptions::default()) + .is_err() + ); + assert_eq!(read_file(&write_manifest(&manifest)), b"retried content"); +} From aeffe9e5a3664e1f8d67bb40b275456fc1f5187e Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 12:58:46 +0200 Subject: [PATCH 163/166] chore: narrow the README statement about wiped buffers --- crates/pithos_lib/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/pithos_lib/README.md b/crates/pithos_lib/README.md index d52d605..d83ff13 100644 --- a/crates/pithos_lib/README.md +++ b/crates/pithos_lib/README.md @@ -150,7 +150,7 @@ The upstream parser accepts RO-Crate 1.1 and 1.2 metadata without upstream valid ## Memory wiping -Pithos wipes its owned Rust buffers. zstd internal allocations may retain plaintext after being freed. +Pithos wipes its protected internal key and transform buffers. They hold keys, key lists, and block content while Pithos chunks, compresses, encrypts, decrypts, decompresses and verifies it. Other buffers are not wiped. For example, `MemorySource` keeps the archive bytes in a plain `Arc<[u8]>`, which holds the plaintext of plain blocks. Plaintext that a read hands to the caller is the caller's to wipe. zstd internal allocations may retain plaintext after being freed. ## Platform behavior From b76bce5c74bd78d9938e3d11c8fda800a8edaf70 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 12:59:03 +0200 Subject: [PATCH 164/166] chore: qualify the key material statement of grant replacement --- crates/pithos_lib/src/archive/grant_replacement.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/pithos_lib/src/archive/grant_replacement.rs b/crates/pithos_lib/src/archive/grant_replacement.rs index 2405a39..af31d67 100644 --- a/crates/pithos_lib/src/archive/grant_replacement.rs +++ b/crates/pithos_lib/src/archive/grant_replacement.rs @@ -23,7 +23,8 @@ use zeroize::Zeroizing; /// /// Write [`GrantReplacement::header`], then the old archive bytes in /// [`GrantReplacement::copy_range`] unchanged, then [`GrantReplacement::directory`]. -/// It holds no key material. +/// It holds no recovered secrets. Block keys that the directory already stores in the clear +/// stay in it. #[derive(Clone, Debug, Eq, PartialEq)] pub struct GrantReplacement { copy_range: Range, From 05787fbdb088e3e5039bc75523b5a5422ab53b38 Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 13:26:43 +0200 Subject: [PATCH 165/166] chore: update chacha20 to 0.10.2 --- Cargo.lock | 4 ++-- fuzz/Cargo.lock | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2afc8a0..51f6bb9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -306,9 +306,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "chacha20" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", "cipher", diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index a5e708a..7362528 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -217,9 +217,9 @@ checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "chacha20" -version = "0.10.1" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06" dependencies = [ "cfg-if", "cipher", From 649157017b98a8d43703798722aeadeddba2895b Mon Sep 17 00:00:00 2001 From: Sebastian Beyvers Date: Sun, 4 Oct 2026 13:28:34 +0200 Subject: [PATCH 166/166] test: run planned ingestion again and after a sink failure --- crates/pithos_lib/src/fs/ingest.rs | 84 ++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/crates/pithos_lib/src/fs/ingest.rs b/crates/pithos_lib/src/fs/ingest.rs index fb1b4e8..21a732c 100644 --- a/crates/pithos_lib/src/fs/ingest.rs +++ b/crates/pithos_lib/src/fs/ingest.rs @@ -758,6 +758,10 @@ pub fn validate_input_manifest(inputs: &[PathBuf]) -> Result<(), FsError> { #[cfg(test)] mod tests { use super::*; + use crate::archive::{AccessKeys, Archive, Chunking, OpenOptions, WriteOptions}; + use crate::crypto::PrivateKey; + use crate::source::MemorySource; + use std::io::Write; #[test] fn synthetic_parents_are_deterministic() { @@ -785,4 +789,84 @@ mod tests { assert_eq!(entry.metadata, EntryMetadata::new(0, 0, 0o755)); } } + + /// Rejects every write. + struct FailingSink; + + impl Write for FailingSink { + fn write(&mut self, _: &[u8]) -> std::io::Result { + Err(std::io::Error::other("sink failure")) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + /// An empty archive owned by `owner`, and a manifest with one file "file.txt". + fn planned_fixture( + temporary: &tempfile::TempDir, + owner: &PrivateKey, + ) -> (Vec, InputManifest) { + let source = temporary.path().join("source"); + std::fs::create_dir_all(&source).unwrap(); + std::fs::write(source.join("file.txt"), b"planned content").unwrap(); + let options = WriteOptions::new(owner.duplicate(), vec![owner.public_key()]); + let base = ArchiveWriter::create(Vec::new(), options).unwrap(); + ( + base.finish().unwrap(), + build_input_manifest(&[source]).unwrap(), + ) + } + + fn open(bytes: Vec, owner: &PrivateKey) -> Archive { + let keys = AccessKeys::new().with_key(owner.duplicate()); + let options = OpenOptions::default().with_access_keys(keys); + Archive::open(MemorySource::new(bytes), options).unwrap() + } + + /// Appends `manifest` to `base` through the planned path and returns the sink. + fn append_planned( + manifest: &InputManifest, + base: &[u8], + owner: &PrivateKey, + sink: W, + ) -> Result { + let snapshot = open(base.to_vec(), owner).into_append_snapshot(); + let ids = manifest.validate_append(&snapshot)?; + let recipients = vec![owner.public_key()]; + let sender = PrivateKey::generate(); + let chunking = Chunking::default(); + let mut writer = ArchiveWriter::append(sink, sender, recipients, chunking, snapshot)?; + manifest.ingest_planned(&ids, &mut writer, ProcessingOptions::append_default())?; + Ok(writer.finish().map_err(|error| error.into_parts().0)?) + } + + fn appended_file(base: &[u8], appended: &[u8], owner: &PrivateKey) -> Vec { + let mut output = Vec::new(); + let archive = open([base, appended].concat(), owner); + archive.copy_to("file.txt", &mut output).unwrap(); + output + } + + #[test] + fn planned_ingestion_can_run_again() { + let temporary = tempfile::tempdir().unwrap(); + let owner = PrivateKey::generate(); + let (base, manifest) = planned_fixture(&temporary, &owner); + for _ in 0..2 { + let appended = append_planned(&manifest, &base, &owner, Vec::new()).unwrap(); + assert_eq!(appended_file(&base, &appended, &owner), b"planned content"); + } + } + + #[test] + fn planned_ingestion_runs_again_after_a_sink_failure() { + let temporary = tempfile::tempdir().unwrap(); + let owner = PrivateKey::generate(); + let (base, manifest) = planned_fixture(&temporary, &owner); + assert!(append_planned(&manifest, &base, &owner, FailingSink).is_err()); + let appended = append_planned(&manifest, &base, &owner, Vec::new()).unwrap(); + assert_eq!(appended_file(&base, &appended, &owner), b"planned content"); + } }

    YI0V9X7@^p{RLdSc-uvjf4csqDl});9Rp>bvEzNpV;bXIL34ze))|* z>-ChEx>tw2rL7Z{Wbru)8e@Q2N+c(yP@$^Wb=Ok>_||ydNJV^VlTJf-!a~vxR8DJa z@Vzrmi@p_fLt{oT+V92=sKM~a9s$?9SCD$B?3V$R^Q$FK9(3~3ISCn!2xbm7^+l0F zyE)`nm>NzUDHb@A`6yY;H7KZbNL4^gnQD<;?aQBDqKp8hE8Pf_I5zSJ*F~tjhA0T; zezG%F=)W-4=`dGHdO--iP~b#M6p+#$B~RPn%pw7zAr(7~1Bw!&WC@^RS3d6JU;$~= zqM}Z49LpTn6r_?nQSJc+ZR;L9@n%tSlr@>0YO37ge`eUnoC*@P*aecHqQ>@k1gGbT zdb0&q!~wJ1NN;|b{I)+f;|dKAMg5&BA}z)fb$p1!Fy3RWMMDu~U|iLES+tu-*RK(s znp8v+2@**Up}BGd4LkNW5G+xmz(ln4sgOI%a9L^w%1hEJaMT=QhJ{4|mI^w`QMgjD z0!r-r%qbtC%P5C@p5T3U{)9bZFdjrL3dH{2V^$5NMDJKaVt4c1Nw(1r8BN#k=b84S zKs5h0sZ{4HVE?|S{d^Jr8K&TSvCQ?*k#Tcmf6(!G)c(0;w_SCzsKcD#aK5+UvA5^2 z&*Sl_TJG@0^O=*~`nUJiTK`v3b=%)ohdqezuFnTHe3spH3a&uL&sAlQCqDh1JN>L( zMEi}Nz`3UXg@=EIXi$iZxvt|Bzsq5s{q@h`ip%Z|@6A5i`%+zJZ1mhc2i^UWvXHh&feLGVf;NxX~W9TqC_AYF5=HIqnvUYgxW?3xWGXYm1C0z zy=^h3ZEGBKXiEG~GMWZURLN2sFc74*F*!bOs)$#f;6Sg<%~3;J4%G*pz+ zq2=^Iw^Np+H(U^kMD-egSPS9s1EsKQ`KMaZ^>&V+1>(b1FZ9ilu&0hM!W97`3QW>n z$OC-*HQWxy$gA%S3W&x?KLIBA8n>8jjtf0WV@Z!3Ls>I;5tYCa2yTwB=}C$an#gW$ z!8t8SieoJfK6P%-9vr;CxvRRS0=*eOOf-O{PIpJk8|y(hQ|teeD4vOB{{;e zDN1*zo#?#&j!7Kvj?qL3TSF_q%$Ig3T*TP^`kQ(YuGT+TZyB&_YH#9xB+a}JN5ogT|bs&rIh*u8W^%Q6|snJNkmkjQ# zUwqS($yCggZz)4-?_2!fS`XYVC&d!V6tpRYoO5IKHxd--xd^fZBx`)Y&h<)`hD9M5 ziug!QMf-4pi_>Lb<}5(J-lZ< zEw%0D6e?K_Nx0qbY>kZUSC0>iFm%^v-JC#6i0Pvkor3B?G?Iz7!<|5?(~5lNtoJ>R zMX?`-%z#kMhj|Cr_j(Qnr*<5=4uPR9U<0lf!HzT>?kPy{GzY#!6c7=oh$==D3`ENc zNm}9Vh>C0$j-DYWOrj)`MFztur;%qum1bLzYaBm~%1JYz4|(1G1g}v< zB89Gzbd)p?Q7uN2u=} z-pQQjj$IoCiEy}nVl5yImQCU*of}zAv!&5XHsp=}%9HmQ*8np>%)i~}xX{jkpAy{5 zY&XSwQr+vgPhyo6jgDsEg7O`=wR!Uz#W)%kOy~LKo2pWM&~3w)ydOSQ63>-a+;a45 zn9wlX_a+_^{Qpw`(mx8|eO`GoJVHNPt{zrkOYkvUv1#bP)iH`P*5gtUyjy zOKrnvON93^n`ryyjmX&e$p|w&-Sd)5-JrC`E1a33c>z9h$slGBJGc@d6%j7hT?qC@ zc-J9<>L(v0=8@2#5-~r>$X1cQ8nI!Pq;_?$R0K|l7WJ8`jD--Z4Yg2*DFL05yk#C& zKRy(3sJ3D>Fu|!*ac&VV0RXWmOcGKt&J*m(r6d~Z&?*F|7ZQPvuN`h|FS z58J=v-;LfM>L@}2Z-};2_K5XaZ!<+i2IeDCB7u_sJ0=A@dN3{wEk$y2V|EfQ%~S(Y zIgTukvWu8KN6v(D66K7m1aSd&94l!Bm#j~8A)L{iqpegH78~hr$N4&tj^S2L$%4&7 zJT8?k(B^Bs==OvGfmSRHXY}kI1qL86BQ$aB!-cU!}`t*~% z$rG;(-E5$PtT&q{!7pz!;PNo=?j!GbLq=EL`*o?&&J>GK-W+agVdDLv!VAl);!AWt zoz{y}vODy020X?Q`C!P(-xsF7+89UgJshPl!BT*R6x2=e*hu`C; zFHG0YCnCr28}HZ296GRa@2l;Q)+Y@uiN%2&|J5JXKm9dp#!+?cMO_NZ)~%T-_@)N_ zSr@*Z*V6BkVc|=t=p?g{ZvDC96>+d}UQMJisW9WR!zPX8*~uwQt3$`&aglSlXC<5s zg@j%Tz0eJ(N8-iS5n0VlGV7}(#|~PXPiFNL=M^@oWB(+E&=lzsexq`pqyb=3vj3suH2LR0cYo7LWa~ zfxnm(-9W{y3K=c`?_EEa_J)4&X@AtHl4c&1@YRIGmdiwAJ*SZMvXkVtb^ANdmsLt|D*T`P_U z8Xu@c;N)kL`jn+vXbn(8bzd#(_mw$jN<*u#p60t5SCLWpGYbx5te2&tQrH&UVDQ^d zoHp$uXTbr)#W04=mjg_lB3sDknYwPZt(=D`YeWHvV%yba&GGAyF0|dCXD_=h*Fq_;ii9E#+q8#ki|y--p2CH<^4-z$n&Nz`eDzb=%9=6NOm1x-rTDZ zxzuWVL+Blf!2Kd+Udwwtk=g3sJ4NDXaddw3p~$-Uju1wr7xxQq>rXs__oo8B16r#d z(g}Yi<$gE|5UYH5M#6XMNl#e!<47>_<9yk*4VSiInI?WW<)Gd{ZMx z-|%Xtvd2iD@EBOUe@_@r9Ec}SRDtj!nwagwPpMgyB4NdU$x%5peY5!7a>6;|}Gm~>>^ZHnGucUyq)`(O2!cDSO^K9l32_3_$4 zKZ6t1f%4zxAVk_^z?ptAk-wPY#E{r^o~+NO5}eedB44`Jm9EQ|$-AW?A1VLH!}OW- zqM2!vF^;3Q+?Mr}G0r4E=N5Q+BXtq$tCAp)YpPeUmg42@ zIw*Jnibu6$@I%H?n&%-E$|@2x#A04ga=xQ@R;%}Sp@S20_O`qiJ&(-L9CsvI`!1Y} z1BglFm-cyxY%#!)st%qFn1Y`gRftPZ)L`C4Z!<*1L_Dssry)v~kVr(q6_3ea{52m3 ztK>r&wU?xh#+IP2q!-2UeyygYh&N(SWm+F=+%g@UaU$~0Xd#aFX`f#6)e&5^wP{U$ zV|t^exv5duOf)u_)`b#*eooc65_l?j5EPpt$q9_gCua#r-lfTTnohWCoiP}_Gf@Wm`luC-+}kBt_xguE zUWB*r5`VH-p$&8WW)=X)NXaV^-hDO=8Bg!Lzdt0?OFb99|B`G4E3ma?+ncaxbcEJ< zGL7UtHxYaO@;3ikw4YCbqd-y7)o%wskG-or&Ve^{B!v=VvdEh(v8yk`!i1$-=nlu} zJROt*{v^a9MHh=-TtdT$rlYeR*1v{KFQO-U#-s^g-%ScF7ATn82%%T$H&GWWq=^cf z_cGThD4~tVRfXG!WD#4D%=0z7t9!|o)I-?q#+>({DuhZVOlex={HZq)X@Dvd9elA)$_}X;^UzK<57vLRzyhxQ`IP=S-|K>l-eqwcEPT($y1VbMI ztW;yW_D(K2lR#Rl`LW}mM9fUUu@8GvqopaDid#rK*R+Y*?&QYfhh1(f$LWwDDeCfQ zW?A~r(CWg?OO+0w83U0l_AwwQzcUkyHd?VvCS8Zyo;R;TwZYdTY9K*7hLPdHz4Yu> z8Syzjwdb#vaF*UW1 zPQ;xuk|@3C1fYgXSDKC(s&~*iakl;!PD$XN+B6k1$^=0G zIf0kH?QK^XDH*H?^aNQ1LU3#N_CXX0S>c?g0x3+qn!0$2`6X=x#x^i!-IF<3|GFQp zE}FnR`kY^@<^(lbFc%;@^>bkN^gc4H9({N=CnZ00M$$$>Zo~_u@onLBnnRdhF7U(# z@9-g+c{=G4?|9~Hcv8tMI;Xj9aIzm&a(!p0_li`J?|o_ShqSfyQuGndZfq@4S^X@w zC|XoCRU>@&8>L#O)vD`aQfz|amb}bGNJs`XT8`%(id--%|R>s{bVw#*US;J9X2Wg-UBc<+AHh-$;;UJU~+1iSez`wUg=^k4R~2w zyvF1&HiRVJ{{pts^rc1fQgOCIL;VYhqR_FjRCb6oX_lF(D_4@D*WKL6Xl}oc9FTXI zc37u30Ijq2vT$umyTukwm8KYFLcS7jB-a(jO7NLeQE1;je#MpY>y$Ecq zMp=rhts+yr2IOG@?Cmh^h-En`G9M7%fFCX`Ah6!!%{8$SI(JTJh^vQ03?` z^AwwKcZ9u3_(s1#(}M323)2>UsSN(%5erKq6H8LMpsb!LqY`qp-~rh1L9Tb?&%uJ1 zk8OCRcJa~F+~xhs>)vBzO4v|!F%ylP8^;%X{C_W)x{4Dr2J9PL4$_!`9yzMh>1xL2 zLn=WP#O{7AV&oOXdm4(z0#LcGLpL)n>rfzJcd^FrkkPa0DxT>@LW9i|S1aJk^T$)T*3Dz!on>wJiQUV?YUr%# z*mm7toaINuk%`TF@3*THtLrhd>FtmvC1W$``zn_GX0nbi4JTsEyb5lw!hX@K>^qA!84bsT8h8_Go*P5x z%Uk`o(_4ho+=Ktct;j#z7P+rb_#nqC2Tk}gD%+>Ly`>G zqU+liz8;O1eql=yv8G>wqXqR8`QWTfTZOqIh#VSi%p&eTNhJl~YV5CeUDJ7-txJyZ zw7=}Q@$OeQwO?5oDd|e$yf%|gxVTewu2`*?Q%J6q$MhZH!ouv%sa7i!URHE7z!)L^ ziS}tm#ps6@e>sAnl)#I5HD8s}iAnY~pLf$%A>%CwE!;Lu$}7xWJ{~=)^?1L-3cRiC zTI(s75@G=vLhC>=)5Z$Z%Ln)_X*7J70^IiNKUb`?Iavt*=psBzVnE2$(c6rU&6L#< zs7oK(s*B^%3QJ$3VTUkMGt)q<1k}lYpL$7EJiWur^bT#^Wor`&S#(rB_z8y4`YVUS z1)qUU>@LNWCkr7|#mE$nUeTbU&UQ$kWpv3mo2Oeh(3Cufa8vGniNRe4{xQ>ZQkmd* z|8^pzWKKn1peH!p7Hi_OM$?c<+4ufsvaatxJ~<6DKbGG_U(AD;b4N7c<>dUc&e5xe z`C}F2U5`q|SDg=ztBpMAs(46c1-Qq>vdr9Lukm!gUFj%e5sCX!i$N6f(m3_7NN~Qo zsG~Em$W$fIi`$^2qUI49`4EZi>`WVNNjFX=Pe0NAJ+cdNRL5~}6-j136 zxOIlNDI{9efrPz}eM)WA-Io!;Za>Wi>B_q9-2|NfPWj!;2#pDE`0*J1 zjp+O~Bj931A}w?Z+<0o$x{j#}#G;LTZC|bZJmb4PB>%Rw$*vdY)@O0}LiaZ@yZuix z_MB1AcS^oAMi$|nZ@Y!(*e|#E7bDHgdnYxgZ@Kls|0z!dJHp7@i2SQ${eq9{1K+Ij z{;HAv;T@?dZA@a4}Yc87vuO2Movx+^oWmXA=Sz`ah8`$w@MwK1t~#V z^N9i?mn@AtEh+IH>Q93&^%VmyD(tnp^P0a?rJ^3|!zQy|U7_9)N-LP^*ChY!DYtAh z6~6AHc@L$w!c^bti%tx{11^1_P1IZxn;@T?faiFL9+3p@W#PU?k4LndM~TY>YXU38 z$SgqPJNH!xT&kY7ckP&$=G zsmd^YA7p`Cu|!+g-=w||7nY7D7~Dz?lJV&w4|@a(Ce{uDVGBxYvQ|?FS)X|32uaFT zZ7I_J)zUmIoLbZkG9-l(8;B?f9*qX8{i%tHE^f$M$%Oc;`<<(}?vF%|&vwa2Bv2i5 zTr)wOjpS8^u7v4jnRSX@A53?HG>SqUI>HdjoPo})LFAYLHC=HVhFE;R5U=$X3 z3zfo?IFB=$Lgdc@gm#7Ce%Zt%YDex>Uaq0VI}pF;rG<|Vi`s&(wtmh4XmVc3ckdNP zMU$W^NhMSfq`h`BYaKtEhV=tYGFxR`B)?wKxLeXLD%E+}p{=30U848!`^kK5(1tYUC=F(u4(VfABB80@tx zt5dA$LPVF59-%U40>X>#WFS3_0?jM~;6p3u=n?PRZ;qMy9heCe6cE!?lRz89$t8)B zrQSkDCq30A6lyYKDqxFUfd>u6im$95y$vx;H*Mmr{2`dClMwK2UN5thi${qkJ+hx& zNNB7S%?^B5ua>#)2dd(B*dE1J+~0og=|gXtT-7rgT6_VDseA1nS-^v}5P9gD)d81@ zU`mV)Z)B@6hEL-nU;A~X_x+Sf7{x6@`!;2V@-s%kde3G!L@K>BA=L4qMB2R=p|vTZ zhe`896a(MC&eL7eR{`yqpiz8yx@sT8uQnzip`LUoKv8M|q`dH0$Qb=%FzW*>1l@nC z<>kix(Dj_`1>yZFYv#u7^bOk<=fU@7$oDPhGERTe{qlz&#Q4A93WP-`ijs*zz9O)S zGLx(-(9xbAbrZ$$#tdJo8o46CM5h->(rhNe65|@3uu%s!ct^3~;SQ;PmX=29UQEID z_wkl*V#y?)1D`6Zq|jP=T&={@{@oFNV`ORHu}x<=-D%!g=U^s}r}WcFe0&09nfg>m>br_X zji6`wm(UAb9beP+pFNGe)`zp9H}Z%1=rXg@+ub5Qg!6N$L6Dl*R2hnjD*`>!*aDEM zrDevqZ+gZaKIwl~ZEiS=An>Ym)aZJ+1XOf{6k2HdYV)4-9h)=;nT5;WX{XXhn}j~k zVAl4h5o(841GAC_zoB+*>6GP+AB|P%IC6Q-r3p{$t?*SYN#~mbk*Q-ss2y3`oM{~? zDH%~Xc~xao(Z?LUusW>5M8ML8xaU$R+xhJ2Kd2FdM84G_$tBi99&6#@B7lBZABpTXR#O)FmVJ<=H=qZEc z;bnrqhQUfzBt={}(0Rg7CSRJra?!`c_zR%zo-B=}cU*=_~UF3B%=xFR*`(+PfH#fbEXbHk#CY+}f{{)MKR#8=mkHi^y*W%<; zqIngs;|J<{%WU$cH|7X1j+Od?9#!X?i z%NURwn`0^%=$=--Bz29tw3c^6YMdP{l78jq7MqwklhR|WFN)RJB(KF9U6*wI^jF9*9)e$#XdzVx-s<}v2A=arv_SoOw$C&DVPd$FdyB` zZ`T)VkW!|`jrasNZ09ZU@W9b7Vy*Q@B-KHnSc{6)vAnt%C8&F>VhADHWgYPS*4GaS8AB493{Bp~gI;UyOvhUzC6alGh+LYPW$+`Ac;slauc!??yw-K+SWqG=@%bL!A= zVAKDsGv~d7=k|6+FrKA*xA>iU*mGuHYh!>h;GVHMa2Nm2-lupzFQfAR-JxM@ZApUg z)3vR~*MUgVR)wO5dxH&945PO>mh2&^d7KIZe^S&yR`4#LzEoG-;7DbUYR_Her-@Gp z?WSN+opaY%%IoNk>o>J0Ql^zI*vv6LLu?3?h{cSC?u8ATWA`O=hWKPFAC&v<$7zb#?Hqjtq~`Do(T~ zx_m^lTfZf|5O`Rxag0jWEvKP?#Y$;VgGZ)hFdyk-3m;UrP$!KB8pF6;X;&nMpl&g| zA+r^uehmKlfg(%<0ot08sT^nF&e+E~b1=GjYVags9@6RTi3p?=nCshSo#T+}^1 zYkJV#?A(0xCbjCkkh1~B6b?4WgR1Dq!!2~?7IC%ex+mY}Qah^4@lZ}0^O@v+HU7lm zEp=#s&T}$tsIS<{=q>N9CetlS1S5iU8Gcgw1tos$<`gSrs~xIP`#h>RZHKhxLUhtlc$rbN#zX+J~Z^29lj8T}PNw(?QtQd*e&f z#@;Q%V~#~X{_q4OWxNkpb*9Pf0u?Z=qouGOXeY;ELo_}VvoWa<7buA5NT4FD z!Nm=PaZM9c+c>v=_L8X~|L-k8)D!qnn;9R+LTZgO=rb}U%;BG+gjWjRCi+{@?zvL{ z2dxt6d*Rl1W;H3}Br%GxljcwCKj-3vbS^811mTz95-Dj zq&lmF_=`-W19bKJZ~bL#8j2gbM2AV!TfdNxnF<~w)25}6mTV%6$rU(dsi|EgaOk>D z@%DxZa%`?kOAWpX%Mu2u*~cQ@rQm#&(A~)uE&q^uu%FG-e(GL#DE!_>CK<1oWFe;* z%TtaPivXJn9F@Q(iW|VTmE1xNwyUU2_S2bFM*+b7+PKh<(ny+h*GkQx+jQ?Z%a8|p zZ~oZQ;T&E2nlQk31QWf;SPI0$MBh8{9e1a)g7TlqHtm-L&_s;#MOZFH>hS*bUgfUA2$A)Zk*)37J{YCGL3IJXX6=AdKTVD)XNQQ$)kJ zMs@tt`2Kk!L#0!A`GGg z4B;;FS?J%Wl0dN#SJam#}%P#3w-+j|>vuWE8-=1ww!`EsNZsbDvf7^0aH#J_w;&k2Se#(Jc zoeJqTWn)V7+W+B8^Ku(g>%8G!kv=z0qu75wNwzHjwb6BA6PvZI7wZ8^cl;B~|EHW=-Ti4?Hg9npo2!uHxsYoq zGTfiWn5+b`AIIJlqqs1okz{g=v%-)2h)ZZ(fhyYK$>FHa(NLmzOJ1mGDqy$>XPHR; z6zg;0IC2<9X6#d{JG!cyb7C;g(I~q-&MXo6 z`KU=;@U?UtBu3vV9+Sw`gFlag(-Lak{?*=2cg<8kE)sr1G8`}28WWh01&M8Gb`W1^ z2BpL6>8+tk*n$`Kp11;*Nxk9hO=1P_!+^p{vl0IZW&mY4ksVX?D|B z+nq?St}U0JqMRw$i;*NR$!|&TaF|ow4lBfWEDXgYgTUq0)R+a8)!H?qUy+806tqy{ zgs>Y?c9ACnX>cJ^X`$JE6TWYsH(w0i;TJ5HDLuLeHR?7B@=2QJC&Q>`li zR6A84)}TbMq2Xn?&Buc~$ieOb|ota`LR$R;53+1Zy-PkTVu z=6z#vwLcc9LJl#Z+o?x6%m3jFm!09jKf{v3RKY_NLr#^qK1u%|t&u*cB7!5G(+M{I zF`1;Qx`zUjHcNI_J{~e;Duc-&hFw32tb2N`s)iH3I;m<7@1Vg>il!A~9LzyefQqGE z$ZG5>V4BWE*9af|frT?ha=M)?BrVk~jxD5cJhSDw*y5wcuw%o_kQ??zFAvS4zXv*x zPC7*-FCV7$r;sM_?(u#2pUR`bTWF#hTTR_UCk?>}ti;SLV+T59k&L2<6fX)dBpkB{ z7~LTyjr2-g3hq)&JI9O(ulEe`X&vF5i7HCe*7FgKNsi`JkO4xTvtxM_=d-fFB+2oN z(e^T2Gm*JvHAF)9Q8UiF`-0QC{B1WBZkg=T00j`vD;+g4nkaoSGA5tM_{G8>E#ii* zkw46f3q&z(0W`HvHgO0QDtvIOvoY8RHmL3>CSh^9KGfuBkqS9#la?sa>{bGKiNo|Z zP5=f|ollielcu15d}*3G@w*{dD@+puAa49is+%#^RaszEYqT@w(yqhHKw+on@XM_u zWndK^y*8X+{Xs$OYkoZY8o z#fXq1QX{DRig64D$f8es^l0>@y{YcVreiwh)YP;+7rG}1pmqN8n_;yi)B}FnmdwD# zRS$c?VS<+xudx34JG2$vOe_3D)dgC#CLor}y6(Ei4PlTMW;-evPl@S$@~HfOuC$Em zSsfW=CgT7@v_Rf$aS7^o_rcMmG8p&N>^8x{0>_n>pn+BRv3A}VcMaV@baLL;;^GQHAHoD_AkJLLHlpI+u*R%M@FcVp)> zB@LCKTm_ot$^~5_D5CjX+z^#gi_Fi&paUF}(ZIh0!jq?@&$^|z=_X1$snpKMP6FRr zKru8H8i_0%LTzrjhRMmLSSAGI-7t*xO}x+UGRb%O4dtSd?0Apk<=TfUmRbgQ^n4h$ z5$%GzOSnXaUs}fka9fc9^qp*tY4K8TQMK}qB9&=`LE`TH8Csk?ZTuX;DOJ-+LoxtZ zNje(D1wr*=x)H%))j7VHL?R5Zlgs=E^lg5FxSJrHE$VpYe6=X88Co@u6Gu?{@fFXS6aD=?`_2I1ElRv8GN zko0c}kR=+7e*BHYd3w_jA0mC)QseaL)Pm0mlPI z?d_QGx|xSpjPFHY7+o}Gf6W~sX@odLCLcDZ!D|^eFBf7^ATP!!u#(U|TECFU!g21+D8i50PH4&r0YP6kM?))DQ(bog*|BWtN>;T6mu>fs{T$<>BE; zfabI4K49IfAjn7^(rR^VLjJ0o92``WZw z^je{R!k1hVc9_JhgCpeS-m3_GJ?P+=&K{MMv6~;N8a|vC%<&OYM5+UiZYw)3CYtt| zja4k%HmJY**M<3iiNsaM{0~WEji@9D5O^rq=QT_;nX-9^kS<2iZ&MDo5K^EaW{3w@ z07Eg>7|SVB;1cUAUfKG&N|tqY5jQ-&)mg$Ogh=GJ#4Ye%e8goN<#jF?xq$M6o3%tK*AY2RK zv*b`kNx@OAEW+mfd7-H_>YX-8H*udFK5&V55)@(Ko!;3>1Ov%&1(9qj*_J3mMMGvo z8a-lDK6)E2_=scLHkJ%6qCr1}@x+R^zsy@a)EQZ^%6eT_%-qCFljXBe^|PAxjIIa^ zY|Y2auy3pwXKDE@@c5%J95vACaknYeH~jcmT2PY?I(Y87T5rnX_Y9x2mdNMQwHdE0 ztd#xN76{4)T|#Cm|CRWBqfkY#w#zThx?VCleWmhXzU33ZKLzoUZ{RBmzK*$^DQ-b1+$ke^Q`YL2 zu~V`Dz=ArCPTyN!YG^3z36x_s)49D9Mc4J9Z{+X8<(r``?>5JG|9JAK(bMS0hIO|W zP+CQ%eJq%RsA1)|2^iwNmls2PE??0l9!9V9cAcW{`xn{i?WxV>YOhJZ^T$Ah@D0|+ z&DMbZ3F)v;2VG9>Lg9y~Gd{-Y-WyBT$lCfVW69-hV11)H9;B0qFeR5Nvy+6p<7CC_ zr;KwI8{4IT>R*k_qe|>O5(R6mp;*Z+00Q=O;L`F!;G2zN=`TW1g{p%nem(7F#tBgWYMTIDI3Ef^UXHdNao~*u& zWB27N8p1$W$DMTveKfLFr#N<4bEy<~q|w%gPrswl{IKMcey?9z4F1JVLj>sncRaa_ zGgL(rS2T52w|qPtEc1;C#op{nY&%E%2!MM($8N$dTOO11Oxiv=4sQ<|TDI9+Eq|G@wyXi)~u> z`is0fmU+tK)EL-4WcOa_88c^UDsVe9^(YrM`T!i0c(v-QU`D_YUvOQ&^1)sB?zGOnsBCLm_8sEaB&--oK}6EGZ)i70E-Wf>6Nh%&+nrE-_h z;7YX!2EqhWrDFn=&O1d`hsAfH6}SHjV%-0zH>VP5u&;^waTSU=Cq9rm_&x;6A&Yx>|%wD=eWoLzI_`C@05ZDiW1lfa-hmT7dxs7^*um zKUoP`f6x5*(KO@Ymct61%!U^Qlqr2LL}+bmZBVRl@lLMJYyf993JM`@)cZEev78s! z_HER_D+rpqQWmJF=`OLaDwfOjl?g2=Ytrnh{5*j3@$Wk|)^?fs&^l)@JKT+7B-d|~07&Q6bo^d1n{gtjDY(Z~L;C}JY zYuC)F{}-NZ-JRdR!3MylwjAsd!NNRqAG5I*IbY(7bGO~EcEufAH))JGF*Hkl zQMvnrsxGf>LSfu!w2CCVwzq48F41+syqC21uP^zQn8j!Mf$w& zFjVsmZsTfkF=x(qu`_x9Z*psyB4IQwje()>n&O+=%pP|lPjoxIY|oMcUD<|5>=O}#(O2QT7&qPtyUbAauCiGJ)`;w zPN^dzP^R+)r{`3p2xL4wE$Il8rJ~7Yy?3)W`ujG}9-ItCt);)=7taL^^;ofGH#Ynp z0gv2g95$ny6RCj??Wv=`u$J-=xpMwK9KOgIvyM#1g6WHjV?uW%zTQD$alKv_gs}YN zR-d2=75zTtLS{!&TY3EG134b63hCnZvO1=vnAhvlN9sR6hx3fM3Qu48i$E-yP82D^lWCq02B}FW z90T-fCZ407PvuBL?E12YCmI;@e)MT$Y`N>$CC3xe(x`xk8Rj4VfC2X22@rbUoA7(V zp4m<^mwpB%c&zgAiZ!RNzU4^f7ObU*PG}uD!P%4XhueXY<_BNla(;Fu?l)4O-GkYP zl%co^DQvzM^zp7r^4||Vdw*JB-AD)iVS9Vn{Oc*4P;L;7GHF zKTD|ONs_ikf}dc78%6q6LQeAl$$$dK`)eu@1>Z8qG0FJ1S~)YVTwIKOhPIJa$LDX# zaEk116w*4wqDUp&2P7Qj<^5^B|V#cV$V)T@W=eZrA|4t#G zT@33Dl%+PPGQe3m>{t#7U7(%JDQcRXH5aFx!WvOOWd76>dZ+(99ep;gIYme0Co z8#=)e9-7IDUg4lf5Me%%-m2pD5uZ6{>&jTxfd9!R+K$ErhzaO>dn0u?jocx1U!n{f zhX|QTmKwX3Z4w$CSBRb1kMtUK;+>_{CkUI0@V|Wvv1Z-QT%L@Ew}so?j+6z1FAz7)yCzlN^1D&%Z)()d>fmtsD|x@agf&6oL1lKmUL@qQd4w88Ua&x zU3dxw>!K22eOAYl<;0~hgojYBTMxoW&#MD!u7is<#diBnFlm^Q@V%f}ng5+k*k$u; z4&FVpDJ;yk!r%g`BcQ$^+Sr59!<^cv!)@9)H5{nVf?w!#@Z9D(KmKMXMjcCspAWtH zIF9V&5<5>(>o8i%8`EgPTW+lvp&G4Hrv#KPmng#A^M>xvsKdLT^C4D4m5FiBNcK~` zhpyw0CiY>lqy|;OYCe5_z5Vm6DW^xCFN_mcf{!|qa{ROO1k5gcQ-#@VOeQLAbIZuD z@Nx5Ua&ndo&@ELM4k#m?lf*dItRWLS{G%x5>?N(bh2#?7PWv)%@Yn|mxEi+R;e0LV z!tX|%AWr~X_Z%DzeCXI`maadP_ixNKE~)pxLy+D8MWc*OA(nfcL?!}}I8lkf(^Zi{ zeQ5(=)nWkfQH6kFZ~Tv}Y)C$TI{B>EWoO8cc4~Sv6bP3!+zcl=>mWxThFq_hsD)l& zdY~g}c(GM&%(+@vi5v|QCm*|s9-^e>|8@AINUrN|T*5*+&4m5%7d{$2^Rub=(d=lj zY-C|3^|_}vld0%>MsrEwuezflV2Rndy_5mUN%1g&xMBMG^HylAw7Ad{f3IBNY!S#U z)}cagg}_nS0%eo|Zp&2iLbiXD!h8T*$uDcf@R81iV5f!#(=@Gbr=vFJ5*f4Yq+I(j zvj4*=F_Zwa(G4zit&u1UAF>%@r{Yy@gqB}3JlwTa(9IVq3A3 z4dtl?M1f7C$c5y?-ceUrVqt?WL09puXf|0nbQgHuy1u`DVM_}p4>s_AGp@`rbQ+y= zoql)-GyP%At}H)n2DJF$_lgkFbFf!hc*9$RAM1nl8W4HUHN3p&A?lQ=su9u%a%ug1 zR<<{Vn$G0HS!@IRHC&WNsYsdALN2J1L+;MggKMpjOug>W4-_+wr2=&9wn4YAIfv`% z7;Zp^ci$ES9-wu7P%ae$Rs(ad^Q+oY+$>FqdJZN$5+xv#Bvm8Sv+|h?tX#>cZVT&D zezidnR_ve`LO!Sy3jOG?R+!qKiBQ4iZkH&llAR@DQ|%el&nv2_zGo57!6Yk!gAfTW z(6)qNsp?Jmv6W28)%IL*YPkJ$^1oXCb5tP3-wnY$JP4()QrFy7uF%TX^Vz;5@h=B~#$)!;H|i zOn<3yGhxa}Vv;Oo!Ou)$9ZmtSO~FBzu(k)>Y>VwqboIyk^Fi5!RG=NIQs5K)#gF%Z ze(-Qd?GetjC%PF7$hck`Rm`h6g<5R8o`A+VkGX2ZouY^+7(VqLaGAJanhQ%6P`3_obfCK41vuEP<@OlB{y7fV>27n}2G9L$mwL-BD+M8|_zJyIvcVQ@u1m=x0A z(##k=38cIQ%~@yDopBr~LCh2ET_UkDhp9sHrI*Y_zPd1ZuO3GgvPz85J6JeF6PfVu zp2UfESq*{6UJvuPe%o%}E#2H=1oU>Q_cxx6Mg$*)0Ccezu2Rj9beD4l#VyEsp=?`m z{btS%G4&LxvbJTS$f+N#m~1Lt6t}cQBMm)r;hY3i8FLb;yPHof0dX10)w-(qyfJ@) zryv<(HJd4RJ>zN#cqDIwIUL`0zYD^fA&Sv-ADt#}X7KUwraAQyJo4s2`vyYF6n3mr zw?ywIRVAGdT;DR1qmg&%HncmimHRB&X&36H_Y3UAX_#T>}&*`NcTF z>U%(Z&D`giO|S4i=l+ZPPxcDHi;hFe5d`s^AZ^)`)*-s!2P1|6Bi!sS&@re^>u~nV z+lX-A(uC4&?c31HtPKOautS~u}4m$XQhJ$Gu ztXiRCN+>Z)B)2V~I5?=eixc@4B$~;-dgNBmQAS2@e_m1jGYWxM5uA}QTe?5W3HtX9 z>#iMS>_?30e|53{Co2cHk!XSvTx8MTiv+t!sfD0VA&|sDBySy+HYSy1a=0p$V!Q+@ z7RF+A36r4PtbtIP?v0?;m&mbp=^T-{R*4Er&iTyGX19M5S5>B5=Ya5eM=L?AUBxZ9 z3J#i%S|NoMn87d=fPO#;j`RZk1c+8vH8u43E)nbsq3@LJR?ZQ03+Y}(b0nQU8lo>! ziDXC9FaSzoc|~z0ceqf0!je?kDaO|O{NGyu`xw=aVIOtY>I2W&O%hp2ohXT85nvI< z0BZo{IE>JUQAFg6~i^{pYf88_3$V9aEj*1fl9GU~;T$Vd0#eoLP=C&e+k z%`&nUPwezyjA$xjC9nLz5^DsFPX$H$(8L6_Hsl_FlKbK_M?2f$Tqs|DK zr_f2m!r%exUbwc?t&FB;vX@oMHA9;f;*wz`4@C}M$I<^jM6n0}MKzYY?oWHeXUutc zS!oKwdS;1Z5I|Wu)eKkKS*?9gbO+SYa6a5S^RPzbZ%f1hZ+tx8{nC}*=E$6{pL=qh z{C}IBEuy5}hB1pk^Tu&fO-0qTnjK;oac^^}n0xPFVqS?iKVkm@GHp9Um6hBAo;`)f z(~1P6y%kj;T>u%4zU;TBwZGv`Z{h#o_UHB*@#UfS#YO1#DB&uh$Hlj1^6x_G_bscp z>pHg`f8pUy<}OY9dM1{)orEj31<&ic-bc2O-s0}3{gFRp8}BB}Z$)GW!a{DeMlsQH zz1JKsXNX3{-gnP`@Hbzddao1~{2n&1FajvHpWlu>uYZzs?go78`aDz9OZ@i#0A)a$ zzw*wdugrS|dx>BA(f^*KZ~s?(?;8*K_y6oCdFE~ZH9z=ull3)r&HwAUqvK@Wd#v?T zRZV}`0|~|&X3KmfNES5KBBdl$3Yj?oidzReB$1?UbLMKvq(5PMTaolq>e{hbVo)e@gpZU}$ zdHow6GQ#1%#w9-i7aIR&3aD?(6>Wq!ai1g$s7Dv0J{UKqT5JsA`A8_HW(_|&g zr{Kc@X&{+Qwz+<@hZ7A&ZE#YNBoTwr2&p}FTcBtWM@M9hr7SG{R$*!=Ymb)^UTTaN z>>n;TKHemZdjv_3QJNrR%y4poyu8J5w83;S0|8;IF|9;NiS(`$`AdOZ*(y7_i89j~ zbg#$3m6v(`qaWrCkA5AoR$P1OC4Tt>zs{Mj`6|BQEpKN~v{X@qA8L}K<>Zkk-=A{x z>J5&r>~iVTPjmI@%N*2mZZrq{`cJ)teE&b>jma@~n-UYm1QwpZeud3b7a1`m?bQUs zfHtae!wL5!W9T18YC-ICPFNLT*n$QFU)-!L8_`N@`R70W0s8O#Z~6YW4PLo{R!{sKKlJkr@A;?i{_=L} zmpuOm&yu|5&72J!*M9XU_?Oqeg@5s1y@mM8+wz^Gy2>X$V|e(jchmbbRuuNu6a2yK z0`GY6)n0ITrXTt5_>tc_%MbqeySZC^>5q>1@W1|1K63Xz;K#oG0)PIpPICI|zK0($ zTX*he@@F8@M|jr{oF&=rUX?rcI++*v*l%Cu)Vux(f9LK0;osPN>VM{aKl>SO_b!Ey z1idZJKk~JF+kf&-?isvxmx-_DYv!)}JAUk6UFPe5;(dJmLhf~hm=#u95Vbxyg|oZLEwb&}Z&&vCF@P%UdFqb;m!5vn4M6w77Jd|tC14M>vY zm9B&}9)v{+k1;jQ2NX@k(iKdzTP)i{9=`u^Y$Hk1h|xw$U1pp;dzRy;{!^NwrYoJ=zIM~@E4kW6t**tNQ zV`t6~1mVgMKwzz5dbr1}YnN$_Ax(QcaQ}Vudnpt-her!~gC5CvKoSqBTf>R1i-f~1 zw$I;>t?#04#~4-e;G6GeKHFn&_a@nV>VPzHc*AZRSFR*tgC36oG?fk z`h>L5^lVI1Olh^lwW`y~lm&})z+m$j&IzpX^hbU6mN_Df$qPd@H$3(9Q=H$}CUgQR z6>+Euf{?1NG0x(BcS{fkh8sg%PjTwpNp`MZBWnxZaQ-wO`uTs$&;IoP$Xmj{$vYo^ zE9K2SZd|^>&Efz%NvXat;}akLAnn1NaM5MvFQdtU>}+3)`cAKKacfM@aCGwf^0!80#XzwOagVPUxPg%>av zzLon^!Kc2QotDq?;a~m)uYdG0&IS-h5ju|lQWw{MQOxC!@hiW-#f8WJtRlv7FMy=;8h2fAj>YlAy>f44Xy9TA2x z0~FqAL@F?*!bb_BYLG~B1zTZEku9kLfoQ>dL6Sr`>lg(Cj-NTklUH7#&Rf!659>Xq zFf8XK{WvDA1u7CuHu?1j$QkqQ>ejJ6c?l89k%KpX`uvzFb1BZd==nC|l8OV4okg|8yel29vxIK)LU!a}H0 z;{Jev-R9(ZNmVzrPEb}k7cQLT!d>?;Uo0r9oTKTK(Qtxo3@=@JnmU^@9!$7(?P+?0 zF?Zej2FkLaX==Q8%w|)BgyC>RUY0az%EVZdjxfd$MiFt6;E|{xq_&+` zyUvs44ih>?n-jvIN4D2+^3-XfNK)n*MLs9cg0wdvuPUN2#tTX7B&pUI0Z#UiBBIfd zEo%mt{C!TtWzC~f3(DH^;z}>`A*4lHlq|v&eS9FJjYpqYZjb3mhg4o_E*UM z{RjBDU;gL3`K}(*xhHltUiaAzOZw%LT;H$w_!m6QvA0n@w~O4gJoj7wJI~&PE0>?< z*mwS2HqM{IJb4{kl_;fgw!{VwsTIqYuJOme_j~ZI-~460|4%*i5Tg%0%d^wJ#_3G~ zl)UiFHBNu^n<<`tmMiAbZnVbT;MwPM&b<9WhO2dntgrFmpZlNq;FCMl$r&E|_P@=0 zA3u)5lVAEB{%aRL&+cTnXoCxH{Hwg@-EZei^dBC(uded@zxYdh=(8^~Gn&!4ui~5E z{hhqyd<@0M`RO0}C>Q?LS99>Yf5_$iBZ}nzWbZt~EUT)#|2z4ddv3nBVpmsntj;-+ zRB{kAD#EC*X$*tDV|K)ZBk1U;6VwqMQBhP75D6*>C?uWIn-E8feh#Npma^+)*LXKy6& zo-gtC`5_1Hy`Ep*xrsyN2(vho)vvmc*R5yNx6)S$&@vq?Jmbw=bjC~)wLgadUCNRT|MC}nTFD8?$JjaNV|@N~U5C9$g8tur znSa@`n$LdXY?`0uS_%f&anp6T@yOmWJS9c@f>XKR!n2vJJi_-r`%^agA=1BngojtY zn}2!7l4pK2C7on8NAB=Ha=BBu@U^?y_}?4Z>72yMCj@JlyS{cg_jezew$EIW;`-10 ziO!3cGVsR-IWST|opB!Td_yxEZu~7<`$nnRi@4yuZ|8(&dD8Rv`|G&zo^AA3BaFff zR-OMA-mtnAO&sEuFa0YA&b)#TojiH#E_!d}@~`dZ^eZl*^PaC9n{0)f`O(L}PvfGm z^ABsD&#xJ=@5V3kUzM}CIJ=oUw+t{^7PPE+Gw*)WVyd@2O|CC{Gc!xiF1K-QE#Lmk zTDsmelkGSE3H991@Ve5s`R)G ze5{>2KL2Dm*^ATvXV>>5d0L-=r^yoJD7@-IhPGb8!<+kg-Ha~eaG3QsUCXVH>|wYb zlW6JYlndU4PW>Ul?%SY zyO$XdS9##upXQp+e~|@xKQznGq9;7#$fQVvuCC0HnH4LP(U8E%-)_C3%HP zLT1-+KRJ?kE+mpVzV8zwkR^EY%U(?;ZLojWUWV%dnxv7pHEJSc^@_z`gjQS!&=Xl(*HAoOc^jXH5`5QzXcaIq|n zl$|6D8jM#RVp%1VZb6GA0^dW{1^Il6iSary3Q;68Q3^;VCBiVK(eMzmjvvVceuH$; zW?~?wTJu3x@S+AN9xYQ6)T@H=i8?`JFU`g2C@8p2gV1*{%p_juBgYX4LEr~S5=c=5 zflQ{@Le&=-**r-zg-t+lY8S$lk%Z5(g$t0Wf~TFkA{0fzbzO8_N09^(0dj1jqGDPGswyKygr=&*LLj67U6qLfAFp1f zd(JG%V}rzj3$Z}aRQ4b0W3*Id{=9{>x3y9#S8=?Edfj7U!X*d;qJ~F}3Wb7+E<(1i ze-tzsO*L`j3Q;5@1wO9lP^$$fs)8E@DD_c7)hAMuSbB`&*0E9@ec-wc;-89CDX9)_KEClV`aBqpRIN~*@uHp@67#!TQk7Sf4rH-&=*CuAZ z>;p_qsBC@UTDA<;!a7;9$!ydpB=fMD?9hp4~h1AOW22K~RklK;5jcPw504i?M1x%H}>8Cv-vF8kOl ztm-yy{Lk<6le=f}=~u4e-5>6w?;D3W`zs&hBwIpMFXoa(8I)iEQC_2-cN9A^M|?Yf*B}{hq+6Xd2DSjZ+!db zcyp%A13&sEH{G(Hm6x1EN*v<1-~9_*e$_K*J8H-> zOgzGUL+|ASpS}p&+r#z$^6%WVb`_sIyZFS8-DkOcN*cB2H~jRs zRZjfK6?|%{LH)oVx%$d$xO3L$cx~a?_%xl$pGrJ*=c62WRTnco#@smQP;J z^o+&uZ=PfP=^h)d{~>>H&*sBdoI!^=#=2|%gKK`?&a8iY`7_ooSMTSJKa>6<_22G$ z4t<~ff)x;1;)$MQ<7+e@)$9moF~yktgBiKNFEVG4D@%W0%?j1BFkwZ8{N8^Xw%lu8w3PeN5>{6G+RAp=9h6rwh| z6K#x)?_jJ_!A_^B0_jAGmvzr!U^rlJ&pwi-Pr2rB-m+6^U(?RlZoGp@Rb~Ubcu_zJP7_kWH7_v3u9y=Ib%lN)0Izr0gW=OcS!KGEuMNG(3C} z;Z*AsvUy@ugrX1&A0a8!eF?*~7%z`wL>~D<8tM_{kqR|eC25(IChDNZG_^^XmW=O1 z;7hoHN8jKi3maoT{j5A=y0;UhL9nWBVt8G#4@^Jp_vIJNg-2c$Mzcx4-K*Ik#)?RIg8G& z85o9!qNwPKg05<03hhX`ie%U%ixRex#PvM9poC_qC>ofig`LV^CK4E?N$ZqpBrJ;04qjm(kHN3i%w>u@Ry`#%|IG zkr?kA;rF-PfgAY9l1{O$1FM*%cmE-r?BqCK`~J_U?K{NVFUpaT9^>wDoejfvYL9P1 z%h`qXEW@ zRV;3~i^uz_#Iv(6h_y3MeJv~6Od!FErJdY(=j7apeGl@`_&nZt{sM9upiSjv=dI@M ze_O}njg_oPyzmo29Cvc!8HaC^&1uZ((s+1i6v2~540<=q7i#=?(=NQ_OE7TRv2iEH zlD}t`TIHd&TgjaAPn_J5KnC&)Udb7=?&o*w4{`p~B$5=v%o90%RsjtN8x9RgBVijT zl1h5|nOyRfGm&M9xcY3AN@SOvz?_?I;gNl>Wk$D(7~aIj;hCJWEdPx69qoUFb>p*n z^{Ww@t~)p3&p4m%r||q}Xw#W<;t8~U;Va_;(n>q4%NN_)~9=C2wp; zMk068`}oodg;df&0y7!-tMWp5D;pGSME zj2PI+rqOAfvMi6-clS%SXC24?4!mSWOrznVYdTV_(rEaEUPLCNA_dr79$I3Br1WtVYCuswcSn@g1!9cA?~LjMi4_wG!1xVMg}~9^N*2HQ!~%ycyJ;faaz&i{`Gu^BfFK zAz>MKjRs!BWn`kn&_o3xR8m=;1A`-Yu0yO!C}=bqE~-44$nAO#ajZ?QIRTbs6Gt)S zks5{~k`4pFVTG!=8GYCxkNvVGSUDlKJ}EMAF-!PGST2ozLg zSt4Z8;}Jy>k}RWZ8mcTKOEIb<6NV8o5`q{{$QbDCMGBpil~N()0ixp5D1A#2u8*SNT@1N z5E8mI!qCGo6{bz=LJ~6hrdB$p&7@LqKrCbHGO`p9gdTy2(RGVh)+m((iWFlgGKQux zeTIpqYeZp0!q6D54vu9xiKI=AaroTD?iq5EB=ekE8#mXhgHMOBz7R?<6 zN`u3!TC{-PbsI^Jj4Gi285$zcrFVb+_-_dT_ICY1ypo*SwD2+R4nA95sv~{$RFSOH@Q7MlP zU?y7d6_vs9K};mVFhQs(pg1V8f;*uQr+qS(!q}LD9|~ksqEU4*4HZ>Yh^*+Zumc#( zUNDpB<{j)0myq7Mo5sxZ=u%DGxl{Ss#@$rUO0#>%0nB-4(y4Ixs!=AB&pr_+FR413 zB#5FIBtdD%@A=hTo9HP8NU}s&FCpd%;%6rW%tf4g{sVmXs?Tz1dpGkJEM(d0)hujF zJmaULJj!U8XGzmvY>j$>LfU8lSdDl#AScPD|9nW4P!t8Ci0Ft!)Y$*0Uvc}R`xtdW zmLnSFh$)dc(vP1Xa*|x`&%q%QnyMm25m6lDjto<(Z{f=SyY8vaC|!(|V|Wn<0?Ejb zv7UThQI*Ndl&3(ad&NSOt2eRRU&2DMlZ`tJmb`Zsn&@G8)Tih1%lN>}PZ=4*m_U#O zWO7Y~C*Hj}>kM8q|Jz*mg->(O?D@=Ju!L1>meTn=*RxV`F()jzfnPqdi#K%7Wn$At zMrOZ)rJ1Kc7vqkOQcJeb^cTEQ3Qgq2Zbm0!Xg&(&I1h2tAd#7UD(`yd zD=1139xZJyAlsype>Ug^Qrr^&03ZNKL_t(lR81zCN3MY2Ay~f9XZ)`(MfuAff=0U95V``*`i_!*jZQJaW@* zJg{>JN65&rM>R~*6#=5mco}kqCkrM>i6Tu&iSjY6wT`uD4sE~O!vXI!*t-vD+664h zZs)!|eMAcddk)%kpWciNc+bIaskoHo9k=kD1;+5WIah`=|bY|Ytgg}dIMr@GH=F8q%iaX zerOWrGDukyJ8Yna4WuYws%>+qRA$5Y7`YZ3%a0MN1xlXvn>J$_K4&dn$)j5yr5uMS zPDn1ZfQ#SuLGHNsdLCT=OQv=e@S-YvAAg)EsFOBjWD#Q9Hl1_kp=L6C^S{1_QyxGb zfQFbnK&NsRhNWPr8g9+S(iAdD8_x@fMNH&3P$d&xHn8msFI&BeLxa8CyJ-{EdPM97 z_;H0sU7)HuC8b6&pFxd7D#F1R9x(x9dmg2yA+RzTlGbEpGg&iH6@^GdXu3{}fg1bh zgh+CTY)l$5UKp}$n}ZNPm`wQhTn|kK&#e*oE*U#Xu3%Epir8{Q z|L_==F5`L+NU%L+k13-_MD2=5yj%ryv^<_id}P zr&lMF$}%^U>Facn449TH5=1g~a~=#C&rOk(C!2&$9Q24HiRPwOW@hq?^z9{LoaW{n zZe0ABE(TY4oIuY3f*pO)SZTQTQC1eYB8B&Rsy)FYaY$-vrs6BXq5vg@J@Vdk#&v z>|n2xXU86i?h9u=wd#&^r=dAdFC0n^UwO(l#b>{e~Fl}KV8j#-)Hdp4}O-HPfZ|; zL2m!*m)UwWk3S0$M`n+NBuPlwQ~Agh7adh8UqzN73<;ioNR0!6095tZKM(rs8S3G8sxarGukevmv(IE-Jj!Gk(qDMT)D}*D&lg!bu!}8Qwyhl26;ru&;}*tGU%=*#!^}KoFC^pE4JF`6P6?ya-9y$I=6 zHf-HT-c<1=gFwx2+AEi_;nBOOjEs>qY;w6Gv2Af^Xq4t+8^go>D8wWzjnEgAMrssW zY{u#i4NqW5F`+L|R2{b-VhEXPy@4!wbac(2CDqBJYabzDf{%t1M#zpJZM72x!zivo zDk$L99grMGN<)m7Mws5!Mci=MykRYK<}D(nBynqlc*LMc92y-Xn{7o@>x>@Ufma*F zv=n@y(eSFIVhOLj1+TUZk!U5GbZ{yRQxl}gZG?ImFtn1+sF8>o7cLMD%uNPrX*7zxs< zO&G<*G9-1CO6cJeVaF=SGWEKPrm9$yzzGEn&%?3|h$js}RRJjmRn@4}Cyk~?y@c;q zQDg}#VG;-zCG?R^g}ju&ZPbz7h}@I{9UZd>Mn-wudzfm)$Ej4QHX5`~%QD#4htn9L zqbZ4`=*(TR60cSvZ5j0J-^bL}BBAFoUKT7nX$7w9Fgi3qI-93n8$%ZXbEnTiQ)3Pt z+DERrm7%dRhx!NUDmJ6U5QG7W65|CPrlex%I!?Vt!fHW{YlM-67brLr74n%pK_rpU zZL+B(*|dvgm{hA3>NOWTktPx`BL3^tDv~nunBVp*9^bKvR9_o&yC-jyrRHvCTMw}F zksN!xnVj8yq=fjVBc{=_pW)16E?VA#ApvoPL%n6h^W_=GAx_Q5NO!RCrpp~+xNOo?OdivD5+eHc_aM!A2d7M(455nM zJ`RoiH9TF&9X)C^DVpB;qJg%`rgUhkGMcflgzrl0Yb8ejnOt@@sl@v^@{xBF#mKjYFdd(v!Z8xWdS& zPi}esfwy*!3n-+hXix(u2#2^O_dQ3m><;+Y^@+R!>VK9Qu$pw&0$2>kZ zLCj=9hqyk>P%Y-^cK`FPRZcNymf)6M>+yQpn02v=V$NXd*n@1_5ivS_B{NhI!_UWW ziNccQOucmrTla@-8lK6?OC~Rm{@U;UIR5wHrR(ezMMQppu4x3mN9cp=2?`mBGc#kn z&D_oMECdygy*nSId~gE|$7QTGOi!hcy$2sBE)Ai|F<}%Ci-=04M8owdR~`23KR}F3 zSgYgh*^WbqQuQ&hs7o46sRB!vokTs5@O=eMRbNg2Ry!g3lo*y^!4wi zqh&5m*i1Fdv-?1WtFF3JP}1{Q^Sbwu=$K9|Zed%nngiAvoVvyQjy2>{ zi}33Ws--c;CS11eI*5=2ny!*cXHis(WHOKEduW=3p-JS^X-r8%7am^S!Sia!N)lHj z2_=I_#3*vi2w-R*U>Z`P16u@j4Vr7rtw5GQ`d6|0v|Q-s0<%Mja>{=Mp6~* zWC|&cNf|0#ZAGSbcG1?}%B)%4G`F-9u$=CCSwj zCHiYF3E!Y3MYyhmTX!dW~na<$tTg?s?*XWqr_umY==Thpo9+lw{4)--^c!q8wmP)kjvxDOj>kmI(bb) zkpxqUEwty01W=|j zXO_#hKW<>Ga{+UYu51r_CW~1;z@9;mI4rU2_rGGJlY&Z#vH(;SRg@VSc5uBX4-TT| zH~j18zr`(&kK@GukQ`_w24j#K8pHetRd|Q!80B`&jb~ zWHPx7VefACJ26r1AotzyFrzk9%Vi=!(G)~wl#!Z)_Y4P^jH*I?beM|cKGS3JG$_;7 za6-!g?z-t=2HY6Y7+~GC-{A89zMs(}Jgd%n1!v6|;f^2tlzX-vWN3JhgWK-mXW#u5 z`&v)utVJ)RyN*ux@`Z@UZ|Ba(EtamBHhI97QY=1kAvw$rim!9i zPX7qw#Ejna9X|K# z+gP)z6Isg9Qk3c6y@y&1_yeq6yPYRFeQdIY7-GZizhzzDU&BlL)VW31vB~Z8K}u%H z%46)?Gfo(LjBdV#-|hs&B&zS!yYkyJt|NQ!Xtk62o;o>uN9{%}_Y^%p0XUJw^ z2KVftDg=#zb=-JQ4~F7Ut^^?EnLEFgkw@-g-H=Pz7~-+tJV38<1V`NSu2p5)oau}{ zaxc4+GnkQ-Kg%R&BhZx8Kry0N6A;WtQyB`oVW$ip|UXK>a^79YMaIMzMuIF94# zc*#uNSVRQAKtjSaOyp4D#UZs&;em(Nb5^&*35!o-Yp=lHd?zcHEM$B_qNFu2b&aHe zqJTh_5phI}KsPlsS0M-j0^dhgG;%4M*e%m9KnX+AW)df%;#bRzj8c90uBNofkpR<7ia_x^z`8`h%|;Z$83^)y{mJDDht&~RKl zzk!`f({KcetfHwZLc~=}YarRJh*Dy4FXK~+Y}*$i$kku^241p~Qb4cBk1 zl2$wLERDq63yCWG$csI!S$IBjIsc--wncoQsYYDQ5F0xLu(CX58NhKnRgxUPq87}T5q*PZ;F2a$w` zd=iNS9aGy5C%XGIwY1@d5eZEttXEL9B&imQMyW!*RzejGgjK~<6q345-)J2ffe-?J z%)u~Cz$I&0BvWZD!@{puFm;7`y+*#N302h??(aiZ6oM!MSz`0nEeIhnEfdQ!n3x!& zr)NKzY=(}h)A0O=gq6ZAjnH#oKVcZrHEkOGgS~jZOD2;h3C(eQ6i(`qZ4~o=)xLybrvK;M!aG z<|luOgH7w)Q+U@qUQY2yJeJ!y{R1DyyzMsb`RTuNLp4B4wK8+*>-oo5y^O9G*pfn< zy_^M&|75LoE^q33BD;z7%D3`?>MyzB>dW|jIY7%!WBF^}&+F&tM;Jw-_=>ml*70k( z z1WRA_dN%&(IxhS4EoAeZtU2=(y0+~gir|=KOAN8`_TSRTozAK$FZS5*%=dLna{D}x z?B%@r4V$^@SC{dT8`Df#dM+2g`vP+R@jb5n%FnRB@OMv{|D7M7@fT0O82x`feg6or zKIYYEOgr~tPTlx#{QQO`T>AD!ob>wh*?#p;xb(v^xvo{b=B@8xw*F17{^3{2T=E%S zcjnu;VDKm0@THIOGkXflUU>#5HT{gi=;>zE^Q;vxXU(B~Yy*R*%sgD0Jk9j!2G`!9 zvZDKNDDLy}`>iZC3M^gT&aZxXh=uQ5^kn$tvF=&NaU4&B6vs!7mVEf(hgq{`%?lnl zkx)=%1zn#!g(yd>+#_7&@>Q%Ssc7G7TqW(XvIzk%Tib#DeZFw)RwU zeT8&3O{G3asoamI7q=C(mS$)(J_~O`-WM#v>7?ph&+Rtot=yijpF$(u>i-Z zlWWc*gdmm5qACWOA|cCyiScnn9HS^Ib}9uzLWCiTB%x|Lj_ZM{pc*=oEMcoUA`I|+ z2USxs%mlpy!wij#QEh}|azzTNLmZByX&GWQ&(P>Fql0~9vq>UFCDYVFd$EfxTka#9 z&Z9{hW?Ds(bfgG0D}iiVBn$2M6`#6oBC=UzKcq#I$(cGa5`G|X8a|q)p=v6U1P#Z< zvg{|q~!8;rlMCWe~~=Ml7fe^r4$7k_`R5`>9sO(R3Nzvd|Tad^XG8 zJv%5Ai-ci_q9_DmMCjER85yCyYc?yEEo1NYt%Om8l}I5Z1xYpV>J_|tjbx#KtZ4Kf z>LZm*U|ANDBoT%Yj$1`D6L0I}8c@08Q5k0w3RXx#9LZo?F3><2e4N5#wz7F<-nf z&HFxo!DL9=-wghiX*-VNIF94^|A%9>JzlcyGohOVK?p)3jv2tgzWVing5(4~-E zwuq%gR3|D_sud#LWcR*<#Fjuy$SmwGkku^i{M9!J>Mk=|+G))Z2#@{TJjr9*-T@@JhN-F) ziW*_iz%~?QwM4Zc6YB;t`s7W#B_T;6x~35Lb#z-smQ@;c2RmWoc^L?Kj5RTQA5kOMi!iybsw!iyS&jUn3Gvdr!7 zWXI0k2q6&#f;iAdpbetOP>U&@>g{`^XZgF-(k&6WcbjtP;yAK`0Z*KolXNFg`knoia&h zQWUZoB1K@NEb{4QoT|s1`71Fs8AXxEr87)Sj8k_e7#bZUnX;(XYUox9TeB$^TZp1~ zvWP*hiSdac(z!g6EYnnMK~-fu*JYwqqPZ=PZCeDPOB_Mqm5Blm6i`eJLpRY>ou*Wt zh8rM+m@GjX#{@wQVxNie30${|7kG$z9knt+7`iA@Ots=5%abmWhSR`lxTF%vmug!$ zj^lXAAwmylcq7-}xRv%ZFJ=1Qkdg4WOxtlB$8j9T|1`X0TWG!NfFx0ICc}0tUB+|D zKu9QjWFRL&JuHDPqX!L;1yU3+SRP{I*8QZ@MGDyhC;>uKsO=xcsn#cRHDZv7A&Qx` zdKJ2>AxBlJ*;e-KJ%ETpVq`LwL7Yg^rex5pEVJ7SBr`UhQ@b#YkPd&GUtjk-Mu*0! z%Le=Q?PvD%X>8fJo-huX)xDH?3l@?{BuFNkal?r9k3C8d_z*)R$(UvWrQre~%NmlZ zku0QeoEk&rDxMotZ#+m-F-ZlBN~w(7e~7fW3s2U-3JJ&VBXXxwn%2qwgO8#+0e&z* zE|KOJSA7FJC79MF;dlX26k?h>g@Q?@C}Dar&50DF6E2Y);>!lZb%7hGB(oCbYL%hB zh>Y#gImO1Q4Kq5Xl1#K>+9{$C{DzON>4bnJ1(OxV!U$QGh{6cVl9@Qz%jn1mg+hT` zE=Rpw#q>j>C?c9D({LL|k{}ilLc~<76=DIS!(+sXglUx-FO~4z2A)5;jyM*iQYkW- zEFy|fWCaPCsL)QST*j&UB+NARTAeWP5HWaOgLFDeY&KC3Vr(C(BEa=s46i|=p|fSr zE)E^o%dGix$aT-5Ty;=n9XT+G6(1+^k%$R=7h6}cbP3n3At@>%h>?^Sp(+STFkTu% zm8;}3Y21c{Bq>A$L_v+Q5r;UGNLWd#u7{>7I3pFDa*6)IeoEysfm3H!3`Yhew~Kv;?(OT5($Ke7#tiTl}=GBS0M&V*Kr#zq3;n%(cv=LA-ZlL zNfLfI+4R$MJ>poPNGi2j4L@)ZB1AA*r7a8(zcUe|NfIgBpwaMXGy;$zmpg969LN7u z2zqY&3SYmwLFdVDbb zSw2%%kc3VcM${EZ>IoW^GCDd3_Vh8((afQdF}4o4D2$?unA!7|ps5C`WMU`^fu10e z476yRNKx3jdpC9n(b6KZca8vU2fabUmW?&?x&i?nBX1G`F^*C0)jp37RLW zXvrK?mKTZp#&POCsfDJP|#NzqcCG0V!pRCC0#z^Tc26@`?YL!p5el&E_V4H1(tET*=$qK`(5S1VY$ zifjZF+ghoW%Zv~CWU@)pR*b->;k$TIjA5kE6G<9wgIx!D2}6%uK8;g%u}V<;-2h-kPj4aXq}L+p%=hy@}N7@0JNqERR|6HBmR(`G95IubDn+r%(U zG)=?Q6KJwdQ*$0!nQUaHt3HKP0%8@XR-+zOL6J}u71wo$L`Wi6A*?mnJ2prVgxF~V zFRUZSF^VM7-?x`^N})2+hwnus5;kpZUHB0|gsG~?64YghSe7YQs>q6lh++nZM^Oxe zMs)%$a!A?|LK0Y3g8sfCdi#2r-93YBI*C)SuxES>KU7KDHeobrL^PZRB7$AJ_hQ?_ z_+CKOnZOTRd_SOA$TKuDN)$s|TPveuW7xKh<2dy74O1*;v2B|uiY5(aKfrfgR7J%L zLW00UQNgxN>Wv0&7-8EswVFdJl|WNW28YHlbQ#Ms2!%kz5wffyLKzXocrk?42v8A< zj0gj=$rqB)eH_Q}w*Z-Gue*$IzwZAx9Y@07GHu6k9LI4S|MT#Y88K2knXs*C2A&@x zNii`Y#0tq|np7%FAZz%pgPCnd?U+sD&~7&G?qS@EP|Q5;#3+rxW8>CM%$(g#CYivm z>vXL?i-~$n&%?hV@FY%|wwQzQAp6@I=z}#fN{sA-8+kN@!JfTCOw>mSbdOrCPB2)* z%%#|QXgd$=*h*w2p)TPL4)ef+_cMKZ7pkH`9I|=yBPg0cmK8Kb$MYn-Mu4T;Byn%`JMTEV$~ynQPd~SpnL9O^OnM_BJrqGoz{*-s6w$T2w%t{1EB1oBuB+>=4J)f6 zf(;c=nt+9pgwT?ZO6p|Vow@z=bDs12;|8Q86HxHQ{+7=_d1dY?-{<+3^L(G@Tk<&) z=_F0pLW(6EKcZ>XL4r&&NmoaSLLpC2Zx^O%q8kYoE}BQ^``9fD5esa`B0AhOtXi$$ z+cnY|6;(CxJ@ABwoHB@o46(%WC5zEi1x;1RCJa@|wq%%o|$M@qnHmauKdLA(`$l&^JJ7>qW5mCE`hY%2k0^4?pL`W1w zNJ7xE96%zDBccf6*d+`>_zITip=$!9l<;x5%doH8* z-$H!lO^8^Wo)M4)RXmS-W!5JwW@qr(Kk#TF{P zGqR*QLR`lHT0o`0CZTI2jTGr=dFB|$;RPYhR+GFhQym#)pnDF_ajtQMN(qC$U&84R7E8e4y95F zKN19vi=yf@Y!5f^F%5%UE>GQQ5qg3!h>-xxvY4z?@LZRPatjH8A`9%sC;|xLkXGFx z5~9rqRf8DpmP6=AxL!ampQF%~r#wE%%)wro%?T7)C7H|+cu~74C;_sf;CU{xq>xIh zL{W^06m-ob67AC)Nsf^f5Tac^+cY#JNp2V5kOaxpQ`0BrSv*NN>-qYJuBGR~PxA7? zr?2|`Q-!$sC%*ELo0<8+k8;kmCqeK3vT*_s4s+|bzRp!!8q9s|#k}jJ%rhtyET*~T zXMFYhzh}rioeMvCF0=oys__3t**Yph?)%;cxZHRf7hf=seLwja|6P4KU;KwvN2sd! z3`RV`O<(vlo0fl!51v(e(s~o#`(57u;{DFDPL>JX zI;C76iOwR9>wy%ZPoItJh3wk&TM~AQ6$1-#Y#YbOU_}C{*HSQYk-8Qrzqq)=;NeeIJS-JIkZ|$!Z>E4TE}Wx*p7wccp%AGj)y9@ofugHNeE;` z#tB*gq|+%}&mjy1GN8zi&!$PGk|fh6bLP*YqmV<07|XKo{06QUP^&q_BEWZT0ym;I z)+7?3DjH!B5DS5(EA5ie2@}_K3EVM+Q1JYSOfJp*Su>GEjAhyQzK`STj82sCf!GVM z>vd#&GN}w&B8L{)=!pcXuF`Ou7zrID4J9ejRqTLRL)8>gNsrEAC$c1wN~I7&U^zCb z6q7VmPz_=|jUpoC!=0CX-vcxf83kE|X+87MlL3<736g>rMi{b*t|(MnVntvnG2SY|G(9XHu|+jbIbS2O+DPut%NPn$j%nvNua5Q0=L&D^=O*?Hgq zy)$Rfv}&wey^O%BVNFiZKeL;~g9}L%I?!X6J)7?$lQ6k!^ERe;cahT+7B84fS4RQg z^9ZAeI0jkLNM*7}nH-f`gWbCyWZ{ZgoOJpNIXE=Jj!icccy;z~y_(&d?z5VRYF`o52% z$V8@zp?4zU7$F28(QG!TH5zD|imIw~6pH`^p+g)-gd)N)G#tl8PbEnt6+|qEqNv@o zN0EtzAPQpign_E6M8Zcxrq-B5k(I-R13D2xAmW%PjEJHL&uiy1mSS1Pn@hjc2%o;`aQ9vvl-FtO|=e&7(tf*_2MWEIbI8ORsN zq!J`m4bS!QLxI(5peGD+xeSKUrZ<^Ppelk!%W4;i$YxO!8lBx9A_UL&aBYW$MwTcF z2}2)78BEh4 z^c5rt3at@TIf>`Hs9Ho2L}bzhkYp;gN$NEVLo*4Yi2Wm@_(6bS=$K{#MUu%RleDUJ zq&UV*I-tf>s#T&W!q7E>P)3Y}s0swGLf|*h zi{AVyrsq`Li3izq)3to#^E-Iu``*QAJvtl8|Cqugsk8hs$zX~ypdv4=_o|P=CZ{m)f z=P+y0(d9n&Ts*Cd>3u1TC-v2g5d!3=RWLkUPogkqfV`DShVCg{O8Nq;xNHj#O*rnI^l=c5c3E zJxiCbV!HOs(3k%Uc-ov8U6WBY4MmpQrvf6Px3d$wQKh4!gM)i_5Qiaw=Tf(9YIO^t zsDzQr`0zn2%VN&-euUj*&GO@!HFE~FYL#pzO)N_onn}G@r_|MnY8WKrfUqM?Rtgy2 zxgIm2GrcQ^sdiJXH(_N+Qj^h;aBUA+OVM4-p{YJ9K0QVeb)bmt**Koh>SaqvB~yfc zh^9+~L4fZyF$|qdCPOxp!gXE3Fhr7+_NhY{5ruNQFNq?^rVZlQLnfe6tB^{iSulSl zK@egl4HQLbSD5lVOhZA{48l+aBod~Ehyy$?#BmygVSpb5n1+e2fs!$B90%928ERG@ zP5@UFnQ7Ddv2BZLt%2z)Bn*Qn2noUv*N)IN4c#zECpBVX9JfI>n?g}!90wEybVFid za_n#u#00+Yqw5CgbOKpXNG1~Ky3%eSsq4gX%%MYv&=mtof-v-PoCpyL5CSQNd2{Dd z>gYfS4MmX&0+->DgUE`8q8Zq&CNl@7BS``;^aw+lP{>HCL^h?-na*N6?IxYB12{A8k56hOpSClL(6OTby2khlXaVh z+alIdOgb*DmW%HN1VMnQ)aWP_NF)*{ameIkg$RYdt`e?m6F4nY8FU5W&?j(x6eR(^ zMJ^@CWo1+)OLtL2Roi^;C>9BXiW4ZPQoAyr-KtaQD&mJdG@C7Qxh$6L5Cjr|>k_yY zDmjvxgBKZ0v|@B4MZ!p7CbLMAKsRKB2v7o-bW%rEbz~_^5IamX_oJ#wWW}UBQNxrV zl}e!+CYJ3|t<-3>s(-x!hz@e&k8h%R{F}MxjVs7LWYc^5nYC~(`7i$)zxvsoEPL0< z2DmYnr!UbAjCDG8#<`}p-0zvT9vWn4K+|B};r#d+(Pla>IF zp+EhUE3dzsy^|hFrkBNMyow9Xo=ZxK*z@yG@%6@8ye5A?H*OwbqAuuOdme9o-Sv+X+2l>|TO1$?I zFJs!~@A1K_(!6x-2*0_m#)%hxjF_sp?%=X3uV&+c7FyR5Ui8K{a9ZIWzVY#^ z$iC$>yms;7#dcid5C8RXZW_3dPrQ8a&x-{q&w@pL{Cxd3hQ&MvWFXwjwg(HWUX$eB zZM$$!Sw#YfD?8XeUSQ#(9%Lda_g}*mzq*|rV-}>knSbI-c-1*e=};mb{OKpS#5#i) zdAD%Yp834(Uj`q=a#6jD?|tD)th3(BJ5S4T;HDpQTzs>uvN^|ZR4!7U3n~D{ZnSVB~ z{rfYSX+FWlLp;I8+xIYS^@llQ;wyaX_AR_@;R=#RxINr@86Ww^?KHxeZ(q2Xg|GS) z?^s#oPe+j(TDS4lkKD$LSI%bJul|7g{EzeQ+|T&HRYm^!)2}=n>J@O;H{ZuE3jf4s z&YQ{ppMQ$4H_qUdiF>$V>j+h^z^Yfjfs^Xj@w40ZFjf`xpZN=l)fBPEL zsr1kr;!i*MF~7ZMKUGg5(=(svy!`)CVc6d7Fc0$%;flbQCY18c#7 zt3StoTz3dx`Vtr3dn)hxIi#b3Cc%Rh1n&iVhro0rMdwjOP)Vv666F%O?OqsO->w*GBpH@NGYynhP4eHiXWzWtH^;a*3u<--@U>6M@0?elkY)zNtL zES@qvZL`_m+k>XH7vz#8;rbqik-&8VG)Ft|F82A(m9XPJb#Kbs` zW1*@l&8EWG*f^!mE}}4M(;%XD-xWonR-YscU2NOMbwlDHWMXoXO0|krrB&F z%Nprq1}UhaDmM9~fs~Dig&?2qqNCJL=;u(}DsC8|7y41eIANqysOOoeNrZ%?Q!ZVl z<7l;7L{UgkAECFWglU?XhKc7}_<=;BSio~>sw?q zIZPu>7|A4)DO}g1)oNlG2`UX6Cz8=rmDn$$iuQ?FSuJB(Hq*LOsH#pFY6MZ8ILgq| zn?*_Fn5cNzjT#CTvLrCHB93FD8zym>##GyVW5#PuR5in#-W7;AreRs6`#N##w!9n3 z6uRNk(=(lVZSt?ye=&S7_l}o1;q}Y@?7%z(wU<-QT*`01cQ;#GCvkFOAJ<%RIeU(K z3m*Q#QM<{RUsZ2-hzATyZGD-1_`(Snjc)g zozUoMI}D0~RKJIfjaTr`pIJaLB~kg)kNCk|NzS|Q^PEr;4FCSSeEENV&cLVN!U|JD zmLqol;XanV=fixqTVmq&@9_B_Udh1hcW{CRh&p%QHpDCb{$re%t8@FMU*f7?Z(`Lu zmvO@4pYr34+j+&}rI=T{uMlRFO0c8&b-sW;ndYrI= zy7;wR{4TiYTLG{A!W&tt*xdhvDda|0kVTn|_s!$gAN(vcb19_zr=mt`%jxC~jK2L{ zeDu{i2Y>x}KL5l2ruzJM@{b>#4m+>pvtPZI4JXg#w8A53l)5gI~sn@Zn@q51eoeh-EzmU(bC=rj}#g$+CE|;b+;=QMLQoU^ook<*l zmT)h>Jes|jKecS7C#IsM<8d;cj<@mS@85>b)_wdPcXQ3Q}54p#PTDbQ&{Lin)IqvPB<)TF>gnD-BBh}K`mH*b~syLW$2)khoaspX^pcC2}bCpds$bp4%JuYL!g`Fn%m8!q9$e)e0It#|`Vm4j2- z_{1Y={2R;e6F&JeSM%og9OBRy4|4XuzMGShHtR1r8jsd;Bp>!!{AGCBoS5$JE^4(J zx~7s&CGmq0&kcy=08N%C6+6ge3Z$|bT6PO1n+0N8wu^|M*=!xY&u3zyOg3L&$L?LE z(<`Aa%P1vPU84TvS~p2m;1N$51tuc2b8#EP-k0D6&i{lcvnyyhEFH^78c zOs8;Mm)*N}qZ>LSl`?}f28lJ5!12i^l9&mPe4&GEp~&&eRw5$Uw*3JlDJGlEQ>|9% z?dj&gfrFIG6R5gQWwJsdksy;vGdes%DxF~2ie*T$jvvPC+qZ{}>+fLawyn&bHG_e{ z>FrY|KcZfu! zC%T9cY9Hxh0ojXj-8xRx!jLuk`={ZD5@RDau8{Gy0*aQTub3ee0fy+rG!yhs3y5%0 zB?VJS(rh)+)HFgwG#d$$sT5um6U!Qgmcy>rk@4s#cF;`5NU_A&=mA38L)P=elEG-D zMlP45T=g-{fL6;Rnf8g}2-D1>${~5RMpDs9rPCyg6qY?sCZp2ZRmAt&d>3&R%azeH zJ)mTXBc1&Rh7fUpZpic&I`M1cSgj#cA;>06#Gy&xTX;c(Aa0P-LuwNv80ie9LW)Lx zl4L5u#N-%WyP^F2HF8FC)M8eq&? z&C>3hc;HZzcy11ek-AoK+LC;GaoaVI*}2~^Rt}(lI~(o|S^CE3GcTtCS>`=&9gA=I z5qIr;H4C)OJm4?nRcFm5s{+Tpm~{)*vup3+K}Zt#9V}b7g5JZ6U;U(i;4>=~GHC+| zn6`R31HZb3Llwb_zV>a&<4)zo?sm|0=jxMK^t11=dAH9A3){cXKJ6u}>@|TTE0+!M z^Bc#hkYw=*$0J|5fd}lRtVqT*H{Zjau2sCE|M7Dy_b*~W_GY&2XmM6on(~gFG-sW~ z?12<>RJ;!XV9UH4Kx`GHdQ$B^e51}#h+Skj+-kX#UA~u&t_4noj>Cr z>zTwzy(>6*ZVCzLn>~PeLycu8O~U|~F^hgSQ?3XoOzoyc$C|$*$nB#cAxSaJK9Mu# zbf5vzuBr5J1^tdotRzU9Dw3kochZ~q%n35GEEu`#cJ@v?gEz0}!jORORlM-jTlmbz z`G%QA9d0e|=Ye}^X!W_Bz- zoRog#69mJXw@{sN3hNf-P$lSIbrx%WcgLSM3LZu~%0cN&VQLbx)Wxz_f0U(?j2zeb zo|{ zs!VO;S3dF~e^c3gB6@o?Ou?hS_?ge*NO;$mCzVP9AViF!C}^5`IPqS^vQ!+~ zK}of%aT$iuZaNr66mmIi+on`1A<13slZ?Pe({wtDc~n&+3&{$95Cqigb!1s)VzNpU2~ruCdp6(4)~#Der;})! zPBNWCRdn*%43&C=eFxj2eAQZoc?%X23x!&}&gj@dL@01vm%BD?!t*>_-(~BL2Z%+C z>$=FYOsP;Jl{PsvHiF|uSeApV#Ps#`&{@h+9vkD%4co{U@-!Mv3dJItqTn1X(_PA9 z1sbxKB$72A9I2qID%RfpBoYRiY7qJ!o@0R!^z`;XB1NnvFjNUC4$wRg!$@J8GblGI z1g=k&6m}5P@>Dcc$M++&LJozMxsJ5K9xNS{7&2CYkY3 zCBew#Aq+#s5C-Kv4#EWyMFcU#wujf6pk-MIWF*zZ_p1~-x)7F$TW*m{rZF^wQEL)e z@<|v9P0ynlPW<%-Adca1ae^cN-u_3DV%&0>X1a@xhr<#ydKRyxhZyVjacb6PF8Qa8 zkG@}-!DKxKv&G&&{DSN5+QWnkvK-;mBl@F1f3f9kp6o+k4oN{(Y3GF1CzuF}9AA2b z8Pq%I$olM^Y~mVK+-x7khm$mQrp@R@-uvgvCB1{vBezuDLu|O}8gAb)hAm{|*rOR{ znDKB}h)hR!$DdCNlpKX@$b)sOJtcCQLgAs@3<*tDk)ntw24nsSEH^G;!xoF>t0T7D zyO-Y8ujYt;qB_Xp`3CE^?Zkb~Ty}07V_@YR4Ao%%tQObr7)MO6vwf#v&dcVYBbX>p zqLjKRKKwkHj#39yCa4N90lA~=(aHa@$KGo%;ietEocHN9bR51{opUCCx9F=}`H2s6 z>)b^wT6{dK*Dhn=p)JUxjx)xcn4p#Jru2xoDW#)CQS4%(dRTDG5~YW?Gi7r59Mo&n z1;=(`yzSh+$!6gzmynhKsgKobdb$4g^^83Cg~z;KOer@l;<)aFiyePHxt5ERI*;J6N_kel z=u9?z`^#K(?+g|!Uc{0WC$MTkCx)10d@`oEs`#+%N!eMfSPc*|J{8GRjrA{{hle1O zE9M`zdy=Z7NrEU6$B?bZe?b0rvik&wb|g&2qrc*@&*JfT+MJl*-?5Rd&LUd6z>d9# zsMIY6XG}-YG;GgB2tgv7LsArmM@E^bPNJ(SC#_wBq3g_>GY2__T4RElGY1(jPonAu z9mOJwBBSd%hlUQ)Y&J=l2}B6#e2!R16gvt8VSo@3k{r{~Q9zPnR8@UA88w+m;JF@= z5J*x?cXwxd$ISOB zOaBfG3?NGq6BA`z*QHSCVEM9@c%DbGSfo;^w8P!Pu$`Nt>r|^%`uqCvY!5#OaU2g* zF_}4ikZd-EENi$e4-vamD-{$~K>`}KO(Y`jxqAcU@hXjGgCMlAT$g6E)lRNYs7N7* zFhGcy-kxqWO~rCt9M55LVi*O1CM(E#lI`2dq;drsZiJ>A96WFUVu>*DNvG0CaZIH= z!FEZ4SYq+=6KFJCTFp9!CX)&c0^dSc73@}oMZV$r9O=i626B%P4Z}lTD{6m3l}dQjCv|&}>$Tq8LRr$>nlrnF4klR8`}^ zSPjEWV~2fA?hOz^!}SCckwd-VkxT1HLZRA>h$WAN>LEoiF*$-8I;gtBaHY)1$PkHS zj=`C8nXJczp+X|5l1OwQt9b^e&BAs9hKBZ1wFAP?<&;A)i5NhBT`o z`cq&P2K^=uJR0p#R4IQN@4fi^Bb3*SnY{J; zeEW|EFMIb#IdysxS&VYs=Re1mCrIZ}KZO!UbigF(F?Yz1*jbA$HeK;eu5FyoyFT=f z%*m?6&D;3OM{bBCD79{Y-iq0Q!mRT=(k<7S!*V(!^HjWy?&j}J9F86+g;%pc%~ckiOUu)7 zkskBEr!4B^vGXS%Ye%^KU6UT7V@f>)Sk46>UC8rxZ{fbpo4My#|HbvUzJT{!cvia) z*UGo?#kZ||d>Pu<;;v)r%p+-e=)NP_i$7m~RN0z}76kivtc)AXIP0JI?Ajr=Z@!<+ z_x+JeFTQ~#Z~72#Sdn}3{CO-Nj&=DFomPp|{Fm^7&z;Kt2kz(o&G&NsB{y)>$?xO? zFVhdd=82cx`16%hkM*yfha(R9*s>LlJ?0O8GueFsy-#FZN1@@UJbJ}~Bj(;ci@yv{ zn-gQ(Hsy&J+iEd5V}Lz-_o16Qvu4gDkxCK+ciwkDn>XFV+0Q+LzMg(cU7Zw)S*&J@ zY$}b2B1)ZEVhOssx==L(MV5)9h=>?nHwZdIQIGa9x*rt&V9X z=qi;^6a~B0A`l^F!Xyks4(#7YBB3KIV4Cfu{`>B`A4yeFHH|oq$!0QimP)Kxxsu^S zhv@3+LPUb`u@Qt2*q%$iP-M}f1=MPjc&>-zGys`$c?940Xw(~6wvDbS4DH*E-SY7R z4>PIK)7wkRP*GKl&Qd2b5_Zc$k|m-jLFMmmvRbHtPwZMumdDW3 zNs{>fp{ zmMyU8d=@2T%uyNW@vRR+U-^98T&S zIxvD@){!45M%lODW9AD^Wp=i0TEpQ3j7Bi<=ZlyepCsaNsnnoCrKwRUq@Tb-Kw)6* zYC8U76B{hfBMt&=F|1-UEI5^%4Gi%7UU#A zrn9RPao-pf0qqLRg2}N7=$KE30@e{Hv4=BW_a1RVDR)vTNH7676O>yq zZNU;$C##U3)eZrg3UQ51w>^My;+y#33uZhbf5zgT%f8I5x9{YfnG2`v`7pKI9GQ=j zEJG9??#czk4eCt+g=5hBr11pium0qpYD}qzD#B~xkfw9)D$bm{iZj;D<0~J#j=K)7 zkBtB&Nq9cFV`ePUEd_XDs#mIO^Ya9 zU~t)U8C>>U&R%*cAN&3t?0)IXDRoHPJz7SPMFHZ*PX2U9l^Lh4LQ&d1+8%aX{5lOM zMnBeAm#smv{*rk(HjX7*e=YuU**%-T`-}B<3Ld>|0ZX6F?SIPfv`yvn&pMMhj_B?z zlFOt?BoYWIW^8<%eS7yJt15SHx*Op~?AW!7OgfFZS7mTUKfdDupsVc?woS{Tvs9wd zvay;?G*v-2O?(j%L=lD@oY4zyAOonFI^w&7r%e7vJ|ur_=QI_MvGS zx~3C}2rZ!#ggzn??Fv~5ozBi8&4!I_dDxau5XHoDgd)rA-Ln_hu_;fKaozT*jN{lC zibQ{ZKYpW0rRtDO2`0-2X<071Zekb)k=uYUU~pO&0L@ktRX308(ax(DxZ1A7glUm}E+& zSjaIp-okQh`nq~(H7BuJ6@1smb9~a7BGu|RjYb_^Re_iuYX*{*WJKs3m~1g-=yXLP zFV5s3@Q9NpvaAt@5t^psco8$F&49pXWcWdb#>eOyn8oZFvq?w_&1!=}I>+Q#h0&3N zh$zHyU5G)8S|pMQ99O{?F@v)g5JfS2_v}PLrYqAyPe&fx^O-;EII2VYY1Zo0TNagO z1KVm)EETXD4*5)pP(ZU`F+4hoqM4YcLA_d|6$tX#H1nrTr&g`th8~)#;ny8@9NdO( zsMt|JdAy8~Opr;ch+LAo8zQ@qE@auYXD>msiJ2>sE2L0NjmgnTlthwLx`U3iiDH^~ zp+u>x8%dJrA9PV=4Iu)`mEphM0LcBUd-Xc*|H@^2>6>NFIpcT+@+$7+9`3&JYJN91 zpYz|px*a+(b2Y1qU*^VNZ(`O<7h#P2o+~cDnd}88bK3m1oY4ITZn|nc3ockocRb2n zKluh%RLE@y@v#iobxlT*)7Pe~^Xidhxbj!_7OQqgzcmM?M#G_wL;^PdpxNY!kn_ z^#F!qQ*Q(hU*>GNm0R~uqO}~;`Zt;lby@rQUXXMU#%uD`(TW+%{t$Eeq&>5NTm1~)C!_8c~aTyn^DG?p| z6W88RW6lef(J39piILi|<7)ppUiae7{QKoUWywXaXZnV(@!3E1^7n5)mu1~LetAE` zEsa9)*v^_Va}8@c{*#-p-OR#smymP!aN~7%6ED4h#Sd*^N(Z^~`ukb(@}=Zj_w&2; zqm))1N4E@O;%=_KX@dFZoW+92tggiP25x^a$C?Y5)BC6@K{3xcWf9k3c?VlwzL4dQ z$|imcjv+T3VGlqlb%9{%%6_FVc& zeyO~icfRHXrX?e42M$st&yuW4*UA%_b^UMnuIIGO7yJ#5fJR zkM!V^;e$;7;zeYCcN;fuUB*in7>wU>J$F?knE9AX{cT)%@ugJGcsp-6eHIz1$)N*f zg8TvsN|u4O3;5N~ujZ!ZujiC*hpoT-5kDJR%LmVr>FDYt-nx?q-Q_GWT=w012YY}8 z#~7=6@LK-uFP#T-N~TOLTkkoI{M5G}6Mwnv$|%~C&|6hS6?MwvHg4x$es=NqkIW+( z;>mls=Eia6yy$EeKAYwLSK(=MVm5BN9aUE7DRq)cB*_>$u_RE;1j9r7(Ix24WhoSk zWOEs0Mdr}RFk>UbAjQbCM6sBsQE%e89@C~xKX5qRXxm=!VrAl{S9@8{&-2kiQ z5=%0%0xhe>{J|db2`H3`OrJi0o=8(JSLvNTNUdp6IXKGn-sy~wPck@b2CA-8uS}9k zWZ1B2E7?pM(^RNb%G7IhEXyZl2<8m-p~y+fBL_&O74&41Lt~@V>>9Q51e>>QXV&Zi z^n@T?X=2yLDNlrCJG&XHj*(Sl4Beny9>WWKlIb)on0j?$3ru9bkigssA#DKnM?-N(=kvn zHHiQjDV9+U4OvqOg^OXxjE)bpZ{IFPE0g3(orHmnEXjC*zz-UP!odg)8s#dQrhC_Uzn4O9V)AMC^tv8|dJ?m%omiZn%NjE0>UHEaTGu{8t{h?_Ppz zA8}(8U-OwiXAwgW-c9H1**Ka>%achb)9gC9m#$2nVxq{Q$^AU|;I4LKP(#Dib=kn03_3n4SVeh?at@U3PuHR;JZ4F&BP$eB%mI-{1>hLIQYg_7njaCcG%yaRxQpuQyFOVi&v`SlyL{vmf5fkS&_zlQviF&9py)=d7e~+frah~@JuO_(lW4!BE-h;2?7&-Wp z{Nhg^A|pWrNT~x{G59!t@wWG|*ioq-crmZJc?21VTj%)ty>p~*dN#XL-`;S{!UOlS z*ncHIJNBJxTq1SxRUEkOecb=J!@(<`YHYv9x_Od$l!u?ii?5&I!+-J{+@CE{-F+k1 zU6SV0KLW4kckz9U-#@<1(SLX!$}+{5{XDB5`6Pe(n;*uB6jFWDT>k1;aa~zL=9#?i zjV`x-;?MZaPdEsp%=DG7H|0ILr8zFXX06PV|jEFoR_ zYTnTNFt`21+qu1?P@KAwSHJ#dhGdE4%KwX7j{h}(`5%6ZbfLuLb=PuLWtN&qp7LCc z)Wi=y4sXBZsVC*B*47WYUccY=zt`GLzW;qIR}XN-)ZN_v$8YECPyRA*eErS*&6Ca% zB+e{fzxOQWjW@F=^TVFtk2g~C4+Kv>_~3(Fam5usaKoeh6;wqq*58oOp%S+=2p^mB1l=rRdYx=8w-xR- zHOc&id1lUEppdc2SO$h|fduttlay7YT&|+%IxDLi?7L_e4?p}h&YYQ{R%=nn=NTC( zZ~eMGI83EdB^Ci)w~Z!i)|*qlM_=|woS`vpePbwzUzym48h@p`^cC234I0M?-8jQUB5%m>7dIt zbF&K|$&8K+p&NoyvB3IT2g|nU_3G@HoZ!(%A7kd+c_zk32!$X@B$|#(K9wS$wh%%k z3b*R27>0pq8RUyu&YwGv>$;eR!SLW9z86uiH_CMe zK@=lLDvHrXhpyMfL#3G3dHBfV42(~ava@)OgOd1IhKi)Bgo%RRZD1QI+K!8=NGzLsG5$ZS!Dal^z{`O85v{$jnAYg>)iUjk5U>dv$rB~ z>aI^Sv(V#_jV|r^S!_dR_f?m%c7BOamFf1{sG5Zij~m}! zH{!`-+_%|gn{ECj7fI*R6mwZNY7MM(7CoIt z_X8S^gR|7Y^}1Bb6?ROIBZ-in)539lDt%Q_cA8kUX?G%q$Ho~@Qp_)%V{~c?A?wKJ zFW`ADL8xHa7NzPCu}DxYlVZl;#L?3nYn(%sRF;=Euxx`&AxqEe(d#(qs?OxtFgvCv z@m!ztGc(+G@7GvZxIikMr@ya+?>CvBzrdxJU%`Xlc!2fI4m-z(QPm0?8!k(Wiy$X# zwp;ApGsW&5Bc#%4@J7r4L#Eir;K(3n&(2a?+QhOgGU*hC zDdQy(PV8dKGFd$$^fn1)2{~=BzScwwKu`DMbsa<$qACh@I>XZZI%~`G6pLkwm3~YG z0_#(aO%uC!YKGY6Qq9( z2u||hxBWQ_SNwQl`e{`U;AsP^?r4ydIX3pc*n>DAVnWU0!F$uAQtb zt#bDKd8Vc(c;?kt(p+C7mr9Wy7)3KozVVGm+1RYnYHfu`mCJpM4UMq2zRv1~!|+I! z2OfHqQn|opeVr)wSzlelHcf2Pq*Tgam^NMz(d)!WahK-i9Fn53Z~p-f9z4v|S6_$c zyJ+T?64>f&0q9__0n+;B%K0^lwRaQWeP$Gj;sX(!qXV;$Hq_R~`A3K5+yTk~d|Kh88;DJZ* zdJd`#C^||4y>5p@Q7Od=J>MgUeR83ouT(}xu-SGvc*$XeEaA0kgi%N;Ww2vnlup|r z>^i8r3HZp003xEWL$4DP^j!Lj{iJh663ryBZ7#n08nUSzzT0DC_8jwbr;*bd{lx-u zD7gHpLtJyy72NaL9C!WAtrRZZ&GUcaFbgX)T>P9JM5h+1pE`lAs1yeVF`|U-Mx9L& zQ5hJe(eCo$Pb}eg+vH0!txzNMHM+G9AN=?inJD$6U2`=*bHmHY+eNY|l_c1tT@2_I zW6G600)<328C>!P6E_wtsAxFbwg-h)gC;A)jZnwoWQzW2dv!Y8~bm z<|$VzOifP{g*~)HqU(5cx;<3GAP9UihJ>nUB$7h4S|XRvkTx~i%^p##kj)iX-&`V+ zWQK-EkO*n4tzxS(v52s;IhHqDJp9Npdfg7TZLwo&ih-dDLFBWzw8qJkr%2g0QUa0y zMV82CZ2~mr=T`tbK*Yb;Ioi*4S6@M`s!*nEeAVoQ!E$p{E$jz zn0*KKBB@zQ)nQIPb{9!+jdFD-4?cW^uGeK~AWx-Op|4!x_{rm}Z>$mp35sTId8{&7 zdTvO^ZJ`-Fy=7D!!NRqhV8K1O4ugiEgS*e*9^Bo6y98%&cL{@qKyW8Og1fuByE`}M zd(XLh^}qhpt9w`NXVr39fSf%0@R zbV*$seZ>=DO<_8gFp@QOuN`7Z#T={~m9H~hrW z1qRt@-~m1wZ9I*|*jTM?pRgvWEt!B*B15bri9JZ8GYXeZI&A;!jQ+euD@=UrnmI>V z`78H8vHhZA_!S|+{)0Xy#eC2BAME-V{uKwpDP`r4ctdr{>R@7~uJF6@QdYcDYP|5< zDJ>m~i(#*@$Ea)9BRDk2^@FM%- zxCN#Bnsib`GRKQT^1_WpsodQB@1fn#&m5VVd;O+y*b!uNt85`ZwF;-C{;>$t1E@4N zh=rJ;W-uc|TYUzQ^i%ncgxNrAvwPs`c&RN)wMf1qz9hgZ9_?zVu2ncDc5FUb3NY0F zGfWu>2U@hxE8fC}pyu|&cFTZvGYTqdjNZB7`P$$b@wN2fKslON!E@r>v{6nrulT{x z+PX{(7_ZnAdbVPk-D)?t{2^&px7cv+E?p)IHH1BvF#o{%Cw%aa#7I(^CUFaEuM8lH zW6S%QOY9qOc#RvVI3*o7So+&h^ zWcyW)9BRnG!txkuk(58j z|M27gF-Yx*h=`JD4$+4>H^>sf$SqxQ3T+Ons$pJ1@(7>@XT+UQgR@FK^k#t*>0a1jK?IiwD`J}5hGh4>c5MLX1Rbc|Y3J|g+epnmlq$|52s-pb$+441C$*%Q9w z!R9}+1%vFs>g6Yp|EUP3}-6y%%N)`t8L(xX;`Bks0{NRClCM(Y*QY zhCp)VBZ#k;<0@JStGeBw>yC!MT@D*58a+mZ3E%TPKRo_pXu^b8Mf)ylMbjn+XryrV z$JIGa93E1TP%G1B_fCvG!ERok;kN_FqWyr zdE|p60m*Leo{oul4;zsF8(}_CDQ#-{(%o59;~}kbV@}EoL2=sS(7^^(cP^CSd<#nh2C2 z>^U^{oA~h?NGILNh=QvbjEYL-BWTKEZ&ZIjQ>wgh=A#JxnJXgW?=)pawz zxk!qR9!X^y`I9Nfyy15n4X7F6MtD598xoWZyw6pt+)Erai-1LhL;xX8_miKjb}_u$ zC=!s;I@Z08@p%#kEv#pjfSCi{tq`6A*e;W+ty^p!zl61K(2)OSX!c|895I?{-tX{ejbK_ht-H%Hp&W@#hYz6SJ1S zzOk7ng2*{|=Tgvr6Crm>3QFUhLlI`dCd^;0VKjNSf8#Cazm1&9Y-~d^RF6bDf>{GI zE%{v0^337z`@*9A^_<3TcF7DqYsuja-PH629sRcU_ix$3>88ibGA%8=^)kce@T3w& znomqI(&Q!jB{Mb&k}RsQ?>|wUwriT$WC|L@TNirRFhY`hq5!FL7T}P`t!~M`se9k? zK+ppG5tltJ?T&9}k7kqB=o&PUfY~ezAg(K3UM4@R%7(KZFLX(TxUI$*}Zo5PZO7MS&YK$l~v`kls?c=OP4ER+JGsmXD9w7EMR6>;W2 zYzm!{FuZ;Ip9QW#?a;L&<0WEE8?Jv8V!R(6i|4MMR#unS_{(k2jKZyHQEK)%v7g31#7b4NSJR4Hr`)03c!6N1-CrG1eC)3(Ld!22 zbz-5E&f(Z`5r88}ndi^AqJeqp@^Y5V7U3dsXmn}PP@&T@rrakq`PjG#YY!A4$`T0M zAxmC3rUHVG%D3q*3A6vGd-L*6Tk{(NWCCpN2EYl1QNhwB|FK07OcxA&$3vCEn5c*k z8xIx7;Zec)DXBQK4`G5Y5;J&JtNi{M{f^kddw*suBG}FRpv7kV` z;RLonuYRp4u=v~~T>7A@iGob-d3I&GbA+(^nQ+PO2vM<4vl+LDX9|fT4!=@REbu&- zQfK;d4R|qFuslb$*CHhHqh|hp^YT@}rZuNYqFGsH+nJ2|rx$qxkUUFP(Lsxy-K_TE zLwZB#motu|Y=jvj>$YCEFZv@xhPO=oLReBW3?4Q_k9>(#boPh${%P_oeWhxg!`XQy z6IRqFg!*dG&2$IW&by`TETOmv#9}5IdXxs^O>c_K@<|MUd{^P#S7Ji6sH^46?5xu0 zIH&7<$j=_Yiek*p4BGRJJ*Ba!U$8Ta5*dP?#|mfDQ+s7qOLcE-X%6$};aZP12meia zhr$V~0tQsuvLRr?SV}lnBA9(aKGl^e3UI`5=;nJNmnQ{Anc48y{BkIGK8}J(u_(ix zMHh>!CsL?hZj`}2GRto{I=b0RU4vpjRcTbw_7TJ~$>7LHi}mdxmQ{>MCBx_=@mF8_dk$2pWC%#Ocn z@{$P;56{X1RVpdoTuaxmPucCadsMzNrP&mS6eDD#0B5USv%f@P^Si{nHb8Rc`ha$p z=P)^vIJ;P`paH`2g(__*DMSb;6jKVM`t+kZS~hyAY+^R+K2Uak3cFf#4oFrKR>Y9* zoC#K_a^o$qL&@1a@>@d*buxX=m{ z2%Ns+xEYG^b5hB#w@;!~*W)D*CfBkR3u;CJzx@BpzOPn#XkqGV6_m&8AJr~x9z#%@Vh0Z))@3%sEQwkW<$DkB@S?8ieA#j?Ma zN`AQ1_|N^OO_@FNHwgn4yxyA#mV1gxLp#Vp>Eiqf1uJx^Zgd{#n{`XBdf~34z48l% zoCIAje(9y?QJ>Rx7SPb|G#FEztt9-)%b7RJ%h~Ep>E(t#fpqP+QxG3frI=>K+)wx= zh0`o|yJ;A5|6u8z+pT`y`_Sp1*WBv^9{0;0YTZCV) zlQ~tAHt#ZlAC~fsa@{LP_HS5|Jo}nyL%k$pT1onl$v)D89YLQH5v#&GlpCFx)T7N- z>QoEdFnv3`&8n{09JRU@g~MOmjcc#k$MO`AiE+0GDRHu%Ye6h(gi>JsoC|v`ai?VG zG%aews3UvVpheP13W+~DHpWAhRMLw9qM*^?3$R zuA3HdfOLgg6th%k=@d<}-GiOmr9;AgF6{ZT|f6M z>Ll&(5z|}~^)CVaa}ojt`6MLHygYB*4o|onGNfaCAQq6{v6KHX96M@v!Vp5$h zE%JJ9jLps&xp+AAxDX;-)c25}ylzRrA6fia9h|m5^s=rk{r%$9?{ri|VecUFYRKT0 z{>bPEL`Wi4B0>y;MmG?pzee(x!5&$Vg)ZA!C^`*+lB>qao%Ons?vuURWhj1wIv zL6R9ah*IjMt#GKB)juT8|A0vD1p713uoTZLI%@!d?=AUT&TUjc+#ERvmDBxVz!Qe> zv@d4IgYKB677}}aK*yD#rq}9Od!W#e;pXhp>E^t6u;?c?@BQZAo91Ii-QNn`c;0qI z{9~YLCzd-yceUfk)R7-N|JoXi&cG`r&f&4Rtc8pGDWNdvt^L4zzifMdOT{rEIBF?; ziE=7bAB0DmOvk#1Sm}+Db~7;iw$gfEAdre&D6;!GPAJ&WyHAM|bcc0!(t4r&`;J@q z@j)j2*8@Yw``eXsRRgapOjXt4hq#WUw+Vml%6`g?(SlEQ4G1%z{KMv6uEkJsyR+gP zm!3p=UtGwq2MrvWO5^A?%>J*IT4Yh;<#N+lnEV*rzvsm$85$0y9d?aXr;94rik3%L zVNPau`$(;)`DK^gEosH)!~LS)9UDk+kT#715N)cMo0Z>C9lPW_t{xLaF?svt$8B7- z%i2s+f>QO@bnn`~N++u$!^dP2hEzPG0AK zzd9PdIrR3@SvUGz>%|2IQg2G`&R?k6&D!9j{pSY?B}!u(J`^)HlrJwY=oktn3j~=q z*of3s+=)EDanIx zAlr3>6u7M(YQ7{15bI`7Ma|#a_?-7ww&=i>1k{Dhcfk%MW<6)0zhirxbak|-& z6XFqcAGmeBE|B>xA;jbilTHM04B#JBSDNn6jlLsIlu^!0m=HpS@N zsdHF5Tf{~1dHCv^$*ZF9y>@un>T~P!V}zK;>5%XGm`HcdRL5wAb?DM4D-LC*By^Xz zRkc_tXXkekTXyj5gUhrGCg-o*owL_N_gTz0pVf;`Q(B`=!moyFU4{YHnQKkv*pSrt z9>ATj>$kVPFaB97D;xdcuirYJ*S}=E&SdRB`C!uiJ=6~vkr}(bI&6Gl3U|Tm6M0Dc zT}eBS5`XP^-Eile=E2$V+{(jIK70QdzB&>vEZBej_Jl5-_oiu3f3NvkahB%Q7e%iE z<6M5k_j^(_Ox60GbhqLi2Az8&__~FzmToJ{8wgz zXHb7xB#DK}P%2=+AqryoJ>)RhhO|Ws;<*6v3p!r3)=NXIB3*ET_& zR@*?aYfij|2RbCrJi#n=MUvp*bw38y)<)^MPKfpPQTp?Bm$f)BP$A zF17|I4i66U4}CiN5h>1{MQg}F1wq*x8!^gaWYN?5SLFDL-A~ON9YeIhy1h_qY$=>( zJp!pzWF5Df1rRx_`0QuY2i$*jWxC*bLK5KdRN~}EI921=%HrQgUs43U;B9E{{Wqx< zs0k3~%!|SH`;nOi#PWW^LUqVg6y+Xqn&CM_$7$+tpg=rjZ z1{~sDvly7}4}M|~_B;~PNrZqS-XB1%IH+;K2&$c$MB2>Arh$13-$L>;eq_qxBK9W) z5jOR=NCC61TNHo@k>zkGCS?QkoxQni@g{+uomWB0R3;Ijo}Fk2PDG~GxGV2wmxMFT zCn7(RL8QOb-b?@SZ3~ZZ-By-zGW8Xq!uc1t208LgLVA!cMPi0TS@%sk|ARXR@znfQ zpX^bgT2LY+GRiWeOmDy~!-toL@H`rxji*-9?QUq`r{C{=o;4{uiz|oXZ_m z^4(MYgHYH&5jt+!MQB7B34d(w~2YpKnNAYLa|=;Ti2`eaR*WP;yVTk1un zzvt0gVo%_gj^LbmGkL@x&IcQe4ln%xTXy{RE7?+cA@j9%H9Ap>-x-a+kG!w#P%+r( zSWN?NqO!G|!S37N5K7P3##(qgKK`6-Oe$Tr)Lue1UT2mWvmv_=wIv48vU!{zTb}Gz zOx;j==l*Y<*YJ~477izfePsCL;Z`1GUMeQd6qT2!BKYhd3HBAM~}s5wiNj1Ez~bm>hG@h7bfZX&5N7kl-;3(F1 zS+CLR_QOaBNC_34caK#ZyavXSSg~M`pz2a4f4bf2guc=_0)~{!eeVQm7xSaPA`=rD zh}32MsgY|bY;GTkp?dk?QvOkmIT6CKJ0b=Q=QSZiKmw580jrYUc%CjH;>S!@lYf8c z1Fg<4ieht<4C2fYB(Zh8D54(12qN_%Gn+mGR~o4jQZ`(Z2+TAT1vY2qc6v0N^Ne)j9m5d0-^9g{LLKIAFgPytg&JOr%z-vujsuW@BOZ@@ zA^C(0f!oZgq=oK7vAHg~^OC#2m)7~bmnarXMCLd6Iz&^Xg-mRQv21`f;Ra^p7ZQ=I zIi~b}E1TlE3~GCeiEDr1xgU&eJ6`elHdMjeZzbKEyV~<-ZJ2G(5NmIgCZ;1D{vm0Z zvsThK!Dv@JGKc4yS}tg}l=ZyP&^PR_p35K0>kPE82HY|SwYROnzCx(m!$5Gq)Mx`=C<|Sh|1d=e|^CpG$)&rN~MZL_zA=A zV{fT2CE_E5Ca@tx7g2GD#f## z>G)&Sp-WBmg3a&|;zYbD>mTV%3ncX1PU^4u-9qrE4)}(1a+a8c8Rft=!`BsU@(O3P zIjn*2v^BM~%3W^fak#(iqQEFh>%_@g3RMY!@5Qi<7QOHlWG+lmci@P9;y;!}*pQ+t z$t#mn6dm1EnT2lTJ3Wto*r4h|acGmKzfJ&s)F()9LY^~q%7^f4{88MCcz!qM!nlhD z1Q&9lam_mNzpwt%r>O1e@aIY7Tt%EZUV~I(4q0hr{IsbbHM#7gFSXi1CC6k*?Si%H z85d6e`f(_62#~$n^n(L-fgC|`1r8%FGy>ZoZQPc|2XkeHHgzml^irVwbFLU_=8yJd zRrB)y?FCTU4lUe{FmTD3Dv7IdzeNx*yyrhw1^@tJ?-_jot_q8&G7{KP^oiG7>5|AW zt$zbpk~%Ts+=O_A0~_VqH%%)94EC)4BO{iGkIP6WE+tJzz#+sVIun54++~wuYw)Is zS&_Z5|Bs|z9)CUm-8kNBrnz+q7UEH1QVdH9g)^zm4l;BrM zF=qeRyWel@(*CCkzqcGeN<{K`EG>E7?KV<^SDB0+6>`rd{TjWsIz3Wu6uKK%(ZD&`xP5R6-N}PegFEY$86Hu1r14$7BuG*g%{fyVU#*_IL8yRkk+xb6AdDA)zB~-u_ATGhD?b9>5^Xt%WXLiWEYl-P1rd0qF~V%uu^r zy^8M1OFk&qh{cs zmQ!9qfkskXI4ge+Lm7yu0k5GF@%+t>yK7a*$GT4EU;b`Q4Mg}mCMPlT^2%^u z%2y^fb@cpZ(kxjNK=w&Df3F7&6iirE1Z0?@YQS|w<(DBCeBdlI)+V;eB4kSjW&WfA zqNvTrOUmQZ-6zPFOi*2u>+w)u`{K)OJ!a*~?Bv{(}5n0+jMfO;Rug)j73YnAa z7R6OCR5Qp>tU=UMx8$mfXk)fGP48=pjraK*>|v6^n&aL2`dig96unUIZD(THQuHV^ zODJhsdV*YAwc@V@VAj75pt(OwDj6xyuP==O57#Jjb-W^YOPeH@`+(cYCY^*Q39$Dv z&g~CINlIskg(M+g2|!`kGP-Lrnkc`& zX~|aAdWqvapA&wY@3~OtfNyPfHsblR>PqhM{CeiRRzRwEl}+ugfF|s=YyAB1zSuAy zmYx(neY$sFnmz*UlK+Wv`pkUP{PO!bktqxTXiq?(i+r^dn*>^zM>n!N8fmJANDQj= z`-KzJ>@k?(DNd6B%^I$WT8_%yaG6D?_tIS+o3Dv|7;SK=wDh&;jGiHir%%UAjYz>o z3tol#l^csVq{QIgv;^FnXSNJG^10d5g{$~%RXZ^#vvWV*7!Zj5e^_Mv2aC0DlS9p0 zQHI5}b={6!N`}5f{ne(~d`^jJFT-OO`xkhE{>Qx58b#S- z-x-7UMD8ql>FBMwkES(K6M&&b^&b_pg@?}j8Fo}DSkLF}=CnjSQZEJ=VUksKivZ^( zJ+B_F;tHYJu~7H{Nz2U@TTyVZ%2Zi5J`nb}(hKZIVz zclRfXCn7|nW5q)V_iyz$4)B}pGUfTZYI84`PoEZ71YhrIqLrd2tZuZj1%^_-81$_x zhvLalc9m65TB}#pAZ8ncU__r0Z44jtwd7zhWiX>sbrMNIn%D%oawX>2>YvyWf%ymg zk+9%GobLCEb2O1O_&~cM00H|?=0Ido&mWOc6Iljld7_4R4nT&w)G31XoJRiWwy}4}t#P+>FQ+KFIiwKgplm>i1o<^8{f; z3Dfb@U_15*JMM*Muhy@}q(2WeW9dtB=5+jjqz2nziQ4E$!0%nTVxqc@b|TFpy;@ej zoxRzX3%(%AcYQYy=$u60=fByau~QfqTrHWfiYF)SoB7PW+9At~a5~PW$*;hjR**ev zGQS8xa2HnU*qi#OQFyF^^-;!{axJ9rOpL#$dz#n3VAOW)nZN!9>mt_U$LX@ACor5- zKuTX};%t^pKRkB2N-ucfLHo8$f%|Ng7K*#sE(1Vdoa>Pyyn@|g%1}5xhhGEmAIB~>+m-tg{Cl4AgOhWa4eE)~(p)yVZv+ z%ZM)V0R@8py|lx}G~h($P3UcgbnAJQ@|h$}+8%aD@FWza$`=(ju$rvP=-2S&AkmulQ>E7%G~iE)Z`RS8f0RDF;(MJ2uuA2X(JwKM z%mTNoO3xdLLKwHbp_pPHDBn<^cuQe@`|ZZ|sUsY9A72(Rk+jUtlTa&u>Gwm7N2xiL zRU=OF>kT1ieRp$ce8=;KlBdJ1w)C^l@5e2TiUlM{9>t{azt^MKpDSBXwwaXLqOUY5yS z69gZn(Bgc)AnxD&p|~~gtdkRc*-(y9yTCm|@6dKaXNLoH%e1lcW0$c_e!kV#N8z%g zyZNO>)0@ATG9z@dfr+SNSG8zFzR85x*Fw%2pcLl}i!oChn^?n_7*@xP&5w4RQ5B5pc8*)Yy7w;;S}2P- z+$E{0Y;H+|T6MFA6_p$}E(?GF>eE@qLV51FQ=|PzgE9ID z+RO@L8|C3bWvI~))k0Oqs``g-2N#Sm1~WI)B=_3D_gcl!Qd{2GFmuMoHtdeGDSIxy zR^KprI;q6&ah2A~Z97(h%6vnQ@7oY>F`^W~_Gl4{Wk>le$Je!Dr--weGKX zo>u-D4rN2VqjX~XNh9W7j~Rr-nOdR;bmF<7gm9H0Fg_|2EFO>30+#`l42&zrA6b0= z@5}VnhyoGXfn7Yv)8in{bDb{TqY6c#&sYfuWMvFin>B^9=G zoR~*qx>HRU@e43zIm#&<5VpL}OUPmfu)0Hjp>SH3`6Qs8I>yP-(QTUd1|Vb=lp(W~ zq?A$Hi$RTqX(u39+@iqu;PmzONz(lJ7H1pU3`g~yN9UCfE1CZ*~W*B z4}HmbkM{mY4~(iL=?4^zZoO1=+M*#ovj5_rjnB0H=d zx0e4#Ox*rRK<=6sd*33K;m!wLsS3furi(?K62siR*Dr6DuiK;|W-IN(Utd=Vd2^nY z9*jsHGPG3%zMk(HJ|R`M*N6G<(5Bin?z8wt_U+OWE1g~LHntD)bUb0&X73GkydW>% zQ&hhG7~SSf)2br%J%9P~FuL^FqT^XRb?I%*)@OSV#CY=deA;=B{deT-vcB>SQKUuF z{9dhNgpdyNVW4bnEA<9@XhLK4faT)SRE@qG_2~^*B;W6*! zlR*c0)$<@N>?k~1@D7Al`b{9;H{1;o$o~e2cvK4(Hms+<}c#ctE964v0&jnaZw)NxAZS zX#X78aV5~v`}Z%qi2n(>O^hEfs3g}rF(sVkyu)`#iQ%K%_M~=2yGP1I0y}CC08-xM z`XFd~R<|8t?|+J0!)Ff~%Ug??fnqFUbzFaif!#;j7>& zRlh`!LC5Jv;Yg5=j&Zq~wRdi2#@i|o)H3WDcSBdJarF)?2qvi4$)SoSBF$M(=%d)} zEJE9Ffu8l*3C^4OJWVYaACb@44yjdOh<1&d-!Yt6g) zwcp`KTD-Rd*{KGNGIX?1iQrWFi#8;k{c2zD1cyqA+NB;e6J>%7`P8jL;grN4D=%b5 zjNerX>k!EAU@RJ2m2x4$8g=Y_DlM6}HEY#JijLC$KcPot% zoG55}x%uAeT~4l(8TkHW9D2;61%qX8mw8E_=Cl2;*$F895rwMH74T|5Qr8KqPAL}$ zS`Z=Avb(~MUpxsHtKQF$bBjDv|H|RI!LB#I8S|y~+r>9$tkwVHM@zf4bcAM`qmVE8zMFL;*tz(c_H{uoJ%9ewTy;Jlwx= z;X2`tG2>nY4A)P5+&|ndm4^Whp9 zWB#*?TX;{}738Q6YfWMU+-xCAZePPMyP4(CrlxH$W_dgq(Tgrxxq8E{q7_D+)dgS+ z!r}~ZS#Q?_A3l6gcjhh)AF9+Obb5xpTBgNeD#fhW-v;VDlmgcKF1{_FeZM)m|}a_Gkj26eICeS z{ychh_VRjW!fTx=l=t;{#{079U+Kt44_kJCZi;OhlFSd=x174+r80-6QlO{h9~n z7X$LSTW~W=b!D}mWt79V@q%?ReF+=-Ka1^@%$==!QRHaGDfU^q+9c=IVFCKY%GLm#9tk7No}y$HxY3~^N1M^RlL-1#b_P&vAhqJeD_+vjL^ zNRWxQcrh1vJ3*pQ^n+w5yK47%{*1lRKFf+Cmo98I|!d2n_HNsCM}F8Ae|&vvu^=f3=<^jg}BA4ibraH3bode1p~a2C}wa z>-vrx>~+$xPW*6Ba7QirGd|PT_lVDd19ZA{60D|r;5}?&#g{e@yUKg5#e&h@_x!CVn2L_7S-V)9-8)p}0`+qt|JH`8C zau^;SjfKIf$71{*ts({_`U5#1F+0EtXU=uPyn98LQnidSKLe@jW0LQxiMvJQK((Md zQ*c>uzUW#d`$a!+$!AGER^fXoLJ`8Cmifnm{9liHnZ6J)*|MeaLdr? zBNC@t((n6DPQi=*Bwx=hB=6<-2n9vBH;e2q26=aD0dWSc!PX>tUdK2b&1B*AbV96~ zQ>-1sK4-s2m+sC?50RdHp3pUz%CiL|Skfe92l^c|evNQt#=yhazE4>vj_p9QNc>m} z-RqQItSCENM2nL9+{ng3s9tDOf*FRFMtkY%baH-s@6#xB^3z$Xs@r0bB?i09IhsM4 zxt*X|W|kZYH9FEP-D{uE?Jqf$iC}0vd-3~|+-)d5o% z;eK(C%6OP^Z9@>@IEVX#-wgy4PuXi&d+2C|Nq5Y3vT;Sa)P@AgG_LeDxarMpHhIf4 zr2rx&I2cJof+_1zUWBoe1{j#-3e-7Je5GLDb!=gUz?^|^4yw#47v65G52thc+_7GV zJ%;r9#6_%r$)8bh$-4%FaLv%PGkpZqJ$SUzbws7vojMeH-5?;e$`yJZ;UM30>VNMx z`4|<4za1GF-%3`1*wK2Ql4sc#Og%mg*{+8Iui}e#n0&fv`#oruh~7Upt#Y(NHW45 zmSo}T8e&rrmL)r3LtAH~Peu<*cN=%jIXFNOzI!mFMa6G3NVBPrFe z7tD)^St28+bUE7ntm+k>+tz^wn=szh`tNZd^Y&c8^n?iBoSlFohbGMm>4Gn9Nmw(m_4JNt7>*CP!99s6wTZ z3k5exvlfwGEA_pktBwp9XX?l+9m#CU>$g9a1nOWBzWy3b4uqx%n5D4QM9}6C&bDu? z%`di&7mCuR7-UYH9u^ayNQF7D<6>50oybcUp{QD4QvtOHV>)yUeBn7`^SIDXrF>&9 zy<`^0s!mE0COAlERv6vzXaPcoZ-cp#E2=$r(ew9=jqv7?jB9(^Za!p_%=X91V@u-3 zTS3LY$7hQF?0|fL@0(5lh@m)5(iB=FqX%gv^pj-5DVRgPR8WOc_UaKtJ31$SiUl0c zcfrEzRycgli@oTWpQIf8FcF*maG3Ef6K< zh$%)@%NTS0Z{bFya}LE)P73@SR$2$Pr|}`k_&Cmyk}j zjrzGYV2ZQ#?9p_Y21kZqS69J5hTJ^iS?y(GPSk=~S5;O0>B2HJaWc+KNMNsdfODWz zC|fc8dKb|ZEYaX`nglUV&uZ&#mg~~56e()3~>1k$rBkOv{pX#gSOxU@tT@|A0VSQ9o7 z=rc7p2)(N_&iltK^-*r$@idwDwy6cDr(a&%3yriz-$3Cf{2^21kv}6J!R9sUx)?zP z+!>`_WJBUQk(qM^Fa&R+f0^R2Z{_QYE|&`or4{cI{iYm`kRFDT8AvBqih$RD$euYF zVG;w3#R2@K66;Sopo=vQ%Ajt*7#;@{WDw~@Sb=zj=gqy#JO7RnOiTzv&nE0<2h1md zOx=V+=v|@-M(IeI2{ir5Rm!+xq0uN{%wlVMSZCxel9(Sf z01L?&U{`Hs?wJ6*7nUm_KzU|qqcYoDf9FcBAMUc3N=!_#^@6oppewLh+wn;2oKzsV zQ9RJ1{iq^Jol_J)b`?72k16q`{A+n_Yh#=>vE3P2Rb!h>tpTl9l|<{*R)P#gpSzuO zZ}I+J%kx0TJ?e`;<(xCa9@_R5pCs1yCsWY=cq9rwYXg$Y>E#O%{c*neBVNc(iT!@ ziX%huOEYV5T##~wk1D5DOSmKGsHd<6X^E5+K@5xMGsE_jq+ejMy!;ex&g3{oV5uA# z8gri%@Nnyg4g3hW{{S;}W-CVIsV{ms$JEmSu@)4=B^X{=0xj%~IXb(19_o-~(h%eP z=)PE-Jvf1j+EqnFnX>_wiXM)|o-swIjxc>ME5Q&Nk9*{e&s;4jU zO!Kx`0}$mZYr#`#6P@GDD5D?vN) zcaF&B2@WTGm~UDnGXNxIZbl@6y5A6+pn^u+fGtE>WCAn5Q$l7Q0_?PyrATLkTXK~I zzul7!(LKeZtWP{n4mT$a=O~Zn9N`{UyzEeUGCQ@Jib;qy*?OjP7cz5Q)qIv zEZ$~4B2r+VMDJ+iN5s3?IRb(9xLza>b&2lxPB2Fhxw%!p7VLE zx75mQRR!AIHM@Cy)aBIT<&26%q^0kgOzA@b{mt{gDJxX9_CK2IkT08}I~SB~CKChn z??wF%IO1M={dS4y!FniD20xj$#LF zX^Q$o0+kF+)oQl{pSDfST%;^g?Sj6?QSXX1X8dj)eZFREuvKA`Nd5ms*R2ISTm*YBVBVtmFIz5A8r@180#K{RR!4*63-^BdQy`*nr6tmT* zRZBC>Zm_q(&h9f^G))Mmclnf^D}*em8Ok1_xxSlYkDfuN>v(?0!-f$hK4=o)my*1p z_CuYGIi~B7Wu+k;Y~*SNH&Q<@&@|GVLM!|-Sb~fZNWYY=pRv1c6k{P`ba0))NK zjt!k04H_1vez~2V?vUM0v=r%&A5@XRO|FNFPMj2xp4a{ak4gDE#v!9l=X0WDJcM|7 zl*Qywe^WAnRa111Z+T5(3e?F4NUI#LXoMC-Kr{GYRdSVIXp!-P`n%XZqxLXA`PjR{ zgY^6E;O&-*mrfFaDB@sz+_kIomv=lV0A%qHlIS2xqW@cm+xESs*!y;L1gOM#RI9ixxCNOKN) zeRa%XOkH)0Lar;{)%&kDX0VhjocEE8la&BZK(N0{ldD~GEc3h*=_v!U$FcrBE}?;U zLMC+PFfdXg1~EdR>BzpKJdL>uA%#vN?#4n-nPv9;%7;(bFA-nCCUgDdU+osU*( zjB&OkPmwAkQwr*edzO1GbZb63CWb>mpzk3{$#}_2_|YooEa1+qLcb%X_y`>fBmYR5 zDMd?552pERFZk*{wyFV1{yl)V5^aiFc({MXv*hR6!*<#w0wrN*x=cdH~jm&mdV z?u{N!c4<_|7lLtA*VZ13-+)kWh6!V1*zmO#v3?!oH^1tm;M1{-@A<1txu0u9Q5w#- zoWa_rs^N?W+Dq#w*U|0fG(+S6ua>upimU0mg@XiboInGO6Fj)PYXboiLh!~365NBk zyEiW3A-KCW?vmiHjl0Xq`<(ZD7vJqa{<_+GjXg@nsI}IdwdSsuEBxs?ZJ{6W;iVqM zySr25nR*>P%TK*^?SAKQMA|J`opKbr{`b%ZAQ@Ynl46TbW=Xb3IESjqzBFsLj;ZSl zVZB`H*s*BG)kx>q#ipkB)lj#NmD!Bm9XY7WpToNWkGe>RfT3>Cm z&eeS4t5R~?jmHmMR!GvZ)qtVy4f;&sGUqnt$IvC1x*xO#Paz2*7<^-T61|4{zREUK zwWRmlAz46`Lxp^a3+}vVTKm;;fi>9?TbcOj@pXvuW!cO3{~ab+Tw`({IXat0g)7A{ zP&S8|HUB!M&^+~2z$;)t_srs`7wC>%^UN{s7~K=Uv6aF?joJ1T{Ti~5#nr|^?Iz<(X+rka+UA5`i3IMrf?A;-kVhUfpC6&8i> zro(U%mz$k$P^teXA*GqA_*v2cWdG3jNNibOVUxUzQ0xND+B7)LT;&QKRFzwVCrXJ> zbB3Uh3;TMJy519O;uh|DWdEnJf8pS^;n zi(TFQVkDF3_KRt@%pEk}Ur5Vp#6+&aYnrQ3GN@t@K`fzh=n+BT=Ynlrv^h>;fhPZq zq0KNAlmT^JIX}q^e3>r=EqI4W^hHHy<<+|ReID8XL?@^?C#6&K3dUOR%y-<-Cy%5u zsCnOp@!*?7pCrogaI-^x#*y!ei>={04gm;{d}G?6D;vrrY63Z zZWn{+8F6!`AMa=rY0StNsy8bF?SQc7H!D028f=OQaSr4fR;dd8M-(6~E)8mIY48k9 z!ny}_-=~;Q!$ltlX+ML7lJkBa7%lOom^pas38%9p!s-_FSx01s(3XP8$f9$+PH_PL zw>|y(w~WAIEcU#u3076Bs&7;gKbZOH7AA#Gn)Ov@bGiB5>MMLHtyqN~YO>W$B%LU| ztlJt{#>%GG7BtCg+&R#FB6h3Ae;@jnx^cMe60Uh(e-5zAaT;NPsrRg|a?WTHDBxAs z)`#QbB_^6|N4#Hb7lU@lm&};BS)*#sG(C|Y-LIf&)@4E;W>Zg|ExF?l1d;GHT*X0+>cU?vS1YHd7ay7vT z$6Oxf&FE&$kr;L>i1lKTjw%m(~i>Cb+nGho80?<-LTj zin?cE!1hIIoSW9B5V_$nxUoGv@7=)|TQPL!YFR`I-RJ60ESe|QZV5x2Vh`)VM z4kK&pNV!^-3XP5Ci5L8eUHiHQm0xdpbP@Wz)6EZY@vcdjiJ4MHoQ3thDm1j7q) zfoRS$xSOcdeHsxq<3*pCr%xHKy!4$3Awc}T0J?B{O(xJl-svqTYoJC)Q0dZsS|PX9 zBJ^fxk4#7%7&lgUU&deYzrwC2OyqIC04*N(J?DaVO6d@K4@du{xA&R_1mgc`F;APd zrnY?`5H&NG#oU~p8uwR4Mf|Y2qG^xG7Ya!_^b|-OQ_|O!Md}MFyS2NpykXqFJU}MS z9|zV%_>f|XD!b#$lZV#apK#;u$w3aE`L4r9a7;7=F4S6&VokR6>DHBmqSXF65QmY( z!@%>LSv-- z(cm72Uh@T-I>jdC&hq<1Q??ONCyF?T*40*8A~}-kf>opBB`H9 zJJ$wIJ+>VMv2on2_^j!wX(D)8F{j}?(Q;5aB39OMTo3)0Yi9WYUFSfQue1YK+k^&{ zk+uiRYdX?EqCV%0@?)>3;Z&c@!W9cQ-%#&7n1Wf7YMCSrZI=YIB;TQtkw)cjb;gE< z2Ke>Er%bFC2g-M{QKpVpz1W$;p+MJ&i+#ow&p@B{?mHNoR9{fXgtkXoj{^F;>#Ews zZ8_qwMC>N)6@%E`74nO8niZcQ+miHjpv7yG95a6qJ$}sXEr$zI{Fvi?wZ%_BbiT6U zutuRKUxgXufHb3cG8s0>GpUMV($7VmBy`+BrH;7t`6pfBlHHbqkHO@7yta~ za^c!Bt~^3dGpDG z=m{}AHt7Tx>XdpNoI~U>yq}l8KmG4*`y||XeQ0B6cVYAcNrf4o8VCiMd~;0r8#D@W zw=q3DS16>V-zn|j$3Uvozc?w`7$-W zRU_@cU3PapfAIxQ5(;#*cxkez>rON_~CPnnbXUm!nFXv;d7f87BIDf6E1*b z0t0Dy$vDC?p4K!Xa3y9gBCzhJ}<-1I^Ces`a5s!bjw5Z6A0f`*OG?% z4?fX^L9RB1GG6~huRMmVZIkRt_K zkx36i6L$3P`sliA=g{yvCSr(_jq_XAJ{L?`YziZ4jM^v(snP3B5h7jHLUs4py|~6 zt{%^ouNfm>*e+Q*@6C$)2i#BF%WO``?ae)c!%NXu-7n=q=I34^%=fye;C(q8cf6ySN*X?fvS639bpzSOcagEA9un*;6&W?u&bu6E!KAe zNv_>b5a4>_%(~KdVCWSnc zzgaLEyueyWNhm0fM!`7gQ0n#ZS?kZB0qRs1 zTzY+(<&q9vhGZvT28f1QjgaZVM}dmad~$2R@VQb@)V#W5XU4@gMXpxFZHJQwGIJrV zw1d%-5;r827BX_2X{~@iolbb{nrU7jId{sz#VIq7!?+U(Qh@8ql@7*4N4qdpJz)!!&Q`Ge8V)!i>|R5%qf)(kpGNfb6BYDkd=t2H|P8p z6J&(=Bfbxl4M5h@WUwV;OrN~*{8dq2(_AkHL~(ByEBC~#T%y}=VBF=}=1!08WdsBf zg0v2U)MeNb>7h08iczJ;;F01f%>B2V8h$3)vu%>27ymsNTgx#_qcOL$NivwAB}y17 z<@{$re)~djT)$1BeKj|U9q&E$W29T)Aj%FAWUoUN;L<-*JN zNqKnmoHPh@mW(z7l5;Mb0&@L$iV1#s1tg>>L}-ev1o#sCtbn4h0-(wa2Y_6q@%#yQ zY1W=RlnxIK4{ta~M)OWxVrm@sbai+4aBX|ctwJ+7fW7?!xnM!$`_sM)&3oxPZ(p=M@yJgfK~nw@_T$|6=8Uu zgAG74R3>*5s5$ph$p}c55Cl!>iZuc2%9G-Mh#4fq&_w>V>K$8NzXYomP@Ew^&s!@w z^*|1%OLeTW#a#Lr^@$loJ_o<2E;gINK?E-q?gfkAuKxEF40!kTxrdNbO~mKzoN584 zZ1a9iT^-D!the<`<)(|Lp0+{v98DN-dIBW>&yGfoj6*wZq= zYHnxs*~__wLJH%bf%@0>cHzi)9^r|n7x6U;FBx;06R*#II@tApm(3p))7fQ^AoN7Z zCvf^9^b+4l0O$R7joGk}ZlLS{q>v4k-kvC!3Qeh9LVIoI@A}iv>B(Q4zgM5G`FO{F z(2!O)jx^O|eSEUR9~4OU;EE#&L)~jTjZDIyJLDD=RAf#!{_z9aZJe!~I?l-}=D!3t z{)V!n1@+GBT0R05o1LEzi1#R(?ziBf515;KCmkP?H@In`OQ+fmlzar{YAh5^{nTKM zscEW;DJ6-DjkdqI!kca;oFRB$q2KIJCvWvrrnWY6?(ovogM=NBF7Aznr+uX($|ab% zF$Gy%d9y~byy#Ed zd}IWwNgm1dTl9Dt#5%_QKY%b38WP+AYAK1Cz>i9}0SZcCbUK6yb;wL)Pd<^VbW^;N zM=s%Qe*=2(;H~`%MY$)`wM?>drtj8UWTFpKk1n_S<@OVytU96WJ@5U1dF+JE{LWP1 zrDaRZmcWsnoiSZa8uZOrED&5YiI}Q9A+9!gs9Bl>Zx!V#kll?Lom?#aIJ~1+gfC*C z*Hfn{;&zUB+Hs47f*}~dX#A5$3b*^a`@kDJ+ zjokDf0iP#axKucfc*po3S8}AEpKcY3Z2~4O?98L?rJeLzU8AR_5ZT3E@zI5cuB{t~ z3a8PK32L_rHAqgx-TfeGU=`Jw<3|x4@37J(x+}sVnUs5^0qr$CW0zy>kn*hodIMAH zQM$csVR<$NE8a(*!ym4HepL9L`P~Q7f>qTn|Fnuzg?VNPIA%w24?f)iy?!&3m0_A{ zV5@%1a^~y(vq~_0n5mu8J>|qs#fvnDbN1}V0f=+#w5=-6gmLxrfG3gj_7K-m5JDhF zjYKgf0gUsV{e@QhT=Zrq;`F@XGMk|YTw#;J;XH}3m_jz*m>(x2%M03jqdSq7wW$i` z-=5CcUx*`u0jU#uMkcMJTfs|pLN4E;F^v4pCy!^(9`u(dElQT92GdH>sX}{z=9Nv2%PJLejC#i9c6NiBiER60C6m&%>RL)+JL;^chZj z+Mz_D*yIS@q3Pq(;WPo+%JjN)DWWPRHCV3CW9n_3IQB2E@u#r42Zx3ApppVi}<8K zl_lbed+Mg9T&#d21Rl%IJ-wxliBnhD_L6z#Td@n}0Q}{vJj`Yf_Uj0)2AyXbx*?A={xkNc^a=-!na)$F7 z=1r|!-w=K(&njl{7hj_apeNeUD=yES1&S?sS-V3*rBQM0;mgW;ogRtM3kuXdoT}Dp zIj)`DbW%7t0B*tn%n^?^!P&e=aR6A+(i0N3OQ14^8;8$cJ@fZ<``CgRd1*)rFtNnZ z(2yuK=M?BQWYiTVj%fjy)W|;Te}Qy}RRAs;HCiw3+AXIvI=&D@Ar?)|DGZ66_GIfO z6HS)^0Km{fY=2BkYL)9&-Fl0sK3lM4^|6vK9Pih!3eECH%UNOPWwGq5sd#X7Y=QzE zZcFcR>+PU9caUV*mUfo7Q^<8P)<0gY+coHbf$l1s00sM(9;aFB4^mcBKTnO2$uAN^ zEg!`aHJxD9Dp6gddvOjKxizZ7i`}dSM)e!)TZac=0x*d{=pVO6Eqoa-2Hmp`5N1K z_WHst-O%*9OLjE=BmK)>Vwms2^Ak8>@%Z#CaF_FCfu&aab$02#I_)C5a}&Bs;`3Tm z`rIl0Z%lsw)Sm_WH1b)al(V>)e!i>@z3ekRgM72YCB4(-rHI~&kz0w>?5VmfkAS^p z%I_5}2Y%xTmaROeos%D0TKt2%j!Cz!jN4pMF#y^rkH=#-`vj(kA2|XMgQgm$+$3D- zJU)bz-x~Lwy-a5mzx`$ocg_KejZJMagh|Clm+RBfm&8y)EG#lFPO#H$;#XGm!867? zaMEU*kkWAUgJ^nTUEQ{`X=PZ;(DMN1ukvzZGDH;_QzegnIR?vmA#L*jJ##-}2pZG- z5j8D$`&;S-u)i~3AX$hJ8h*q0{5-Ih+liG;4C3V!T2gk1HnQGF%oeAX2@?0o3YN$D zs;B)Qts77dT&A^C|HXhhg+eXmfJENYy>+|@GywXsq&3%J+VarZ8v zl(hrT&m2PfHDL&0(pIQF&Q;A1g_BK~EhrwDiPJPHDJQFoDs0CG;tP9;>#sP>w<6_2 zVanaRMh|>LB8_{mP%VCh=N;gRIiR@y7GaSAi(E%4p;l8n-XFl5??X05IG>@yXGjy} z?JVDm<=iuh?9(x5P!O=@DJ5p1S<%3B$~Ra@0N9SSvHfk?=p`d%tiuOchh#8KmxB0D zSNp4P)q;6O=+f;i`N&|^a`fU7txLI@wz-OF>1pe&ukeW~R)R0uX!<60o=F>1h)@O` z4{vL-Xk@4k3HR2=520bM>6=;JdH+ufko|jo{SqCUTp)JMI$<1}0FHi8E(F=$7znVT zk#_lk2gqDrXNbwi_YyI_Jr;0p+Dw$vi}>L$qu+NIW@O5k4{cF3a!CvQ3=)LdJ{Wmijmj9c{N-! zg3`N~3EeLXSiJ3zQG#ua#;BY`2ov)%UKKdE2l zmsTLA?r|p@Rw=0|9(ViHUS4LP5!kWIehUSy2>}crA5OQmwS}xk1aD6E_D5LsdHz{^1B`aE=k- zALkq^JoUVT1hJ9#2;@^f)E(&O{|qcn-w_;}RIp z(1W`sd4lHhYFbA|XIb-#r?B-+SrujYGVE|i(i9Zvs2gnYkg4s~sSCUCg^Kg0oVTRiQ)#E80hDf&yuG zgGfDkW%}fT72Eh>tjT>l<9T!*6`DF+d9;(2*51V5B?6v*^Ne(J%|JSL3ECD=X7p^3 z(Pz!f#5z^4qigtrK5=Wt=Je>o*((0g?R%WUcHD*!wO8xh#yIq`aE7Gim@TAehL;EY zL6R@^eG0YVGod5KGNPf0P?DT9Qo4##egPJeCw6~INh+dl5p~rO?RY8wUZrWbMH*KJ z2wFZZOjg~Qudp#;0I;fjTsl{@;xwSL;x)!c&(yY*;p%bnvaze2g7XdSX}Yg)DN>`0 z4!Jh>^&3QXoEk5X6z}bp{D==jsa>evv|Qu^ zor`2a&45vVA&-vK%=;L0DoF(%T6#1|%K%0qOHzRp1g_P}*l-gWPysrYz5*Q{7X}A_ zV1(;tZx+T0IfA4iVsSXfRd*$Jnudk5EI-|NT}`}Du!Dhq0~f7p+uu$ZJ1<)qhA`u< zg}rafF|2C+9ZvSjt-1I#ARo^04F$hQNl*LSiFP;*tS^5tG!tW@THQEktf0z%WhM@J zHGk#wntM&aX`{{JBmw*!8TaG7=mQw*rY8Uf4|har!w?D=76gONVmB`J3Tu` zhS;Hf&+Aw>^w|Yfw{h2=sB5G$72|9C1!PYZmzw4X|*6{ zsQ5T%bMM7)ZKLJaJC)8#x~2r7|Ez!p&rx3n&tysd1KzjrpuuoHRTvi}i$c{q$h zT-WTlWosmIg~PD>nlIbzU9(^lXo5AJW9`!*;E#$UE8W|H$Y?5lz9 z#(tilPQ4I=TF&Olb7)=02h|*5S)t7B@#{=yPum1^)W#V(Ze$(C1--TPp7t135gw(1ngsQm_m96cO_{MfsII+#=yf9{qdff?hZHs4>|k$7DQ6agss1 zZ?(f&9e{a?n3|>%$nH?=>?PXFnxDV_2~rQTP1hb`4O|fM-`jlr0iJW&E zOMDRnPIzzHm}_u6l6-u;a>3UUvKGqD%`$SWwNLH_$!mNuw1A3o$_f!FJAezJ+LOKf_` zoW805mM&MT$edmnmrlj)iWanfwTzz6NM{Bwv`wUn3M+VSp)m~#>$-k0nupo>5s_#Y zhsRMPDzd&UZeEK**nGw|%~m6P9qUrT8u%j_Vu{EqO&dX-&}kYRsAcq%uIZ~Ct{UM~ zKJ$)wM3h$1rj!^p{0s9H1_4Nguldz!PK8q#|I)Nn>PyNXT8`raZQ>2%7uwxKl$y`Q z_{4&oXE~@G)X3et1plNO?zpkj$B)UkUAD(IX+B|0L_fLr5}Fqei5Qrb{|GbriU9k> z-+&gX#(9F4A}ixz7K@bgXsaryPqg~N#=myL`MSySG%g$1-+p?&3w^)auNoK2vi#%K z#5L4#9kVhjKH`AY`;LuLPvxR7)#9bQg~)bWq%z0u4Eo!_=N40b2u;!U-R3ZtwKVbF z(-*^_1M*dmQ~KCqIYHegco)3ZXN#67prGyHckgyRw)K~_je~kn(F8diQ`z$4I%#q> z@f|cZ((-!1`|bs&IsOHILhUL49e={=!N8OK(C*dVMCqFX#)$kU9dC2$-ScUh;H2Ja z+R$(p=tA>Xxd@~--yQi`H#d)+i%VFD?4qkZIM8nYVilEA!}>JOiZl6vA}|oC5K0(% zOKBubbg!44nj=244MQVuR4$>kfBb{ASkG12aU(>w)H=V&*xW6$(xJA{CP(v1;pOSlD6b$9kRdS_kv?DbPG+4VL%?;Cxk%FCt>oA>9B_)IuxCQU{a z&$kS@BFQ7yuV39$i%{Lt#2+yh@~jf9$hZfq?zY8fX-%$L$+Jb8`!*L{RQ27%i4C7c z4W9OhwO8h)zM(%DH2H46@8+S)0N@o%)B+I2C5TC-P8z9PofS07yiL} zOKm$npBO42=z=oBS55h2Fn%US`tpJ2KsbntK9dDcF%V5*k?2@1(XMRhVJ0&S%G+Xx zD(V_e#*0$J6w$}}`x{g~mUurVdJ=vkF*ArWZyBP<+I{;2an7QoO_?vIb&!*z~vAK|qrIG|4|ZqTkwGzCpz@@@i+8O@Q>rc0m=ACO<-hh$feG%#Ca!Dxb>4 zjn2nSDmu=w#zkI1C8|)m6p{=T;vBSIT;~=xF6^p${SX)@ z7LUYhA%!9pCpb~y>KrMD18x6Cq(}QN-^%D%tE0AmUz`WYM1cUeLaDpiR}8+E2#AC= zekx*!EF|^%G*b(*PhtEbjjbwCzf8sL?LW1R%`ev1GHS1IqLu#b4N1FF^VjvR?l^`W zp>tF8N{u{z4%wk6k|jko2YZ@ul(9w#8l{A8 zPkEKucyWVY5hs2ss5&0mTfJ_ZSte}22!x<$FREeHY3?T@02AX((l|56e+KKVu@>>O zhB3<8sa46z-kTvx%QxRiWw>q<&(AUh42z02EkJIHz6i#tf5CevtDa;SJHZOOYV#Au zq2v~J^J`k5z7aiH>5$&{6mh%9|E(0g;uu-HKW+8L%9+%Zn?z3Oy<;1WLI3ezvZo@Ys0f`d;Sn~a2bF2kqCKTWh8*kTjYwBzprSyio0ZCZSSLMIVV;N7< z;4_7y-ts9}IJtLY-`cGes8bESOhB^ z4*GF4#ugTtsuN~DH*e(k%+XC{8ds$GZ3?5%JjK)~k2g2t2H%|*$vjX;AKNzYYoQ0m z>9qHsCl187^n0zn5*QQ-IEG;Gp-pzkh>%p6rhQDt3Nkzt0(jvCgY0CAShuq37a&? zr3`ASRxFv4{a#*efBVCEhK>Z~&L^^Tf$1OJclgbKJjlnD(wRfALdUmNM!9Jg-@Ick z#xc0JJX)ZBO0Ci(msyYs^^jtzV0Y zC;}Ycak__`!?2ZiP|ni|^=g4aN&T*3D?%T5VR4sfN}}eAtePrBf_(UASDwKYtdmc4 zM^C?`-^33;-9-S?=PO6D4Abw)vA#{g;?JZ5k>V_p%n|vF!xmfI zn4Asixm&@1NrRgL11JV06h{BH^L64yaD>cSrFCxWVxjix0pZlw9%)3; z>0y~EOv-M>r|UYC|DN*YYR%#B#CCci4UTllD(3Acof$=P8NBu)a{9WiPb1B6V-knI zgrFh;l3EWeYCf{9OHa#&aOC9e-KPoBIiK~2t+Xf)g#F}`K8QvMUVf7fMp--51hIZ( z#;U%$Vp0d@q1x&|iF_KPO@Ltvj>n5Cs!%a0p`CS)1i7^>_ScJ6chP7WA!UenLb+q| zZRhI`W?fxpQmmkN>z-_toj1&(*yGjCw;1<&A@`aL?vKy6VWv?KGGqroTp^vFp3_@S za>3`geb$3e2e+d%y=iUxwH5Bim!@)L&T#|dEAgC&Jds@vPa z|HMITes=AeXHIPh>exjV;{-1xrhy;_>m;tED*YHlD`RH(MKr|tOsh{XZa@k$syv3X zp16%_yv26D&{%z{WWm90|F3rXUV%RBoBO5Q%Wa;?Pe%S96Jf_MFC0q7D5y@O*`i7X zi(SVafV~=6i|o-gywP)6Ux$H*H9#r&e8V9i#?YTbDoDE7|e>) z%=2szDpwLse0QWZ{5aQ{faHDG?GjKGn};b!-#@g=rPq8Bw4(HuB^hGvmgo)ft8XW# zU}u{-ZZ*X~V*OzIvHX;!Xp4sD^A(DUX^_5>$iepa_;1L?gd1-yk zm<*IFm+j$0myT_37d0lK4ln&&Mn~}V;2JxYLKMb1ozS25L~Oz!B~fi+Q^kCBc}P&g7|cJ^qC5h$2t8lh7!*FLqqdO_n;B41zA z8ysozvwdxEsz;MYls>*;FzVZ&!q!ta0Z3F}RD$%SO#J1GP10f8^NlWYb|@tCE_-Bf zAFbXKlRUhFOvaxsNz4I6pTdIEuWFo1pD3*f;u=uX(9? zIJ`g}?FVt~%60+EX~LK&#OH-@qWw<3Fp-F|<5qt>TTGsLr>eP8h301CCU)^5WJfMOb5iw<&#Scz36)3GI1#)*djQL#^#v8r`_Ju`TA4d zXqr|kjg4`+*avODoKfjq(^DfodMTGUv=M!vRKk`3J3Vu@_VZj8i1V_@KRx~UW@6zE zzd&oH?Cg%M5HY{Ilpic5Gy6|ItL(NQG8Qt(p}HE{t1&$WMC&B;uihgt#N^~^Gf!Lc z-7Sx>b8&O`ZS}oB5qA%wl>PLLcd?wWD32z^_VT9nyE=$9j(uBzgI~IS)yV1`9V!64yJ>E@NB?(8Vfci}?mQ!l1fE zTkwdfFpaO#EjfMPZ9kE*+y2? zY+>1g1$yY%nLC^_Anm*wqDm7;;hgeZ0;-KeMSW7lN-0`_NU zp|PNi?NSp!Z1Gtj7eqdela4)`hqL_6+ypI5AjY0L1~)@-EtG1N@`@3F#3c2Ui#aVd zqXJN|I95+LZ=I0fly09XS&0lY##*_YB1aWP)J>Fm#_e38P6it-Djuwe{b&rO^rQn} zuHA+ihi?>y2g%hg(55crDE@^n^!tC{pybt;SF&HBC_7Xe+IG}y zvquW5vdSc$FmT?FYnXK#@vNq}|kO;rKn?g5hIrWVlF*j7M} zK+oCQ^2zqT98daL#ac0fgyAf;K1_0mSTDF8Z3)hL}1YKWN zS~d&gG#L&#Ufm%pvM5R&YB0Hlb7QxcKQ6Dm<#vRl=DsUQSLzts61GlKjWyySZuW6O z*xN+F#?$!PG}hu0$IacGQ#(*oMH9vb@Qc-Qv}#11=8TCiJI2W zW*gm_zmT0X-Z77HY4w|yw~cjmy=nJ}^=yxsbV0d9&nPFlmLnVE;rQUyxl?boC7{@{ z!p5`|&=d=JKsM-t9-caG72IkIxP`O(z$3w607Q^Yx=2JQz?IgBqZE?KX231?m*eat z)&Mpc<+xpX=HFw`1eu%c^GVW`@@I^Ws~Q*8@GNn57=~&~^Yh>rhY-7m<)VgksR9Qj zjNxJ-Sfxq+irG;n1gHoZDTS0WJS`OZ>@pF4b~6L`)C=Az{>I_IPpG7k6wMt;v~F}P z+p(=Ss_T7114|4x&w{_8%jV>Dqu2R#sWvJVUTfdo(W+*(Bb!H<(hQvW7dZceS*OEx z=~Tjzi0W{tqD8q-Jn|kCvPTp5Qe@;-um(*uw!vK}AuBihjfC0L)jk_xb_u?c!0n zIq-7!q;NtlCdp);a`M`jlJ8I8z?MM624(fScmZOtw{9M158gc-6S>lK!?s!RNXhvN zr_O!fS0cz+h3Q#9|M=rSHb#LW0_o={zk#uyx9PTLDhR^yrT@5`aLP_0;F?Av-QA`P zjp0BLr^~BaKJ|%t-wja9k!71SIYf89=p>6Vg~W!Ue|bug z_Spmy8wyBY9OE>-^nd=vuD`uKLX2Lyvf;J%6gD`@>2vjdTusMM-r736WQbFBVtdkO zT)Rb6RHq0!4!dSCN0^Z!(F)qR|P;ubItzsg%BjXzWP zo>^(dhz)e!rB3|V71sE-i!ajp44VJy9sjXZ&BgtVi*Gy7|BM zVQ$%A;D7Dv*-Xr^|Joxi!0Oa;|FtJsL|}0K*Pbt;`6*2O|HBZz-@~SOu;yHRpiwnp|pW2yTP zlxF<@zJ{IGw?p>um;76lr#F9_qO7V+g_Kdi{{c`-0|XQR000O8b!o*~T1l}nOvwTO zcJ%@PHUIzsNJdg&WMOhtR9{k0S28d%G+$6`d2MfXbZB2`Zg6FCb8l{6ZDnP2X>Mm< zVP|D-WMM9FWM-_m1yChTvo?r33^2&x3=V_4ySux?!JUIU4DRkWxVt-p!@=F%ebB*{ z_xFWNvJlwzmfoGc*0$ zFK6UrEoEnB|7qhN5nxLE2?Ox+GsppeMy8(ziJ3Y7gH*Q!xB;9Lod9M4CxD&FKO=Dd zI|e^LG6KNP^j`oqK+3;YnEzLw{`G}H#NNdYNX*0_V`=KFL(KY*mLBo{)MNP{dU7gK ziUzVq#sC`wE@m@ECKEG8I#W{?E;?2dBThO_fa&Lr(Ztx4nbnNV7+`9kVhIFDnNqPc zGjW+QF>%qc060Fe01h)cBNI~qoe>)c8zUzpr-><-@h1)cZ>@jwU}0}cr2;gvGc|HD z{r_M=%+AQz24G61`bq4kx{%AKK_@C%`%e}*YMhvF$DLa9geo78tMxT<=r}TeD5`(dTv4gSww?RvLJ5eJb zfJ&5`nUR^5k%O6wiJgswiRlL;GbJM<a9;Kz3sGe7vaum51g?d|?CB+f+4`ERc{GcnVDqFwyc65D?yR$`|A=!&xu zGjaWw<|iX2DgdAk!)I19r~=%9|8YS0KM#@rJf!{!^3O~%wKNj8ch@2QXIM5aHhLCj zVop{jdM;+Y&o%mV;d7$S#Qzyh!pYvn;lGLax51#opz35~=j`wgAtoLSA}S1`09Q*B zfU<<}r*%~$W9LuEf3o{Oq&X8a{!f0{{s+I}mNr0u6N9+TCoiG^6MIttgDk+#9B4tz z#>m9=kCrp=GiYp)5!|zy^mLrpS}}d6WZxl)iY7BCF8Ae22+a3P=h!pnUCoJR`Kj11 zO0Bi33r#cM|L%fq6L7@uUPhV7g!YC)Zm&Byxck=eR9`^WmyYq%X;D)>q3cfC_{`FYVLguabqe|S6x z=Bw61i@ORGEKl+Nkgr0~-K znOVYo9JC4EQmfY{ZIhKy(K%G?XmH%la8LnGTni`?3FYo`K`R)91^R`N~gm33QdYXCUv(m{K*`Z@3mZ$Yp7p$9npY$)Nx*!1{BL_PLokgXX@zf*pMJ%8_yYzAJQ~Dx!9C@+%*aoUF;! zP#|X5{J6VxNyz@Ny}yg29V=;QsYx**7gl&>$_czP5uzVRrJtsmv$Mr6*bJL-_iYe6~DNsh+M+f7zGX98zg4RP$$uEO7xOD_mbntP&@be zJj&eO+@AjZ%|JXtN<6DYj73ErjJW$EW4I0KFxU*)spHyKqNn?^A82l}s>IXD1L2#2 zHjS~b=(TW&G#wcg^gK^AARuU8ZFJ!K!%bQ9aw)Ip%XObySq=Jh?qfJvLWJl$y^GX$ zweV>SgM`Pt5=naYL^CE%f?3JnK1!cQSLDhdfY5&eVFzY>wpDQ`26*szYw7=r= zTv-Tcdq!G3rH42UPuf5&1=ca8_xHCYeH-P{d{YXffH?I2D_%I!24h0Dtt*oH2o2qIGNpDT^gKpnWcg4f0ok~mPY?Kjt329y}MEx+rI7(X29MnD~nuw*jv z7}9S9{8{J9Bv?rq91u91C2gHfY0UQXgXD1V>hOeW!1Nx@yf=g2X=U90S{9}69bj&` zlXdacn$IcX8vjzF-{HG5JslB+GVm6SRRX={F46o@X%Zo)bW3W8OhwNYgKB}2Z8rM5 zbl3GyQt)*<6f0CFO18!6CFN7Atd=Nfjmq z*k$v~)^!ocWX~uYAjGXt3Z_I44ULgdhxO?kGNicGC1H-!#d<|d)d}E+VO?mU%++QO zI#iUHG0hSUx`KL84k(Plujfo8qWVNo&YPc!M!qEdma$H>E%?sleXhu;+sh?67e*EFYJmh8N^+QW~&%iNoNF@GYX4QRHM!)vDF5xVa&wggK1$tPy@*O-zl2 zEx4JmQyVQkX^a<>y`_<-Y+j&jvl-u2sK}Tp2zrSb$(N4Ez9myRRD-KHU9!P4G!Q%0HRbj`M(Pd8WJ;8fp1Y) zhCkdAU-grTDN$2L4nCkb$@cW;w#bOyVJmsa)8@4gN5P3~a@R$T)LrSwmt0Ps{Bq0h zNb*uOK{;1Nm9#xI(HQ-$OXP}yrt4DnC^hYra=6~V$chx!G5?oUu#G@*ARqHfu%tLR z4b=d_5IDl1$Pk#ZmKiKY5jdV$f7}wo5{6kK3W4^;13TzxU0)8RBxq zOgp0rVjlHd=*BM+>x{8JS1UH*OgYNkmjpCv| zr(mgqNwi!1T6_DsZzEQv|W|J6gABzOk9n zR++}#1u?;rcb(A7&pM}8p)RCJ%r=1E#pK0GB5u=18ltmOYYN#QE#twG-P&8m?kI zuNPHE&co_sM6vR;vY>Sw&(Q>6kky7iC*U)-K9&iT@7RwssFwk$psl9vd7J~GDX!H$ z3Eloy=P?KTK-8333bXO^n>WF5AoHM0=YhIRiKaNVZD$d>0-FjRkBhLHI7#e{KScd| zpmCiy0hsub03xa;)#ck^t|}HBPz~M=`R12WyGbQ+Vg-n4h#ZvnacG!rN6#O$PY&r) zsy?IHjnZk23Zec@oKS6;V)RUp9z z;IE6-mjte4d8TZHqc}khzN36csX2lFXmkB&wY#LyOSipt`Y^6d7h-=UGR|uWxm`8|0Y2$Qq8HJ7zo*aj>V>@xB`?xd|KiLGQg;9`adF0t@ zLz-occKsj$F=p$p-ARI`Z%cUnA|b@0Z`)|rkZxcYNPNiBgCutL+AU`i6Xyd`?T}4u zl)s@f)2$$g#W62^HPf9-nu?n9)@a46Zh<`{D@Emin)XgD4pg3_qSDK#&Mvt;Us@0G zOB?<^{#HUCFa(j3S03!N--6U37+aZuNXV~a4mhD3l~;A`z~zQyiruf?!sO{_y_f()YtB)u$6;={qcB* zNF(U?KdK)>`OhrM>elZc zv=9Gwct(zHC08$3?OY?3BUQTe=0(jAiuE24a{bWUFu_52&?hL%_zBnDi*qhSOOO=y z=Z$3BGxjG9bC@b}zW92?4+#A`O;J{{kwCboB%|$bAI?y%{t~<>_#n76BS-0~SPLw% zVDevqREB{BJOL%s{J=0tJsMn+h@BZ)D;OEl(XoQGKUtU5F^hRGHP;>J8n7m%2wiOp zXz$|o4seRp{TnUJB=^;668V@k0)Fc@6TccJBFG(>mGRq*m0S@H_O}06N25|s06VTQM0jnqoUP~z?P{qusj^8zPMvfVhH~?aN!eV{sZBwssD{?5 zj#}*58#n?ZP2xGfhq?sRP;DTt>}vgABHE0OvA?UnFmo4M38kHUROR-LG8R}ak~{AO z(d0@Etg#$sX5pCB4DtZvUp9ntqwuOoxW>^5pav?&iMDJXIxj|;b>8+$L27g?2PHtR znPX%50gU71Hp+1pP1?JxEGvea5LMhGrp09DI1|t!ew9GLtk#_5*8v=KLhn%i+zNu@ z4T45%){J0tU_RW>aXwHZN)ee9vF>K$9%8C>xnOIj-Pl)pj?diC(Y24B5HVtGp~?#) zj2D_irDo;(HQ;(ymM5lc03-?PWecT<981^S`Gv3VEF1>B;#!C`ZY3}|F|DnHjMrws z=qg5qc?8O6JYShS07TDIM>usvd4&@ z{23{P8g|CHB$4baVwueCUh!?zpF);n z+`*CY`?)Y->M_>-w@ZV2IhXU?ej-;QPz?Og=cT*|<>j?lpBx;mT`jvqQ!H5a8;bcX zcVxN!1+6Q)5m=bH)AN@bSZ7XIdoyqt4~{Y}gXVOg;Q{-0`8+DBdSo z&_J+*P34fzkY(W8V3^SXy!>~pZ=;n;CA!7#4zbwa8ocn9_cI?tNanmw; ziS5w9cIl%7W~Zs(^fL4r15EQIcO#VZ1nF4(OacrcMgx3otu0$TW3*1*Gnoq*M=|Y& zG4^tyX;PvgRSVY5z{b~)Nq5rmS)TkNj4;&z2l=iZrWa-+| zw&WpN!*1y53Zd@P4qz^181SGbFM8-MAn#Weupy(;PSk}#vh6@Cd zWvKJW-Q_;8wdbFTe!!No%c)cy3tU(OL9T9?Wmo|X5vn443cWQ;CYK9KjRg{Ynii$ie%+U_bLbX;Yf9@CbGF-XVNft&89Vj+d&jwHdlY zO>WGQ^x=j}p9%0(rY+Hglp93{0h00v)MYD)RWYo|Xsh5aYK_r_ zvhFh}`ewy&D0`jM%`+gzG3wGtD78cHzD4lC*JLbWWLp`F*eBs<^^|Q(3C*$B>Tr5z*U=NBmplHKX`)Y#3h-(@=gR`(+SW&P^R=NNHv154ChXdb8>)kwR@aMeY+=t$YWx~vEr|WpG)xi>;e0_H-=Ovcs?XmQvr*#Aw@1SvgX$ESOTsdv-5<$h($?V<@w;PskiU~$nTb=;)M z*1`p=PIi2EB(*Fso$HV@WtE53U@PQDCBFRRWqZ!F`>xgC5#{a2qvGG(O}SS;`=PgC zygG$D3aZPgD@q?A?V~PGP8}XSSU;Znj&N#ms!k+6Oq8r2l|A#~X!Gax^6F|mkFItX zrsUvVd<5fEMz0dA_^55yfCSxgfB8(*uf73TJ*G$0rAu~$uqpv;Jum9&cYR+2XO4ie zI8OF;fk$~!67+aZ$ybQ*MjabI-*Z=cPut{88V#Iabh^+QQ-^(PBzPCS&Q1z46=;kM z$@1xr^_*NU6H>HrNqn+?F|y1JE;-ZrM)`#%IXC3VSfjE+a(Yhuq9G_E4|Z_M*e}h8 z!`*P>;^Lp9JOvMre}zw=#Z}Q(X7RlUdb>v+4u#p{W_mNWQOto-^l^Fd^W$St zrRz=gYW?l)Mau7)^}dR9B{!Rz+qdh~N!Uc!%-of$yxPRmBB8#vA6vS~KfX#YYzOE8 z6t57iHgUrIgRyyxfjOffYtZGZZO|W05=eE#!{LJms5acE z8qkhvX%v#+9D-kh;1!-;`B^VEFRl#rG3--Tto_r2=R(h9s;_tbb=RJUJrU^bXz9!5 zEAv_OZ!IExhSu~5u#9xt2M7FSNzq+%#LPL9#x~?N)5eaCoDI8^o_fx2@x3;NaCfDu z19+h<0gU?FP)z1{-hv+oOQF6+Pxqm*ImfX?8!vN|W%l0b@MK)BU8l;zUpes&^?Z(y z5E^#bl+cmm2EIB?aB#E`m#A2Sw(EO^s-_43V7yO2;K}F`Zg)o&ot-+)u2M@0BF`yzuXeZHa}%F{iLS1Cc8X#isw0ckyV5rk z0%_V02SBHL9ObdP)E~FTE9T}A0=)4b5Apc^6I-8aKXvr@@oITWB+&gRnp^m8nfZ?N z7bEFmYwa#|>I(9V|FwGiZ2X}1{kZX=@mJrhC+oiT-W)3x!8ZWwnut%50O1R}leP7D zL)*cP&3h?mX`dfGe9NX6`pawELVxQ_BcUP?(L~Ge_BAA&Eajv(4ysZ155I?jF6IQC zv-v3gD=%U~4Lq;yw~keg=J}W``(p~#W@AL_p*fj&crHGrGTV2gv+1_k$8M0m2w%lB5F5j-d#=-sz|BG^9yaQultTk&C*|?F@?;fbF2~B zv{`~_khHH4$T6&StFt?ddeZ|k!&i2czGXW&8>DJc3;*FA=KRg-51v+dGNi#)@Iya; zsqq>mpv+)D{aCp|^N5j47<6k-MI!2%?tcbTO;|FAoD4yWT&RH_oI|P1!s%oXG6?CK zZ+tY=WKN37@IKmZ8q-a3u5??Il<>^i_$?z@Yr;RrW-J5@G9^{R31}zs32w z9nB!LV(*>&_E7g4tB~yN(RM2e+&Cy)e9GE{O)gLiJGn$-w#ymZ{{d1TUS!x~y$LQK zxO*Tqk7zzy+@QeHAgm@yDGOX11qq22nzx-*W*TpblZI3;wfNa>ybL|G)rMdw)=)w6LIaQ07)MNg6o zGc4PLCJz9hTH)he8Vs$x!IKIX<`RS9%D1?QhEH0kltv4erGPYMH0uEw?!Lhan%joq zvaC7$5jrCB)hM!xp5i0frGx(MA=od&)FNpMO8Ceu<7ryk{ss{E)T|V(rfw|Kc2g7> zlqqKyF}b_o_|f0rPfh!J-kyhCeV)_~Z)pT{#kS$@6k>VdXm+21XI)Z&SI{pZhg*C= z0gJv=4Ktw{VE`N*N=LQA*`7FCjeU&@L*X|lma5c04!ZLVd4&;l>_><8!z^N9jeKAP zb}hp@z?YX-cYv3;YbYXraG|C3k1Vb^liZC!%zo*vwe1yPJ1-bhkM6af!Iud>gr7x* zYyL{C&G<6LE=UZ?O|m7a(d!_I7}*)wl-eTs4zZt-0pAL&zIH6b4$Zi3h%-8eKKVq_ z#4p}>&JQ1n|CB^!E&>lPo^O;Buo=Kuj5<<}HcQE$l?m#Z6J|CM|KfU?IefOKf zhTOQlJJSDK))j%Cuu<8zzS-zHdB!(o_5jg}`LrlA*r!A=h5jaqO{Pw}*p>G0D!2gx zPtccOo=we}nu;9h%2DmfE6(is_h)BKM7b62SNp%+6!>{lr@2E6T2}QhSX$xYA<)E0 zo-wy3%QG-)q-fUfPOaC`^jQ(?v7UBqmvb}F=o<3TC98pyWu){~l+Sdg zCS#g*gr_2#x~72E47c8vh$s^MU5t{ViuJM9&>cjm=uV=gF^!7&(MUQkxklkmxR7>F zt1)t?)De~Xw^`_NhmZ-8DdzaZc@zwsZIxNpWHqiFRaT7b6)&c;x3%LT26_!Y_u|(s z6~I-t=dsuQNu$P9E_F}VYPi9E_v}p@8VR(q1c^|#^rhJm@iVzeKSEDBB>l&9A`;PrvM4@MnIh!0&eNMHW>u#?LmTy}rljV% zx@d`N4L{GzowY42u*mnFB_&1vn=F#&Sc}0PG5z_r z(T1;o@mbINd#s^4?z!?atfskV^UE%Ys1xUgXMtQ*!-$%Ao$l11^sh83?UI=2BkQAD zL)Y`#ON?#J*3G5hiRuv%qo)D9aLc!&ZMSan7(FhX#}YnA3u{}LAEjT(ub@s|65M(i zpu6_Gd>+Y0GIcyyD*an-PTuCgFQhZSOsWU+-<{!?BHYb9m;32e@Z>JXtMcm3h=0q{ zB~^yfty{dI8BTjW4FgjDMYGA2x96(0aXngGM2P`NkVGraV+}i-+4vXA0-UNN==7W; zlI$Kpam_c!>f%$K%Jx3l!w8=l@78+(XD20}I&A7l)npbR+A*mXw=}>Kxz64Vy@X~f&oFi-MZyAqT0@C^M#`! z$unem_Tp}i*>OFLhZ3mIjJhbZfugyS(8YQ934}hnx8l^nXE)WCo6(oxtNghy=4i^8 zjDthWA!ecBcUhaZRcETsawaP=`BK_nufAQXxS?)|vf;Pi9?wMzqSvwtaA$p)3_fzkObvnI$cJ&Dv>;04q3InoH**A1+UopuessRMZyrHfu95INUE0&XG z2Rx7nIN|cfog5kUj!D3t(DmALncXTN4o}j`odlQ%{8X|I3 zD}ox4gLP`(Oi>c0Cyh%NU#D(#=E^AsHKJ~9=Y6xc@a042a9fSJcH{b�VuVwXfu$ zIk`7P5zW@LcI~*0BYO zLKS@I-A*pam8&7L~U$^ZK@VL=Pmcz^2g(z5hy`8T=UJGogr*i^!1w1zVW>VxC7&Yeihw=Hs@FXo)< z6V|SbB5NtArofQ^`|8U+`)(V$o_lDnXc$HqBl_s+dS##VFJZzi5F;{PjpbJ17H)hV z?n6Nt<*s(KK$hpKXM;0jOb_x=VkBFxZaEjQ>D^uTEcu1cEi%OUtoYRHNa7kva?Brn zQg7(w<@d=dYZ|_BTl|m`I(0uhiI+vhW~4q|?=)=5@wU|v+u|B^pSx+>jTAD78MM&bAo6W2L<(nMuaX?q;)`MITDq0{K@Sdv-bz`Ye|IY^dF1Wju?@wqjwN~ zz>oztdFVuy8K%^wFolZ#b!x)MNxbe9HXymlwYT=iNm9;Nw~p8$H{Y*IH*%G!EWWGW z1Z$3D)qj?=tFJCyb7e0(()hn6b0jE6OqLE}l7_jw$>@D;M1#QVq185~(iE;zwPn}h z@TN1V+2@q_cMaihGr?wuv)uL9Gx;SZbu} zqC8e)NI@(j$}5y_`6B_P3^R_N$C5RltU5MNQM#f-<0TCRY~*8SkM~Q@tqp8U`#@P97tu*ota3^ z6xrCqw)8+kukaN{ckFjvDQu8AI!{mA!fVZ0G*~V}&S{*%b+N+rNsk>XVfPG>491bZ z=*4x@;gHk@aHM-h7A=S->||Qc?UvO%!t=U#04`nzRFCf;!Gb7GGHpk4sV$u8F+1ewA;mJ6C|YX~L$|!KP-<(Yk6?kti!OYB#yqncy@gddX-9-^nPEG> zqM0J!I)y$B7W7+16=oOu*TAzrI3TNfoO|iQ75$#E_&zc=L-GLyhodk~hx2t&Ug|Hl znxYiATma%ddHCJ(Qk6`l^MqrRh|K9Bj@Cipw+A%oiQ8n9mz?N*OeDl@3)UR~RRr$m z%kw;W`}`a88%#`o=vTC9{3Q$D2aMlnU=I(8Y7Rg1MVG;-x-yBEr~bah-qNv`h}$Q+ zKj&!yMx`o9Ch*M-6Bbz|>20&+YWmkFMY4OAoIkVu8g%%ZL+90YFpr@?TPSqr9t+Nq ztod!CYg5PLcUkgsKbA3R0yFRLoRU#@<*vSS_e*E`n*7yuh#VVEz6azM5V#ZH4~}KV zWy7tswjcvxqgTwNe?C(6NusMQ(0R0ktRZDr^fEi>AjCBbT?~)8OCA05lI&b%e>k_? zPzTWD&Vr~3zM{}o^htM_`a;DZ@=7gbK}h$kjmB85G_5bQQ>ZEnqPnw3Vk5#*qSID1 zv|~RHU2;KB!^Gij4Xt~@z&zc_^B@`qrzHouH1fFd73SB`utS>ct!aUSyY#t1v%|=P z*HU2*hWf1gURzUeGC13SZ{W#u7>X8$*vLo#ejJt>pF`NAc5Fr!^4d@W7GwvpV+}@y zb7W8X(ARcA8z^B8L%^=hk1wbjKbVk&y!i9oA8wf1V9ATxnEpCIQRv-6b%E*h27;@wc;M zwsl4cMFX9_*C?Gl7~Vm{wi@)iv*tBB{%?6AaT^^rlrM>gzZSb zH(AkCo9=pEjsyeuPv!%z68^@J-{H$0KeLxTp8M=Kbbe=B(yJo z9_p$Fn~2bB_nWiKdCUamX%yFtv^e3^5_dQJJ3}QPD)MHv%J9iOW7gP&&IYGv$S!&1 z#xSh>RrTj?Nz!gX){W55Nt^qM7PJuQ@#OQ$Tr|ug^1q}VWctIo2(3cq1O}mkety%j z7OqT<%ep_ogL#x(4pgjA88#6$xoPo)m zIeE^gl^x@AM*=nnsO4NiC-oIu36y7)?GCuEYS%E84uzSB1)^3WzolFqcNNhe7pz6wCIxks?5f>49NZj z*KOFy-PZ%|t!sBx`94K(cj42z?J&-|h*@oxKzo^Hs)3@|pfl=L9;`1zsQm?y%E<74{UXSgkm}(>|kTEazK}(b}?()gGXBs zpZm}Fa|3a;j>=^OFA=nyic}IXFa!xAYJi2kz=PoO-s!h9#a0lvCM!v$Xk^_PcOGh?Z}rBHhTl!p ziahN{$@yzYFp$SBxz%Z&6&<$fgxuTENt;t*^kqhg-cx^bSCE!zq6fwPGzCyzu`cx? zqK#gzLLq>Y!;I4OGT@VUI4^}DgP}Z4;1qwy8~t)?0w%JIJ+kqz2_V)%jo$|AFJmCY zgsHb#CN(52toxRc8eOYaLBb+Z7#xk;%ar(|E*x_?sy$tJ0G~{n;4AQs*Dt*+sdR$f zz#hp@NJmFU{0v6JIITeHr8sSn0FjA2v1C?ps~Cx}CzRKP$c5URwjC-{HQH~QRb#eE zm8Eqxz$_;2sX-MdxngHr&dQ#}B~1u=2B3>63KTe(zy;g>0EToYBK$^$p$ zSzK_T0c-Ln#Ai+-SBL9R4d^-W4cc6gT>;RLtAUl=z^-|9!{{8ZmNfqT@8w3}#6v{; zO)V9B(SiLHW%q+$O0!kHIp^ z5>~tklnMqg>&nG>8|-pJ9`UHuLG&rg=bme}LxDx&i2*f%mXTUc{wUV}<{^%CEz5(h z@JbmbhWwsL2A4C1*MS#sZ8LNoEeteX0Lmkl)yK|88|`S2;fHUUy8Ly6eb*wOuN&)} zP>+d+auJz-acR)Fohg}>kY6X_cO=r6=k zGFJokc9xQA&AJA0R^PS{I>65*3;jNFO9i?xKT_Sk_Wu%iXSv+^K*yvf@p<;~b%Dg? z=k~94&)BrssA`4$3Qy15>iBg9hEW*)d#t6hC#T9;`x|h<_uOm!&#$Aev-Dt+idc}U zP+kp>8<_P1HB4|G17EO=Ew?~>pXd1iOE8h90+m(Dcw!TunT=*{4j!q9JpCI(C-1J^ z$5SM*ENWa#7&>U`<}Knrgq<9c8QY)i1K{z;-nPOAtejc4xPMWS5QMWO#eDqyo{T3& zZB~G0bGm9D(hM;y5*q$^j_thf83^y9wzUnSp1T<~U{h7+26AQO`gFZHlehxOCrblm zv^B4w!GCtt1LJB8d9mc(JsSA%ro#D<;3zXEc4O)uleSXpqC)d~Bci5CKf1h7A8T^H z`CoQcLPu8eJmd{7Tvg#&sx7FG7MJSKH&*aF^{I^fs~8|&%pK!8g9dDw&h~xG8)fPn zvjbi2A4Ld*!svs}NNf^zFsN-Jf>3ILCQkIVE;71ZEAPq*>#k(83VD!UPriJF`Mms@ z|AJzUFQ{R+#K={@5wR98`4%bNS`nxNhrZ7&U65E&A5-LK7|FOZHZxVz@To$4gDjPDqq?k7eCsl-Y&?|K|w@zZLQP@ zTeZJ-@Kfel#oW1*Zm5?e7ov(o<;V}?QkXb*HAmU@k=^$>;LlI4hSZ3+#+$OFC)Glk zFH)&p86DdE*m@0VoAOn-X7`{7GYguGY2t-tLu(Nbps8B4_-s(30cJQw{{Z6Aili?7`Ox- zKx{$LlUqP~9%6=Ul$a!(a!88@@!$#GF0hqM^VLvOugWy!PIdPgMNKyY-lEiU@xCdr&E_4IQz- zPls=iFxAj_W}GN$&9bu2sc@Vwk(b z{H{#gf3HKP$V7y63zMpfUvyD|bMhrzXZouxYGK^=uB#3d`e>4!d}%>}%opYv@^wB5 z4Y%Whcj<5_`C{rqg~M!DEIH$q0`YV4!f#L}5!GwI@gd@0Ntv_Wg>dA*EWU->kSAsa(i)5;Ul!sL;G-S zgmzO{mFoEo&HxG;1CEyJe%f&uvtYW!EEsW@zhtV0 zYWe+_dFIW`9VO|0;G#(ac^0+=af<(i_4D+CP`XWsFy)!a& z7DMFkDkfV=R`#6!epEb-XHRAw8ROC$R!lQ7u5MZ)tt4ay$#~~k--Nhffi-7kq}AYx3LLW|BfoUSbKER4=MXQJJ6G#&wKuMaybc?~%h3R-DNk=M7NT0ZNS z*dFn>chK6$ee5#i@$t&9CYKw#!Qfy?Hy%xpJC2B%k66#lVJ<3|-GLx)(@GH|H+eg9 z`Emje#(6v;u?GZb@)^;+4r^rJm#+8^G%fEsd1d7_v|eS-MBUCwnK#fZsW+rGpT zjsa~6Qhlsd(#Qe!^E&JfG|MZpM_8ixkRwsPxz|lgeom}wx*zxif)vxNcf#QGD4N>@ zfv(e9MsEV-*CwG-CBuhJwuB8E$n|#aABby*cP`;SYi#*8m-ZkzwQ4JMd%)G5GxDzt z6xp^v13t(v#@=M8kjMDe7yxkN z{Qb*uVreEFu~N>%HIZw(z*-_LR1j=bAVgp(G0RPTHT zQO>SvmbKrEnCQ9}u1gfEg1soZ-%9~5OIg1F;_4jaWkc?3^_?t`QE-9U&Gt{+7&ods zU*W5>Dgz7!61jSVu85wr1FZ>f2QnF!&Anp%BZ`|^)12_aP4G*swqc^vk~w5$@-eRT z3&a3>zmf>$kG~1>iuCMkvRN<3OU7AD)gya-6ceJBSy^;hX7P!lgDR<)pgiNp$~ky` z&(b->`WKkjcigL|y5YWqYTqbYh%6l3Aj3Q;eU(&v5wYx*Q3LZETF);lgL zm(|S#dx;cz1Jmb6=|$mDp}5TCBrvrK+mV+$Yu$GY^uCOqIo{h;<4=P68i$|P<0WHj z(K+Rub@yD+xI(DgeJ!+*G06)2LM3cCyiLAREHxHGg7B_`t^xyOf~i~NXMwg2w+M}o zOEllff4*Q~v^-9`_ssrnUDpd^ZK@7!$8Rr**jUScsO#HFCxWMGRlOp1it0j?_E*@c z1)I*8?u!s&eB=VdnOq{g_{M`yT^$YGyXD0aAgP7e0=8D3XLUZ;xqR&GH{KT<0gQWDt1W`=#vygu_27goZkK&Eos`)j;y?=0~5 z_-PIoxv{u@XmUzyB%&gO1J7vo`A>Fr-u?=nJoXy&)or zX};BL6s#U@lD7dqvQTO(zp1W^KDyDvVvjK5&N zJ$Nya^a+LhZF)R}ZX)e_=kA;vzmLc~s!o%Cv0BAfhpMl-w9~fZrd<(8xffnAFj~^@fuD2^Q-X^ zF-<*nQ0E{3f@fI^PUdMjy=Pw52j97X6lawD#L?9X6aXf&JJVT{9-*yoY;!o zMB0N9Wk+N!*McZU-0?#P8x6fw1%iMX27VZDZzgRyI>V)?-muAv#Om_gxKtGBhK^p> zr`i${E6(p~=W7SF4urjt26bv2Oq9I{oI-%gI0SV~OrH7_yscV)NJn?-Y90#vOm*bR zGb&zEVmQW9=4!j-2Gig7bWC3u8}IjSUkR105M%q-d_(%xo%|(TSH@}4c|WLg2t$%9 zNTHGCuJkZ|#8pKXe>%wxnr-?a?^}=t?iD9lv`jH_PI5Si{Sf)*?|{(2zDf-$JSJkO zu1i3Ne7mP3KtT)2EmX3$klH(lZr$ext*|BJczV3c`o21U<@h-gLR%QCBd0*v0_;vEW$_ptC80CwcC)HMBFo z7*fyfv%j*L1No265CXm7;Hr$-{;6|}6eP*alB72zIypm(@>JR%m-VZwvSQ=7NUUq3 zHrv0WrlBvdyf`bXtJrd!&??Ka%gb?74qd|>oWTxVsTOEFC| zXr{?WC=&*ZMkm&0$y6b~(H>WB!03c5)X!{YQc0%;u#${6r{11TSHnDZNMTF)i^7060L$zl;+_WJkkB+Hf)p+1U>Kf1~Xk!z^pMbltRV zR;6v*wry*rZKG0^m9}l$wry3~_Nn*%dbhsbea^Xh@AWHU&d?ao93vvupIcEfqrhw{ z2*FS>yTDQ7N7WA*)!YW4FvccZt6l+gtP>(sVH+b!Xzw0s2oWF?5}Z9+>&bHN+dR~+ zv#{9f>*b~IdjigB20IEWD(^bFe&J#AQN13FAz|0CmAopr>Kt{0&@o+3Vxl;Sm=6(G z&Cn<~7?(>97dWt$M~~B%6YaXYdw==msr5^0UrMWe=a+Q(`8}2=WN){uvQ*#^kH>@b zi_`1_Y)M;5xza_QhsP$Zix}v{F_Qaq9L(#6eZgZY0Omx~H9&#`jErh$#Ki(Asw780 zO3qu*a+a2h{Ckb`Thth9>dU+^31$P~f&<077BC|Lcs<3B0a=YYDD02LC)E^0Jv!y=SgW`F->0ea*#ss(A}Pnro_Ao>^Ss?AKx3G1;Pvt z2N&rFgmE!|q@be}^puF{o28+YhLts3dBHsX3KedmT&>1=6<|8Rz?}(o$A@AJUq{kD z7^Qk7mPe2U3Wt5Ze8FY#tEk+5I`y&%g_pI%76Tn2Pruw(UH>yc1L*Pd=*ohS$sUGE4 z!R*gD%Eu;Rxt951hvi94vVxD<_(UiLt~!@E-6~8Js{H-6Yz={IWn-+8S`D;})oyy> z=1~$c!n*;c6DeG_w#_Y;O3*MY)J)K-7#&i>U}SPeuzVyub9JIr+Z=s!4--}+^JPPl z;FdCqyhPP_ztrHQ#vuz$LFeXxq4}so3sy8(X~6jjqest%4`t(FW)fXfyqsP5g_-;2 zVw{}OvcAcfO_^PeV!Wt1O`9!9v!ic|S`k$w6l<7BV;+f+B>z5I1&;1`uPr0V)FxNVq&lkp zw5*Lz5B8ka3pOBd!j*6;b(xmx9KFAW>H;!N(%(!~n91USAy>7Z3FIae$PMM(fAh9o zloP)X(nJs^fT6<+!6ImWjm1YffPd!AMDSb5PEAF8Q4!5C=0T)X|Jbi_h7G-6B=`is zbRVh90o04Kz6)s%ty7!fxLc+#HBMRCkuOdrr)(a+c&5eSBD=W#Iz~-FR4H(yY?O&S zP&ASFlSBcQJ6aGqc?zAND&9lkD$G>h%}zC#>c?vKQegvV0@&b~8v*lWq0k-=P`w^M zJVko~_V}oc;H1VXwYUiAE-^9hf84c&tY-E_vGJ8a&5(5k;F>$3*|kOPvpM&B89VLp zG!?M1u1*1L+C~BA_hbO&%o(yEnwIWT#5FdSdfA_wbQH(Pz|!0$We!bQlx+D&fe}YI zWIy0n5PpT?BZz%H(Y=L8s08$~+R1CR3*@=XF9C4}GrvnR+|XubBV%{UZOM~_%GzxM z6G^&^-^fl?qCg}cjb3G$%GtzJIAv)c0mo^D>i|#5x7!NU=9jVqSlC^+|D>dAE<}$| z07zG`8s#D#Ed|CADH)ddv&s0xGo^x7Fq`oYiEmxG&(L`x7YvOxh>sbV>E~6QiFO=VAKUHc+Q0pH8Bw=K6VX(5eJx2iYwPa{ zmVcgJ;D+a0Oq5r>4}cQoY92oR&=qf)TL;^__!ZEO?wka#4AY%yLQLtTIgMa7$5Ap`Wt+Qr^MGEB*cXbN+FV#74*;_g;+-Ss-i8k z%ql~{zb64v1HP6HnSk^Tx&fI-EM2yzmh!wh?;9h+z^*JRj6zNT=LMKJKQ)U(uhW8b zPGC#hIwdn(5L%&;nhTmsLze`N9M=otnFI*a}!Fs+NsIfyk@)9T+ zcBiM5@P!V&rP*j{dCi=32yNKKNKA|M3<#(zFgFfNu=Y8?Cp8j~Ut8(ffDGo{z$buz zkf`%d%|_Q(juE1CU}QpoGme!&5YPZl!n0wOwOs3x9k)a!a0j|Upvs^DXyZN2uecP!T$=uBlVpXVU)*cVJ(KWnFlTf44 z?$Mn1=PFCHtH!%Qx+;=rOVZN8Xz4JCm|bvffz0pqgG{X~GSlG=ZJ#y1bJ3YD zjX!}O1NoMvF`sVNgpty9O+Kr!<6G%+_dMcc#RZ`Se{RrCe5fEp1J$>Q$xssu9VTL| z&^YHhSOr|aL@_j4Lq1>$K@LE6%i`b*!nWmQl$C%fjKL%>DVQtJ~ z+mI99((p3TSFWRsFBhA>Vk_L7ENB$QfiCaTCGK$v8Awvo(s0is`A@(@YG9YXpW!BK-fd& zziShD(aj6|rsz=JQlSsmUSuYs#ZP9PrqUow-EMK(r8!x`nvqV(*6SH~QlQ`P0_X_A z;?cglZF5BxGmwW)#HjU@EYclXtuRcAMdt@DSV7GYFChbo=cp51v2`vd_W6}BZU}*G zAebhHgPaQ%Sz(Ar)$B`!Q~4vBR(b3adlE_-ay^=uRBXxMCKt}H_&|C(k-xelRRf&2} z)-r~Tkz+uQjUQ;(lr%DVRt2_W$KkP9LR0YSFx9X^mNh?u2?XOjOWSvy5Gg`YdN&7J z-$JKeONwR0T?HFCE!-qO?d19xD3Jpukah0L9;D1Rd9eaSS`L!Wh9^&#?J<%Iwhskmrl&WbuC6+St}qP$n~Wha zCAu~oNFuO)(#j^gM<7KPXt4_01yXg9uM*^zMIL!a7-$MCR~GfT)Fz9hAu;c~pwu31 zDG7(|xdJf)x^ZMZ&w@X}tkCw2ttk zV#P$qlV`2A?NPy21N-PyI;_AjeRPjb%V~~F6c=TR;)JaV@qR=Az1gInuQORjS@9Vn zCgT`JQ6+Q$EV9m9BWSw z({Pw{qpP=WC?W07J;cA}gF>50zL*(kcTzjd@q+T3rAShJ1Y2vmMf~yw2nTO4O3FNS zQwYskLkl8RK%iRUJ*t+Kvb8%~gSS7_N~jvVq#e(*Sp4n?Pg$g*JQdl01qB(0$FqF^EwUy|xv^KFBjt>kb@s zNi)aA+*p4l0Th0QNYTYJn`f}7LPBB`6gU4tNNxm|c+#ZKgJ`@UhyY@jt(?~SRP{v9F;EQyTHo&o^ z&H21wFk1q&Pts-=Il^Sr8!3IqDk&axxI1c9PFb=+y40i#wc&Bv?gR<6Z^x#}Y`+kM zi1g}1j*y_-P)U)$v;+w4K720?11RfprSfbLP9wR+t(clkjuKwGyRMT!30Cw43t&TH zuHA6rCV)%5qheDAFe#hEQM0k~ly;eo2m*GH6mt!G>F->*|M|q))^nS=E<$Aj7+=F zIsE0fHM~8TsD^k7f0>+dE%tXzMoLH#@7B3m(eRPpDf0xCO9p1b0>jb;_ZD$LUk?wb zZLdc_={EEsON^6DBR!b-6RokzT5L2)@z%vW0UtLxj@y*GTlBzNKW})pNgumEEm(EE>J4pTVi+LCLmj$ZlOce!=yid0sv5+h&Eewci@HaU+J56beoc-7x}w7yKy z+vJ*lY5Myw41EzTUB5p!6BjK?P#`7tQhV91Xewd_O_R41?6O%(aUIv7Eyqd-wOigcIY*fyhFgf98JuUOo#)a$jLXx)sCjGJ6!#9 zFESUMXBS!&aKWGR>#W!BLz|S@8-j`;xLl@@fLK|o@r?mdmy;+FEt5Lnz{)5WOy?ZV zcn0D{q(qnD8U;(%4Mb`=H5E0OCI_}!T6$sn8mn%YOff##AI)x{SzaGQSNiKS``bgZ zv+jCX^Ob*RDa^P}<8+}j6i}dz9ZK3uCSVjsA%@95E6(+z7D(G3`yk}uq*MS~+}7rv z2scc?5^a?%9hLzhL;`d3AY_k_0(gCnzR)hNLMOm2Xu85S4z&pQIO%DzCts*&6=PjHE6cFIH1XP zGF$m#2#uOUCD8bWR25HVjSs?dhn)-kyZeKuWM#Ad zbg!4ZD#XXN!3M94VvR`x6Ye`2(2Vma=`Bx9Mnrenu!%po2BaMETg~@gIQ(%SYL|6N zxp8a6fVoU(7qw)hcgW@wEK&ck9rdXwyFFDnY4H`Vgc-&U1YMJ|Z}HrEwb4aOHkoRo z-7Qi@T-^z44z1qp925j|{b9jp&+Ma>ztQe3^z_8V4O}3R^$0Xdn^Ywhic}LFgaa&b zrQ?yZh|1F~A|&b>8cv8EJ7EY$8d?TWslv!K(Bbo;Q)#<)4~wf!eeQ+#&d4u$zGA+d z!fb`Uu9I{>viQ%xrZ5$cKw+7r+q}eo7dmdPtzF8pjoglL4w4nsXrOwv#9rKL#&GrAC#=LoQ+CyAVKXr3zZD+ zGZii6tMxXJdxUOr%2KOp@|Ri^P^_6R1j!I>*9ia^TN6&>T;RpoL$*27_7+O|bzX>i z3Sx6b-$-E?L<O@*oL-*lqrOa-W~nLzMdgHtqGW>`iRcFi0HLn*;+zattt z3L+u|4ISdFiA-46Q~{f)6TO`+HJzqCTxP(OJ`#@CudCG z*bvP$mX7*exD8c_cz`{cAAKTy*7P8m{KV$ulnh=|MG^^;k`z&rlF9MSm0mz`gQm*D z#jV0i$K+}?f|w{*zyKsNa{O2Wn`U%{q`f?LYlcfvkvl!tA#w0dr#|YHA_xtYtJGvT66m#K-AI=Tld-&*4C3 zHy$hq%*V;5{}I4*peSGlGX~=GA-j3|gX?#czMc?SYi_1cCTdw@kuLqX7XhRrK|979 z&968IHCID(6QWV}g6ECs^Q-A%pBLjf@$I3mC((o5*uJfI^=s;3+fH|qYrCuGkB7?i z^wX#1X6TPM!!OKOh_3f(?dtD{e9n7QyYhXpKFBWb+iv_^=wC0flG_=rZl`-6*;S^Q zcaxoX8t%jmBn)MHa^3)ED`O)wYb`x-HA#8HMFVWhf;Nwi7#?WHxiw+yi&O^{$7vbm zBWK-p71MnS#YqD-OFJt)aTzfud!b5sk(WX35dN)^&r_YXn=4cVpKs6@_vQjfGea;O zY=ao(Td%P9*}+}TMECppO12^i+w_-1r2dvv(WA3mpnIbUucG6(exgSX1dzA<1g!ivi}I4$*>dmC?JfJ;19%@adoy9V^!c3KE_YAWdT*QK>V8kR zjoY2K8WGXedSRd6Uf1>s%eK3H_gj~Jpc>zPY+rx$r+4{zu3t@ygM32BJgZ-qlM8LX zle}lr=f(Kq-+hj@>=*CJ@3irgYO=0YWxr>5?+q6q`bv0TnQX+_U3NW)E?9pL>%KQU z*57XbILa5K7hnfIn#T2aS}$7ZE&6)$WwXFQ(0#nkf1&uqqzVFev8(d3cwrr%2+{$PFkDD_$jn6SZu+QqLm+?k?VjlQFR%x z+Vo9J`AtQ39*d^#eDCr=#?hTbNl2M@(!9k zEd1rbgI415Jf2cLWv&BDD?4ICYln65&1fdosCBGnZ>JR;lBf^J{m0TbPb=T$$&%A} z^}1MJ+-$`a_`Q4F&J0k_pI@9eTlIi+sqUuFnYxC&dHOLlt}F%4;a@Uy?x4mMk6qX( z9(65d6rDNDw|e>dDubF|?{;(rL3ensG<$>lgnpbD_kGBakTIZZ9XnBM%^UfXZ@3o5 z^L`q@kgt1?Gu$)Dd@Xll5!_*Xv9HTd5!<>`9pBU->gl>2b4ug6ct%r?B>rqff0AlG zcvD*wf(1_TEmazs5%cJBA5~XthReZvFnd?H$3=b0^wj3}w1cV(L+D?B77n@^cfPNI z3-`SSA!sy^t&2bnDKOU-Qs$GBB*PZL-><*j%<+mO@ zEv#gT^n}%TIbI2=yek6JP!ZhHK*zNg8N3y>xE;*DDh5zB@Ej^=s0&G)oG~w&g?C2CVr{iIV>SI zdAhi>C^FdNfy>Y#d#D)Flte?tvW2@~DhO?#9pZtlKfYl^hst6v{Sq?Q7ZZL^a zs*6dQ6ItqGsnIiu(F=*m3yJY7Z-u#d%{5pJYQkt0wwUE+<&jQi-u^oRer4xgJnb>e z(nPEL(TF-KguVlM3%QwNk z@z+VJ+aGiI=lEcSfkXql09&PEB=^?k;B;zMvX(ZGd0RP~ zb;hlP`_W35Q&U@?X~RwCGf0nH!$ijoDo4rPrFk8P6wD5EOzjj5mG7z++hO>6^4%Rj zQQG;XKQDEoZio89%6cBH7vvW!b+!F+;;kd2>@F`$WMb$wyh&(T+OoS4KsXTSz$6#Z zYN#MNZ0P(jz5nXOv~k;c-#n%BJqel-&KCRj#`YPnuOzQG#e=J`gR6)fx8~km(%$6a zXc0JxP7z&1dX+y8B&hX~CjR#CT-u}4!dr0@E254uB!gy$^JbLTZk z^KafRq*uu~b?LPajGFkXiT#a$g-hheg^F-$6=0MA7kTtxHFfnxt7sWAH`UIqny$~U z06;iI-u%h4H*n-&C=3`#rd&6myca>L@KRV4>X!xJ)M5Omikxs>UZ~BYy~~!m*3=y7F0z+aj!XRG!-3zn9(+ z>soX!FRO$mHWtnf9p#&WboI6-VPxgCb&xFqfX+6Ly?Fgg;DkR84xEY2jjI;oueDw% z&lc+xvtY^1Mt*%gVhro*c$>b8@9IeLAS>Y@NJ7h=S6t)R;vU+wnIS(Kh81dP=zz(W zp;(-7Fo|~-{NgBXbZaUzrRp}rS|Fus2i~K=wm{v4ptCU{H&^!)YOD0JX$*ZM;GiM=KJ~SDTGA} zFU$o<&CJSz_m!TfiHWoQNahUpuAA}x-7V}eIhhPuyt&TYsdJ~#hvsT4quYLUf z#hRtLd9zmK)~Uuq?dodl+}!Q+a{;$u9AWNmoFZfC(;PV&nFHq522G;W_~@w9ThaNC zK;VQ9&UHEyQ&Un#%ERGoUwqv36Q` zf1d4zVpCCp;B0p+DGlS}q)wo?m)4#kPshb8!iM82o$JB0*5CopB+WDnA2tUgp6|#l z4f%w&y~0YLaS(bN;tW!T?MK_fk_DO)HkQKD;Y^8@C{nhN$uX2xv%YTjxxc%c`1!LY z+m{a&%blBtXiV|!43`z>!cL1zK5!~QMLpJ0ug!b$$qa~ql=g!UO<;HbOkG0-PHfi` zO^@y;;lA(hMQgs#Pr;txzCNQKSpUPmL23c=8K6qF=0#CbB}+|@!fd~p#qn$S!0j_h z7j{RlF5B6Dd`>5~tO=`KwVj>w1`L-6l$Qej5-lO~{GWB!A|Qm-RfM9VgDX;AM{>_E zu*nWW+v%U~G>`2-M;lLlSGr9&AMEsW)U0B;D&xPYk7Y(VqPO{{Y|paG3FQw}5sMGp zwj}{=$eJI@0<)Ny^;_JTR(I%jZIRIWH=(v%_A$QMo_iBP@qFNz{Zv7}aP}SBZmovW zJk7*Qy<>n}=l4##j+(Y|bNFy8g5OiP61_w&3kLxePSbRbj& z^XX_Sv2e3@>|2Q9u*S@s$jHc7@uuc}rJHoa!t&%GU9xj>y12VznMf8A$q;h+u<>^v z-zJ`Qb$xDb@|xAB+i()-7Mh~39JyX+syX6TLA4{2_M-Q;D4-Z9*K&((yUCyAO0Bhj@6(Wpxsx zT&fVoF{RRITT&)i)+lEn(cr_)_+sXizUe)8j?3-G%^1wuB;ug1L~c$IJx%vAjg{ia zO$-JyadUsc z%AXTHT6WoPL!?wpOe}MX1_^5RLBGQpeWX>NK%Qt36qmWa?$9yhh}Vt9Zliz7OPqwf zml-T?FBl&`a|=oI%I;vc$Xy?+`HkClBN<9kdnI?vB#sgumgs>-n@te&QQm$H0;>Q) z%b|bZStj+T(lXG^YdI+dyl^bTwMWE%gU$ z%c+t{rk#$lg49{&eavx1)$@B-!d}1|I&o{kP2NyxU6Q2|W+xzgk$)677p55cd@dXv zB6GH`?i76%#MbbLGpo!eI9*#lVhpz>ZU!TDUfg&zTM1|`1_`>Nw79vt+YjiYao_d+ zjkfY&l{0iOB&kNjCIy8C19CV|G&xy)iaE8a5+x&8EIlAoLq|zXNo#3lMx=O}maeV3 z@YrOwXF9WcVP?gK;haFp**6RPY-gi2Qdf7=HoXTf6M^`6nfYj)n+p>&J4e#9b5~Cb z;**MkMy+gPC1@ojA>(zt`pD7oE)!0SEezL_90cwMlk8@7FUWLwa9XtjRY3I07`|M|0;jTrluz75F_%o!ygj@!j#OP98| z+1WHL6x)K1JeWoKO11V2hZ%Z>Qk84f4G`Ocwjx(=ZxGx~j3_?t<)x*mBUpooe6?hC zl|4!qXXgO(y#t)+B0qzgx;hwLEL#N{47xjMJR3h5oK>am{O(?U=BMuDXE!em-y^mGrG5Lr`(I| zxzNN@Y4!L5vJ>;NDXT;Fb*}qW3a{1tSNHP*$WPZ`iJVpBxY<7&oPh&}Ej<+#FmFj| zrPD)j{Q6puZAeee*yQBX9W0O6B?wfg@`EsbNo6H0Y;4EwzT26rg}@GN27Jo=hR~9g zOVuKDkIUid!*#gDiY&Jk_~c~ea@AXKaHsv_Y=H|+CwjXDDdNMsq#3OhD4>1azV3{=(G`zUsH3=w) zoqqczF1m}GxA#Wl#l9;LrQ&RD7yCy*Mu1>%PN1+bvE21xu!Orm3l1$C9U%opOJPU{ zAjk#X4oQ*%4H(rHGOSN2COqoq4iOHnBU4`iD%{>DNVvbB0x^a86h&|80k?~zth9Bp z6toe#v}%i=07@X(4d0m>iypxT&&oj2fijUxSxqe>axd4A_;ljqO|t|j%MLPpVPjZg z;!2kCbecRGn9Ea)6D>5DeCaegX)&_`_Jpb_!jXW@zrFBqQFOd)w8O?(5h%pe1O5$Y zJ3O3}-hVbpACoLapeWm5L@Trfe6nGnu6C#5lN$P>xIe9{RrN^~>BPD`E7}r(XMigs z@4(>PA1iVlO{Rb^E{gBu8ES024-DLVUDUEcuBBsU#?~Teso7di-ceeLeTBoi@nk08 zIpm`P`YrovB~RAO$;Ac!5gGv+^r+XRxQIe^k(ICOb7pFaF8&0%DT)Y~Wu^@_Qffy1 z^4QGD@pdYMH#_kniY-QnXc0J@nAmv6i^~@BXC99Qf(@H_xp6VYyv5XxPz&s^99~9E z;s%7%)or3RWvOMR?j}lvNi4$mKT%l@Y9gQ`h%%F)z|g?Gq3$g-_ho%0lQB2)$oom( zRpwT_)zKC&Nya>ySusRAoS&?R`ixJ-@SeG^pVL2=pUx!)~3PWV4~}txrRzOT}%6gi-w5is1y!% z`X>hvvE1L+*sy8iodI`A6zs>N9_J+_G%PK}n4Z7{lBs-e^Dh#b5^Pj@RJXOs8AloY zrPUb`)3ef}Apn<6{Iz)69yc3|IoeVrv5UMfV8G@DCDJDb)?Rljb?zT+ypLMrVQ*>R zkKfKcKppDhN+$hFhs~2d|H>EQ3>xrkdMKhtT-~8NU?Ay9h8FL%9{EGJam&yJ4`2L~ ze1^XqjE;|^)3V?Zn2za#@MY>CyxJ%XfDH(PQ6l8vo7fl7sL4CwqV= zx!;~5cr5KqTWsEY@^iI@(9;?cJq*l`zH+hrm!}98R#y18-ez50yJ#`gXW<)>RwG*W zW&%o1bYGmdyO^BNk(6s9=o|cQ7&{1=WS0Gih|=p(Je%ZEH0FAcIuO$(UaPIym67Me z-fD$U#UL`xnW*HSR-7uCpho0mg+gWu66ks|WXgty&?HGJQj3aM*a*pyS$v-FaQ0P| z00GC^Zo5crdLM^x(5o;*Bw#QjLM9bCg*sI^87gG)K@CXgTp#8qSWsX@0oZaR^{aDB zO98+q=jTmYPkU#Bm=R!{FfNcv$=#dsebU3E_plUq!mta>fyZa4T9Y;bBQTn6y_bW|qAJ)Cwe_AN`F$lgs`nfcNE z-k?nBi2q&zuFMZ{hQD?n+i=p(G9^E_QQ)ZvFU{9!!^Oe#mcL9EClZLth$TZn+R-;h z!96TX@|@!+jBeSC<@yc-F(Y9o#3gTuRRZ-TssWen#&z>zkjG$LV6Yn>JtG}eWH!K2 zNAyT5MUv2w-xUz^9icAF7)g>OO%+ZL~-*aeSa7dm-Rs1Keof3K3hmu?I9# z3;7}81H0F4?OuesIqef>WeIH_5waw3(r#{u5LWi~P(|VInH9%!_CVR=L^Dztcrchk zg-X4SR8-am>lT2546&jJXE^v84L{6i?=B!E46KDyXF{@yv4fR43GycTVE#>GN z0UCDrYn8=K*iltwelrB$c3==+WeA4oo=aFj~LIxG>LN z3nu$-t0JO-KbKZIlA3LhC?`-BzbTcmW(otG0qGm+r&Zzrd+GSuRIG7YA>bZ_ zT1sl10$z;kXT~oz0ctiJ+2xNU`wJc%pmtYQYuH2s0zRREu)V-IVm#uI_r1xwlEX+5 zM=u_sSsiUJaSu`S1w~&$!q#VVTi+~@)BSy8o%WqyWpG?UqldS7WQl}C0d1qB^x1sU zKpP-mL$9YRA~0Z6XQ#Lhxs1OVP()b(Na+bVE^=j zxMEeSa6f1|gIQL7 zX5hlJ1vf}$XyfdOI$@dMzlc8iQB;q+1|4W_=g05OP4qj=0D4M}KY|3MbhyfT!jyro z+jsuqd9?m_5?lNJnD%Z`Qo#xKs=vQfVV9gDv1%~S_}J9^$%CRB6gmYj$LDy1GY5FE zGG07GZN$NiI|M}7%uZlnU)AL|1$^f*dWt>;aTI89s6IZ^L)DQ4WhH&!&0G;m6(JGN zte*)e8MlM005-3U7f~Cc%uMjIz$Re69W8bihCV9%D3GuIATqvbupmV-F;{8n_2*}j zx!SZePXm64IKA#xM`LG8N|?ADV~3UJa5TN2!uYQs?QAongb5c2Q?|@2bscOs^-^2B zYa;erp($-IfJZWpfO!4@nNvujRH9o-qQbmfh`ZA1+uzK9ON^V#kXbSf{`ud&l!EE{ z3gh6})*(m_f`!M1iyQG=4rb{Mo(ijz%MR)^Op^h=S0gTH9Nnw8{?&T6q9l%egv89^ zevA8BT6-iY=QV^SaiL-26i+v~q-EvSH|YF*I-+@;B2FpuW2aUNhrG0v7m5)L6qPlKM)kDLjjxjX8-QP_hsE92w8Z|1H@{EWE zg>_<#UYFpVI%31BQr3Cti18#=z_dw&fLt*%y(mVT-DW^&rXTLkmA^mB$1ejjm(F+w^-`>oVXMgqmzO|glTBLoOIHl7r0pM>^T;}Bi+cCu6 z7uCb=&u@P}lM^YKm_lhL`5`?PO@`VEzTC`iq3b6|qzH2`AMw?EyZ~c1)JODejpch99^aAS4$JjqwsHQSEDjAV z=|7=KfTU;Gj8+|d^XF#QTju4ZNr!o(LJ22HfL3U3(p$AhFlTxVJh#Dv!-Kzy?Tp+i zwPf2YY)L@xF9OL$!7o_;Q#e3<1(+dN9Is^~cfH*LCA0T^xLWad+;#t;D0EA}yo;+V zZnT(E#!i%NGp=mNj<`f-#-50(;PuRpvD4w$?8T);&=lVaOgytEBs_5)qKTQA zm@eO-O@Is~YVt{cs-DcKgNtHiUvE;leevqi4yIJlbMyz+_shI@L6sAeEmx;i{Lz;AZ}!EmqlAI^LAZcN_OE*yd;CmcQ6y zuDfbJfh8>v$5P~MS;ll(2udE4ym)VgUD5*jsLiLt|g28##@Dvs67)up;( zj6(XgQA^j5@K)By)!IzVKl2P1%hD42=fJ>y;~lPb!Tx-tY`nRY)ozuRDjB^3>XS2L zZ4u+MV?gXIRnzu$&leog@%?-L$F;ZhbaTm#$vl{byPRz=ffp2A=*oau_nl`c4m4RpvbZsm_Tv*Ec^T~( z)xV^o1La#V4Uk2kDnq)qDU-(nx!!1CwaTe%MrTC_%n9fQGb1!BdUinMQWL_!SV_R! z^Rv5$M`3FrK6zjDuV2bH{SE5a@bJzJY>)ZX3Yh8X>AamjGcFvMRL*ja*Jebm&&;P$ zKh=sM$!A%i7&2fPNGNnQ`;Vquc(b0(LR)q8TwZ`D z%e$StyN3To9&2DPPSw2X;`%A8PZ`MZ7aDmxwgMNoY{bQsLDYnFz{iy@u+jpc z8zJ$7p|duZEqyYfy*61S??^zeX#T{AUkdsRMBu02jXQG>NKC8=?X5*zwBTz{1>~Hm zr6x>&5)JKsv-Ubp`Xq*N6WCP9KK0U4NFOI%GCkT46GzUqSTWz5vHb$NWa-K+5!aw4 zqAubP5y)71xhi376oc}`wx*(t^z`8qGc#}jA|v1&oaWrzptCcn{849J_T3dozQl1up0F%upPI_*D+ZHQ<@pro|o$U;`Fy_m;h64|D5} z#K4HbDo80fN>78*)VqJfcwR$sPgfT*(m0E(ql{Io(q`0nX~?ca3Fks`b1pbuz=Ate zp;Q_d92zDxCvKmi{Xl2+@_Z6Mxu31hiG`Y$@y+yRM~&IsT%@hLZNKq5#8y}?P>_G6a3IX^4MyV%iDoAJ|`P4P-qT}DN! zoSJLQ0M1p#8gn6sJGn~@20+)ztCBfMY|3`ArPjiNyjVYZwwxB1bw?WyiE>@vM(v~s zdSTU_NvCA0}o%T@LK=gZh_#n*Pm&#L*RhDxzEn6}j=+&!s}o)j}$q zIi^;!$GOkEA!DxVx9Nys2t(rM9Ud-r&JS(1>^!V<7lr6{g>1Z_Ww-n_*ZH%y^t)uoSh@U(1zaFkbv>u1ARIEbEm&B z#{bIs_zN@r_ft7Tz@Jx)^g@PCrhm5(5*85^6{Qw7w6QdDw4@icHMTRcv^A$!x3m?s zb+Y`|`d^JjOr4A!E$yA{90@r80YA2MbaEE9FmxnfW}=rd{FgE#6VpGCB2ENM?Egdq z=lri0`fnruUHyBN|7m4nHiqU-f6XK5{O5_Wpsl%$DFFk$pp)@mAQE;~2Kv7S`KwIJ z^rs?hX#X!TlfPaGx|;t3a>B~QNH1&X{*MYHI}7vQ^Qr%%B?|)sy~e-v`fChjXH$UM zpDus55&E0fOv}o^{PzTZtYl&MYiwPDzh8@4+L$u^v6uNj+xs_H`u|lC*wbX z-2Xd0nf`;GV*lnfS|K|dlmE}6GBW+QsxtnYsx1FxZuii(X)OARwT=oF{nzA|x8n|DE3#gh3?`@PC=v82<_D{y*`TxTB%R zAHkiSP5(hr|4Xj_A)$zciH+qy`Aj=PjcBw9FcKj1+XF%|)f5E@85kHiH5d*17uc`a z*DrZ@awfsb%k{;xhr?5^W~XY^#)d#V*MS(PM^3?y$8<1g6cUM_1cHJ8o5J%|u#9i; zIj;VmKfPZK9yujWfYQ!3D=?EgG#E&{-%b2s@R)@YN2XlZW`&~n!@TH-y_7Xe`lJ|b zwBH5ph<)N@wuCuTsx#C1Qrvc*z8p4Buf;jl0W)_#dD2zHLz0Rd<+gG{apg`M>Em{g{NlyJJ)uei^GqL1FFb$wzI@Hqe0V-jpM04|=fEvzVd|;9;q&9mzHIt3 zmxf09DRj#?4<7UKeLP#D^+Pk*a6GOuf6HL;M)B>l2YQ5*j^GP2yvDS+j=i7taR?N^ z{0y5Y(J_o!DqjEaTmw9=O!~546`bEsp9VD(prLUXp(KBQXB6%7Ld7DrY@D)S!fCMj z!STtRqR|3DN5lX}o@iC6SFJifAfE)`6RK9GBK4y{CT@E^>`jR(TF%ZNHt~y)old=t zsdLb_8IR|Hn*QCl+mrH{g}+okM%gN8@Ulib8YfK&y8e92`F#Lr9l008izHln2o&LuVhh zv#qW(*+~+)ZsJafX2t84i?|ks(f4fcG{~RhwEQ5jKx1ZPjZ>F z=vT}~5L0@#aK@VTIDNq*KtCr!qt9D`lb};ee?#$&E=VUn+BfOgHR1XWsTjHD76`tB zwEl*LHS_9~4a+q18n}wR-*t@tdOS%K4q~?L@_*wgnwHazi3r7P;P;8rYIp=w;(uLr zR_TtK@noKyHR{y7gJl7RV?J!w_I1A!zCTRh)Htqv_iQvESB}xQd%jT~DknYoDL=pY z;@bQ>^rP2ejP@CR<@6$u5zgdb4o~d8!^NsBuGsROV)y!Y zhw=wU6?rbzVp<5pXs|JhL6-(X4Q%0aNYd~zG3x?nl2hYv78AoV*&dlSP6r4}{U9^R zz2}+RuI2)sw;LtAO<{MFuB=t$1_6v~(1hqoPMs++DMy!g_Dvw#jfY%+5-WD@-67Yx zXR~%K{ia$udj{Ro4iDnJp}d$iv2S}SqE&P1`Wle!1OQw>qrc*KFMrX2A0(5`#U!G; zLpQZ#R$0)N52fhCly0q@!JrDKSeU0*(H;0ef0{-`8dv31TyM>Kfu|TgqQqYKr{mW?U|7B_aoxFTAq&W_<@OT~B{HLl90xU=rChQ-C5GIo5xDt9ME|72 zRyK)em}g*Gr_24&rE%M_GU1~j6ba6*1SV8H>M?~NWOF^K@_VG5%;A{xOz4hhNXhhm zJJmf$0AKu583Si-H%-@d`$a`&XAO?br3N~nK$F_C*x@js%f58^1mR$w309k*lk(kN_N`h4%(|L%SJV0Wv& zI@6<`kw$7wYo?}uQq`Gh_t&5!gSZuXI5gB0oBn$@WtP5C)hw3rn-Oi1-@wd}J96M{+(nDx|zc1j$a z#IXeH;@Iy3<{pO%Y7QHx7j&R;Zvxyv22L%Zm`}ZtFSII#FvVC8R+%Z1R}56|cuM z$5Pl&D$N0^RofYQMv$~5X*?#`UWWPIl0RFn!6>;t24eiuZtKM}-EXJxxx-H1=MGCU zw9Yf@IAU*!OlA?hLh)zhds8s7xeDf@6H6}heroYoyX9m^#40~SK5}#%BaLvZU+r7An$2(`BrAB^zUI)p^&Orf7Q< z*SKp1M^f(v^H6pfh2MP+wu^+A0$=Mr9niaiHEd$$1xHN_E$JY5WTX4ajkthjMHJ_Qq_U-^kkl%pT^{Dh3r zGu(;cy~w4w9F|4IUw}48{}z?M`N!oUTaI%R&hXr)N0 zw0^+IyG3-$Z9d|A%;Tg+oE){r9ivP)w2-gE4Bw>6=9Zse6V!1HzpCH)l3>l*Y7^fS z(!s^Lm+U&xOU<^rZMYyoz%kg|A(tmdwr#(}>gPt)(WN01YN_MU!yfrH{=7F(hsVbV z5g*gFcF;^iKIoX&$^(T2Ve=0E*^Z3b8Q5Q_31R7%g4^d;QO zei2l%87|*0>l?Q%XD0C)uz0)HiQ@9wJO&T(I*#x*@SorJX5R&3cw5Qp$SsO;gV)Gw~#yjQ%1kqAePRHtxj&dWcu|y?pLF9nP?! zU=OxJQl-xZ9vY=pxnGupF`J;0m$(P#JGkoJX_;-H1-Q(#TkHMS)^B||l+FTKlkvbf zFxPj%;7^cAI>M@TJazP3*aA3Cz$kyI7)%U=?a+e}3i`I+h|OnhKW>F@gA{hV?VL?q z^mSy<0udL@S@u{D$JPuyKNWA9xX&6p=a{xR`H}U0EvNd~PW78~A~H;SW+46yyZg2} zvPm=WQzEhNjb~OReE@!7JLsJMPl9@_ZX&^~m!jTxX#!|_GEoMbGPfTUAn8+RmGD>S zSZiNUjijCYk8UJ50TD*#bo6XuPm9@Nb#BIGtU<^KHIY`)ztRlmfq!toje1h31M!V7$ zF2r(cw-^NPedhbs>gS)DZ=pSB(<}V28Z_={w;b6gVtVZig9N69@T#^pEY*p`!tioSfA>UYr+ z)tL6<*dPGi(Or3E)s4XuvXvVvieL2?{WA8RiDN)-cWvb-CJ)c_t{t43%dZEq{Tn?t zh!W+m9vM+s{LO0-`}i~ zxrNx78F9pD^#w0x4Ahz?4Am`Vk%^_ddA3C$(r+H_b*kBt9C!5#CO2w$L6EJH=G>;) zJxyxaye5s0y`u^A3G7pknQ6?B=PRRnbq!WI)mA;y7(mjg;9Lpuao2F6e0`jkix$3#G4( zNY#@ucWGxHKPtA&iuUyV_zD_pb&+rW`k5RA3V*UsrsoTEr(FVm4`&%ad z?5Qs_>eZo-2zSFbu3t7QtroLD9HJqc<%3XzusOGOhIXxMi%We5%1HZodGi~ySDHpCH$l%E>&=k%g^~m< zozkI+VwK}W*7gA=3T3Oi>Eu3> zFD(VWk^7U^ZKBK+gScn!*pk=duqA8CD#NU;_c>Ji406=r`4vRW2 zZR5&rRdRQr$BMV>10I2PZdN?V6Pf;XwC~9)-?K+9Y|Os~27ggDJ3-#Tv-%>#{yA-7 z`M)qkroUr|Vj?0`N>-*$hIaPU|2Kxn`rjBLE6ZOPA_M(@F+|pXplSc`ch&!fA^wAb z|DGXouyg!{A<{Gal_4^*{DmR1GyjDlGI22eFBl^GUl}3;E8Cw9{6WE4{;2!EF+{e1 z==DF~{(t8s=Ku1NfSswXg|y=T-ZSj~hVC-`1=+LzgNgria^asj=KtCBKRAP)$o^l>U}pXctY>8X7)k#I)=L@c{?Tg!CZ_-J3p4Xy{qk}3pKTE^($mxb(Tpa+ z-*xeyPD}hNFR=XGvm*ZAaQ5H&{J-`KBmKXl?tgt^IR3k182@2z;om$l|H90F>M9xr zhW|T#XZwq@=o$ad^qrN1;Xmj*6VpGLSz)LDhu;5#&t&z?6dWw+WC$3UIsSG-2^bmw znB)1Ez?X@ch2!sm@0f;k^eQW$4SLXf()Rvl7M0JJfS(m#9C?7=0^A+uj=o`i**{_* zGM3@T%X9fpb-Wc*iArM+)h^b2vTAYG)-cpm2l5{F0PY=jDMs-x!KOHQTUJ> zN03kp``aokf~fkk4`oo$3*X!LWs4w52ET239AZ9QgfT2N-(x#4j12DB+$EMFxC$ z{fhPY_0@%-t5*g-V??(5Ax6-nJX3emV4X2>R&NISH5xz|WT%7TFF9d?s+;Ra=YTH( zFA|xde*LO@5firYg@Wf~f~->wP!{76xwr~I^drvFmdhIqfmh_8oQ|WU`$3hfN%WAP z#s6eV88~>b3x|Z-Bx_cL`^?62lLg_hBW{g&&>d5rn>jJ(gS6L-Rb-eEkiR_z5ewJK z@oN>*X~>j>Yy*a(_S=n1P-{1vZ8z3dyOda#t-XjZ82ZlgL8k)}Cn=@M!Hr*^9L4)GC|gfHlPZS zMIbwN1F_hG?k2be$ph+9jNl0*uVk&_KL{W;{e)q4gX>$4eGZXe={wf#ZYjtHcQ8xJ z>qn?=uDx0C`s@VGUZY?ymw@JO$V1x5gnVIAgE& zb4M-Cs&D^Ww`=F@XMceiMzcUIBe@9=xD^_dFR75 zTEWppmqoNIQ6#@90@d0^6^Vq=2*Ht_lfB%kPgf#)cjMdw$pV8V{M9HwO;VDaNf%!n zw&q^quF|`HA{)(TzxpzQ6bm36L}qtgf_VZ_7fB{vg)|M4gux%dfCO2sHcH}RVGPBb zF)_YTTvJZIqd{S$5^O^(a=peW-5I*s6p-#jR4Bo<$#a-A0T0(hPQ8IP8#WY@!w&%d2Wrp!Qa%4^aGt1R#SQC|b2 zD~m+erqt7)Mtjq=H__}Xf&N-)r@wR=$DB61k_WjE=LmR4vT*Xduzoa}>LG|Lu)}6f z7ZQ{Ix`bBeUmx7DDzN^>O6Z8QXQY{BZLn8#gJ7mpg0o{(y&;ZK^xL%oIiAnnBMHq1 ziipfE3*(#WwMOjd`d$jEnRr^Tyk1_YKMfr}n42{1jTxM9dTWB2;L4~O5ss^?@7n88 zX%C8wZvW=t=F;ZUImSZpzW1vaH+O5h>;2W;bLsXJ{18`;K*T4A3UwW_CC@gQHR7G7)~_EFOEDmW`YK{if^X()V$O=Y;i= zf~?a1X99_Hq}q6el$Xw%U3C9!{!_Wit4jycJoKJ5(kLD2BR2?X_=n(r2>yoEmoNZj_LeYm?+zughK#~XX3f=c!fJN$49js3O&l(#41?{{r6lp zt(WzsiiR{#j|wwplHHQ$COpz#)O6n*-EI>@XUl)N_w#nT*g7&_n>9N_acSpo@hIjv zVb0^=dlpEzfJnt(fF$YHLU--7{2|~3$+@}ET?S=;6NfI+?}&pYQ;Z^}@SrOn`w^JQ z1i3nzGK&$^K*9*%sf59IK3R)+6HT`Tr8UcVlT`pFEXT^XJ&VOa9r^J<9lHs{Iqj@AlVj?f)B9sm+1M<7tjzAs*o+a_6{ut815KDpv>`~Z^l>l@0`BmAm2%`B zOl2(5VVc=&8F$jTRi%Y#3Ad(%s=~1mL1BU|g2I{>BK2OrLAtaAqs2)_*nTCz(=PD( zmS-hrTcAp0z+Wdk3c7pGbrGlFi>Vf;z$-^lMBtg=wN+ujGos)Ty?OfSUfO%|Ps=4c z0@UvfzxEQNq&UOw!SMmZQLxXw8RPGY6b(jh$QENa1dCf+0#$u|{yNtyAL73Ja)g4g z8q#lNFkRf8XH_Ba`A|&63D@w*EB0&Lb*gcAG;}vQK9b*m-gPHq35^6E#P>LE7}j&1{}5c5vAuTaZqksyi1_lG)!byL$%!zt z3lSF2XN1g)OqODej``lLhcu6vohWxq93w#bD@MTBhe%6&q<;v~hmQP_ zeu(sE4B-K;LeGTaE;Kau$PjTk-j<{Ebt>AYvT`erC~(!zu8#y3r*;C1%e#J6^&^nF<|-?sFl-6tJ7UZ? z#8}Ut5#Gd)qvtG*VVfPvGJJIkljNemc(L_PBD@WFHo8>1}1|X{NT`#)J$OEH(cNaDW@;#?0{IKFMMboN8C%2-fl(<^Bq?Mxq-8Zz{1f+ zVDV_fuWGIvQn%C}{+BW|lJLp59`LE`An2{lwWX!K9Y2>Q=VkW{4`if!8Ta*Aq@3;a z)R8^`{b|C3jcQ?{eSdGsl9r0i>8#CZWCwaU&M$|8UNB*W4Bhk;8MGJBJUb}Nil_x= ziJKkcPWvz2whdw@2->tT?jtHmR|SVGSm;kvRhlSLs}ob{m+HaYb3F4cDe)mu#OQZx zEFX8C`omha4{K{atThZxeMa#M+Wlk0`w8nNLE3AT7~IsZB12_u>`0BVh&}Fs-0=IO z=gsy4i=+4ai_dL>zcn~vu~Okbwy%A1Dolb$69mrB4;_+}M~UXEW4~gBP)=8kc|*o| z52V!$+el5Q$?`ht3hFEeXZO{dJP?<^mwOop7R9TW{ylT3>d|q^VFa zNXI-<_RP%T-NlWAWy}-RX zZ*!A?$fJX*(Q~+Dg|x@}&E@69N@lb-Le3-~$nledJ2&?p-@0~ZORH;?igwx?H#b*D z8_$bNn~P(cT1B#P``h8u(bdz#$rz;hAbv+P)UUqN-9zzQmo`J7HBzTWTN{U`&HE|#%X9aub0UUH;R624#mUjt+@-&BqTR*vacX#D zFg~qzXRCk$t@Y+L-|*(j%=d(+vD*>0!qlbuS;Xk|_SbC{RjS!&>i37O)5~b>#FbmV z+{t%rz2i>KhX-%OhqC=bzGF;zo~g&nOL3mEqJcYJ-q*r`C0<^%!htqkUYdg7ru#d9cMHROE1#Wp>DroBvG3`QYyX5awPvUQ zp!&2F(>`2y0h4ft($K0;}ci+z6pC4Dx8?CQSu1C4f+l-CIlR0X&ntl)vVe?8UW@N^ycr3puIp+1f zNT9ZI^BL=%A%*8A{6v34maB4?klX7z#>02>5w{R2y($SrlVnU$ktI>Lp@@0BDv75` zb5CgF2{_o}j>9j}WOUvPLb}xzmShSkJG4JunANg?Ul(&AOL=eUtSHEN+W%_aHi&^Z zFv-bY=IosKIF;cxPoq)&G=C3YnDYXhYD>QjFQwe`TsFrJ{jM$ga1)4Dns`%NfGFaL zyOj=auB@oxIe!W?KCj_@uz>?Ozb88C^cdROrCL6z!`yCG?x#~zt|c+D zvQx=H05<(zAlWA}=_RXoX1+~E;kY7jtp8C5-Kll&x6{~Po^myk-4jW;u4*FRY(|o&K0^#we0HPb7s!n&~j*km+clA<-?v>IQ_hc<_*fZHkWcOz7fPy(CsRwQR;Y zm3T*-Vjk7Dj%^G0`wV+XO|TNVCHpzuw0<%nhK>5t3enivrRz<=gySPyqEYoAa>!T<9{C(u@*rrL%^1jf}^H7gL82?P>G4>n#Z- zw;c90Px|XJPuwiUK1Q5@*})0dX8mXVsO-EASeK6V$88r(|2M5CG27vrWAdfnZZABu zDstAZP!hc3>!}jnREZ6-S9KG`TB#Nns^#(hAp=E)PLQ`#9XW1tB4~QLD=Xv+;Y*6v zDN&1bpLRx^7=s^Rci-ZOd4Y)*xt2u1Z@7jg*hPsA$vDgO!yHT1uZ>t4=78t-6Di$sZNnFO6r}J*U#Un3c-UqQ|2Z5oy&NGI>~s7k`Ke)$Gi?S#ZB;IwwvB)2Dl&| z*4yU2&7P1s?1VnjnOE{PCwpP zg7A9RdFr_iJWjbewI^~JJ)zR+ta7ipH0=o!Z=on>)j`am952TnMDGOt=UPx+TedIEDp(tOj%R(m%|J6s3pi*cQ z_8k#lZUy&!s2=i8Gxv!%KF#>AOG7uX$H>Y14HTVh^sFuA~CW$agO!~tQu*yNqg z6vKQ2ED2)rR8a-m$lR}rQiW3hb;*@OE3<4KJs(zT24Y+OtslLcp~NVwp&aALg=5M4 z8e1BN8r#25xXE?{wWTckJ23(+WQMag-z**nSs|AwO_F2A76?}eh*ydbmy`)sFhyAf zS*M&P`Qkzv$d`oNcE2%n3)QOh&c&bXoa>zFT=1TJ zufU&`?TILq$J!%^CDqxh$JTg_d?yORhdKL_1f9R(#U90 zRl=WpwJwufB)Dm`knvMT3kVddhV*tAK!yj?K7^%YC+(SQe9k+kA17F!}Y#=te^N@KC~uq*ru_ zs9ZOjFkyqB_Ui(K;Z7^uaQfP;T_wdFi<5=6nv+tO+*G&Rnd2{|Nsj(wYPMf#>BOrc zWjv(QV=-rU;=1-xc`=?$X^*sp_kG-nwMubl3H%@5hVh*3(4TmQCJPjEbwmg$U}@4G zVI5s%D#@Mq+~JkyS#}fm3AOE2O)97wN$aekSb~EDf&^UMn-iGBwQ>Up>$Dz;Q&?(L zcMaR!B)mlGoy~N=dS&oX6whABRm6;7#~mZE+Hvd;@y;}l??tpr@7FIE)o}B)Eh}j< z)b^IDWXZ*5tk2fZ(6O1}voM*>j{K0yxh( zFQkoS1J_aF^ti<3hY1^GQI6tCZxDETAAbHc&>pvRQ~<)Q_5T9UaurXX5jz{X6{}Ds zvbZZrr-{uom!#b7?N*SumR^m#c2XS!+9Kn}J1~&i=5qVXRn4!IETIIcEO!)t|WkMpmrx;Lr2 zf~ky2C@q>aMdj9}(qhH|x@2fUr4Ph;M{&@QJV58NvT4Ja-_>_{($?U@x zrU?@fqV9Mv9ldJJacH<^SCX}37qb1+f2O?R-WYS)^09DWyus%NimZ)Smq^HV4+qK$ zVf6sd_55Ze71DtfSSrQv6F=O|LUB^l%(vfU$nB)bsi_#w8r~@=&J?$vsA*j(VCPD* zcDFF8#T!dk%;E;V?A~oH>X!@@nB*ErgPNhJkr7_6?sGb>^lkIkaamSzPJMZGP4UvHmHJO@EjF;@oFSHqmLVlY#%U`D zFZCg}XLZ+8lN16{)O~_@QI0X%>iViBng;av_>=0-9F6FS56s!S3Hx!Z_rjV(cw|~2 zK`OCYhjEaF)Pqtp->SY`-tk!Fh~~~4ut(S_l@jzA%kc^*m18|ycw{SX3tFYUm8eT= zY%5QZK2V>S?;L2onZTP_)a-%&3J-{oXfXPTa)QXxN^^$n++;LIpFh3pzYp3Q1Y-t^ zp{#2yI&K=&s>IA1+;r+5AvN<76B!cM(eH%sM%%+l^6S=RBO{miA}+oz0OblhGh{F( zsE|GzaRPab>~Y-e@ui%q+?d?Z{oQn3V+mu%pGpxBO={Myqgs%f5 zV`+?dw)!N)n|RxnWhFme31mYpu+tbTN>GpGfx?6A)-a3#4@?9(T_(f}*jP%NYNa-q z04odJ$o#cw2iK{euKE3L^{%F6<^gQv@%AYSyMc_7sX3b9kz>&2Jj{mSdB``BVG@h_ zVxKXncE7?QicOSi6qQgqUopuC)Q{4If8%mDtJ+VMH~$o{{2EUZoL?BACe%*&`UAOB zKKM4)J&FU^R2Ro(9oO&xZey9s_K8R0ls^6qDhXFHGS;*E9Nj@Fh?jH*wPBih!OSAP zW=6C?N722UVw>{9Y5Pw1qNzqvKm0dD&kM(0HJfS_H_mkwO?LgM(?UufVoavLSq~R+yLHg6YUP{_8;e>QnN`=dVRb|8=B#|l&Ua91eD-Zu`@IpBd~=Ul zR*qF2U7Hyfj5PxFFbG6W&IH+?wJnOG*&_;(?I&#h!odWODXc%)mSp6Bn{=0B7FXk^ z8fE5m=;9mnNz>62$X~*Kn_rfVHo6~Q8k8Wrils@*q?!jvdVPr;5Rcm?t*3+Mesek6 z$uTXnOHka>75n9PRp3YJDqLQ2A)ERxu_khuF0ZEv!_6&D61;7hc0^6*n6Pa^2`BmU z=ZR{hjxo)6aRuBXWzz6=vbqtD5@pH`u?%~0+;V%j3&Wi9Y}3yRsLNs85M%ATsU2-r z18Xxo7S~9}?>;(tvVAq!WU!r*)0${3S1_7XvXd;LD_{-_Nw~>l8Rx&$D|}ayB&Bc~ zIwT(D=17nZYKLbSP~@biq{la<7yXPkPjw~AQ?#N~;ys8tBEM_uiowRx%Ops|gn$dl z8xYWeE~N$@j@9}pM7c$*_Z{L`Fx2E2&q%Z>D1}sK^s}P+p)O952i_RJtpD@FDrYyuQ33yAI~Hi~0NN@ms6R zbY83?Zy8NDult|0(O_pG$ zv<(7a$EjBKasp*k^vVI|ajNCLlt5+5sjOZ&U;7b)e4hMECO;))b=1VinBd*lq?L)9ck74?Duh7xghI^a)qk_A`(&x zIp0EyC8Xr^Y5=iuNkzR7z8w}uy6IaY>1vmj)D5jJ`A%HF9pDTsa0M!z`2>7N{O~K`~Vppc3jH-y{~=8oyZFiwHcE-;x2IecN&ZS}ANP1Fht?jDS}1 zTcW@QDqDZR0+nq#V1d#$5}-n58xK&Sv<(C#QrRX05-Dv%0T@)aF#rroTVKFFRkd{J zbsW1&C@FAUPOlX3J#MkAmlQZ#Emou-{cQj*JBmjT*FAvq4#x51^U)&l|B#_t5Wx8k zrd&p!af{FR>p z_5!6z%8V>Ft8lb5#_Ij!`5nDDlYoLgsc?1!LIs`~FztJfNsj_oDMu;$WHZmzYkn4u z5~C8m5`z*QO5>Cs=G2w3t!b$OOesUDd?}z*A%nE>6+Idg8iTYxr9sLtbD$~X3`s$# z6iQP6^T{^~jCd4xpeY0! z{&-v;bA&1R46+Psk}0w=<{-0AF(288!O_6!6Qd!0KGUu4(b#M_MWjL@W4_IJh7@gq zflPYu&W;me0iIN-)N1mH)T+lSsecP2@S#N-)KNFg*ceBFnoB<^Cpo8SydQOF9EOxi zfqfy6A&@c9COwcMiDGcUk#g|lSRICu;qYX(i6m)|V5;At7v^(t^9|Juom6nsnYPAL z@?oYF9|P_U5Y^x7W9R$Uo!`my+I7RT;5m?Qos3BD+2g3q^ctD2&EPq7qf$V}zGF}8 z*`F`le1lOSo5Z`<5}m|run#SD&eT3|LmS-Ili!%UcIC){a8pn@;xLIVRatw@QlOHo zrR$a~q@}k!F^-X}rQ?=y$bsmZaER%|J9dLQ6D#$cl(8{5(0ub{Myo)q`RsE+@X{ml zjK35T{i&|m?%=9MOXeYOfu2@F2)+0h-tmc&E6g4Sdw##5b^5sc_p_1{Bk zXF4tE$5#3+@rURIsmWG)EitBCGYDxXZsT^iGxZwI2s3k1Dh!%K2o)RCyHbe^n!}EX zHVNU$v|BCk$+SA0nWoq?t~jo=i783ry350Lfu^-|6<~Ek7%H%wsiwJf73dD=Gszw7 zNJUc^!DFgvo8j1S%UyLDsg>JKg-qqQeMgw-v<4-{>Rrs4rg)8|`Y9tkrMr79Qgb+JeTqYh-HPMP!KSQ8Zs_+Gq%)uJ954%M zcj4pd)B9J`$1$#K67&j)3QVN}J5~2yjTmpY z_dFlrFr^OD%8(SSW#&IdFhx8?9|q94m@GEjp45JvL}qV?`eP9Prj04)icK%ZR#3^b zL?`byPDMD{svQIEW9nn*W9*|BvG*sp($P02QGkezr${62VP}j>CzL`j&?z8@Vz$o< zP3Mg5{}{rn=xT>Q*a2$0>^N$t4`@#q8^{-Q4_-m%nO(A70vlov@1QTZj(k}!JQuhZ zG&Ue_);^VT_iS;UU>y)|a2}r-7A{?d)3=#66gKF($SyoTy}>*nKY;Ascb|ivgPns| zf5%{7t3kABgZ1iBKX+M8IdVa8L3=_oZ`n570P64^X$PD`9t#d!PfiH%!M{d0c5Is- za}OxYa}LDLH%EDzJVRXZAaX-Bt=w&0%WZt#*u2P(3jUSIf%i%K%!R-sP#nr**H zil}R$ea?o!wIg=A8>azMo<7@#jNL`@8XRL+EFR-Clbf_nwezJa{dO{oc9`$h+I+gN zhsVxk-q!70oa7Ga#^w7rxaK_O+@;*5&Sh8JS(XLrma*=6#=B8a(Zz%fx+4B>`u>Yk z1XozX?@-xt{^i|ES;3VT)YsJO)uTcq1+?ZGdzwa^eXS?khFCNx+2xk*-xHSHQGHgA z6VxmrC3(VmB%P@5s7o{tlA{sFM^cW9R~Qx)Qk71Nm*h^1hf8i04dUJR492cT*BC~a zm`X>P^u>}3nM-@^v9=10(cHPY!gxf~0y9jx;CK{jNZm<$_$bW%jJKj-GcMKidA6M* zZX<>$FLv}VW>@D|moyuP5T(wdGJa_8+%&eH;;(czrVgIfpVuEx#jN72)~&^8ik!t& z=-DnXwP;N$MN7F*2-uU|jFX3)T*)U!w|Fe{p>Cf|w`zm-t!$-Vx~1WKf63bd^RK;5 zg8szNe-z@Jk1pIFI>;`Tmhi5bwkXwN11IncYMLfw3Alv=`P3u$*3HKRYfFe$C~Hzb zSqG~%GK;BIJ(ns^!nYb|xRzC}brm;8Zi>G3BELOB;-bHcPsuQx+F+U6n+Chqn*+$~ z@O0OcIfSszowaJrUdba3h%1=TTC3u2@Aae<@x;0HjKjMxIt8YpV^R2xQooqsu}EogO6#G!8VMAJzto9`^#I zkN-100y#thn1atx7l{rLJ?Iyt2=IO|eIGCa{6TO#UmP9g4X7Glxvt_a@eNcR2zs1B zupm$&U&1aO9b|gsFCaJ2jJ{s{+63U@&`%)kzFPcPKj88}ctA0Jd?Epr=>ok00r;Nz z;2uCZKmvSd`MHsyd%+m^nUNtEeZ&dCs=zk+zaYZ}fS?n6fdL!!F?GP^b>s(!;8&#C zKoRrl>8k9a>muCH?waW0^Ud_}?~3l?^XUZV16c=O2VDn)_W5`%`F4V8gKC3$Lw11h zLi2+2LcW1{fO&lKfbf9v0BJ*NgKmRw18+lUgKUFs18qZWgKC3o18e)#25|zd1)&9! z0i6M!0hs}s0hIxk0dWT^3la?;4eIVI<}2nC>>KO@?F;S0!%tO(-GtwSvR_ILBpJYg1P(ruT{Y}UHAX7*nhUz(Y1`B`V3~St#b-r=IL8cS9SO2 z$@kj(&38Mh*IKR%ZL2yD*I@tTx&V&aQ1S=29O6WWr&KBhQ(l@v(HDyXO9xmUiE}xi zWk5?2gTf!RWP$#UtyputsD`DVqD@BN2$TDI*F1sD}q*>I#Ml(eIm)Pr~ z!m;_gQ-%)fXpLzju4yb(>1@&OrFWEV@5sR`h@0`@b}Qf1<{S9MuxPwsI^G7`#jN;@ z5aEg=j3%X0M0OQ@FI5z9$9hJov)t6h!y~4NX2<%BX(a5C15f)V#d6VS06LiESy1Ys znZU`BSD;$sEIHdG7_bY zXG?g?e0#4(2bos09Q#A=-}~2xe(SRUn(h(za6R5E znzC#g$uW+J-S+<32B$(dsHr@Xj#W0&c$D{?G`;XSVB_%f@>+5(r2y3*AetDO(9#DJasZ5DB z=NcVl5&c@C-`<%|S9n|nX5Cwurc`X@XXNOg&Gl4i9ldH5XI=39K>2t03POGoPou!P zjI)mAx(JsEnOD?NRM;b4%Sw~ImKiC2{Uq)0p=VRJdaSqgQm&ZJzVrKf_q$2(8-k1< zVkUG>DFc_e+(ee4Tpz;$BmE0oN z0Ln!w2UH$-1&L#sauAXI3-8eM6ARN8DX*Q)t+?iUR;XetD_&OL=k98G6U;;y5Pxvl zCOS>IKJ<4SnB>`fD>|e@m26%1nl|=x$Mss!aNa3sI3JAAv~PZ+bmB<&Gi8^~8FC18 zR&W~^iic=8=Y6d`JDaCjsdpEjWaTl_C_QMU z;&Q>3*?L6RAdsZPfLs+qT zD|C}(dF!UwU~!IEJmF;Cy>b*oLdey=@^iEW+@mM-OpBjsy8Xf%9_W6)qGC-$!z6j5HW^|t{)!O2q7jG zct4!#>SC|qoSf^ffwRxa4Lfsa#$~Xq6P0fuk$=D2cAlMKseK057#dok+$nBp`Rt|g zMK}qNjeKS$pTD?MIQQ_0jd!(OIIL_O1|`<5MP9+{6B4|vxQs)fx=0&E+Aqt^<@O3O zO}b_Wl_n}~PFwZ5fs^c$laqtZ9wsK^v9KdXZa7A>AE#YiMFQYY;Wuh;joxl8tE=D? zJqliB=Sj8t84&SBJkp#s8+-Y^1JIbx)Y7uk#11NUW=E5{e%1qlX!+K%> zD0=E158;n)CK09XdJ=h6FquQkEakzqvW+k^hg-y#55<>vFtK{QlMPP;6)#5G9Id@3Cu0VYl=|8HmLWjPr%?yZxYr@>hpyv8yT=J zY&791FKJdern;23z$Bzt>Bh7eA26-`hJ<}<5?bh{-4c-Yb1@_fQP%id+Pi7|gsp}J zk~HtUotxBV$`ar>!~UrWVM3d}_~1%qd@w~@S*J0{F%~!qV6YjUrkH6|Lc*4J+>H1- z0uL<)6%2LQY6&~_xw?Jkq_6SCW;L>Njf-h~X#nr@3=L<4yOg-Pk^0--zrk;!&d0+M7x2iq$3SuPrMQwQ zP@N{2?r^%*8KZn&uRdw)fgM2NfzB;ZfP65NQl%;+t)Fu`g)WwISjHR-T*kw;jIO~+ z#@9Q>Ls=a|_b~7Q4qJ_?tPr+;xsDgbj(miRx|~pn@b1uixklU0to-us(WldKR!b@$ z-K!H(1SBmm|5kc1_fun0iB{-6Be|K*%q3VYG$#9gj+j*q>b!(4r+RVuzJI&Ftbvz> z%8j8P-vfe;^ou@o-3s9NB;f~+d3OZrmo-m@RffgtZdP}V2p-Cvi~a6-omx!%0{0yQ z620F{%N;D`tXveSyg#;`$ehmcl7INvk}VSLLW`y$W6`4Zrx0W!Q1PUYrkS4AT`}{} zO!G;`ys3rVRCbHMV@2Fd$mhMF{7&}o(VG`pZ7rg9%Lu>c!3D>LsmBLgw zQV4rYA4X=0KCyUQYjmm)*_hUDki7lomfXt?SChZS&0VK-v!=w?My_CVHD@fdKVYWq zrZ|9kT89;k82&RlS);t4TR%)1cfN9{ZrK1932`E&z1pJtU3XV{o#eko=#{N8wQJi@S|>vFona8@cD6-nl%;waylb1Um-Tc9RS;l0O17^t1O>4>-`>qU!xn79ss6)T_#1@qg7o zJw#(%a3?@!>Z3P(0^#+U80a!iD48Ot(o?>t4({yT{5@R+R^UeHPv)bYC28=DPUi9J zKun38j2WFCPSWk~8WTTKw*uyw&jnPQG%HU_9(K;Kra9_7XYUzrE>j6*_5g)7_)9BnJRWK()VW>M!fNmPsv$>+UWi3>U9> z;6sO?1`Pc5Yq}YhHj*rCPQD6u&F*LNAf6QLiN)CZ$dND+P-U@oha`TgVUiSaVOB)6giF`8bdtQ{ccK zVEQ~tS3Ez^cZmsCGsN?=#*-jwE^<6oF}T$9Qa}(5A&^mBbYPbVMSKgh8u$xxwiSRa9%y3DYD(+y%*HIexfj_p`-0!4fjVHMG3jA3`4A@zMtExJn(fINN zl+^G)6=H!p9EgV&Ax94Sa4vhga)b(+kOT2I3^P@Vp5H1FxF1^mGvE6(dBCB)oi92R z?r+A$Djv4(PPNm*0v9|tTX3VTLI6nKk0;s1p=4{WFERHWY%FiG?{DSi-VO^TBWGS_ zBaRbmEDlSad;y(%;QG7pAn`KPL7r$LBLp-^@d?2aBm8-W1k+dFOksq>H~gR+1i)Kd zWc>FvwJwt3t@3*Vkni3>hg5JhAN8WqiAFmtbU-Y02V;$ zza$MT51$k=O+bKFAz#Su{;xMYs51C$a3WUOaZ}aMwrldS`PPZa>aTuj)t0^>`QXOK zu5ByUmZ9?PsGqT9*55cWd{u+?+?ipFRr58iiwxLa;<3 zSfUUtQ83=D8uL2qoVPehXHqs{QZ`{yMoUsgOHwvr5`HU6?^A1PAP_8LA*vZdMgt*p z>LfH(dJ#9}$e9M#Itm(DU@MTAfp%F+qiIGM(jW}+r|c)8-O|8p3fW*0x>D03F~YbN z_%dLDQL8!Hc=^u#9rX`SmjHIxzuuQK#5$^0UYm;=mBsHY$J%ukk3jn}>)PDa<4^qk z@kh@kI+n_t*aLu2p0AJh}C{XYK~*`;m-3iUsIv!g{bDAuGH2 zTA^79Ow$al+>8*_46WV_&D;#^bv%{@FJl=2+7=uMvW*29rh*JpLAJ2~x4SxPc_3bo zZO`J_tfd{GYu-0vk!cWV-a^^X$r2k%hCW=Ig-z#0(9j249uwuJko^!e(-q_wA#2GC zFlx5ANy*%##b7q$4dF;QEJK!JX=BiB_ZeAgx2dM%vexTM=#5!$Lw%=x@cLDeK-c=T zx1lC#+^JO-&-JXfWl|6PbI){_2N0nWC}f_)>l-FAfpfnqqZ+6}1*usvKG)s3Ww^zt zjkm3;Fa9y;BJap=voLfq?`vHRqEi3)0yzzEz7mg0Ic_INZFL}7?Uc>gDVrHRbLbhF zwR3@}&c>5jgAvasvjWIylffkCu)?+u7>*n~5C4HC4(K|^lSDm?Nk<$g@;-aSCO;dc z=VLl3Vz}C;@Ceonq+~eD3f^YCIm>c*UVu9x)bOZTXco+EP;z%V6tOWgm;o}w0(TVz zn6t#=lYD`P6tk#m^wI~5)Z1JO_os;h5S!uSBI-2;yFD(Veig9eG9iQ3ak=Y7YNkpP zNvd(w(w9gUX>z)I_tz#n=O$V$ERdl}ZEE$7ffbY8!Q|++YqyT3TDLuLS$txs%|KHG zNwW-_=$UM3THRnzj$L)_Rbwgq>MQP@PMW=bYsh1E=^1}C;BH=>YF^b^pX#`5$MDF_ z<25>)$G{3!gC69N&Vb8R*A;47)s{@PkL`d8Ne5W|mtdaxv0IQrvSwi}AV9kxg^K7y zQ$7?H!ueYcXye1QOOG-lSBb}yi|Np!UMx`(%7lK2 zvb}>PMFo!JY>?cKazT^@z5kUm$~P$mr@i9zMJia z&x6@}-k$8;2T_kM?xZD9ASA?|!Oq~TS`M=I4O2^c{ zSnOJgRBGAxPf#ta4p4_u$G-nO)v5-^3a#A&IGG1n`M*>c3t~}h2ejh6)e_;tngs&S zsw`nIc%Kcc7l_s@kA*|7s>r)AE11=9Rcw{B3YhfawuJT6X)tN@X*>U0=>XOjdC-E3 zyelrYkuwRH-7P6bMSRTCVibjnQRJ@~k`McQA)}IRBcEY2thMmNqjSFU_e6e^zlyQlWL4$mdusL*k?Y%*0!YYWTZ)UjzTXPI)mm7ROd# zn~@3%r=B86jKw^J`H0EHRxc3kV7#n7@`w6*CiDtFlX{!kQmUB1;Bo3SI5^FpPD)kO z`p6$da3&vmMdW9g)Y~FByHp)P0Rx9A>hjtM2oU6=nlV%eo3A@ly=$gBi~qwI3>xw1 z)J@~nhG0`5K9u(V)l}2l_O1DJXVlc{Xddf%`ge^zDHon<81GH`d6$p;hR@~ineL4A zwA5&oRo!dxM*}TUbMYBROIuoa6e5FJ@H zXMR87%D4!Ze*xD6V%|)=>#eV=C+e#g@Ww;THduC>ETDfNjGlrew_`5z`=U!g4b-Br zxt@4etY_f)hegm}S+#9RwtlH<8+p6Z-mr4ASv-7mU;iyf#Kgovt6c@8Kf{JIli9xO zN2(L!yH~bPv`00xQbB&v<+J;ohW^jJc-yT%yLW(h`s@Lp-YzITL3i_(hbK2ZJd<(< z+_d0?S_GotEI@%C^I#oPo!vmBF$}X4#;i(ZJ)@m*oKb8kiFv79&S|Y@MD2{iaYj^Z zSt{NG6~3*MsApH+`Qv-uM{X4E{BO7Sd_TJ8GvWjHZ$5B&HQ~AYX9qeZkM-_<=BClR zx3rvnqki@a&^Pe@v=sa`#kHj>AhW+w73y@*RZaZ;+{AygW| z_P>j$aQCjLnW4cfq$@!A5$bhXiec$~iYHw!ZO`WcIBfg#H;2=1;FIa;B z?m45=k-rIgi$|5b&7ik~d%sL64OU71#OLzCdl%2YLB2sHu`IS7MI5(VXNBLk!lJqb zVntTn71F}2dhs$tIcav;(uU_UaQS#Qd-K^ zc)r+DIwt>C#`!e<<r{!RR=;QCRlVe$GdZD7_4xmkgr1||%; zkHD{~YG5wA-D^W zs%BU%%kdVS*)1@fP9p?cv&65;H!tDf9&nc{unp2(28b(<*3_6+q@E_)fi7hUBWA|b z#B^4JnWM9QR&dS=Wjv%29Qe1ArxP&BqKQ&@lDxK6Y2OHb5h%!F3x!%72f_9pr6FK* z1}z#z@umw9F-PmnUaQ@2Pyt!4Dt;5+O)F{A$^fn)VR2G8_gCcwjw&|dKLPKO@LihK z3T*NEqDraNsAUc(eg&*xE&0j-X91&`1yfvmh}t6g>Ll`&W|o5Q9u}!BC0~Ugn<08G zV;lS@e#PsG9%FFvzM(Si4FZNDe^$D!v$ryWL-K=S7I5tf>?%~5t~S*~03^<5RenuE zT~p(4P{UUO=5L&-F|(vAJmcER%Z@;j7EMfjx*aYMf%z$*EIF%NetM}kxV$nbtGf@X z44O@fS;pYC*t~j%DBeW{q9&(aMHU|+7`@kK^XQqdRrFN*tSUT4;YrTsi#a#jf{U|e z_u2g%2f#{``_6u@{O${WFDwb3Yao8?uClXUe~F*o1RiO{@`#^&did2zCsntA{}e#c z*)+>mSzuHlw^Tw=W~MA#`S}tQ(og_Er5_0xr4@-~qcxb#mQ*dd=uGlWzx{eom@mE- z9gg5Qfir^BY<0oW?h_nFL0OEi0U{s#(R!!F>JsQKzt`g<*uh8h{(->(|GB3t$Ews> z`C@S7YopOMYsRDa+o)2YgcWlF+kE~Fss}_ZSO?9)aX*6@iB>==Ztx#gFLj4?n*$5D zPUerY;6Wuz-BD4u8I~+{Wg_`dCCi@9n}4`x&+)x&UAO#j&#w7>*~7kp8`iG9aWLQ= z1kZc&KEi$5|9oIo&u4ye;HH!Juj)DQ{C#U56x*_M4~|^%@HMSn+aH2D4S4J-fM6F; z>#HQKPCo_MLjX7523#gY-c~5M&^x9XZfj+^v7{Ch5fES%oGJ8a5wc!ag7WlnpB*0REl)5HGQEkk>b z-&ppYKGfuIyrEO8E&glSiz~tN-n@Uk$!)4_&jc*Np3naDz)e2|p8U)U`@8pEy)IbW zX`(~K;M#-N0Iz<2_4+S}ZC%?Rl)d^G@M;PusTqkI)r9G2Js%eu;FtTutwcdU9XRJCRw`v+r;{)TOaB`R5Bhkk@QMh|^rqht<#R7yu!~%zU{}t@EWjic85#MQ z(iCz!d?qzHt_#+6Hf$-O0LVrAm3LfG=W5K?J8D8c{&F?*7gOC}_Mv+_RwZo)29Ob{ z(z1W6>Pgs(!(~+b)aMHKZRv!$8qfObveEx%w-LV!w8d@3@7faZw^&!6e}gy+@G*$( zm+aC-=;y-?;RdY>{+0oz1yrTUs?r^2T-}QJW+aD1PP$y_g)BA)}0y`1ElwSCN>KAdgb@RPrsi}M;&nO52 zXlJ&jZ)00czS$G+TQ|9`zp7#VzFgI4cfA&Uu3}WN_R&-%TWziGU$EN%*g0IS zcQ|>LH}OW7QRND_%(2#RtUVE`O7(8+%x)Wq@n)Nj6|B4g;vKb-XtHMm zp57vq8#vpqaylbv35mYx^&f{ z(-u#w^gf&2V^kJTmr^0(HRx;dHzD8I+sar7h zsu^Gw0G+?i`z>Cz#RO8JteUmBX3Yv&5$iz!MYSQRFodc?$IG8v(Fv7Bv*o2SSRf;= zrPP+N$6saTsN$DsN*DBc0!D_!lLSx%Dx=@!@@Xi>Ep0o}a^!PnhX!p+I{P(py;=h+ zK@Kyx_vZ8O(G0lvHta^U`k+dvIjpU&j!Q1!Z)VkPoW-gM1p=IZfiPzUE7v?*HCq>e zWxmB^1y=b=BeEr6oy*Eg$NIEh7Dvn5U93DvUZgx|NEzhvpnL>JPrX8E>`}KrZe_{e zlcy-o8ufVNc3_JC!{GX`*X=Vfc=MN81KRd|kg@0^0u zEwhmN?6=7=Rs-MFsLuY2bWCX-7GSL=;OPf}wNltF38g_|1!l*h1eR4NZKc287KYVqpgA323S8gThdYQ?W~+$lK$F0x zGWabvFnNndIYt-uyF(@wb(Z)KFaZKie@Lrf@rR0K&cSaYhNWsKNM-8qmy|3`Q92u( z1w9x>h5$Flq#XSR7*A|G<~4!`7KT4Rr1m0`w9Tl)h=*XBl0;;AS}rd^#`N0YS@R+| zO%aw@2_-H`i95@+CCHe~_>%29Qa%Qx7Pt*?tLLt_dMG8Y!T(V7@jTS5gvc2AV^aN(+90snhei$x+j4CgSo0*STA>s~Ez2}C15v(htK zC~H|Dlv$%-c@Asf8;I6t8*r=vZ)m9PtXjaq)PKp3`~8&b_1b~b@c!rJpz363&jqs3TwxXq16V!rfOf4 zyztZ{FGMxTR)?>V)teUZce4&t9In8}y>O=}^kN*Y6vuhAfs{~ZRhUWL)aauWK;|fp zg$Epcd_D~h4=ILFCS>Vgg5+JVj1J~<(M@+vM89il7|(h; zvb~X>ecc^vn{D{3yPn$LCxlyK*D9rIrBbn?l+>O3Q*1?mUv>NUclF*j)2^@TN){g( zn`oQf3)RIsU}G=&BG!og7-=6TT6dLpM*Uvi5%t(WM3g3W3Y zT3l;;)st0ga-QGB1kbka+Yl zU`n9a*{ppkUYP68v<$Zpb@^ALNh{odJtn1WUWz!Cg$ra~X7P=1SQh!z#m#S8;v};r9G2Zhny4H4 z?;BjVchIlWgZ#;3WGr?4^&NYAm8k4vP_hAC`@{-Ium<@`6H1SyWnZnyR3TqM&$bRu zAkSvSdsmsN4Hjel=0~^1dYk+jvT1N-`{vJWJomDag)@Xz61t%cJ;AkW&fQgdhx*?H z;c3dnGDCG*!7fBR?x08VaRB+)#M|_oE#yRK*#Dy&(hTG5uI{?-vU&x>YBloAqUQky zc4NOr7#Lrby&Ph|6V94YTYG3n>iSiXta=a7;8L4J)v5M*^aQ$Kt7Kk() zD4BW2=yNy%RxMrJzs&#s#G?Oq-TD#ykFR0$How&d1fEt`d=j7I)M%>=$!Jvg--?>$ z7+A3;@o_M4_P6+_)EX5@GOUWT^2H~MA;Bc!Lh+1Z4!cSE=EzZO_qJgY0wdQvtOe=O znJk-XPYr{|Zew^IBvbS5%;@kaQ8$yFAx39LXQn1z9oUzff-_fj-B8L}(6;9Qt5EYb zL)`^sA1cZLKqM<+fr>%WS3FYu=_LRBsnh%k0pY=}8}*D5*bF}dK!Ez+H}R?nu5cZ? zghm9{Xtx$baFrT(84&YohLqg}5nKw@_W&}IOYs3YqU=_n;bfn}T8g;>29#X*mGxd( zdwgi@5Y1*|?G?9;jeUAF{tDC{0{=>LpC#xtDHTeZWVFsm($PPib??^cDV2uVZL8^u zMZ0Qk?mCr%&~uv5iZXeWh`ur`0PqZ(;^a|BPrPexw6=Eq)-@X#!EOk8i|!p$Dz!?X zwd&n|jfQ1HgV%4u-}43yc7Yjan`m}8lYLbyMv_{+tr9v?h%i`{R z;uBw12Iw51dpdpX*}jOaBy)BTjSt;^Xw%N8_V@R86Kv^=59fM;!r8Ph+jHA2P(0oB zz`B#bx{m1(=xhB#8HKM@_zx-kFRl{9r)zG@t(CpPn`vawMX1*u> zjW_|ngftH|WCJA&Budsn=rxO;CGk%|d;5IVFWf@iPZ7^j_)8Q{QO?B6;Q{OGS8BIw z39agN=a5{2Tl{v7BsG5u9HW)=yQxAbtw<5l*}ebf?z(x`=6cEukZ|PSxw8 zQ&K``-#u(MSzhJ7pbU*F*;2qNMeykWf7WH&Kdn_gqkajea1=-9KcXXg)Y?d`e#OKT0c z9jwjIHp6HbI{zMV07Sz!?6as6Wagp3M8eV`>v%^=zdK}s?~sA(fIs$GR~1?~|58>D zzt0d-7n=I*;X+-`o9A;V04AaSiJwTlC27bM+-8y$n$!gmoLLtxi1N88lqEA4tdUnz zRCz&ak(Zdr30{s!Cy5%k#8QV>ghDt0c-c92OwDCwc;k7j9$ ztOG8QB+)H^AYhWz8&&EdaR>~hqE)H7dP~RZTHyy$DEPs0$5c%mn6$4*#VlG9X9TYu zeij?Zt&D8Cb0Yd3Gfb-5J9{JD`+7RouDDoIMLkE7zhA9xK@T5fEgi zmk^PZR+dw;VnhNVjs8m;Y2@uE-p6xyKUNUol@9DHh|90lqD5h$WvC8bF3XDWg0``O z2rh|GBF6^_E=hTZ!7|6mKP@G$0gT?yHP%Ef4>Pr#Q~sV z)>3F;86YK!kOI>06vAF_e}pl>1RY^d4X|eh*wa4=NP!Y`PnNJ(6qlngn7g{ay33>n z{Giq-P2Q+|pbbB@3>k!f-xuX!-Y&= zz?>+gbFsWL&ml6FAq5%vMCt^DR0=JM%Q$kbQx-(nHZxohVe53FAeP%(7x9rP#jld1 zK9QGwi*HlULc@E2;oD!7sN%(s=&pky@NT@@1T-b7;Jr>IG~mZJ{Km!iWFOEphyd4M zk0BqVQq6D^$eE+--0p5j0*9}uX$DWnhWfi_4Y0IIXH8oNrurgR^Ff!(H?vI@Eaaq3qzQN1)=6$vg$oM}lwM)eW*>B1$m^%+iRiC4u{p|F?8OYw- z$HqE_CSU}gerUP@v3Lj9F%2Hu6Wa@Wa#!Ui^Fuj{J)IBb z>ws&H+XsZc{(_=J-f$VFNjOKx5NGcTf-|T`U|3>%+~Tn zD!6(%HUHx>(Em4%_7TO4P;&o$1DEd`^eOdHfz(>NvaVxq4*&%G;vkX)HO-%hmJd?a zi_!3hC?EkoR;xZKA47HU9(fF4%7Cqrc99T!H)$j$2Z1HNl;xavs&GF?^HQSRYyfnX1$|BLEQqkTm1_$k?5{gr5X(I-iG-k`{p5%C z$p^THx=2jC&}WLbRF+DgS&+}rtSVmpkLHl4G=*X|6e|0GOKSqar3yl=Vf41Jqqh~f z)c&Deda3E6xgdyM+~dUuUro_htwRnHn(c3Ac7up9R(5gVS}9< z7C|o(yCzF61^rywtMbknNgIFSS!e6mI&6#uoveZ+C|XUcY%!0k#-;ltv+%vM>vxXT zf<`Kq=Xt$@XH;rK&B~Qz+eey{#Rug%c^~#1nv=KPw&C6>IRCEp17FPnUyXnl$=Xrq zDDQn6IyZDCcg)*7DRvUTA&`wEFwEe@i80GCd5;IY?+ugKu^o|+f%iSbBLmJC zm=Fkr5I*KXAYVVoz$0Nv``vr1s=Fmi&cax7x2n79R^4;XJ@=e@&ixOxkq;AJ!16LY zoExnE3-}|km9Ud>LQ6!UwUxUe$1IW3c_9aXwnmo7*^1XSzhi4}ZQp2hYi+(dJvmry zo76^G|0Fv`jHnqXjRmEKVTH}A*NQ)D5tDtrH3>E|zqKY^-jK7|lDgWoJjq;+Z#2AG z@K@Q+)Mmsc6xVQuUCywG;u>E83#>tB#nBT^TfbXnCaGhL`L4Y9))sVELb0IfMfB-k zzPCy|L%(X^5q=xHWH}aIC?|%y|o=91G#E#wz__@Ycf_98g5Zz zx-OR$*QjYlLsl=hqveeS38m>912qXXzT>?$2{k`ktx3ynAiGPA*}Hs&aM`;ykyoF} zTg>wn!lv`;tYLRk9Pn%o#evPq;_$X@XR8Ys_QC{XFzgw4>`utxpaU*d5eq#QqgbOlFtPKMohPJdvdvfgLqxymfd+oNZ&|h;6_Nu>qd81uc=Lrosof>)Y6i3`6zoQ1b_GAivFoO zcqn=B*#FAG0~wZVacW)viKAncMMQa};y0Rq!ZC&iusvKEN6QfG{z!!A+`pe4+0j8D zO)K*3G_rW?yQeDEj%r_@n5lV3#|gG3R<$@_Sd`7-a#@}&As;UwHd@pTn5TE+2HaAk z1#hM%p*BReCgE0`3IpY4(#x|OS7A+S&_-p>TZWjyfy;t4qX)-hBYu{lD6Njx*-%WU zBglb=bSUd2Xwm%w!3$WpUv^i;nAgt0lQeqXpUR1&`zk&vyG9sCCVy+x=Xr)jJM3BJ zc_ngem{TLizA$grYhx2frb)BR740wKDEkEQ9ju=Q2i?;s_H{0?YnMqfBM;R*<9`30 zXo$6$4w~HXC-EVokWRe6DpijhDO^|EHacCc)_O+n^q3R1LNzpLo4f~kFN~VhR&2Vc zWqIYOB`e_arK70Q^0w=05^B2#PIS2SLZT+g?KAga&xUDaz7?A;tGcUE^A0`K)p-2Q zSg-x1o)17~jqI!VuVmArSY2Y+4+rJ4&>lDu9YsOBhzZGuUdz~g_zNf>Xe%gc0p%Oe z7XBAlK5S4vw3=&L8G7ZhI=iZ{Y@eUIXM4(QRz8Lo;E&(4whSSRf-dOa-ba0l7$l~N z-9W?|+HT2=qCA079SdKAycJB2c9fScy@3L&WhFxZzQ24PeN|>=;GY!}X91Hl9u1eL zIv57Uvw0ka{d&a&|HyPO9uL!z(V)m}r4xm2cc5GB2=ef<9c{6S5x&Q`JVW)4|A^i4 zirv1am-^K}wJo^*h2HUNUI@-8Tbs)=^xyMx7&6^)5ru3WPzn)>iU2?4FQ(ydT7AK$ z7jBmw|B=Mn?OR@vP~!vCuO!rb@A?;{-tpiyFG#@|bt80HPITqJ*$7HP0Z}dPctQ(3 zG0~x5@2HMi56W5T?ol&WL{EA=~;)#TrQjVzmtlaGpZ*lI!(>?F;SR6y$ zuWneF&34}Kg~fxXu5IT-gEaJeSi*X@bN z=GISrU~5LN3)NrS_1<9bSaRoefu8D{^L=Gf=gOp#_Mr{F{2R*X<(DJzl zN8gM7O-P#%W-~E@vJ^~7A2HiIRvg^q*F=U;g|f^z^mVCokxA#Pw9Ga+Vu*d`pIyK@ z0%s4ITCU8XOV3J~@d&!OqL(6*wDVPoR+u8k|78C-wwmADwQNH=>1Yh`%kxEgq=%)# zd$x4jbZ9LDE%c84&2VM!`e15&bTpx+kfcU8j;7QI@nuOQv4f}fv>PpU6UW>IG8HWbiYdJ0;V|41(q zMHw~jY`GJ(H9bDNdbE1_|Gm%F>F?RKp z!zr!l$7+OxB<{ZSC$O2$5q2U(d{M?@jLbw1M<`LrpcfT7;bSX)$4W?4G?u6kR(z6N zB)o)OrlwuF%C3BCSIKUNq&#m3RN$4UDB@WcKZa@ed@Zfy*(f7$+CbY)bAzg{ZIc z4?;R&ZGeMQMyL(mGw6>DkL})qsJ3$|J62k$4W2JTrwBtuHxLpb@xt@9V+eL{CVH*4N)Og&XY&g&l(b-{wK&w->*La_Cfwci9GQL&J~_1 zA~bNEP7mMz1IczwO)En%;Yi41h;?a}P|j;XE|q1!0bz<*YH+!TVit{qEab;y$-tNm znT0Yl;b@S#h;ygp5rM|TS$It{i-!cffVN>fPvP4)(=xy4Wq@bm^1X!12FhlCW#4Ep zz*Q^2zLm7wY(#5qZ@DW=vRJXWrnyF&Jdl%W7sxh%iSPV0ELPLOcjg6CRl~5Ya;{0Ny9B>G6Z6>Jb8#=Bb^-cjx?6M@G{p0 zI9=p;oF`X&S=D`WaLl%z?Nm;G+n^cp5)*Z6bBXD$mD&kx)hT(`dK1dKK4e;bQK&g} ztiF(=>P|>eJF&Te>U$WY`mT=R+Z+>E$Bw8vwu$09^6nIE^I7GmqMwC!)+XS`eD+un zn6?$o>~V}Km%xB>3$tM4a&&PSNwEcm{MQXT!S5FlnykMlqxfPxUkv4o%>}fA_4p=) zvbbsRa*%hq*~`G-p9F6}gM|m$GZW)coaH?n1)8=?q_o0zx%`vne0s~a8Fy|d zSr~4!LZfCd4cWj^1(#BrPB%fVc_7SrFHOI0M=+nBgjj%P+5rxQv_|f6f35El|$E zY<&Ir`4U3p>BgBYX(Ku45}V9DD*L4 zH^eS)lEm93F-EJrUvkx5l4JbLn!6;gyh##X`Bs}GR}s6lH%Z<wtDC=i=O`cUjMN8Jt*!Z$>e{`(Z7^k@yz7yh zHr~E(zzTi4`lq(+7}$5KJi^H@;=RIhPUJ!th;s@_n}rV)(sByTIYe5E0>TfKGl6JK zxFN-36@7X<&e?-w_DMofvv^ugH+{L&X?&tVYM$v-H0X-V_0`5ICYqM~BB|BubPiwC z?#g%dMOKc~*ic`|X9`7qtcC*AwW43p>-9QYcCzQqM=u|%cXkgaI7(+Q=*kY`fVJq0=froL}UYUJ_CE4sxdJL}$n}j{WfIm6RgB~vOkdo!>bY@@Ip|a%#;EH{9}v(JWh+RsAAjz07m;R z-VFvY`AosSI-CWI8 zy$@OBdRe2kZcXFSVqggef}XozkyimNa{4*{5`t!ufgd`LRc{K^B>6_O6Oc2r{A_R* zat7b)K*!S(yxYk3S0s2p7gv*9RsDU~(AO*ac=;Z+v>@9fts90{-HAr7{neUQ{JY#@mv3I1c81z{L}z?EH}N^Z!zBR(DSG4$cC@m3~ZSwu3MZWwf>Eutg8 z>&~sSw{C6w1wuqtgBQx;v>*ybBd0D;x9(IcjHD{#-k^UK>FB$%Z1wHk7}si@)%g0u zo0n~mYc5bD6qw(T2Q^%tj;gaW3gw`dkWO13gi)4b*NCfLd5fn? zWU4PxJvbKLfc8n@5L`NsBP>tLakqa}r>lCpg{*7yfO@>SjS7cUEXFl9U9|?_Lc*F% zLa3UME7Wi392o~+A_#&ZW8^=e!hHDi7okKC$(aOaazi}wEYp5}nnPNWrMl8XRX$zo z@2aBJhR0%4dU=E8MX38g4yT2p6*J5c6FjvmU6cANUDcQk@2SM+<2{u)8yDvfiS~Cx znnB_ckf%asTDyJExZ-#oyKZ5KbLmQ0QWYbUBlr-hp#yP3uiw_(OE1w-VFU zp=y7Bu$r%u)g9(^t*dH512s0at7U*AS-vnQhbk|iuySMnvO+qS+ze6WtE7>tcI_}D zLP9cH6f!cqmJNR-(mdKBxms*eylju;Y7}ik#tN(H3C(wPf*d6-`|Cq31Sctm^Cwp) zIry5on1pvRTg<=g9&JROuknR=vq7PH3o`_pU^1E3Vg~>!5PtpA>iUM+1NF_{5zk;w zU3+u`;?(@@2^1Jwf75#2jaU|s-x1%uzAG+@@viloY2y0%V~36%y53N`cl7q^(dveT zd%ku(LaET4*)fGuT+avTkCEToX(v2Iw*HDuhOldiAs z7{#pIF*ShNKQAdakTD6;a=5vyKp(%IU7f2Bowq%pTG-!+Of!2lv|h{Daf~>?HTH#C zLUPo(y@i{M5K62f9XdYrma+k!L(%HkP}ua=J4@FXv~P1Pw+atdt2iXBXhiK6tP#)3 z8j&Vdji?=hBmaQB#(5?%ID-}8f#L!4fw{Q@X38^-GKAI_&>GyCm}d)`cO3hss*{7& zLN%QZ_U3!Z-WkGEi&Zs91=tl+QBeS8d9FvA6Uz`59$Y7GA8yJoHQOS3`Ch0FdSf$0 z%u|zMRl5dj0lR7mHg!NFzmmjK-aB9Os!6_@O}mIX&oMXy|7 zj^YicP56NoD9;V_ffZq+?q<}8`V3?7*&RMXJM|D=wo*euj$AcAcx)qM3!p%dzOkBh z%eLu(_uh9c8E(XQem}E&Lu|)3@}_3Q4G`f=f1-{-Dz*cYf@po|jfw*;{swTBkyrru zYd01`Y%(jc6KAFK<5Xc&ahpXCsUhCmR1 z8U)cGNO0ih5D1~I^LoKv4F#dU43A!^=%JAvLYN2W2=Oa~+GpW~q2yS|=pHjps!(*8 z_C$IPuX?5BUx3!WWbU9kCuz?)A_RCE-^d7W)M!J-tV_e;rYL8oQ2?$b(yyZP`AO;+ zAStqb5#t#8lP8jzwuUq9&Y1N|8=&<^;Z9Zc~`fEvA53{d>}AZ&r>l#kJS zg+N#Yey>^yj85exBZrOawGiI zF^TQR?<1PeT_~3?LYm8*$;oRL1g<;NSe3*qsrZJ*I6lCmTWMJNJ{5eUP3?ffH?hE* z#M`^uNfL^xM!pT?*!K~LOL{lD6SP0)=j4q?+6L=Xj71rcauqJN{7K)Xq3f?xCP&gA zXmhdDQzZEaWpbzdfdo9N{}JjoXrFq+(C~bLB>y+5x4H(m9E$c5)sIA=M_AQ#TTfpGYpH@VQ>%hnwm@2L&{-1!f5dLo z{O|`Fqdn~P#RQ-S&id;*U`+&l5t~7C@mY;Q2zY!k3#qUFvE6KCX$s*#?yr9uLC?~x z)eIg7Uo=}y8cJ(m>Zies7FAF*+PL~W(rfh-)U7z;x+qi611Zskbdm?8Jiv)VT}}`; zcbmzC9=OpmU7s7cdJ$P&ATZ`KSjP+#ni*n32{ECHNoiS~Ahm*E-&#*n0!N4xAxfr1 zk2Q|4(>oi`uU2`Bl#L-f-lr`T-9drW-m2%R`qy-P)b9`5^fUm}8(JY8^hO13{TwgQ ztj!Ec8jFG2Wp|osO2?Vr%#zPrjWn7Qk*3&55abW2$BDFzyZse{hdmWh#kn}HJC}p! zbn1uoq&_Ar0rD|dl}q4|!9)W9bM7M4n2@<=VlH&bISHOaC*%;p#5+X4^poTdv^uly z&FAeNB=O+m^$+n@jozduHAaDD&}a3Vz!!AT9~eP^m)=m=EQ&6kJP?W@Fk!7(6oO`_ z!_B|>5W{;2k{BkxK~`uFkstmSQP*ebV!wlw36Q zJRspV6xbjInX*}zCA=moWpi<@6_k>!c>E!EY_6|;qC1eBkTz^J1v=uffs9{gvY7k# zts9;zxj!;-<7gj5r5a6H1$-yfi}AhA;J}GqK#J z1H-Z5u3)PBsca_DxdGw7!k#;67m+7sm8ws%CD>!mU_a>#f0A3IPPHdL!yIj`t&U4P zRYaftq~znmpOm=8cIuSW&Llr0F-Mmbcfxh98XRp|7}v^d#XTmSq=pG!-T#q~rYEi+ zw55}Ahfzx*^~4yG<j>H}H<>ai{v*Y=Fw)%7IU0bQ94xUa$sfrBnRm3&lOEPk48k@b>{ zu&r_vog^k7mBdB1_-To473fmAiCyvbV6dPNf4~(IL{6LAJFxzmk~=uGyIhz}G92cp z_TJ=ZG77y*)?bWA$Fk(V%N!ia6>>9wGca;wCLNE1EUnW}a89J_Te8_;=Y~jhq$`x} zLNUTo*ylrVyvK+vaVyT3$wIH_p0gf7@WhwE_KJh>SWlSE`t0!_0-18A?hh{NA9Nm7 zD@H9U%;fh?k2YgF$Bwrwcv72Oms~-!MD^9ie+P8;2vEQ7fq-+^)pB zHqpqjR=3w>X7%*_4~{HOwVnN3zq8bb6HhHh4noCAx{L@H(``LYT!Ru%jZh z#lyOX@`u7F#glD^y+_Q4@%lwg-qi3~@q(NTZ_Dc*lJbYz!Y8FR5&oCY#0wFv;oipD zsJPl5pYCR!J9vWBXen*lfxNw&ShLA^i`MM1*&(y1jZi;LI|}1PN5QVAN&4T+76WNw zJ#D^Tr`PAKmm%vli2vY!I(=Sech78>Tc_8VYy?HLf!E3XaLf)6lf*T0mGQ3;o53)V zBrISSPH7v#md6We_!IN2TLN3oFvJv6W*%ZIxT=zjX-zb|e&ZGkB{s|IN8{ zseu!7<^1mM%hJ2yf9CS^Zmm{1BKMp06Bcu5@T#sruf<@B2Or2zc6uXYhbOAn4f)#> z-eANny269oJG?pjd86^Ez8+7?W9sYlrah)?SMHvO02e^$zjIj(UP zQsYU8ks#*`;@V>5x5?Y!jHKx0(@6{BpqDU0apnlGkaU<&rx(Kq9Y^RRYSM72wbThb zBW*q{HO^5pgpm~JD)=kd9rrUxF07Y5drwPN;{T&Lw_SZ=yPKm;;e0j}&gMhSXC|Fmot7lw%j0c+ zf2u9uC&)`b1g`@YK4Tba(+S1L2R}NKVhJ>jIH2e6J|Fij*iO+mBmIe=^Md@K@%ANT z8B6hsZ-KU*l4vZ~%HuYJ$K7&lv3vCg2ZSY4?H02ld_JR2tLN-OEXr^mE9wjz&gDXp zY%WATouBUZYZ->M@?Hl;Q<)Okd0B5{Y1#{=sgu}=`Z8oB3gBa?@)jhx2s?HTT8Rd* zgz9ghEWriuK|z%(MXclsTsZf)^^BH0!iH%>ts+i6%{)TJDvIP@#^M!4ub`*Dw^F9g zY77=PPW8~6KmTo`u0^;q^?zGMu);5B4O$BISZA|5T7T8T$vq~12YcL-eZBpxfj@|Q zrDk6rlzXtM6#Z`Hb2Xo%d2eN;Qon}t?g()RkJRo6oNUiz2LvQ2USb5!yf%aR^w47P zbZK#*J5^jr9dWc!SINQ4{j3Ck5rG{;aNG_pmV&3H1?-8~aXw z4LgRVsk1A%A*touR=bB|ie2ILjbZMJL>$HW!q_&arz4kh_DvUTYY+96$E1^m|JHY8 zdve~m%NSiZ+p9?8F-Wa;rG8UXXzDdH;Wa;+Sd2P?3o3!+tT&|9Flo#Q^P^JB87(<( zSD;3-_ei51LkzXTu6FlEMw2QLT#=BgxTm9kwy;c_jiD}`xU5UKGf_$l;_vE+uM>3RRC$T$cNn>dk31nPBoJpNk%)6uQR$hlh#{(w)&ei zs<5Pmc2o@wPspM|J~wj%_I1*oPi3;QuLETdd>4DfBRC5`>?Sy57_)A}g~Vc*v-=n9 zN17R1uU(Mwv`q=a1*!S{+sM%XgxET^oNe)l)f^HTXpetS zaGDufOBlh^(!R0o`z3s>U^tFb zq@cbKXz_l9$1(&hE(@ z$I7W-X>-xvk#HMn_zkTUqq#s48sFpPHtLr2`i#@aaje5`va&R9;lc@T$l*v1r zQF;+?SikVvH@=U(TzlZKHE~jMSu0lKNy&PcNuN@jT%2m6qG(|UXQlnW%tt`(F8T~S zqBV&A5VvRRw9#l}r?iUq_c3^6{8+Hf9oK5KG)eKI6Hfb@UGpH0?EeqaI$EQFFCWC- z!T&83IgQbRW%?WBZL|$qDRTW>eMEM1eMp}Q5j~Y!a4ZBLZ8*6N@OZg6k#YhXe6+>A zZA|FPz1x;Bi?}SxLf+D-rf^XywZfEI zRh?UTU2FBtm7Pwk-T#(9soS7PGL+`?l+GG*dqQ?Sc?gjEDZLFH*$mWw(NK)wwmG~4 zL%yFRZw3tSvfEu|E%jlNyb`mH+heIKOs z(6)64dF_|~s}^uUx63E!sG~KNu0S+mnl_&7b^3_CS}bX{P4Wr z27id~5_q9v)Q5yquESj9Q}m5+qWU`gT~=nQVh*{4QxZIbj(kdj=Rh_5b@;2!K#eta zYoxKp4(DMc|5)|@9@Q|ef0TowX<%wUH=41Hx0@FX+^_$Li!|yun~4$aP6NmBMlk92 znBlR-;j(&d^|Q3ehO8)pdK~X@H4-e5RI4s8p4I9pgpT{tMcKJHtEVa$?biL$#par~ z?An3N>K4-aBbtlKhPV1@)V3xFCg{7Dw|xt4d*6z-_pQ^zK+>fLH>uh9)Dx_QrL|C9{<#!NdyGa; zIuy#dj7C>R-aY&WYCC+wPNFKWV4~hz&bi%ODn&1WXU}k4*AltE!j+@ZfZp~PP3QH) zeJITUoX+F27(aOiMJf<50Laxk(60braZONk32l2!!WUKgrt;{rGg4lzapA^SVn_is zjLj(1AMU)815B>0oOKzl-}d6ww@opTgxzn^X#vy(7IA1-$sMfhUEjC8k}@zxoknXb zZQk2?!>9J;>lYxA{J|swR!au`kZ9CTcig+Dn|_VsP;?X69X>0Q8s1eb?cNx7`JHgQ ziB6X_;Qs%_eFvQ^ImneE*~*F z*EMOzQX=Bmx!H5z2)R~R<(i_zG~gRB7pBK+Pti;n zT=UHsq!aU6$sRd{Zt_;tMbd)qFEZs5S6cYPQYg&~Rz?C?3Fos}J)Bf7H~8$ffKe_t z25fepK@O6r-v$By!Dp3>f}&`R@`B4AG|{vvXt#&V3WYfYKl3fK=LnqKfYo5#MNRo_ ztQMbyYLfy#JjFJ{pNOr1Yk3Zf1Ol}C{ep}w^ERrk zah&GiYls{yXT*SH>8qx>xh35N?F;rkO2wHu)~EpF-L%PPG5d`4gU&=OYW@KdihI7jeP0dvW$W_%v}?IM-EG?5?US;CoLi9A0xbrGXOhOzw=3*o9aP-QzWaScJDP zWA(C2`Rvn3nz%wpnuJ`=;zn8YJK=JbqfJom@arc19Yjl`(Xxs4{bP7Y>1wdh_vK-^POKH$HVUd>c4 zzQ-^~`>dTkM<&VY#eLD+gudu)LSJ;lG0ix%KMY5ywU_oqH=M|8#?b*m!?8Sk|C;-v zC+qgUc>5ivZ%m`>fx?aHW4`X2hkADPg?!yN4fX8q3*p?h*FQHp`0(pD<+$e>q9G6Jh~HJA4WFr?8mr-3_*-oCvb|9F5s5q7^|cUpy{}63@X)J?&2Atdp+WF zC2vFToX87rA){|jFceKasYFr?qonFJMAW-4kW3d%?&YtovXTB%u% zaJ`gP5UU=cNtMIInpvgfC7i+mlK+U?)$-LQt?nQnv$%pT31c&;4UCHM zc}&6X>9)lB%{R8aBrkRa7?_zM_CWfKVhs{Y|ne^aQxKh#8^p5L2KV`{FR zt2fmYfQeTu8;;F-Jk+vrQvsNdQWM2va24ljnd0FqK^+#;X`$I8XnSGyGGV$ZKkSLk z=7k5Kp5JE3x)b7=w~B~(>g`H3JA}sQbNL%rP?Z8b^$vpAlfC}g>DFBi&7&eDVc`e(X_cUXi4@bv+KL66cDYGI4RR-hqvWM?;VZ08pblY z?L!r}4?M7`&fs*?D!s$NnKcr3z~yNesZOoTd7zrqb7n^3&8