diff --git a/.planning/ROADMAP.md b/.planning/ROADMAP.md index 1da554e..9fea66c 100644 --- a/.planning/ROADMAP.md +++ b/.planning/ROADMAP.md @@ -156,7 +156,10 @@ Plans: 3. Empty directories within the input are stored as TOC entries of type "directory" with zero-length data and are recreated on unpack 4. Running `encrypted_archive unpack archive.bin -o output/` creates the full directory hierarchy and restores Unix mode bits (e.g., a file packed with 0755 is extracted with 0755) 5. Running `encrypted_archive inspect archive.bin` shows entry type (file/dir), relative paths, and permissions for each TOC entry -**Plans**: TBD +**Plans**: 1 plan + +Plans: +- [ ] 08-01-PLAN.md -- Update format.rs (v1.1 TocEntry), archive.rs (recursive dir pack/unpack/inspect), and integration tests ### Phase 9: Kotlin Decoder Update **Goal**: Kotlin decoder extracts directory archives created by the updated Rust archiver, preserving hierarchy and permissions on Android @@ -206,7 +209,7 @@ Phases execute in numeric order: 1 -> 2 -> 3 -> 4 -> 5 -> 6 -> 7 -> 8 -> 9 -> 10 | 5. Shell Decoder | v1.0 | 2/2 | Complete | 2026-02-25 | | 6. Obfuscation Hardening | v1.0 | 2/2 | Complete | 2026-02-25 | | 7. Format Spec Update | v1.1 | 1/1 | Complete | 2026-02-26 | -| 8. Rust Directory Archiver | v1.1 | 0/TBD | Not started | - | +| 8. Rust Directory Archiver | v1.1 | 0/1 | Not started | - | | 9. Kotlin Decoder Update | v1.1 | 0/TBD | Not started | - | | 10. Shell Decoder Update | v1.1 | 0/TBD | Not started | - | | 11. Directory Cross-Validation | v1.1 | 0/TBD | Not started | - | diff --git a/.planning/phases/08-rust-directory-archiver/08-01-PLAN.md b/.planning/phases/08-rust-directory-archiver/08-01-PLAN.md new file mode 100644 index 0000000..ebc2a9d --- /dev/null +++ b/.planning/phases/08-rust-directory-archiver/08-01-PLAN.md @@ -0,0 +1,316 @@ +--- +phase: 08-rust-directory-archiver +plan: 01 +type: execute +wave: 1 +depends_on: [] +files_modified: + - src/format.rs + - src/archive.rs + - src/cli.rs + - tests/round_trip.rs +autonomous: true +requirements: [DIR-01, DIR-02, DIR-03, DIR-04, DIR-05] + +must_haves: + truths: + - "pack accepts a directory argument and recursively includes all files and subdirectories with relative paths" + - "pack handles mixed file and directory arguments in a single invocation" + - "Empty directories are stored as TOC entries with entry_type=0x01 and zero-length crypto fields" + - "unpack creates the full directory hierarchy and restores Unix mode bits on files and directories" + - "inspect shows entry type (file/dir), relative paths, and octal permissions for each TOC entry" + artifacts: + - path: "src/format.rs" + provides: "v1.1 TocEntry with entry_type and permissions fields, VERSION=2, entry_size=104+name_length" + contains: "entry_type" + - path: "src/archive.rs" + provides: "Recursive directory traversal in pack, directory handling in unpack with chmod, updated inspect" + contains: "set_permissions" + - path: "tests/round_trip.rs" + provides: "Directory round-trip integration test" + contains: "test_roundtrip_directory" + key_links: + - from: "src/archive.rs" + to: "src/format.rs" + via: "TocEntry with entry_type/permissions fields" + pattern: "entry_type.*permissions" + - from: "src/archive.rs" + to: "std::os::unix::fs::PermissionsExt" + via: "Unix mode bit restoration" + pattern: "set_permissions.*from_mode" + - from: "src/archive.rs" + to: "std::fs::read_dir" + via: "Recursive directory traversal" + pattern: "read_dir\\|WalkDir\\|walk" +--- + + +Update the Rust archiver to support directory archival: recursive directory traversal in `pack`, directory hierarchy restoration with Unix mode bits in `unpack`, and entry type/permissions display in `inspect`. + +Purpose: Implements the core directory support for the v1.1 format, enabling pack/unpack of full directory trees with metadata preservation. +Output: Updated format.rs (v1.1 TocEntry), archive.rs (directory-aware pack/unpack/inspect), and integration test proving the round-trip works. + + + +@/home/nick/.claude/get-shit-done/workflows/execute-plan.md +@/home/nick/.claude/get-shit-done/templates/summary.md + + + +@.planning/PROJECT.md +@.planning/ROADMAP.md +@.planning/STATE.md +@.planning/phases/07-format-spec-update/07-01-SUMMARY.md + + + + +From src/format.rs (CURRENT v1.0 -- must be updated to v1.1): +```rust +pub const VERSION: u8 = 1; // Must change to 2 + +pub struct TocEntry { + pub name: String, + // v1.1 adds: entry_type: u8 and permissions: u16 AFTER name, BEFORE original_size + pub original_size: u32, + pub compressed_size: u32, + pub encrypted_size: u32, + pub data_offset: u32, + pub iv: [u8; 16], + pub hmac: [u8; 32], + pub sha256: [u8; 32], + pub compression_flag: u8, + pub padding_after: u16, +} + +pub fn entry_size(entry: &TocEntry) -> u32 { 101 + entry.name.len() as u32 } // Must change to 104 +pub fn write_toc_entry(writer: &mut impl Write, entry: &TocEntry) -> anyhow::Result<()>; +pub fn read_toc_entry(reader: &mut impl Read) -> anyhow::Result; +``` + +From src/archive.rs (CURRENT): +```rust +struct ProcessedFile { name: String, ..., ciphertext: Vec, ... } +pub fn pack(files: &[PathBuf], output: &Path, no_compress: &[String]) -> anyhow::Result<()>; +pub fn unpack(archive: &Path, output_dir: &Path) -> anyhow::Result<()>; +pub fn inspect(archive: &Path) -> anyhow::Result<()>; +``` + +From src/cli.rs (CURRENT): +```rust +pub enum Commands { + Pack { files: Vec, output: PathBuf, no_compress: Vec }, + Unpack { archive: PathBuf, output_dir: PathBuf }, + Inspect { archive: PathBuf }, +} +``` + +Key decisions from Phase 7 (FORMAT.md v1.1): +- entry_type (u8) and permissions (u16 LE) placed AFTER name, BEFORE original_size +- Directory entries: entry_type=0x01, all sizes=0, all crypto=zeroed, no data block +- Entry names: relative paths with `/` separator, no leading `/`, no `..`, no trailing `/` +- Parent-before-child ordering in TOC entries +- Entry size formula: 104 + name_length (was 101) +- Format version: 2 (was 1) + + + + + + + Task 1: Update format.rs for v1.1 TOC entry layout + src/format.rs + +Update `format.rs` to implement the v1.1 binary format changes from FORMAT.md: + +1. **Bump VERSION constant:** Change `pub const VERSION: u8 = 1` to `pub const VERSION: u8 = 2`. + +2. **Add fields to TocEntry struct:** Insert two new fields between `name` and `original_size`: + ```rust + pub struct TocEntry { + pub name: String, + pub entry_type: u8, // 0x00 = file, 0x01 = directory + pub permissions: u16, // Lower 12 bits of POSIX mode_t + pub original_size: u32, + // ... rest unchanged + } + ``` + +3. **Update write_toc_entry():** After writing `name`, write `entry_type` (1 byte) then `permissions` (2 bytes LE), then continue with `original_size` etc. The field order per FORMAT.md Section 5: + `name_length(2) | name(N) | entry_type(1) | permissions(2) | original_size(4) | compressed_size(4) | encrypted_size(4) | data_offset(4) | iv(16) | hmac(32) | sha256(32) | compression_flag(1) | padding_after(2)` + +4. **Update read_toc_entry():** After reading `name`, read `entry_type` (1 byte) then `permissions` (2 bytes LE) before `original_size`. + +5. **Update entry_size():** Change from `101 + name.len()` to `104 + name.len()` (3 extra bytes: 1 for entry_type + 2 for permissions). + +6. **Update parse_header_from_buf() and read_header_auto():** Accept version == 2 (not just version == 1). Update the version check `anyhow::ensure!(version == VERSION, ...)`. + +7. **Update all unit tests:** Every TocEntry construction in tests must include the new `entry_type: 0` and `permissions: 0o644` (or appropriate values). Update `test_entry_size_calculation` assertions: 101 -> 104, so "hello.txt" (9 bytes) = 113 (was 110), "data.bin" (8 bytes) = 112 (was 109), total = 225 (was 219). Update the header test that checks version == 1 to use version == 2. Update `test_header_rejects_bad_version` to reject version 3 instead of version 2. + +Do NOT change the Cursor import or any XOR/header functions beyond the version number. + + + cd /home/nick/Projects/Rust/encrypted_archive && cargo test --lib format:: 2>&1 | tail -20 + + TocEntry has entry_type and permissions fields, all format.rs unit tests pass with v1.1 layout (104 + name_length), VERSION is 2 + + + + Task 2: Update archive.rs and cli.rs for directory support + src/archive.rs, src/cli.rs + +Update `archive.rs` to support directories in pack, unpack, and inspect. Update `cli.rs` docs. + +**archive.rs changes:** + +1. **Add imports at top:** + ```rust + use std::os::unix::fs::PermissionsExt; + ``` + +2. **Add entry_type/permissions to ProcessedFile:** + ```rust + struct ProcessedFile { + name: String, + entry_type: u8, + permissions: u16, + // ... rest unchanged + } + ``` + +3. **Create helper: collect_entries()** to recursively gather files and directories: + ```rust + fn collect_entries(inputs: &[PathBuf], no_compress: &[String]) -> anyhow::Result> + ``` + For each input path: + - If it's a file: read file data, compute relative name (just filename for top-level files), get Unix mode bits via `std::fs::metadata().permissions().mode() & 0o7777`, process through the existing crypto pipeline (hash, compress, encrypt, HMAC, padding). Set `entry_type = 0`. + - If it's a directory: walk recursively using `std::fs::read_dir()` (or a manual recursive function). For each entry found: + - Compute the **relative path** from the input directory argument's name. For example, if the user passes `mydir/`, and `mydir/` contains `sub/file.txt`, then the entry name should be `mydir/sub/file.txt`. The root directory itself should be included as a directory entry `mydir`. + - For subdirectories (including the root dir and empty dirs): create a ProcessedFile with `entry_type = 1`, all sizes = 0, zeroed iv/hmac/sha256, empty ciphertext, zero padding. + - For files: process through normal crypto pipeline with `entry_type = 0`. + - Get permissions from `metadata().permissions().mode() & 0o7777` for both files and directories. + +4. **Ensure parent-before-child ordering:** After collecting all entries, sort so directory entries appear before their children. A simple approach: sort entries by path, then stable-sort to put directories before files at the same level. Or just ensure the recursive walk emits directories before their contents (natural DFS preorder). + +5. **Update pack() function:** + - Replace the existing per-file loop with a call to `collect_entries()`. + - When building TocEntry objects, include `entry_type` and `permissions` from ProcessedFile. + - Directory entries get: `original_size: 0, compressed_size: 0, encrypted_size: 0, data_offset: 0, iv: [0u8; 16], hmac: [0u8; 32], sha256: [0u8; 32], compression_flag: 0, padding_after: 0`. + - When computing data offsets, skip directory entries (they have no data block). Only file entries get data_offset and contribute to current_offset. + - When writing data blocks, skip directory entries. + - Update the output message from "Packed N files" to "Packed N entries (F files, D directories)". + +6. **Update unpack() function:** + - After reading TOC entries, for each entry: + - If `entry.entry_type == 1` (directory): create the directory with `fs::create_dir_all()`, then set permissions via `fs::set_permissions()` with `Permissions::from_mode(entry.permissions as u32)`. Print "Created directory: {name}". Do NOT seek to data_offset or attempt decryption. + - If `entry.entry_type == 0` (file): proceed with existing extraction logic (seek, HMAC, decrypt, decompress, SHA-256 verify, write). After writing the file, set permissions with `fs::set_permissions()` using `Permissions::from_mode(entry.permissions as u32)`. + - Keep existing directory traversal protection (reject names with leading `/` or `..`). + - Update the success message to reflect entries (not just files). + +7. **Update inspect() function:** + - For each entry, display entry type and permissions: + ``` + [0] project/src (dir, 0755) + Permissions: 0755 + [1] project/src/main.rs (file, 0644) + Original: 42 bytes + ... + ``` + - For directory entries, show type and permissions but skip size/crypto fields (or show them as 0/zeroed). + +**cli.rs changes:** + +- Update Pack doc comment from `/// Pack files into an encrypted archive` to `/// Pack files and directories into an encrypted archive` +- Update `files` doc comment from `/// Input files to archive` to `/// Input files and directories to archive` + + + cd /home/nick/Projects/Rust/encrypted_archive && cargo build 2>&1 | tail -10 + + pack() recursively archives directories with relative paths and permissions, unpack() creates directory hierarchy and restores mode bits, inspect() shows entry type and permissions, cargo build succeeds with no errors + + + + Task 3: Add directory round-trip integration test + tests/round_trip.rs + +Add integration tests to `tests/round_trip.rs` to verify directory support works end-to-end. + +1. **test_roundtrip_directory():** Create a directory structure: + ``` + testdir/ + testdir/hello.txt (content: "Hello from dir") + testdir/subdir/ + testdir/subdir/nested.txt (content: "Nested file") + testdir/empty/ (empty directory) + ``` + Set permissions: `testdir/` = 0o755, `testdir/hello.txt` = 0o644, `testdir/subdir/` = 0o755, `testdir/subdir/nested.txt` = 0o755, `testdir/empty/` = 0o700. + + Pack with: `encrypted_archive pack testdir/ -o archive.bin` + Unpack with: `encrypted_archive unpack archive.bin -o output/` + + Verify: + - `output/testdir/hello.txt` exists with content "Hello from dir" + - `output/testdir/subdir/nested.txt` exists with content "Nested file" + - `output/testdir/empty/` exists and is a directory + - Check permissions: `output/testdir/subdir/nested.txt` has mode 0o755 + - Check permissions: `output/testdir/empty/` has mode 0o700 + + Use `std::os::unix::fs::PermissionsExt` and `fs::metadata().permissions().mode() & 0o7777` for permission checks. + +2. **test_roundtrip_mixed_files_and_dirs():** Pack both a standalone file and a directory: + ``` + standalone.txt (content: "Standalone") + mydir/ + mydir/inner.txt (content: "Inner") + ``` + + Pack with: `encrypted_archive pack standalone.txt mydir/ -o archive.bin` + Unpack and verify both `output/standalone.txt` and `output/mydir/inner.txt` exist with correct content. + +3. **test_inspect_shows_directory_info():** Pack a directory, run inspect, verify output contains "dir" for directory entries and shows permissions. Use `predicates::str::contains` for output assertions. + +For setting permissions in tests, use: +```rust +use std::os::unix::fs::PermissionsExt; +fs::set_permissions(&path, fs::Permissions::from_mode(mode)).unwrap(); +``` + + + cd /home/nick/Projects/Rust/encrypted_archive && cargo test --test round_trip test_roundtrip_directory test_roundtrip_mixed test_inspect_shows_directory 2>&1 | tail -20 + + Directory round-trip test passes (files extracted with correct content, empty dirs recreated, permissions preserved), mixed file+dir test passes, inspect shows entry type and permissions + + + + + +After all tasks complete, run the full test suite: +```bash +cd /home/nick/Projects/Rust/encrypted_archive && cargo test 2>&1 +``` + +All existing tests (unit + golden + integration) MUST still pass. New directory tests MUST pass. + +Manual smoke test (executor should run this): +```bash +cd /tmp && mkdir -p testdir/sub && echo "hello" > testdir/file.txt && echo "nested" > testdir/sub/deep.txt && mkdir testdir/empty +cargo run -- pack testdir/ -o test.aea +cargo run -- inspect test.aea +cargo run -- unpack test.aea -o out/ +ls -laR out/testdir/ +rm -rf testdir test.aea out +``` + + + +1. `cargo test` passes all tests (existing + new directory tests) +2. `encrypted_archive pack mydir/ -o archive.bin` recursively includes all files and subdirectories +3. `encrypted_archive pack file.txt mydir/ -o archive.bin` handles mixed arguments +4. Empty directories survive the round-trip +5. Unix mode bits are preserved through pack/unpack +6. `encrypted_archive inspect archive.bin` shows entry type, paths, and permissions + + + +After completion, create `.planning/phases/08-rust-directory-archiver/08-01-SUMMARY.md` +