From ff5a96ad1472a02e9eafafa139a11bf3ea09d98f Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Sat, 14 Nov 2026 09:04:32 +0530 Subject: [PATCH 01/20] feat(browser): OPFS-backed BrowserDb mode with close() (#441) BrowserDb.openOpfs(name) keeps the database as one .graph file in the Origin Private File System, read and written in place through a FileSystemSyncAccessHandle in a dedicated worker: a 256-page cache, no in-memory copy of the pages, no WAL. An OPFS database with exactly one valid meta slot opens at it (native opens keep STG-033), since there is no WAL to tell an interrupted commit from rot and no repair tool in the browser. Imports stage the new image in .graph-import and publish it by its meta page, so an interruption leaves the old or the new database whole. close() exists in every mode; IndexedDB mode opens in workers. New codes API-026, API-027, STG-048. Refs #441 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01H2a7QjCHpikfsg5qwAV1p9 --- Cargo.toml | 8 + src/browser/backend.rs | 116 ++++++ src/browser/indexeddb.rs | 11 +- src/browser/mod.rs | 656 ++++++++++++++++++++++++++++---- src/browser/opfs.rs | 564 +++++++++++++++++++++++++++ src/browser/opfs_tests.rs | 1 + src/browser/udfs.rs | 2 + src/error.rs | 27 ++ src/storage/mod.rs | 6 +- src/storage/persistent_facts.rs | 117 +++++- 10 files changed, 1415 insertions(+), 93 deletions(-) create mode 100644 src/browser/backend.rs create mode 100644 src/browser/opfs.rs create mode 100644 src/browser/opfs_tests.rs diff --git a/Cargo.toml b/Cargo.toml index fa64d37ab1..70edcc0c3d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -44,8 +44,15 @@ wasm-bindgen = { version = "0.2", optional = true } wasm-bindgen-futures = { version = "0.4", optional = true } js-sys = { version = "0.3", optional = true } web-sys = { version = "0.3", optional = true, features = [ + "DedicatedWorkerGlobalScope", "DomStringList", "Event", + "FileSystemDirectoryHandle", + "FileSystemFileHandle", + "FileSystemGetFileOptions", + "FileSystemHandle", + "FileSystemReadWriteOptions", + "FileSystemSyncAccessHandle", "IdbDatabase", "IdbFactory", "IdbIndex", @@ -54,6 +61,7 @@ web-sys = { version = "0.3", optional = true, features = [ "IdbRequest", "IdbTransaction", "IdbTransactionMode", + "StorageManager", "Window", ] } diff --git a/src/browser/backend.rs b/src/browser/backend.rs new file mode 100644 index 0000000000..f959360291 --- /dev/null +++ b/src/browser/backend.rs @@ -0,0 +1,116 @@ +//! The storage under a `BrowserDb`: the in-memory page buffer (in-memory and +//! IndexedDB mode) or an OPFS file (OPFS mode, #441). + +use crate::browser::buffer::BrowserBufferBackend; +use crate::browser::opfs::OpfsBackend; +use crate::storage::StorageBackend; +use anyhow::Result; +use std::collections::HashSet; + +/// One `BrowserDb`'s backend. The buffer-only methods give an OPFS file no +/// dirty pages and nothing to release: it is written in place, and its free +/// pages are left on the file for the allocator to reuse. +pub(crate) enum BrowserBackend { + Buffer(BrowserBufferBackend), + Opfs(OpfsBackend), +} + +impl BrowserBackend { + /// See [`BrowserBufferBackend::take_dirty`]. + pub(crate) fn take_dirty(&mut self) -> HashSet { + match self { + BrowserBackend::Buffer(b) => b.take_dirty(), + BrowserBackend::Opfs(_) => HashSet::new(), + } + } + + /// See [`BrowserBufferBackend::is_dirty`]. + pub(crate) fn is_dirty(&self, page_id: u64) -> bool { + match self { + BrowserBackend::Buffer(b) => b.is_dirty(page_id), + BrowserBackend::Opfs(_) => false, + } + } + + /// See [`BrowserBufferBackend::release`]. + pub(crate) fn release(&mut self, ids: impl IntoIterator) -> Vec { + match self { + BrowserBackend::Buffer(b) => b.release(ids), + BrowserBackend::Opfs(_) => Vec::new(), + } + } + + /// The buffer, unless this is an OPFS file. + pub(crate) fn buffer(&self) -> Option<&BrowserBufferBackend> { + match self { + BrowserBackend::Buffer(b) => Some(b), + BrowserBackend::Opfs(_) => None, + } + } + + /// See [`BrowserBufferBackend::mark_dirty_for_test`]. + #[cfg(test)] + pub(crate) fn mark_dirty_for_test(&mut self, page_id: u64) { + if let BrowserBackend::Buffer(b) = self { + b.mark_dirty_for_test(page_id); + } + } +} + +impl StorageBackend for BrowserBackend { + fn write_page(&mut self, page_id: u64, data: &[u8]) -> Result<()> { + match self { + BrowserBackend::Buffer(b) => b.write_page(page_id, data), + BrowserBackend::Opfs(b) => b.write_page(page_id, data), + } + } + + fn read_page(&self, page_id: u64) -> Result> { + match self { + BrowserBackend::Buffer(b) => b.read_page(page_id), + BrowserBackend::Opfs(b) => b.read_page(page_id), + } + } + + fn sync(&mut self) -> Result<()> { + match self { + BrowserBackend::Buffer(b) => b.sync(), + BrowserBackend::Opfs(b) => b.sync(), + } + } + + fn page_count(&self) -> Result { + match self { + BrowserBackend::Buffer(b) => b.page_count(), + BrowserBackend::Opfs(b) => b.page_count(), + } + } + + fn holds_every_page(&self) -> bool { + match self { + BrowserBackend::Buffer(b) => b.holds_every_page(), + BrowserBackend::Opfs(b) => b.holds_every_page(), + } + } + + fn partial_page(&self) -> Result>> { + match self { + BrowserBackend::Buffer(b) => b.partial_page(), + BrowserBackend::Opfs(b) => b.partial_page(), + } + } + + fn close(&mut self) -> Result<()> { + match self { + BrowserBackend::Buffer(b) => b.close(), + BrowserBackend::Opfs(b) => b.close(), + } + } + + fn backend_name(&self) -> &'static str { + match self { + BrowserBackend::Buffer(b) => b.backend_name(), + BrowserBackend::Opfs(b) => b.backend_name(), + } + } +} diff --git a/src/browser/indexeddb.rs b/src/browser/indexeddb.rs index e134462a1f..1f1ccbaf75 100644 --- a/src/browser/indexeddb.rs +++ b/src/browser/indexeddb.rs @@ -72,10 +72,13 @@ impl IndexedDbBackend { /// If the object store does not exist, it is created in `onupgradeneeded`. /// `db_name` is used as both the database name and the object store name. pub async fn open(db_name: &str) -> Result { - let window = web_sys::window().ok_or_else(|| JsValue::from_str("no window object"))?; - let idb_factory = window - .indexed_db()? - .ok_or_else(|| JsValue::from_str("IndexedDB not available"))?; + // `indexedDB` of the global scope: the window, or a worker's (#441). + let idb_factory: web_sys::IdbFactory = + js_sys::Reflect::get(&js_sys::global(), &"indexedDB".into()) + .ok() + .filter(|f| !f.is_undefined() && !f.is_null()) + .ok_or_else(|| JsValue::from_str("IndexedDB not available"))? + .unchecked_into(); let store_name = db_name.to_string(); let store_name_upgrade = store_name.clone(); diff --git a/src/browser/mod.rs b/src/browser/mod.rs index 07645af604..fe6cdd8209 100644 --- a/src/browser/mod.rs +++ b/src/browser/mod.rs @@ -1,23 +1,34 @@ -//! Browser WASM support: `BrowserDb` async façade backed by IndexedDB. +//! Browser WASM support: the `BrowserDb` façade, in memory, backed by +//! IndexedDB, or backed by an OPFS file in a dedicated worker (#441). //! //! This module is only compiled for `wasm32-unknown-unknown` with the `browser` //! feature enabled. It is **not** compatible with Node.js, Deno, Bun, or any //! server-side runtime. For server-side Node.js, use `@minigraf/node` (Phase 8.3). +/// The storage under a `BrowserDb`: page buffer or OPFS file. +mod backend; /// Bind values read from JavaScript for prepared queries. mod binds; /// Synchronous in-memory page buffer with dirty-page tracking. pub mod buffer; /// Async IndexedDB backend for browser WASM persistence. pub mod indexeddb; +/// OPFS file backend for `BrowserDb.openOpfs` (#441). +mod opfs; +/// Tests of OPFS mode, run in a dedicated worker (#441). +#[cfg(all(target_arch = "wasm32", feature = "browser", test))] +mod opfs_tests; /// Tests of `BrowserWriteTransaction` (#824). #[cfg(all(target_arch = "wasm32", feature = "browser", test))] mod tx_tests; /// User-defined functions from JavaScript (#180). mod udfs; +use crate::browser::backend::BrowserBackend; use crate::browser::buffer::BrowserBufferBackend; use crate::browser::indexeddb::IndexedDbBackend; +use crate::browser::opfs::{OpfsBackend, OpfsFile}; +use crate::error::{ErrorCode, err_coded}; use crate::graph::FactStorage; use crate::query::datalog::executor::{DatalogExecutor, QueryResult}; use crate::query::datalog::functions::Functions; @@ -25,18 +36,167 @@ use crate::query::datalog::parser::parse_datalog_command; use crate::query::datalog::prepared::{BindValue, PreparedQuery}; use crate::query::datalog::rules::RuleRegistry; use crate::query::datalog::types::DatalogCommand; +use crate::storage::StorageBackend; use crate::storage::persistent_facts::PersistentFactStorage; -use std::cell::RefCell; +use std::cell::{Cell, RefCell}; use std::rc::Rc; use std::sync::{Arc, RwLock}; use wasm_bindgen::prelude::*; +/// OPFS mode's page cache: the storage default, 256 pages (1 MiB). +const OPFS_PAGE_CACHE: usize = 256; + +/// One open database, shared by its `BrowserDb` and every dependent +/// (cursor, prepared query, transaction). Its lifecycle flags live outside +/// the `RefCell`, so a call made while a query holds `inner.borrow()` (from a +/// user function) can still read them. It derefs to that `RefCell`. +struct Db { + inner: RefCell, + /// After `close()`: every call but disposal is `API-018` (#441). + closed: Cell, + /// An OPFS import's body is running: calls are "busy" (`API-027`). + importing: Cell, + /// Queries of this database running now (a user function's `close()` + /// is then refused). + running: Cell, + /// Bumped when an OPFS import publishes a new database: cursors and + /// prepared queries from before are `API-018`. + epoch: Cell, + /// OPFS mode's file and where it lives; `None` in the other modes. + opfs: Option, +} + +struct OpfsMode { + dir: web_sys::FileSystemDirectoryHandle, + /// `.graph`. + file_name: String, + file: Arc, +} + +impl std::ops::Deref for Db { + type Target = RefCell; + fn deref(&self) -> &RefCell { + &self.inner + } +} + +impl Db { + fn new(inner: BrowserDbInner, opfs: Option) -> Rc { + Rc::new(Db { + inner: RefCell::new(inner), + closed: Cell::new(false), + importing: Cell::new(false), + running: Cell::new(0), + epoch: Cell::new(0), + opfs, + }) + } + + /// Whether a call on the database may start work: `API-018` once + /// closed, then "busy" while importing. Callers check `INT-001` after. + fn admit(&self) -> Result<(), JsValue> { + if self.closed.get() { + return Err(closed_error("database")); + } + self.refuse_importing() + } + + /// [`admit`](Self::admit) for a dependent from `epoch`: `API-018` too + /// when an import has replaced the database it was made on. + fn admit_from(&self, epoch: u64, what: &str) -> Result<(), JsValue> { + if self.closed.get() { + return Err(closed_error("database")); + } + if self.epoch.get() != epoch { + return Err(closed_error(format!( + "{what} (an import replaced its database)" + ))); + } + self.refuse_importing() + } + + fn refuse_importing(&self) -> Result<(), JsValue> { + if self.importing.get() { + return Err(busy("an import is in progress")); + } + Ok(()) + } + + /// Count a running query until the guard drops. + fn run(&self) -> Running<'_> { + self.running.set(self.running.get() + 1); + Running(self) + } + + /// Close: release the OPFS handle or the IndexedDB connection, keep + /// nothing to save. Already closed: nothing to do. + fn close(&self) -> Result<(), JsValue> { + if self.closed.get() { + return Ok(()); + } + self.refuse_importing()?; + if self.running.get() > 0 { + return Err(busy( + "close() was called while a query of this database runs", + )); + } + self.closed.set(true); + if let Some(opfs) = &self.opfs { + opfs.file.close(); + } + if let Ok(mut inner) = self.inner.try_borrow_mut() { + inner.pfs.discard(); + inner.tx_open = false; + inner.withheld.clear(); + if let Some(idb) = inner.idb.take() { + idb.db.close(); + } + } + Ok(()) + } +} + +impl Drop for Db { + /// Freeing the database and every dependent saves nothing: each write + /// was stored when it was acknowledged, or failed. + fn drop(&mut self) { + self.inner.get_mut().pfs.discard(); + if let Some(opfs) = &self.opfs { + opfs.file.close(); + } + } +} + +/// A query of the database is running while this lives. +struct Running<'a>(&'a Db); + +impl Drop for Running<'_> { + fn drop(&mut self) { + self.0.running.set(self.0.running.get() - 1); + } +} + +/// `[API-018]`: `what` is closed. +fn closed_error(what: impl std::fmt::Display) -> JsValue { + JsValue::from_str(&crate::MinigrafError::closed(what).to_string()) +} + +/// `[API-027]`: the database is busy, because of `why`. +fn busy(why: &str) -> JsValue { + to_js_error(err_coded!(ErrorCode::Api027, why)) +} + +/// A thrown JavaScript value as text, for messages. +fn js_text(e: &JsValue) -> String { + e.as_string().unwrap_or_else(|| format!("{e:?}")) +} + /// Internal state shared by all `BrowserDb` clones. struct BrowserDbInner { fact_storage: FactStorage, rules: Arc>, functions: Arc, - pfs: PersistentFactStorage, + pfs: PersistentFactStorage, /// `None` for in-memory databases (no IDB backing). idb: Option, /// Free pages whose bytes are kept because an open cursor's generation @@ -58,18 +218,37 @@ impl BrowserDbInner { } /// `INT-001` while a transaction of `db` is open. -fn refuse_in_tx(db: &RefCell) -> Result<(), JsValue> { +fn refuse_in_tx(db: &Db) -> Result<(), JsValue> { db.borrow().refuse_in_tx().map_err(to_js_error) } -/// Browser-only Minigraf database handle backed by IndexedDB. +/// Browser-only Minigraf database handle: in memory (`openInMemory`), +/// backed by IndexedDB (`open`), or backed by an OPFS file in a dedicated +/// worker (`openOpfs`). /// -/// All public methods return `Promise`s. Use `await` in JavaScript. +/// `execute`, `checkpoint`, `importGraph` and a transaction's `commit` +/// return `Promise`s; the other methods are synchronous. /// /// **Not compatible with Node.js.** Use `@minigraf/node` for server-side use. #[wasm_bindgen] pub struct BrowserDb { - inner: Rc>, + inner: Rc, +} + +/// A new database's state over `pfs`. +fn inner_over( + pfs: PersistentFactStorage, + idb: Option, +) -> BrowserDbInner { + BrowserDbInner { + fact_storage: pfs.storage().clone(), + rules: Arc::new(RwLock::new(RuleRegistry::new())), + functions: Arc::new(Functions::with_builtins()), + pfs, + idb, + withheld: Vec::new(), + tx_open: false, + } } #[wasm_bindgen] @@ -79,23 +258,13 @@ impl BrowserDb { /// Data is lost when the page is closed. Use `BrowserDb.open()` for persistence. #[wasm_bindgen(js_name = openInMemory)] pub fn open_in_memory() -> Result { - let buffer = BrowserBufferBackend::new(); + let buffer = BrowserBackend::Buffer(BrowserBufferBackend::new()); // Page cache capacity 0: `BrowserBufferBackend` is already an in-memory // page store, so the LRU cache would only duplicate pages and add // HashMap overhead for zero benefit. See #275. let pfs = PersistentFactStorage::new(buffer, 0).map_err(to_js_error)?; - let fact_storage = pfs.storage().clone(); - Ok(BrowserDb { - inner: Rc::new(RefCell::new(BrowserDbInner { - fact_storage, - rules: Arc::new(RwLock::new(RuleRegistry::new())), - functions: Arc::new(Functions::with_builtins()), - pfs, - idb: None, - withheld: Vec::new(), - tx_open: false, - })), + inner: Db::new(inner_over(pfs, None), None), }) } @@ -108,10 +277,9 @@ impl BrowserDb { let idb = IndexedDbBackend::open(db_name).await?; let existing = idb.load_all_pages().await?; - let buffer = BrowserBufferBackend::load_pages(existing); + let buffer = BrowserBackend::Buffer(BrowserBufferBackend::load_pages(existing)); // Page cache capacity 0 — see comment in `open_in_memory` above. let mut pfs = PersistentFactStorage::new(buffer, 0).map_err(to_js_error)?; - let fact_storage = pfs.storage().clone(); // Opening may have written pages: a v7 store is migrated to v8 (#411). // Store them, and drop the free pages (after a migration, the whole v7 @@ -122,18 +290,82 @@ impl BrowserDb { idb.write_pages(flush.puts, flush.deletes).await?; Ok(BrowserDb { - inner: Rc::new(RefCell::new(BrowserDbInner { - fact_storage, - rules: Arc::new(RwLock::new(RuleRegistry::new())), - functions: Arc::new(Functions::with_builtins()), - pfs, - idb: Some(idb), - withheld: Vec::new(), - tx_open: false, - })), + inner: Db::new(inner_over(pfs, Some(idb)), None), + }) + } + + /// Open or create the database `name` as one file in the Origin Private + /// File System (#441), read and written in place through a sync access + /// handle: no page is held in memory beyond a 1 MiB page cache, and each + /// commit is stored before its promise resolves. Call it from a + /// dedicated worker: + /// + /// ```js + /// const db = await BrowserDb.openOpfs('mydb'); + /// await db.execute('(transact [[:alice :name "Alice"]])'); + /// db.close(); + /// ``` + /// + /// The file is `.graph` in the origin's OPFS root; OPFS-mode and + /// IndexedDB-mode databases of one name are unrelated. Elsewhere than a + /// dedicated worker with OPFS this rejects with `[API-026]`, creating + /// nothing; an empty name, one with `/` or `\`, or `.`/`..` is + /// `[API-017]`. While the database is open, a second `openOpfs` of it + /// from any worker or tab rejects with `[STG-026]`; `close()`, or the + /// worker ending, releases it. + #[wasm_bindgen(js_name = openOpfs)] + pub async fn open_opfs(name: String) -> Result { + opfs::check_name(&name)?; + let dir = opfs::root().await?; + let file_name = format!("{name}{}", opfs::FILE_SUFFIX); + let staging = format!("{file_name}{}", opfs::STAGING_SUFFIX); + let file = opfs::open_file(&dir, &file_name, true) + .await + .map_err(to_js_error)? + .ok_or_else(|| { + to_js_error(opfs::storage_error( + format!("open {file_name}"), + "NotFoundError", + )) + })?; + // The handle is the lock. Every failure from here drops `file`, + // which releases it; a refused open writes nothing. + opfs::recover_import(&dir, &staging, &file) + .await + .map_err(to_js_error)?; + let pfs = PersistentFactStorage::open_wal_less( + BrowserBackend::Opfs(OpfsBackend::new(file.clone())), + OPFS_PAGE_CACHE, + ) + .map_err(to_js_error)?; + Ok(BrowserDb { + inner: Db::new( + inner_over(pfs, None), + Some(OpfsMode { + dir, + file_name, + file, + }), + ), }) } + /// Close the database (#441). Synchronous and idempotent; it stores + /// nothing, since every acknowledged write is already stored: an open + /// write transaction is discarded, and an in-memory database is gone. + /// In OPFS mode it releases the file's lock; in IndexedDB mode it closes + /// the connection, so a commit whose flush had not started rejects with + /// `[API-018]`. + /// + /// Afterwards every method of the database and of its cursors, prepared + /// queries and transactions fails with `[API-018]`, except a cursor's + /// `close()`, a transaction's `rollback()`, `free()` and `close()` + /// itself. Refused with `[API-027]` while an import runs, and when called + /// from a user function while a query of this database runs. + pub fn close(&self) -> Result<(), JsValue> { + self.inner.close() + } + /// Execute a Datalog command string and return a JSON-encoded result. /// /// Returns a `Promise` in JavaScript. The JSON shape is: @@ -143,6 +375,7 @@ impl BrowserDb { /// - Rule: `{"ok": true}` #[wasm_bindgen(js_name = execute)] pub async fn execute(&self, datalog: String) -> Result { + self.inner.admit()?; refuse_in_tx(&self.inner)?; let cmd = parse_datalog_command(&datalog).map_err(to_js_error)?; // BrowserDb takes no options: the default override list (#869). @@ -156,6 +389,7 @@ impl BrowserDb { if is_read { let result = { + let _running = self.inner.run(); let inner = self.inner.borrow(); DatalogExecutor::new_with_rules_and_functions( inner.fact_storage.clone(), @@ -193,12 +427,13 @@ impl BrowserDb { /// `[API-010]` for a query with `$slot` bind slots. #[wasm_bindgen(js_name = query)] pub fn query(&self, datalog: &str) -> Result { + self.inner.admit()?; refuse_in_tx(&self.inner)?; - let cursor = self.query_inner(datalog).map_err(to_js_error)?; - Ok(BrowserCursor { - vars: cursor.vars().to_vec(), - cursor: Some(cursor), - }) + let cursor = { + let _running = self.inner.run(); + self.query_inner(datalog).map_err(to_js_error)? + }; + Ok(BrowserCursor::new(&self.inner, cursor)) } /// Parse and validate a `(query ...)` once, for running many times with @@ -215,10 +450,12 @@ impl BrowserDb { /// ``` #[wasm_bindgen(js_name = prepare)] pub fn prepare(&self, datalog: &str) -> Result { + self.inner.admit()?; refuse_in_tx(&self.inner)?; let query = self.prepare_inner(datalog).map_err(to_js_error)?; Ok(BrowserPreparedQuery { - _db: self.inner.clone(), + epoch: self.inner.epoch.get(), + db: self.inner.clone(), query, }) } @@ -227,8 +464,10 @@ impl BrowserDb { /// /// Write-through means individual `execute()` calls already flush dirty pages, /// so `checkpoint()` is only needed after `import_graph()` or explicit bulk ops. - /// No-op for in-memory databases. + /// No-op for in-memory databases, and in OPFS mode, where every commit is + /// already stored in the file. pub async fn checkpoint(&self) -> Result<(), JsValue> { + self.inner.admit()?; refuse_in_tx(&self.inner)?; let (flush, idb) = { let mut inner = self.inner.borrow_mut(); @@ -255,12 +494,16 @@ impl BrowserDb { /// no WAL entries are missing from the main file. #[wasm_bindgen(js_name = exportGraph)] pub fn export_graph(&self) -> Result { + self.inner.admit()?; let inner = self.inner.borrow(); + if self.inner.opfs.is_some() { + return export_image(&inner.pfs); + } // Free pages hold no bytes (#440), so the trailing ones may be absent: // the file is as long as the meta says. let page_count = inner .pfs - .with_backend(|b| b.page_count_raw()) + .with_backend(|b| b.page_count()) .map_err(to_js_error)? .max(inner.pfs.committed_page_count()); let capacity = usize::try_from(page_count) @@ -272,7 +515,10 @@ impl BrowserDb { for id in 0..page_count { let page = inner .pfs - .with_backend(|b| b.export_page(id)) + .with_backend(|b| match b.buffer() { + Some(buffer) => buffer.export_page(id), + None => b.read_page(id), + }) .map_err(to_js_error)?; blob.extend_from_slice(&page); } @@ -286,7 +532,11 @@ impl BrowserDb { /// queryable and all dirty pages are flushed to IndexedDB. #[wasm_bindgen(js_name = importGraph)] pub async fn import_graph(&self, data: js_sys::Uint8Array) -> Result<(), JsValue> { + self.inner.admit()?; refuse_in_tx(&self.inner)?; + if self.inner.opfs.is_some() { + return self.import_opfs(data).await; + } let bytes = data.to_vec(); if !bytes.len().is_multiple_of(crate::storage::PAGE_SIZE) { return Err(JsValue::from_str( @@ -302,7 +552,7 @@ impl BrowserDb { // ── Sync section ────────────────────────────────────────────────────────── let (flush, idb) = { let mut inner = self.inner.borrow_mut(); - let buffer = BrowserBufferBackend::load_pages_all_dirty(pages); + let buffer = BrowserBackend::Buffer(BrowserBufferBackend::load_pages_all_dirty(pages)); // Page cache capacity 0 — see comment in `open_in_memory` above. let mut new_pfs = PersistentFactStorage::new(buffer, 0).map_err(to_js_error)?; let new_fact_storage = new_pfs.storage().clone(); @@ -314,10 +564,12 @@ impl BrowserDb { // Every page of the old database the new one does not hold must go // too, or a reopen would load it next to the imported pages. let new_ids: std::collections::HashSet = - new_pfs.with_backend(|b| b.page_ids().collect()); - let stale: Vec = inner - .pfs - .with_backend(|b| b.page_ids().filter(|id| !new_ids.contains(id)).collect()); + new_pfs.with_backend(|b| buffer_page_ids(b).collect()); + let stale: Vec = inner.pfs.with_backend(|b| { + buffer_page_ids(b) + .filter(|id| !new_ids.contains(id)) + .collect() + }); flush.deletes.extend(stale); flush.deletes.sort_unstable(); flush.deletes.dedup(); @@ -364,6 +616,7 @@ impl BrowserDb { /// ``` #[wasm_bindgen(js_name = beginWrite)] pub fn begin_write(&self) -> Result { + self.inner.admit()?; // Before the mutable borrow: a user function of a transaction's query // calling this gets INT-001, not a borrow panic. refuse_in_tx(&self.inner)?; @@ -378,6 +631,106 @@ impl BrowserDb { } impl BrowserDb { + /// OPFS mode's `importGraph` (#441): old or new database, never a mix, + /// whenever it is interrupted. + /// + /// The image is checked whole in memory first (meta, length, `verify()`; + /// a v7 file is migrated). It is then written to the staging file + /// `.graph-import`, with zeros in its meta slots, and flushed. The + /// publication point is the staging file's meta page: written and + /// flushed, it makes the new database recoverable, since every open + /// copies a staged image newer than the database file into it. The image + /// is stamped with a generation above every meta of the old file, so a + /// copy that finished is never redone over later commits. Then the image + /// is copied into the database file (meta last), the storage reopened on + /// it, and the staging file removed; a failed removal is left to the + /// next open. + async fn import_opfs(&self, data: js_sys::Uint8Array) -> Result<(), JsValue> { + let db = &self.inner; + let Some(opfs) = db.opfs.as_ref() else { + return Err(JsValue::from_str("not an OPFS database")); + }; + let len = data.length(); + if !(len as usize).is_multiple_of(crate::storage::PAGE_SIZE) { + return Err(JsValue::from_str( + &crate::MinigrafError::invalid_argument(format!( + "importGraph: {len} bytes is not a whole number of {}-byte pages", + crate::storage::PAGE_SIZE + )) + .to_string(), + )); + } + db.importing.set(true); + let _importing = Importing(db); + + let image = image_of(data.to_vec()).map_err(to_js_error)?; + let staging = format!("{}{}", opfs.file_name, opfs::STAGING_SUFFIX); + let newest = opfs + .file + .newest_generation() + .map_err(to_js_error)? + .unwrap_or(0) + .max(db.borrow().pfs.generation()); + let generation = newest + .checked_add(1) + .ok_or_else(|| to_js_error(err_coded!(ErrorCode::Int048, "generation overflow")))? + .max(image.generation()); + let stage = opfs::open_file(&opfs.dir, &staging, true) + .await + .map_err(to_js_error)? + .ok_or_else(|| { + to_js_error(opfs::storage_error( + format!("open {staging}"), + "NotFoundError", + )) + })?; + + // Before the publication point: on failure the old database stays. + let meta_page = match stage_image(&image, &stage, generation) { + Ok(m) => m, + Err(e) => { + stage.close(); + drop(stage); + opfs::remove_quietly(&opfs.dir, &staging).await; + return Err(to_js_error(e)); + } + }; + drop(image); + + // The publication step, with no await in it: nothing else runs + // until the database is the new one or closed. + let published = publish_image(&stage, &opfs.file, &meta_page).and_then(|()| { + PersistentFactStorage::open_wal_less( + BrowserBackend::Opfs(OpfsBackend::new(opfs.file.clone())), + OPFS_PAGE_CACHE, + ) + }); + let swapped = published.map_err(to_js_error).and_then(|pfs| { + let mut inner = db + .try_borrow_mut() + .map_err(|_| busy("the database is in use"))?; + inner.fact_storage = pfs.storage().clone(); + let mut old = std::mem::replace(&mut inner.pfs, pfs); + old.discard(); + inner.withheld.clear(); + db.epoch.set(db.epoch.get() + 1); + Ok(()) + }); + stage.close(); + drop(stage); + if let Err(e) = swapped { + // Old or new, the file holds one of them whole; this handle no + // longer knows which. Reopening tells. + db.importing.set(false); + let _ = db.close(); + return Err(e); + } + // Cleanup: a failure here leaves a consumed staging file, which the + // next open removes. + opfs::remove_quietly(&opfs.dir, &staging).await; + Ok(()) + } + fn prepare_inner(&self, datalog: &str) -> anyhow::Result { use crate::error::{ErrorCode, bail_coded}; @@ -527,7 +880,7 @@ fn commit_stamped( /// `execute` and `commit` fail with `[API-018]`, and `rollback` does nothing. #[wasm_bindgen] pub struct BrowserWriteTransaction { - db: Rc>, + db: Rc, /// `None` once finished. staged: Option, } @@ -536,9 +889,11 @@ pub struct BrowserWriteTransaction { impl BrowserWriteTransaction { /// Run one Datalog command inside the transaction; see the type's docs. pub fn execute(&mut self, datalog: &str) -> Result { + self.db.admit()?; let Some(staged) = self.staged.as_mut() else { return Err(finished()); }; + let _running = self.db.run(); tx_execute(&self.db, staged, datalog) .map(query_result_to_json) .map_err(to_js_error) @@ -546,10 +901,17 @@ impl BrowserWriteTransaction { /// Apply the staged facts as one transaction and store it. The promise /// resolves after the IndexedDB flush, and rejects as a failed flush of - /// `BrowserDb.execute` does. The transaction is finished either way: a + /// `BrowserDb.execute` does; in OPFS mode the commit is stored before + /// this returns. The transaction is finished either way: a /// commit that is refused (for example `[API-019]`, a valid time that /// ended before the commit time) discards the staged facts. + /// + /// If `close()` runs before an IndexedDB flush starts, the flush is not + /// started and the promise rejects with `[API-018]`. pub fn commit(&mut self) -> js_sys::Promise { + if let Err(e) = self.db.admit() { + return js_sys::Promise::reject(&e); + } let Some(staged) = self.staged.take() else { return js_sys::Promise::reject(&finished()); }; @@ -563,10 +925,15 @@ impl BrowserWriteTransaction { ) }) }; + let db = self.db.clone(); match committed { Err(e) => js_sys::Promise::reject(&e), Ok((flush, idb)) => wasm_bindgen_futures::future_to_promise(async move { if let (Some(flush), Some(idb)) = (flush, idb) { + // Never into a database closed (and maybe reopened) since. + if db.closed.get() { + return Err(closed_error("database")); + } idb.write_pages(flush.puts, flush.deletes).await?; } Ok(JsValue::UNDEFINED) @@ -601,7 +968,7 @@ fn finished() -> JsValue { /// A transaction's `execute`, as the native transaction's on an in-memory /// database: stage a write, read the staged state, or register a rule. fn tx_execute( - db: &RefCell, + db: &Db, staged: &mut crate::db::Staged, datalog: &str, ) -> anyhow::Result { @@ -671,24 +1038,43 @@ pub struct BrowserCursor { vars: Vec, /// `None` after `close()`. cursor: Option, + db: Rc, + /// The database's epoch when it opened (#441). + epoch: u64, +} + +impl BrowserCursor { + fn new(db: &Rc, cursor: crate::Cursor) -> BrowserCursor { + BrowserCursor { + vars: cursor.vars().to_vec(), + cursor: Some(cursor), + db: db.clone(), + epoch: db.epoch.get(), + } + } } #[wasm_bindgen] impl BrowserCursor { - /// The query's `:find` variables, in column order. - pub fn vars(&self) -> Vec { - self.vars.clone() + /// The query's `:find` variables, in column order. `[API-018]` once its + /// database is closed. + pub fn vars(&self) -> Result, JsValue> { + self.db.admit_from(self.epoch, "cursor")?; + Ok(self.vars.clone()) } /// The next batch of at most `maxRows` rows (`0` counts as 1), as a JSON /// array of rows encoded like `execute()`'s `results`, or `undefined` at /// the end. A batch is never empty. After `close()` it returns - /// `undefined`. + /// `undefined`. `[API-018]` once its database is closed, or in OPFS + /// mode replaced by an import. #[wasm_bindgen(js_name = nextBatch)] pub fn next_batch(&mut self, max_rows: u32) -> Result, JsValue> { + self.db.admit_from(self.epoch, "cursor")?; let Some(cursor) = self.cursor.as_mut() else { return Ok(None); }; + let _running = self.db.run(); let batch = cursor .next_batch(max_rows as usize) .map_err(|e| JsValue::from_str(&e.to_string()))?; @@ -712,10 +1098,14 @@ impl BrowserCursor { /// `timestamp` or `anyValidTime`. Names the query does not use are ignored. /// /// It keeps its database open, and each run reads the database as it is then. +/// Once the database is closed, or in OPFS mode replaced by an import, every +/// method is `[API-018]`. #[wasm_bindgen] pub struct BrowserPreparedQuery { /// Keeps the database alive while the prepared query is. - _db: Rc>, + db: Rc, + /// The database's epoch when it was prepared (#441). + epoch: u64, query: PreparedQuery, } @@ -727,8 +1117,10 @@ impl BrowserPreparedQuery { /// type, `[API-022]` for a missing slot and `[API-024]` for a bind value /// of the wrong kind. pub fn execute(&self, bindings: JsValue) -> Result { - refuse_in_tx(&self._db)?; + self.db.admit_from(self.epoch, "prepared query")?; + refuse_in_tx(&self.db)?; let binds = binds::read_bindings(&bindings)?; + let _running = self.db.run(); self.query .execute(&as_slice(&binds)) .map(query_result_to_json) @@ -738,16 +1130,16 @@ impl BrowserPreparedQuery { /// Run the query with `bindings` and return a cursor over its rows. The /// answer is fixed when the cursor opens. Throws as `execute` does. pub fn query(&self, bindings: JsValue) -> Result { - refuse_in_tx(&self._db)?; + self.db.admit_from(self.epoch, "prepared query")?; + refuse_in_tx(&self.db)?; let binds = binds::read_bindings(&bindings)?; - let cursor = self - .query - .query(&as_slice(&binds)) - .map_err(|e| JsValue::from_str(&e.to_string()))?; - Ok(BrowserCursor { - vars: cursor.vars().to_vec(), - cursor: Some(cursor), - }) + let cursor = { + let _running = self.db.run(); + self.query + .query(&as_slice(&binds)) + .map_err(|e| JsValue::from_str(&e.to_string()))? + }; + Ok(BrowserCursor::new(&self.db, cursor)) } /// A new prepared query with `n` as its derived-facts limit; this one is @@ -757,6 +1149,7 @@ impl BrowserPreparedQuery { /// Anything else throws `[API-017]`. #[wasm_bindgen(js_name = withMaxDerivedFacts)] pub fn with_max_derived_facts(&self, n: f64) -> Result { + self.db.admit_from(self.epoch, "prepared query")?; let n = limit_arg("withMaxDerivedFacts", n)?; Ok(self.derive(|q| q.with_max_derived_facts(n))) } @@ -766,6 +1159,7 @@ impl BrowserPreparedQuery { /// Takes its argument as `withMaxDerivedFacts` does. #[wasm_bindgen(js_name = withQueryMemoryLimit)] pub fn with_query_memory_limit(&self, bytes: f64) -> Result { + self.db.admit_from(self.epoch, "prepared query")?; let bytes = limit_arg("withQueryMemoryLimit", bytes)?; Ok(self.derive(|q| q.with_query_memory_limit(bytes))) } @@ -775,13 +1169,15 @@ impl BrowserPreparedQuery { /// still overrides it. Takes its argument as `withMaxDerivedFacts` does. #[wasm_bindgen(js_name = withMaxIterations)] pub fn with_max_iterations(&self, n: f64) -> Result { + self.db.admit_from(self.epoch, "prepared query")?; let n = limit_arg("withMaxIterations", n)?; Ok(self.derive(|q| q.with_max_iterations(n))) } fn derive(&self, set: impl FnOnce(PreparedQuery) -> PreparedQuery) -> BrowserPreparedQuery { BrowserPreparedQuery { - _db: self._db.clone(), + db: self.db.clone(), + epoch: self.epoch, query: set(self.query.clone()), } } @@ -815,12 +1211,12 @@ fn as_slice(binds: &[(String, BindValue)]) -> Vec<(&str, BindValue)> { /// A page that cannot be read is an error, never a page silently left out of /// the flush to IndexedDB. fn take_dirty_pages( - pfs: &mut PersistentFactStorage, + pfs: &mut PersistentFactStorage, ) -> Result)>, JsValue> { - let ids = pfs.with_backend_mut(|b| b.take_dirty()); + let ids = pfs.with_backend_mut(BrowserBackend::take_dirty); pfs.with_backend(|b| { ids.into_iter() - .map(|id| b.read_page_raw(id).map(|d| (id, d))) + .map(|id| b.read_page(id).map(|d| (id, d))) .collect::>>() }) .map_err(to_js_error) @@ -838,6 +1234,13 @@ struct Flush { /// (an open cursor's) may read (#543); those stay withheld. fn take_commit_flush(inner: &mut BrowserDbInner) -> Result { let listed = inner.pfs.take_released(); + if inner.pfs.with_backend(|b| b.buffer().is_none()) { + // An OPFS file keeps its free pages, and is already written. + return Ok(Flush { + puts: Vec::new(), + deletes: Vec::new(), + }); + } let in_force = inner.pfs.storage().pages_in_force(); let withheld = std::mem::take(&mut inner.withheld); let (free, withheld) = inner.pfs.with_backend(|b| { @@ -851,7 +1254,7 @@ fn take_commit_flush(inner: &mut BrowserDbInner) -> Result { /// No reader needs them: the caller leaves out pages a pinned generation may /// read, and any other free page is written before it is read again. fn take_flush( - pfs: &mut PersistentFactStorage, + pfs: &mut PersistentFactStorage, free: Vec, ) -> Result { let deletes = pfs.with_backend_mut(|b| b.release(free)); @@ -859,6 +1262,102 @@ fn take_flush( Ok(Flush { puts, deletes }) } +/// The ids of the pages a buffer holds; none for an OPFS file. +fn buffer_page_ids(b: &BrowserBackend) -> impl Iterator + '_ { + b.buffer() + .into_iter() + .flat_map(BrowserBufferBackend::page_ids) +} + +/// OPFS mode's export: the committed state as a clean `.graph` file +/// (`write_image`), so a native open with no WAL accepts it even when the +/// file still holds pages of a commit interrupted before its meta (#441). +fn export_image( + pfs: &PersistentFactStorage, +) -> Result { + let pages = usize::try_from(pfs.committed_page_count()) + .ok() + .and_then(|n| n.checked_mul(crate::storage::PAGE_SIZE)) + .ok_or_else(|| JsValue::from_str("database is too large to export"))?; + let mut blob = Vec::with_capacity(pages); + pfs.write_image(pfs.generation(), |_, page| { + blob.extend_from_slice(page); + Ok(()) + }) + .map_err(to_js_error)?; + Ok(js_sys::Uint8Array::from(blob.as_slice())) +} + +/// The OPFS import flag of a database, cleared when the import's body ends. +struct Importing<'a>(&'a Db); + +impl Drop for Importing<'_> { + fn drop(&mut self) { + self.0.importing.set(false); + } +} + +/// An import's image, opened and checked in memory (#441): a v7 image is +/// migrated, a meta counting pages the image lacks is `STG-044` as for a +/// file, and any finding of the integrity walk (`verify()`) rejects it. +fn image_of(bytes: Vec) -> anyhow::Result> { + let pages: std::collections::HashMap> = bytes + .chunks(crate::storage::PAGE_SIZE) + .enumerate() + .map(|(i, chunk)| (i as u64, chunk.to_vec())) + .collect(); + let have = pages.len() as u64; + drop(bytes); + let pfs = PersistentFactStorage::new( + BrowserBackend::Buffer(BrowserBufferBackend::load_pages(pages)), + 0, + )?; + let need = pfs.committed_page_count(); + if need > have.max(2) { + crate::error::bail_coded!(ErrorCode::Stg044, need, have); + } + if let Some(problem) = pfs.verify()?.problems.into_iter().next() { + return Err(problem); + } + Ok(pfs) +} + +/// Write `image` stamped `generation` into the staging file, all but its +/// meta page (that slot and the other stay zero), and flush. Returns the +/// meta page, for [`publish_image`]. +fn stage_image( + image: &PersistentFactStorage, + stage: &OpfsFile, + generation: u64, +) -> anyhow::Result> { + let slot = crate::storage::meta::slot_page(generation); + let mut meta = None; + stage.truncate(0)?; + image.write_image(generation, |id, page| { + if id == slot { + meta = Some(page.to_vec()); + Ok(()) + } else { + stage.write_page(id, page) + } + })?; + stage.flush()?; + meta.ok_or_else(|| err_coded!(ErrorCode::Int048, "image has no meta page")) +} + +/// The publication step: the staging file's meta page (the publication +/// point), then the image copied into the database file. +fn publish_image(stage: &OpfsFile, main: &OpfsFile, meta_page: &[u8]) -> anyhow::Result<()> { + let crate::storage::meta::SlotState::Valid(meta) = + crate::storage::meta::MetaPage::decode(meta_page) + else { + crate::error::bail_coded!(ErrorCode::Int048, "staged meta page is not valid"); + }; + stage.write_page(crate::storage::meta::slot_page(meta.generation), meta_page)?; + stage.flush()?; + opfs::copy_staged(stage, main, &meta) +} + /// Every error a `BrowserDb` method rejects with goes through this, so the /// message carries its code (`[PRS-001] ...`), as on every other API (#673). fn to_js_error(e: anyhow::Error) -> JsValue { @@ -924,7 +1423,9 @@ mod tests { #[wasm_bindgen_test] fn take_dirty_pages_returns_written_pages() { use crate::storage::StorageBackend; - let mut pfs = PersistentFactStorage::new(BrowserBufferBackend::new(), 0).unwrap(); + let mut pfs = + PersistentFactStorage::new(BrowserBackend::Buffer(BrowserBufferBackend::new()), 0) + .unwrap(); let _ = pfs.with_backend_mut(|b| b.take_dirty()); pfs.with_backend_mut(|b| b.write_page(3, &[7u8; crate::storage::PAGE_SIZE])) .unwrap(); @@ -936,7 +1437,9 @@ mod tests { #[wasm_bindgen_test] fn unreadable_dirty_page_is_an_error_not_skipped() { - let mut pfs = PersistentFactStorage::new(BrowserBufferBackend::new(), 0).unwrap(); + let mut pfs = + PersistentFactStorage::new(BrowserBackend::Buffer(BrowserBufferBackend::new()), 0) + .unwrap(); // A dirty id with no page behind it: before the fix it was dropped // from the flush and the caller still saw success. pfs.with_backend_mut(|b| b.mark_dirty_for_test(99)); @@ -1175,7 +1678,7 @@ mod tests { expected.sort_unstable(); for size in [1u32, 7, 1000] { let mut cursor = db.query(q).expect("query"); - assert_eq!(cursor.vars(), vec!["?n".to_string()]); + assert_eq!(cursor.vars().expect("vars"), vec!["?n".to_string()]); let mut got = Vec::new(); while let Some(batch) = cursor.next_batch(size).expect("next_batch") { let rows: Vec> = serde_json::from_str(&batch).unwrap(); @@ -1395,10 +1898,10 @@ mod tests { } fn stored_pages(db: &BrowserDb) -> usize { - db.inner - .borrow() - .pfs - .with_backend(BrowserBufferBackend::stored_page_count) + db.inner.borrow().pfs.with_backend(|b| { + b.buffer() + .map_or(0, BrowserBufferBackend::stored_page_count) + }) } async fn idb_keys(name: &str) -> std::collections::HashMap> { @@ -1815,7 +2318,10 @@ mod prepared_tests { let binds = obj(&[("min", JsValue::from_f64(2.0))]); let all = rows(&pq.execute(binds.clone()).expect("execute")); let mut cursor = pq.query(binds).expect("query"); - assert_eq!(cursor.vars(), vec!["?e".to_string(), "?n".to_string()]); + assert_eq!( + cursor.vars().expect("vars"), + vec!["?e".to_string(), "?n".to_string()] + ); let mut seen = 0; while let Some(batch) = cursor.next_batch(1).expect("batch") { seen += serde_json::from_str::>(&batch) diff --git a/src/browser/opfs.rs b/src/browser/opfs.rs new file mode 100644 index 0000000000..a4bee9a404 --- /dev/null +++ b/src/browser/opfs.rs @@ -0,0 +1,564 @@ +//! OPFS persistence for `BrowserDb.openOpfs` (#441): the database is one +//! file in the Origin Private File System, read and written through a +//! `FileSystemSyncAccessHandle`, which browsers offer only in dedicated +//! workers. +//! +//! [`OpfsBackend`] is a [`StorageBackend`] over that handle: every page read +//! and write is a synchronous call into the browser, and nothing but the page +//! cache holds pages in memory. Commits are the storage layer's copy-on-write +//! protocol on the file itself; there is no WAL. The handle is exclusive: while +//! one is open on a file, acquiring another, from any worker or tab, fails with +//! `NoModificationAllowedError`, which is the database lock (STG-026). +//! +//! The database `name` is the file `.graph` in the origin's root +//! directory. An import into OPFS mode stages the new database in +//! `.graph-import` ([`STAGING_SUFFIX`]), a name no database maps to. +//! +//! The handle is a JavaScript object: `Send + Sync` only because wasm-bindgen +//! implements those for `JsValue` when wasm atomics are off. A build with +//! atomics therefore does not compile this backend, which is the sound +//! outcome: there is no threaded wasm here. + +use crate::error::{ErrorCode, bail_coded, err_coded}; +use crate::storage::meta::{MetaPage, SlotState, slot_page}; +use crate::storage::{PAGE_SIZE, StorageBackend}; +use anyhow::Result; +use js_sys::Reflect; +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; +use wasm_bindgen::JsCast; +use wasm_bindgen::prelude::*; +use wasm_bindgen_futures::JsFuture; +use web_sys::{ + FileSystemDirectoryHandle, FileSystemFileHandle, FileSystemGetFileOptions, + FileSystemReadWriteOptions, FileSystemSyncAccessHandle, +}; + +/// The file of database `name` is `name` followed by this. +pub(crate) const FILE_SUFFIX: &str = ".graph"; +/// The staging file of an import is the database file's name followed by +/// this; no database file ends with it. +pub(crate) const STAGING_SUFFIX: &str = "-import"; + +/// The largest byte offset a JavaScript number holds exactly (2^53). +const MAX_SAFE_OFFSET: u64 = 1 << 53; + +/// `[STG-048]`: a browser storage call failed. `detail` is the exception's +/// name, or a note such as "short read/write". +pub(crate) fn storage_error( + op: impl std::fmt::Display, + detail: impl std::fmt::Display, +) -> anyhow::Error { + err_coded!(ErrorCode::Stg048, op, detail) +} + +/// The `name` of a thrown `DOMException` (or any error object), else its +/// string form. +pub(crate) fn exception_name(e: &JsValue) -> String { + Reflect::get(e, &"name".into()) + .ok() + .and_then(|n| n.as_string()) + .filter(|n| !n.is_empty()) + .unwrap_or_else(|| super::js_text(e)) +} + +fn js_failure(op: impl std::fmt::Display, e: &JsValue) -> anyhow::Error { + storage_error(op, exception_name(e)) +} + +/// The byte offset of `page_id`, exact as a JavaScript number, or an error +/// before any I/O. +fn offset(page_id: u64) -> Result { + match page_id.checked_mul(PAGE_SIZE as u64) { + Some(off) if off <= MAX_SAFE_OFFSET => { + // At most 2^53: exact as an f64. + #[allow(clippy::cast_precision_loss)] + Ok(off as f64) + } + _ => bail_coded!( + ErrorCode::Int048, + format!("page {page_id} is past the largest OPFS offset") + ), + } +} + +fn at(off: f64) -> FileSystemReadWriteOptions { + let opts = FileSystemReadWriteOptions::new(); + opts.set_at(off); + opts +} + +/// Bytes moved by a read or write, as the handle reports them. +fn transferred(n: f64) -> usize { + // A count of at most PAGE_SIZE bytes, whole and non-negative. + #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] + let n = n as usize; + n +} + +/// One OPFS file and its open sync access handle, shared by every backend +/// and reader of one database. Closing it releases the lock; so does +/// dropping it. +pub(crate) struct OpfsFile { + handle: FileSystemSyncAccessHandle, + /// The file's entry name, for error messages. + name: String, + open: AtomicBool, + #[cfg(test)] + pub(crate) faults: std::sync::Mutex, +} + +impl OpfsFile { + fn new(handle: FileSystemSyncAccessHandle, name: String) -> OpfsFile { + OpfsFile { + handle, + name, + open: AtomicBool::new(true), + #[cfg(test)] + faults: std::sync::Mutex::default(), + } + } + + /// Close the handle, releasing the lock. Idempotent. + pub(crate) fn close(&self) { + if self.open.swap(false, Ordering::SeqCst) { + self.handle.close(); + } + } + + fn check_open(&self, op: &str) -> Result<()> { + if !self.open.load(Ordering::SeqCst) { + return Err(storage_error( + format!("{op} {}", self.name), + "handle closed", + )); + } + Ok(()) + } + + /// The file's length in bytes. + pub(crate) fn size(&self) -> Result { + self.check_open("size of")?; + let size = self + .handle + .get_size() + .map_err(|e| js_failure(format!("size of {}", self.name), &e))?; + // A file length: whole, non-negative and at most 2^53. + #[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] + Ok(size as u64) + } + + /// Read `len` bytes at byte offset `off`; fewer is an error. + fn read_at(&self, page_id: u64, len: usize) -> Result> { + let op = || format!("read page {page_id} of {}", self.name); + self.check_open("read")?; + let off = offset(page_id)?; + #[cfg(test)] + tests::count_read(); + let mut buf = vec![0u8; len]; + let n = self + .handle + .read_with_u8_array_and_options(&mut buf, &at(off)) + .map_err(|e| js_failure(op(), &e))?; + if transferred(n) != len { + return Err(storage_error(op(), "short read/write")); + } + Ok(buf) + } + + /// Write one page at `page_id`; a short write is an error. + pub(crate) fn write_page(&self, page_id: u64, data: &[u8]) -> Result<()> { + if data.len() != PAGE_SIZE { + bail_coded!(ErrorCode::Int051, data.len(), PAGE_SIZE); + } + let op = || format!("write page {page_id} of {}", self.name); + self.check_open("write")?; + let off = offset(page_id)?; + #[cfg(test)] + if let Some(fault) = self.faults.lock().unwrap().on_write() { + return self.inject_write(off, data, fault, op()); + } + let n = self + .handle + .write_with_u8_array_and_options(data, &at(off)) + .map_err(|e| js_failure(op(), &e))?; + if transferred(n) != data.len() { + return Err(storage_error(op(), "short read/write")); + } + Ok(()) + } + + /// Persist the writes so far (`flush()`): as durable as the browser + /// makes it, which is not a power-loss guarantee. + pub(crate) fn flush(&self) -> Result<()> { + self.check_open("flush")?; + #[cfg(test)] + if self.faults.lock().unwrap().on_flush() { + return Err(storage_error( + format!("flush {}", self.name), + "fault injection", + )); + } + self.handle + .flush() + .map_err(|e| js_failure(format!("flush {}", self.name), &e)) + } + + /// Set the file's length to `pages` pages. + pub(crate) fn truncate(&self, pages: u64) -> Result<()> { + self.check_open("truncate")?; + let len = offset(pages)?; + self.handle + .truncate_with_f64(len) + .map_err(|e| js_failure(format!("truncate {}", self.name), &e)) + } + + /// Whole pages in the file. + pub(crate) fn page_count(&self) -> Result { + Ok(self.size()? / PAGE_SIZE as u64) + } + + pub(crate) fn read_page(&self, page_id: u64) -> Result> { + let pages = self.page_count()?; + if page_id >= pages { + bail_coded!( + ErrorCode::Int049, + format!("Page {page_id} out of bounds (total pages: {pages})") + ); + } + self.read_at(page_id, PAGE_SIZE) + } + + /// The bytes past the last whole page, if any. + fn partial_page(&self) -> Result>> { + let size = self.size()?; + let tail = usize::try_from(size % PAGE_SIZE as u64).unwrap_or(0); + if tail == 0 { + return Ok(None); + } + self.read_at(size / PAGE_SIZE as u64, tail).map(Some) + } + + /// The highest generation among the file's valid meta slots. + pub(crate) fn newest_generation(&self) -> Result> { + let pages = self.page_count()?; + let mut newest = None; + for slot in 0..pages.min(2) { + if let SlotState::Valid(m) = MetaPage::decode(&self.read_page(slot)?) { + newest = newest.max(Some(m.generation)); + } + } + Ok(newest) + } +} + +impl Drop for OpfsFile { + fn drop(&mut self) { + self.close(); + } +} + +/// A database's OPFS file as a [`StorageBackend`]. Every backend clone and +/// reader shares one [`OpfsFile`]. +pub(crate) struct OpfsBackend { + file: Arc, +} + +impl OpfsBackend { + pub(crate) fn new(file: Arc) -> OpfsBackend { + OpfsBackend { file } + } +} + +impl StorageBackend for OpfsBackend { + fn write_page(&mut self, page_id: u64, data: &[u8]) -> Result<()> { + self.file.write_page(page_id, data) + } + + fn read_page(&self, page_id: u64) -> Result> { + self.file.read_page(page_id) + } + + fn sync(&mut self) -> Result<()> { + self.file.flush() + } + + fn page_count(&self) -> Result { + self.file.page_count() + } + + fn partial_page(&self) -> Result>> { + self.file.partial_page() + } + + /// The handle stays open: `BrowserDb.close()` releases it. + fn close(&mut self) -> Result<()> { + Ok(()) + } + + fn backend_name(&self) -> &'static str { + "opfs" + } +} + +// ── Opening ───────────────────────────────────────────────────────────────── + +/// `[API-026]`: OPFS mode is not available here, because of `what`. +fn unavailable(what: impl std::fmt::Display) -> JsValue { + super::to_js_error(err_coded!(ErrorCode::Api026, what)) +} + +fn is_function(v: &JsValue) -> bool { + v.is_function() +} + +/// The origin's OPFS root directory, after checking that this context can +/// use sync access handles at all. Creates nothing. +pub(crate) async fn root() -> Result { + let global = js_sys::global(); + let get = + |obj: &JsValue, key: &str| Reflect::get(obj, &key.into()).unwrap_or(JsValue::UNDEFINED); + if get(&global, "DedicatedWorkerGlobalScope").is_undefined() + || !global.is_instance_of::() + { + return Err(unavailable( + "sync access handles exist only in a dedicated worker; call openOpfs from one", + )); + } + if get(&global, "isSecureContext").as_bool() == Some(false) { + return Err(unavailable( + "this is not a secure context (https or localhost)", + )); + } + let storage = get(&get(&global, "navigator"), "storage"); + if storage.is_undefined() || !is_function(&get(&storage, "getDirectory")) { + return Err(unavailable( + "this browser has no navigator.storage.getDirectory()", + )); + } + let handle_class = get(&global, "FileSystemFileHandle"); + if handle_class.is_undefined() + || !is_function(&get( + &get(&handle_class, "prototype"), + "createSyncAccessHandle", + )) + { + return Err(unavailable( + "this browser has no FileSystemFileHandle.createSyncAccessHandle()", + )); + } + let storage: web_sys::StorageManager = storage.unchecked_into(); + match JsFuture::from(storage.get_directory()).await { + Ok(dir) => Ok(dir.unchecked_into()), + Err(e) => Err(unavailable(format!( + "the browser denies access to this origin's storage ({})", + exception_name(&e) + ))), + } +} + +/// Check a database name: it becomes part of an OPFS file name, unchanged. +pub(crate) fn check_name(name: &str) -> Result<(), JsValue> { + let bad = if name.is_empty() { + Some("is empty") + } else if name.contains('/') || name.contains('\\') { + Some("contains '/' or '\\'") + } else if name == "." || name == ".." { + Some("is '.' or '..'") + } else if name.contains('\0') { + Some("contains a NUL character") + } else { + None + }; + match bad { + Some(why) => Err(JsValue::from_str( + &crate::MinigrafError::invalid_argument(format!("openOpfs: the database name {why}")) + .to_string(), + )), + None => Ok(()), + } +} + +/// Open the file `entry` in `dir` and acquire its sync access handle. +/// `None` when `create` is false and the file does not exist. A handle +/// already open on the file anywhere is STG-026. +pub(crate) async fn open_file( + dir: &FileSystemDirectoryHandle, + entry: &str, + create: bool, +) -> Result>> { + let opts = FileSystemGetFileOptions::new(); + opts.set_create(create); + let file: FileSystemFileHandle = + match JsFuture::from(dir.get_file_handle_with_options(entry, &opts)).await { + Ok(f) => f.unchecked_into(), + Err(e) if !create && exception_name(&e) == "NotFoundError" => return Ok(None), + Err(e) => return Err(js_failure(format!("open {entry}"), &e)), + }; + match JsFuture::from(file.create_sync_access_handle()).await { + Ok(h) => Ok(Some(Arc::new(OpfsFile::new( + h.unchecked_into(), + entry.to_string(), + )))), + Err(e) if exception_name(&e) == "NoModificationAllowedError" => bail_coded!( + ErrorCode::Stg026, + format!("OPFS file {entry} is open in another worker or tab") + ), + Err(e) => Err(js_failure(format!("acquire a handle on {entry}"), &e)), + } +} + +/// Remove `entry` from `dir`, ignoring failure: a later open retries. +pub(crate) async fn remove_quietly(dir: &FileSystemDirectoryHandle, entry: &str) { + let _ = JsFuture::from(dir.remove_entry(entry)).await; +} + +// ── Import staging ────────────────────────────────────────────────────────── + +/// The staged database's meta page: valid, in one of the two slots, and +/// counting no page past the staging file's end. A staging file whose meta +/// was never completed has none: the import did not reach its publication +/// point. +pub(crate) fn staged_meta(stage: &OpfsFile) -> Result> { + let pages = stage.page_count()?; + let mut found = None; + for slot in 0..pages.min(2) { + if let SlotState::Valid(m) = MetaPage::decode(&stage.read_page(slot)?) + && slot_page(m.generation) == slot + && m.page_count <= pages + && found.is_none_or(|f: MetaPage| f.generation < m.generation) + { + found = Some(m); + } + } + Ok(found) +} + +/// Make `dst` hold the image `src` stages, page by page, meta last: its +/// length first, then every other page, a flush, the meta page and a flush. +/// Until the meta is written `dst` has no valid meta of the image's +/// generation, so an interruption leaves the staging file in charge and the +/// copy is redone. +pub(crate) fn copy_staged(src: &OpfsFile, dst: &OpfsFile, meta: &MetaPage) -> Result<()> { + let slot = slot_page(meta.generation); + dst.truncate(meta.page_count)?; + for id in (0..meta.page_count).filter(|&id| id != slot) { + dst.write_page(id, &src.read_page(id)?)?; + } + dst.flush()?; + dst.write_page(slot, &src.read_page(slot)?)?; + dst.flush() +} + +/// Finish an import a terminated worker left behind, under the database's +/// lock (`main` is open). A staged image newer than every meta of `main` is +/// copied into it; any staging file is then removed. +pub(crate) async fn recover_import( + dir: &FileSystemDirectoryHandle, + staging: &str, + main: &OpfsFile, +) -> Result<()> { + let Some(stage) = open_file(dir, staging, false).await? else { + return Ok(()); + }; + if let Some(m) = staged_meta(&stage)? + && main.newest_generation()?.is_none_or(|g| g < m.generation) + { + copy_staged(&stage, main, &m)?; + } + stage.close(); + drop(stage); + remove_quietly(dir, staging).await; + Ok(()) +} + +#[cfg(test)] +pub(crate) mod tests { + //! Fault injection and read counting for the OPFS tests. + use std::cell::Cell; + + thread_local! { + static READS: Cell = const { Cell::new(0) }; + } + + pub(crate) fn count_read() { + READS.with(|r| r.set(r.get() + 1)); + } + + /// Pages (and partial tails) read from OPFS files on this thread. + pub(crate) fn reads() -> u64 { + READS.with(Cell::get) + } + + /// What an injected write fault does. + #[derive(Clone, Copy)] + pub(crate) enum WriteFault { + /// Throw after writing the first `n` bytes. + Throw(usize), + /// Report `n` bytes written, having written them. + Short(usize), + } + + /// Faults for one file: the write or flush after `n` more succeed. + #[derive(Default)] + pub(crate) struct Faults { + pub write_after: Option<(u64, WriteFault)>, + pub flush_after: Option, + } + + impl Faults { + pub(crate) fn on_write(&mut self) -> Option { + match &mut self.write_after { + Some((0, f)) => { + let f = *f; + self.write_after = None; + Some(f) + } + Some((n, _)) => { + *n -= 1; + None + } + None => None, + } + } + + pub(crate) fn on_flush(&mut self) -> bool { + match &mut self.flush_after { + Some(0) => { + self.flush_after = None; + true + } + Some(n) => { + *n -= 1; + false + } + None => false, + } + } + } +} + +#[cfg(test)] +impl OpfsFile { + fn inject_write( + &self, + off: f64, + data: &[u8], + fault: tests::WriteFault, + op: String, + ) -> Result<()> { + let (n, throw) = match fault { + tests::WriteFault::Throw(n) => (n.min(data.len()), true), + tests::WriteFault::Short(n) => (n.min(data.len()), false), + }; + if n > 0 { + self.handle + .write_with_u8_array_and_options(&data[..n], &at(off)) + .map_err(|e| js_failure(op.clone(), &e))?; + } + if throw { + return Err(storage_error(op, "fault injection")); + } + Err(storage_error(op, "short read/write")) + } +} diff --git a/src/browser/opfs_tests.rs b/src/browser/opfs_tests.rs new file mode 100644 index 0000000000..179adb7b4b --- /dev/null +++ b/src/browser/opfs_tests.rs @@ -0,0 +1 @@ +//! placeholder diff --git a/src/browser/udfs.rs b/src/browser/udfs.rs index 00f72d82d0..8022eaa1bd 100644 --- a/src/browser/udfs.rs +++ b/src/browser/udfs.rs @@ -108,6 +108,7 @@ impl BrowserDb { /// when `fn` is not a function, and `[INT-040]` when `name` is taken. #[wasm_bindgen(js_name = registerPredicate)] pub fn register_predicate(&self, name: &str, f: JsValue) -> Result<(), JsValue> { + self.inner.admit()?; let f = Held::new(function(f, "the predicate")?.into()); let desc = udf_predicate(move |v: &Value| { let result = f @@ -144,6 +145,7 @@ impl BrowserDb { /// not a function, and `[INT-040]` when `name` is taken. #[wasm_bindgen(js_name = registerAggregate)] pub fn register_aggregate(&self, name: &str, aggregate: JsValue) -> Result<(), JsValue> { + self.inner.admit()?; let member = |key: &str| -> Result { let f = if aggregate.is_object() { Reflect::get(&aggregate, &JsValue::from_str(key))? diff --git a/src/error.rs b/src/error.rs index d9162df634..b59ba9dfb9 100644 --- a/src/error.rs +++ b/src/error.rs @@ -243,6 +243,7 @@ pub(crate) enum ErrorCode { Stg045, Stg046, Stg047, + Stg048, Wal001, Wal002, Wal003, @@ -279,6 +280,8 @@ pub(crate) enum ErrorCode { Api023, Api024, Api025, + Api026, + Api027, Int001, Int002, Int003, @@ -1476,6 +1479,12 @@ pub(crate) const REGISTRY: &[(ErrorCode, &str, &str, ErrorCategory)] = &[ "WAL is from a later checkpoint: it was created at generation {} but the file is at generation {}. Replaying it would leave a gap in the history (was an older .graph restored next to a newer .wal?)", ErrorCategory::Storage, ), + ( + ErrorCode::Stg048, + "STG-048", + "browser storage operation failed: {} ({})", + ErrorCategory::Storage, + ), ( ErrorCode::Wal001, "WAL-001", @@ -1668,6 +1677,18 @@ pub(crate) const REGISTRY: &[(ErrorCode, &str, &str, ErrorCategory)] = &[ "query option {} is not allowed on this database; OpenOptions::allow_overrides does not list Overridable::{}", ErrorCategory::Api, ), + ( + ErrorCode::Api026, + "API-026", + "OPFS mode is not available here: {}", + ErrorCategory::Api, + ), + ( + ErrorCode::Api027, + "API-027", + "the database is busy: {}", + ErrorCategory::Api, + ), ( ErrorCode::Int001, "INT-001", @@ -2478,6 +2499,7 @@ mod tests { | ErrorCode::Stg045 | ErrorCode::Stg046 | ErrorCode::Stg047 + | ErrorCode::Stg048 | ErrorCode::Wal001 | ErrorCode::Wal002 | ErrorCode::Wal003 @@ -2510,6 +2532,8 @@ mod tests { | ErrorCode::Api023 | ErrorCode::Api024 | ErrorCode::Api025 + | ErrorCode::Api026 + | ErrorCode::Api027 | ErrorCode::Int001 | ErrorCode::Int002 | ErrorCode::Int003 @@ -2758,6 +2782,7 @@ mod tests { ErrorCode::Stg045, ErrorCode::Stg046, ErrorCode::Stg047, + ErrorCode::Stg048, ErrorCode::Wal001, ErrorCode::Wal002, ErrorCode::Wal003, @@ -2790,6 +2815,8 @@ mod tests { ErrorCode::Api023, ErrorCode::Api024, ErrorCode::Api025, + ErrorCode::Api026, + ErrorCode::Api027, ErrorCode::Int001, ErrorCode::Int002, ErrorCode::Int003, diff --git a/src/storage/mod.rs b/src/storage/mod.rs index 20753c1ca5..dcc040c702 100644 --- a/src/storage/mod.rs +++ b/src/storage/mod.rs @@ -24,8 +24,10 @@ pub mod persistent_facts; pub(crate) mod pins; pub(crate) mod reader; pub(crate) mod value_pages; -// `Minigraf::verify` / `rebuild_indexes` walk a file; not built for wasm32. -#[cfg(not(target_arch = "wasm32"))] +// `Minigraf::verify` / `rebuild_indexes` walk a file; on wasm32 only the +// browser builds it, to check an image before importing it (#441). +#[cfg(any(not(target_arch = "wasm32"), feature = "browser"))] +#[cfg_attr(target_arch = "wasm32", allow(dead_code))] pub(crate) mod verify; use crate::error::{ErrorCode, bail_coded, err_coded}; diff --git a/src/storage/persistent_facts.rs b/src/storage/persistent_facts.rs index e20d905608..fb3ac15eb5 100644 --- a/src/storage/persistent_facts.rs +++ b/src/storage/persistent_facts.rs @@ -9,7 +9,7 @@ use crate::error::{ErrorCode, bail_coded, err_coded}; use crate::graph::FactStorage; use crate::graph::types::Fact; -#[cfg(not(target_arch = "wasm32"))] +#[cfg(any(not(target_arch = "wasm32"), feature = "browser"))] use crate::storage::btree::MutexStorageBackend; use crate::storage::btree::{build_btree, cow_insert}; use crate::storage::cache::PageCache; @@ -17,7 +17,7 @@ use crate::storage::dict::{DictReader, Encoded, Encoder}; use crate::storage::meta::{MetaPage, SlotState, slot_page}; use crate::storage::page::PageAllocator; use crate::storage::reader::OnDiskReader; -#[cfg(not(target_arch = "wasm32"))] +#[cfg(any(not(target_arch = "wasm32"), feature = "browser"))] use crate::storage::verify; use crate::storage::{LegacyHeaderV7, PAGE_SIZE, StorageBackend, freelist, page}; use anyhow::Result; @@ -59,9 +59,31 @@ pub struct PersistentFactStorage { /// Page cache size for the integrity walk: enough to keep a value page hot /// while its values are checked, without displacing the readers' cache. -#[cfg(not(target_arch = "wasm32"))] +#[cfg(any(not(target_arch = "wasm32"), feature = "browser"))] const VERIFY_CACHE_PAGES: usize = 64; +/// How open treats a file with exactly one valid meta slot (spec §4.1.1). +#[derive(Clone, Copy)] +enum Recovery { + /// The base generation of the WAL beside the file, if any: native rules. + Wal(Option), + /// No WAL can exist: open at the valid slot (#441). + #[cfg_attr( + not(any(test, all(target_arch = "wasm32", feature = "browser"))), + allow(dead_code) + )] + WalLess, +} + +impl Recovery { + fn wal_base(self) -> Option { + match self { + Recovery::Wal(b) => b, + Recovery::WalLess => None, + } + } +} + /// What open found in the meta slots. enum Opened { Meta(MetaPage), @@ -96,8 +118,29 @@ impl PersistentFactStorage { page_cache_capacity: usize, wal_base_generation: Option, ) -> Result { + Self::open_with( + backend, + page_cache_capacity, + Recovery::Wal(wal_base_generation), + ) + } + + /// Open storage on a backend that never has a WAL beside it and whose + /// commits are its only record, such as a browser OPFS file (#441). With + /// exactly one valid meta slot it opens at that slot, whatever the other + /// slot or later-generation pages hold: a commit interrupted before its + /// meta page was complete leaves that state, and nothing else could + /// recover it. Every other outcome is [`open`](Self::open)'s with no WAL; + /// in particular a native open of the same bytes still refuses with + /// STG-033 where this opens. + #[cfg(any(test, all(target_arch = "wasm32", feature = "browser")))] + pub(crate) fn open_wal_less(backend: B, page_cache_capacity: usize) -> Result { + Self::open_with(backend, page_cache_capacity, Recovery::WalLess) + } + + fn open_with(backend: B, page_cache_capacity: usize, recovery: Recovery) -> Result { let mut pfs = Self::unopened(backend, page_cache_capacity); - let meta = match pfs.select_meta(wal_base_generation, false)? { + let meta = match pfs.select_meta(recovery, false)? { Opened::Fresh => pfs.init_empty()?, Opened::Meta(m) => { m.check_features()?; @@ -128,7 +171,7 @@ impl PersistentFactStorage { wal_base_generation: Option, ) -> Result { let mut pfs = Self::unopened(backend, page_cache_capacity); - let meta = match pfs.select_meta(wal_base_generation, true)? { + let meta = match pfs.select_meta(Recovery::Wal(wal_base_generation), true)? { Opened::Fresh => MetaPage::empty(1), Opened::Meta(m) => { m.check_features()?; @@ -199,7 +242,12 @@ impl PersistentFactStorage { /// Choose the meta page to open at (spec §4.1.1). Never modifies the file, /// except to restore page 0 from the migration backup meta (§9 step 5), /// which `read_only` skips. - fn select_meta(&self, wal_base: Option, read_only: bool) -> Result { + fn select_meta(&self, recovery: Recovery, read_only: bool) -> Result { + let wal_base = recovery.wal_base(); + let single_valid = |backend: &B, m: MetaPage| match recovery { + Recovery::Wal(_) => check_single_valid(backend, m, wal_base), + Recovery::WalLess => Ok(()), + }; let mut backend = self.lock()?; let page_count = backend.page_count()?; if page_count == 0 { @@ -234,14 +282,14 @@ impl PersistentFactStorage { check_wal_not_later(m.generation, wal_base)?; m.check_features()?; check_fits(&*backend, &m)?; - check_single_valid(&*backend, m, wal_base)?; + single_valid(&backend, m)?; Ok(Opened::Meta(m)) } (SlotState::Valid(m), _) | (_, SlotState::Valid(m)) => { // Refuse an unreadable file before probing its pages. m.check_features()?; check_fits(&*backend, &m)?; - check_single_valid(&*backend, m, wal_base)?; + single_valid(&backend, m)?; Ok(Opened::Meta(m)) } _ => { @@ -516,8 +564,9 @@ impl PersistentFactStorage { Ok(()) } - #[cfg(not(target_arch = "wasm32"))] - /// Check the committed file (#373). Never modifies it, and holds the + #[cfg(any(not(target_arch = "wasm32"), feature = "browser"))] + /// Check the committed file (#373); also an image a browser database + /// imports (#441). Never modifies it, and holds the /// backend lock only for each page read, so queries keep running. pub(crate) fn verify(&self) -> Result { let backend = MutexStorageBackend(self.backend.clone()); @@ -719,6 +768,50 @@ impl PersistentFactStorage { Ok(()) } + /// The committed state as a file of its own, one page at a time in + /// page order (#441): every page the active meta counts, its meta page + /// stamped `generation` (at least the active one) in that generation's + /// slot, and zeros in the other slot and in every free page. Such a file + /// holds nothing a later, interrupted commit wrote, so a native open with + /// no WAL never finds evidence of one (STG-033) in it. A larger + /// `generation` makes the image newer than any meta it replaces. + #[cfg(any(test, all(target_arch = "wasm32", feature = "browser")))] + pub(crate) fn write_image( + &self, + generation: u64, + mut emit: impl FnMut(u64, &[u8]) -> Result<()>, + ) -> Result<()> { + let m = self.meta; + if generation < m.generation { + bail_coded!( + ErrorCode::Int048, + format!("image generation {generation} below {}", m.generation) + ); + } + let meta = MetaPage { generation, ..m }; + let free: std::collections::HashSet = if m.freelist_head == 0 { + std::collections::HashSet::new() + } else { + let backend = self.lock()?; + let probe = PageCache::new(0); + probe.set_generation_bound(m.generation); + let (ids, _) = freelist::read_chain(m.freelist_head, &*backend, &probe, m.page_count)?; + ids.into_iter().collect() + }; + let zeros = vec![0u8; PAGE_SIZE]; + for id in 0..m.page_count { + if id == slot_page(generation) { + emit(id, &meta.encode())?; + } else if id < 2 || free.contains(&id) { + emit(id, &zeros)?; + } else { + let page = self.lock()?.read_page(id)?; + emit(id, &page)?; + } + } + Ok(()) + } + fn refuse_if_failed(&self) -> Result<()> { if self.failed { bail_coded!(ErrorCode::Stg046); @@ -745,7 +838,7 @@ impl PersistentFactStorage { self.meta.last_checkpointed_tx_count } - #[cfg(not(target_arch = "wasm32"))] + #[cfg(any(not(target_arch = "wasm32"), feature = "browser"))] /// The active meta page's generation. A WAL created now records it as its /// base generation. pub fn generation(&self) -> u64 { @@ -783,7 +876,7 @@ impl PersistentFactStorage { /// by an abandoned log writer, whose file is deleted next (#431), and on /// close under `wal_checkpoint_threshold(usize::MAX)`, whose WAL keeps /// the pending facts (#476). - #[cfg(not(target_arch = "wasm32"))] + #[cfg(any(not(target_arch = "wasm32"), feature = "browser"))] pub(crate) fn discard(&mut self) { self.dirty = false; } From 6957e05d24a2591a4af0606d8078b07a6a4ea7c9 Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Sat, 14 Nov 2026 09:22:01 +0530 Subject: [PATCH 02/20] test(browser): OPFS mode in a dedicated worker, WAL-less recovery natively (#441) The lib's browser tests also build for a dedicated worker (--cfg minigraf_browser_worker), where the mode-independent ones run against OPFS and the OPFS tests cover the lock, close(), failed commits, open outcomes, imports interrupted at every write and flush, and the read and cache bounds. Native fault-injection tests check that a WAL-less open after a crash at any point of a commit gives the previous state or the new one, and that its export opens with Minigraf::open. Portability fixtures tie a native v8 file and an OPFS export to one set of answers. Refs #441 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01H2a7QjCHpikfsg5qwAV1p9 --- Cargo.toml | 5 +- src/browser/mod.rs | 119 +- src/browser/opfs.rs | 34 +- src/browser/opfs_tests.rs | 1031 ++++++++++++++++- src/browser/tx_tests.rs | 20 +- src/browser/udfs.rs | 28 +- src/storage/persistent_facts.rs | 402 ++++++- .../persistent_facts/browser_access.rs | 6 + tests/fixtures/portability/README.md | 12 + tests/fixtures/portability/commands.txt | 5 + tests/fixtures/portability/native_v8.graph | Bin 0 -> 32768 bytes tests/fixtures/portability/opfs_export.graph | Bin 0 -> 57344 bytes tests/fixtures/portability/queries.txt | 6 + tests/opfs_portability_test.rs | 101 ++ 14 files changed, 1714 insertions(+), 55 deletions(-) create mode 100644 tests/fixtures/portability/README.md create mode 100644 tests/fixtures/portability/commands.txt create mode 100644 tests/fixtures/portability/native_v8.graph create mode 100644 tests/fixtures/portability/opfs_export.graph create mode 100644 tests/fixtures/portability/queries.txt create mode 100644 tests/opfs_portability_test.rs diff --git a/Cargo.toml b/Cargo.toml index 70edcc0c3d..6863f3347f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -149,8 +149,9 @@ inherits = "release" workspace = true [workspace.lints.rust] -# Kani proof harnesses (#627): `cargo kani` sets cfg(kani). -unexpected_cfgs = { level = "warn", check-cfg = ['cfg(kani)', 'cfg(minigraf_shuttle)'] } +# Kani proof harnesses (#627): `cargo kani` sets cfg(kani). The browser tests' +# worker build (#441) sets cfg(minigraf_browser_worker). +unexpected_cfgs = { level = "warn", check-cfg = ['cfg(kani)', 'cfg(minigraf_shuttle)', 'cfg(minigraf_browser_worker)'] } [workspace.lints.clippy] # Panic prevention — use Result/? or debug_assert! instead diff --git a/src/browser/mod.rs b/src/browser/mod.rs index fe6cdd8209..26b5177a4e 100644 --- a/src/browser/mod.rs +++ b/src/browser/mod.rs @@ -37,7 +37,7 @@ use crate::query::datalog::prepared::{BindValue, PreparedQuery}; use crate::query::datalog::rules::RuleRegistry; use crate::query::datalog::types::DatalogCommand; use crate::storage::StorageBackend; -use crate::storage::persistent_facts::PersistentFactStorage; +use crate::storage::persistent_facts::{OtherSlot, PersistentFactStorage}; use std::cell::{Cell, RefCell}; use std::rc::Rc; use std::sync::{Arc, RwLock}; @@ -640,8 +640,8 @@ impl BrowserDb { /// publication point is the staging file's meta page: written and /// flushed, it makes the new database recoverable, since every open /// copies a staged image newer than the database file into it. The image - /// is stamped with a generation above every meta of the old file, so a - /// copy that finished is never redone over later commits. Then the image + /// is stamped with an odd generation above every meta of the old file, + /// so a copy that finished is never redone over later commits. Then the image /// is copied into the database file (meta last), the storage reopened on /// it, and the staging file removed; a failed removal is left to the /// next open. @@ -671,10 +671,13 @@ impl BrowserDb { .map_err(to_js_error)? .unwrap_or(0) .max(db.borrow().pfs.generation()); + // Odd, so the meta page is page 0 and the file starts with the + // magic, as a native file does after its first commit. let generation = newest .checked_add(1) - .ok_or_else(|| to_js_error(err_coded!(ErrorCode::Int048, "generation overflow")))? - .max(image.generation()); + .map(|g| g.max(image.generation())) + .and_then(|g| g.checked_add(1 - g % 2)) + .ok_or_else(|| to_js_error(err_coded!(ErrorCode::Int048, "generation overflow")))?; let stage = opfs::open_file(&opfs.dir, &staging, true) .await .map_err(to_js_error)? @@ -1280,7 +1283,7 @@ fn export_image( .and_then(|n| n.checked_mul(crate::storage::PAGE_SIZE)) .ok_or_else(|| JsValue::from_str("database is too large to export"))?; let mut blob = Vec::with_capacity(pages); - pfs.write_image(pfs.generation(), |_, page| { + pfs.write_image(pfs.generation(), OtherSlot::KeepOlder, |_, page| { blob.extend_from_slice(page); Ok(()) }) @@ -1306,12 +1309,13 @@ fn image_of(bytes: Vec) -> anyhow::Result have.max(2) { crate::error::bail_coded!(ErrorCode::Stg044, need, have); @@ -1333,7 +1337,7 @@ fn stage_image( let slot = crate::storage::meta::slot_page(generation); let mut meta = None; stage.truncate(0)?; - image.write_image(generation, |id, page| { + image.write_image(generation, OtherSlot::Zero, |id, page| { if id == slot { meta = Some(page.to_vec()); Ok(()) @@ -1418,7 +1422,72 @@ mod tests { use super::*; use wasm_bindgen_test::*; + // The worker build (`--cfg minigraf_browser_worker`) runs every test in a + // dedicated worker, where OPFS mode exists (#441). + #[cfg(not(minigraf_browser_worker))] wasm_bindgen_test_configure!(run_in_browser); + #[cfg(minigraf_browser_worker)] + wasm_bindgen_test_configure!(run_in_dedicated_worker); + + /// A name no other test uses, for a persistent database. + pub(crate) fn unique(prefix: &str) -> String { + thread_local! { + static NEXT: std::cell::Cell = const { std::cell::Cell::new(0) }; + } + let n = NEXT.with(|c| { + c.set(c.get() + 1); + c.get() + }); + format!("{prefix}-{}-{n}", js_sys::Date::now()) + } + + /// The database the mode-independent tests run on: in memory on the + /// main thread, and a new OPFS database in the worker build (#441), so + /// every one of them also checks OPFS mode's parity. + pub(crate) async fn scratch() -> BrowserDb { + #[cfg(minigraf_browser_worker)] + { + BrowserDb::open_opfs(unique("scratch")) + .await + .expect("openOpfs") + } + #[cfg(not(minigraf_browser_worker))] + { + BrowserDb::open_in_memory().expect("open_in_memory") + } + } + + /// A new, empty persistent database `name`: IndexedDB on the main + /// thread, OPFS in the worker build. + pub(crate) async fn fresh_persistent(name: &str) -> BrowserDb { + #[cfg(minigraf_browser_worker)] + { + BrowserDb::open_opfs(unique(name)).await.expect("openOpfs") + } + #[cfg(not(minigraf_browser_worker))] + { + drop(empty_store(name).await); + BrowserDb::open(name).await.expect("open") + } + } + + /// Close `db` (from [`fresh_persistent`]) and open it again. + pub(crate) async fn reopen_persistent(db: BrowserDb) -> BrowserDb { + let opfs_name = db.inner.opfs.as_ref().map(|o| { + o.file_name + .strip_suffix(opfs::FILE_SUFFIX) + .unwrap_or(&o.file_name) + .to_string() + }); + let idb_name = db.inner.borrow().idb.as_ref().map(|i| i.store_name.clone()); + db.close().expect("close"); + drop(db); + match (opfs_name, idb_name) { + (Some(name), _) => BrowserDb::open_opfs(name).await.expect("reopen OPFS"), + (None, Some(name)) => BrowserDb::open(&name).await.expect("reopen IndexedDB"), + (None, None) => panic!("not a persistent database"), + } + } #[wasm_bindgen_test] fn take_dirty_pages_returns_written_pages() { @@ -1468,7 +1537,7 @@ mod tests { #[wasm_bindgen_test] async fn execute_errors_carry_their_code() { - let db = BrowserDb::open_in_memory().expect("open_in_memory"); + let db = scratch().await; for (datalog, code) in [ ("(nosuchcommand [])", "[PRS-011] "), ( @@ -1492,7 +1561,7 @@ mod tests { // #869: BrowserDb takes no options, so the default override list applies. #[wasm_bindgen_test] async fn default_override_keywords_still_run() { - let db = BrowserDb::open_in_memory().expect("open_in_memory"); + let db = scratch().await; db.execute("(transact [[:a :edge :b]])".to_string()) .await .expect("transact"); @@ -1513,7 +1582,7 @@ mod tests { // refuses it. #[wasm_bindgen_test] async fn query_memory_limit_keyword_is_refused() { - let db = BrowserDb::open_in_memory().expect("open_in_memory"); + let db = scratch().await; let err = db .execute("(query [:find ?x :where [?e :a ?x] :query-memory-limit 1000])".to_string()) .await @@ -1526,7 +1595,7 @@ mod tests { // #861: `:max-iterations` is in the default list, so BrowserDb runs it. #[wasm_bindgen_test] async fn max_iterations_keyword_is_allowed() { - let db = BrowserDb::open_in_memory().expect("open_in_memory"); + let db = scratch().await; db.execute("(transact [[:a :edge :b] [:b :edge :c]])".to_string()) .await .expect("transact"); @@ -1544,7 +1613,7 @@ mod tests { } async fn reach_db() -> BrowserDb { - let db = BrowserDb::open_in_memory().expect("open_in_memory"); + let db = scratch().await; db.execute("(transact [[:a :edge :b] [:b :edge :c] [:c :edge :d]])".to_string()) .await .expect("transact"); @@ -1640,7 +1709,7 @@ mod tests { #[wasm_bindgen_test] async fn in_memory_transact_and_query() { - let db = BrowserDb::open_in_memory().expect("open_in_memory"); + let db = scratch().await; let transact_result = db .execute(r#"(transact [[:alice :name "Alice"] [:alice :age 30]])"#.to_string()) .await @@ -1660,7 +1729,7 @@ mod tests { #[wasm_bindgen_test] async fn cursor_batches_match_execute() { - let db = BrowserDb::open_in_memory().expect("open_in_memory"); + let db = scratch().await; for i in 0..20 { db.execute(format!("(transact [[:e{i} :n {i}]])")) .await @@ -1692,7 +1761,7 @@ mod tests { #[wasm_bindgen_test] async fn cursor_is_fixed_at_open_and_closes() { - let db = BrowserDb::open_in_memory().expect("open_in_memory"); + let db = scratch().await; db.execute(r#"(transact [[:a :n 1]])"#.to_string()) .await .expect("transact"); @@ -1729,7 +1798,7 @@ mod tests { #[wasm_bindgen_test] async fn empty_query_returns_empty_results() { - let db = BrowserDb::open_in_memory().expect("open_in_memory"); + let db = scratch().await; let result = db .execute(r#"(query [:find ?e :where [?e :name _]])"#.to_string()) .await @@ -1740,7 +1809,7 @@ mod tests { #[wasm_bindgen_test] async fn export_import_round_trip() { - let db = BrowserDb::open_in_memory().expect("open"); + let db = scratch().await; db.execute(r#"(transact [[:bob :role "admin"]])"#.to_string()) .await .expect("transact"); @@ -1753,7 +1822,7 @@ mod tests { "exported blob must start with MGRF magic" ); - let db2 = BrowserDb::open_in_memory().expect("open2"); + let db2 = scratch().await; db2.import_graph(blob).await.expect("import"); let result = db2 @@ -1768,7 +1837,7 @@ mod tests { #[wasm_bindgen_test] async fn export_size_is_page_aligned() { - let db = BrowserDb::open_in_memory().expect("open"); + let db = scratch().await; db.execute(r#"(transact [[:e :v 1]])"#.to_string()) .await .expect("transact"); @@ -1786,7 +1855,7 @@ mod tests { #[wasm_bindgen_test] async fn native_fixture_readable_by_browser_db() { let fixture: &[u8] = include_bytes!("../../tests/fixtures/compat.graph"); - let db = BrowserDb::open_in_memory().expect("open in-memory"); + let db = scratch().await; let arr = js_sys::Uint8Array::from(fixture); db.import_graph(arr).await.expect("import native fixture"); @@ -2148,11 +2217,17 @@ mod tests { #[cfg(all(target_arch = "wasm32", feature = "browser", test))] mod prepared_tests { + use super::tests::scratch; use super::*; use js_sys::{BigInt, Object, Reflect}; use wasm_bindgen_test::*; + // The worker build (`--cfg minigraf_browser_worker`) runs every test in a + // dedicated worker, where OPFS mode exists (#441). + #[cfg(not(minigraf_browser_worker))] wasm_bindgen_test_configure!(run_in_browser); + #[cfg(minigraf_browser_worker)] + wasm_bindgen_test_configure!(run_in_dedicated_worker); /// A JS object with `pairs` as its properties. fn obj(pairs: &[(&str, JsValue)]) -> JsValue { @@ -2187,7 +2262,7 @@ mod prepared_tests { } async fn seeded() -> BrowserDb { - let db = BrowserDb::open_in_memory().expect("open_in_memory"); + let db = scratch().await; db.execute( r#"(transact [[:a :n 1] [:b :n 2] [:c :n 3] [:a :s "x"] [:a :f 2.0] [:a :t true] [:a :k :kw] [:a :z nil]])"# diff --git a/src/browser/opfs.rs b/src/browser/opfs.rs index a4bee9a404..bc5c721a60 100644 --- a/src/browser/opfs.rs +++ b/src/browser/opfs.rs @@ -396,10 +396,14 @@ pub(crate) async fn open_file( Err(e) => return Err(js_failure(format!("open {entry}"), &e)), }; match JsFuture::from(file.create_sync_access_handle()).await { - Ok(h) => Ok(Some(Arc::new(OpfsFile::new( - h.unchecked_into(), - entry.to_string(), - )))), + Ok(h) => { + let file = OpfsFile::new(h.unchecked_into(), entry.to_string()); + #[cfg(test)] + if let Some(faults) = tests::take_armed(entry) { + *file.faults.lock().unwrap() = faults; + } + Ok(Some(Arc::new(file))) + } Err(e) if exception_name(&e) == "NoModificationAllowedError" => bail_coded!( ErrorCode::Stg026, format!("OPFS file {entry} is open in another worker or tab") @@ -479,6 +483,21 @@ pub(crate) mod tests { thread_local! { static READS: Cell = const { Cell::new(0) }; + static ARMED: std::cell::RefCell> = + const { std::cell::RefCell::new(Vec::new()) }; + } + + /// Give the next file opened as `entry` on this thread these faults. + pub(crate) fn arm(entry: &str, faults: Faults) { + ARMED.with(|a| a.borrow_mut().push((entry.to_string(), faults))); + } + + pub(crate) fn take_armed(entry: &str) -> Option { + ARMED.with(|a| { + let mut a = a.borrow_mut(); + let i = a.iter().position(|(e, _)| e == entry)?; + Some(a.remove(i).1) + }) } pub(crate) fn count_read() { @@ -500,7 +519,7 @@ pub(crate) mod tests { } /// Faults for one file: the write or flush after `n` more succeed. - #[derive(Default)] + #[derive(Default, Clone)] pub(crate) struct Faults { pub write_after: Option<(u64, WriteFault)>, pub flush_after: Option, @@ -540,6 +559,11 @@ pub(crate) mod tests { #[cfg(test)] impl OpfsFile { + /// The handle itself, for tests that write raw bytes. + pub(crate) fn handle_for_tests(&self) -> &FileSystemSyncAccessHandle { + &self.handle + } + fn inject_write( &self, off: f64, diff --git a/src/browser/opfs_tests.rs b/src/browser/opfs_tests.rs index 179adb7b4b..d6f8920494 100644 --- a/src/browser/opfs_tests.rs +++ b/src/browser/opfs_tests.rs @@ -1 +1,1030 @@ -//! placeholder +//! OPFS mode (#441). Most tests need a dedicated worker, so they run only in +//! the worker build (`RUSTFLAGS='--cfg minigraf_browser_worker'`, see +//! `.github/workflows/wasm.yml`); the main-thread build checks that +//! `openOpfs` is refused there. Termination and cross-worker tests need a +//! page that owns workers: minigraf-wasm's packaged consumer test has them. + +use super::*; +use wasm_bindgen_futures::JsFuture; +use wasm_bindgen_test::*; + +#[cfg(not(minigraf_browser_worker))] +wasm_bindgen_test_configure!(run_in_browser); +#[cfg(minigraf_browser_worker)] +wasm_bindgen_test_configure!(run_in_dedicated_worker); + +/// The `[CODE]` an error message starts with. +fn code_of(e: &JsValue) -> String { + let s = e.as_string().expect("an error message"); + s.split(']') + .next() + .unwrap_or("") + .trim_start_matches('[') + .to_string() +} + +fn rejection(r: Result) -> JsValue { + match r { + Ok(_) => panic!("expected a rejection"), + Err(e) => e, + } +} + +/// Every entry name in the origin's OPFS root. +async fn entries() -> Vec { + // `navigator.storage.getDirectory()` directly: it works on the main + // thread too, where `opfs::root` refuses. + let navigator = js_sys::Reflect::get(&js_sys::global(), &"navigator".into()).unwrap(); + let storage: web_sys::StorageManager = js_sys::Reflect::get(&navigator, &"storage".into()) + .unwrap() + .unchecked_into(); + let dir: web_sys::FileSystemDirectoryHandle = JsFuture::from(storage.get_directory()) + .await + .expect("root") + .unchecked_into(); + let iter = dir.keys(); + let mut names = Vec::new(); + loop { + let next = JsFuture::from(iter.next().expect("next")) + .await + .expect("entry"); + if js_sys::Reflect::get(&next, &"done".into()) + .unwrap() + .is_truthy() + { + break; + } + names.push( + js_sys::Reflect::get(&next, &"value".into()) + .unwrap() + .as_string() + .unwrap(), + ); + } + names.sort(); + names +} + +#[cfg(not(minigraf_browser_worker))] +mod main_thread { + use super::*; + + /// Spec AC2: on the main thread `openOpfs` rejects with API-026, + /// creates nothing and traps nothing. + #[wasm_bindgen_test] + async fn open_opfs_on_the_main_thread_is_api_026_and_creates_nothing() { + let before = entries().await; + let err = rejection(BrowserDb::open_opfs("main-thread".into()).await); + assert_eq!(code_of(&err), "API-026"); + assert!( + err.as_string().unwrap().contains("dedicated worker"), + "names what is missing" + ); + assert_eq!(entries().await, before, "nothing created"); + // The instance still works. + let db = BrowserDb::open_in_memory().expect("open_in_memory"); + db.execute("(transact [[:a :n 1]])".into()) + .await + .expect("transact"); + } +} + +#[cfg(minigraf_browser_worker)] +mod worker { + use super::super::tests::unique; + use super::*; + use crate::browser::opfs::tests::{Faults, WriteFault, arm, reads}; + + async fn open(name: &str) -> BrowserDb { + BrowserDb::open_opfs(name.to_string()) + .await + .expect("openOpfs") + } + + async fn exec(db: &BrowserDb, q: &str) -> String { + db.execute(q.to_string()).await.expect("execute") + } + + fn rows(json: &str) -> serde_json::Value { + let v: serde_json::Value = serde_json::from_str(json).unwrap(); + let mut rows: Vec = v["results"].as_array().unwrap().clone(); + rows.sort_by_key(ToString::to_string); + serde_json::Value::Array(rows) + } + + async fn ns(db: &BrowserDb) -> Vec { + let json = exec(db, "(query [:find ?n :where [?e :n ?n]])").await; + let v: serde_json::Value = serde_json::from_str(&json).unwrap(); + let mut ns: Vec = v["results"] + .as_array() + .unwrap() + .iter() + .map(|r| r[0].as_i64().unwrap()) + .collect(); + ns.sort_unstable(); + ns + } + + async fn put(db: &BrowserDb, range: std::ops::Range) { + let facts: String = range.map(|i| format!("[:e{i} :n {i}]")).collect(); + exec(db, &format!("(transact [{facts}])")).await; + } + + fn file(db: &BrowserDb) -> Arc { + db.inner.opfs.as_ref().expect("OPFS mode").file.clone() + } + + async fn raw_file(entry: &str) -> Arc { + let dir = opfs::root().await.unwrap(); + opfs::open_file(&dir, entry, true).await.unwrap().unwrap() + } + + /// Write `bytes` as the whole of OPFS entry `entry`. + async fn write_raw(entry: &str, bytes: &[u8]) { + let f = raw_file(entry).await; + f.truncate(0).unwrap(); + let opts = FileSystemReadWriteOptionsExt::at(0.0); + let n = f + .handle_for_tests() + .write_with_u8_array_and_options(bytes, &opts) + .unwrap(); + assert_eq!(n as usize, bytes.len()); + f.flush().unwrap(); + f.close(); + } + + /// The whole of OPFS entry `entry`. + async fn read_raw(entry: &str) -> Vec { + let f = raw_file(entry).await; + let len = f.size().unwrap() as usize; + let mut buf = vec![0u8; len]; + let opts = FileSystemReadWriteOptionsExt::at(0.0); + f.handle_for_tests() + .read_with_u8_array_and_options(&mut buf, &opts) + .unwrap(); + f.close(); + buf + } + + struct FileSystemReadWriteOptionsExt; + impl FileSystemReadWriteOptionsExt { + fn at(off: f64) -> web_sys::FileSystemReadWriteOptions { + let o = web_sys::FileSystemReadWriteOptions::new(); + o.set_at(off); + o + } + } + + fn blob(bytes: &[u8]) -> js_sys::Uint8Array { + js_sys::Uint8Array::from(bytes) + } + + const PORTABILITY_COMMANDS: &str = + include_str!("../../tests/fixtures/portability/commands.txt"); + const PORTABILITY_QUERIES: &str = include_str!("../../tests/fixtures/portability/queries.txt"); + + async fn assert_portability_answers(db: &BrowserDb, source: &str) { + for line in PORTABILITY_QUERIES.lines().filter(|l| !l.is_empty()) { + let (q, expected) = line.split_once('\t').unwrap(); + let expected: serde_json::Value = serde_json::from_str(expected).unwrap(); + assert_eq!(rows(&exec(db, q).await), expected, "{source}: {q}"); + } + } + + async fn portability_db(name: &str) -> BrowserDb { + let db = open(name).await; + for c in PORTABILITY_COMMANDS.lines().filter(|l| !l.is_empty()) { + exec(&db, c).await; + } + db + } + + // ── AC1, AC4: entry point, persistence, files ────────────────────────── + + #[wasm_bindgen_test] + async fn acknowledged_writes_survive_close_and_reopen() { + let name = unique("ac1"); + let db = open(&name).await; + put(&db, 0..5).await; + exec(&db, "(retract [[:e0 :n 0]])").await; + db.close().expect("close"); + let db = open(&name).await; + assert_eq!( + ns(&db).await, + vec![1, 2, 3, 4], + "history kept, retraction applied" + ); + // Dropped without close(): the handle is released and nothing lost. + put(&db, 5..6).await; + drop(db); + let db = open(&name).await; + assert_eq!(ns(&db).await, vec![1, 2, 3, 4, 5]); + // An IndexedDB database of the same name is another database. + let idb = BrowserDb::open(&name).await.expect("IndexedDB open"); + assert!(ns(&idb).await.is_empty(), "no fallback or sharing"); + idb.close().unwrap(); + } + + #[wasm_bindgen_test] + async fn the_database_is_one_file_and_no_wal() { + let name = unique("files"); + let db = open(&name).await; + put(&db, 0..50).await; + db.checkpoint().await.expect("checkpoint"); + let exported = db.export_graph().expect("export"); + db.import_graph(exported).await.expect("import"); + put(&db, 50..60).await; + let mut tx = db.begin_write().unwrap(); + tx.execute("(transact [[:x :n 1000]])").unwrap(); + tx.rollback(); + let mine: Vec = entries() + .await + .into_iter() + .filter(|e| e.starts_with(&name)) + .collect(); + assert_eq!(mine, vec![format!("{name}.graph")], "one file, no WAL"); + db.close().unwrap(); + } + + #[wasm_bindgen_test] + async fn invalid_names_are_api_017_before_anything_is_created() { + let before = entries().await; + for bad in ["", "a/b", "a\\b", ".", "..", "nul\0"] { + let err = rejection(BrowserDb::open_opfs(bad.into()).await); + assert_eq!(code_of(&err), "API-017", "name {bad:?}"); + } + assert_eq!(entries().await, before, "nothing created"); + } + + // ── AC8: lock ────────────────────────────────────────────────────────── + + #[wasm_bindgen_test] + async fn a_second_open_is_stg_026_until_close() { + let (a, b) = (unique("lock-a"), unique("lock-b")); + let db = open(&a).await; + let err = rejection(BrowserDb::open_opfs(a.clone()).await); + assert_eq!(code_of(&err), "STG-026"); + let other = open(&b).await; + put(&other, 0..1).await; + // Dependents still referenced do not hold the lock after close(). + let cursor = db.query("(query [:find ?n :where [?e :n ?n]])").unwrap(); + db.close().unwrap(); + let again = open(&a).await; + drop(cursor); + drop(again); + // A failed open releases the lock: a foreign file is STG-002. + let junk = unique("lock-junk"); + write_raw(&format!("{junk}.graph"), &vec![0xAA; 3 * 4096]).await; + let err = rejection(BrowserDb::open_opfs(junk.clone()).await); + assert_eq!(code_of(&err), "STG-002"); + let dir = opfs::root().await.unwrap(); + let f = opfs::open_file(&dir, &format!("{junk}.graph"), false) + .await + .expect("lock free after a failed open") + .unwrap(); + f.close(); + other.close().unwrap(); + } + + // ── AC9: close() ─────────────────────────────────────────────────────── + + async fn all_modes(name: &str) -> Vec { + vec![ + BrowserDb::open_in_memory().unwrap(), + BrowserDb::open(name).await.expect("IndexedDB"), + open(name).await, + ] + } + + #[wasm_bindgen_test] + async fn after_close_every_call_is_api_018_but_disposal() { + let name = unique("closed"); + for db in all_modes(&name).await { + put(&db, 0..3).await; + let mut cursor = db.query("(query [:find ?n :where [?e :n ?n]])").unwrap(); + let mut cursor2 = db.query("(query [:find ?n :where [?e :n ?n]])").unwrap(); + let pq = db.prepare("(query [:find ?n :where [?e :n ?n]])").unwrap(); + let mut tx = db.begin_write().unwrap(); + tx.execute("(transact [[:t :n 99]])").unwrap(); + db.close().expect("close"); + db.close().expect("idempotent"); + let none = JsValue::UNDEFINED; + let api018 = |e: JsValue| assert_eq!(code_of(&e), "API-018"); + api018(rejection( + db.execute("(query [:find ?n :where [?e :n ?n]])".into()) + .await, + )); + api018(rejection(db.execute("(transact [[:z :n 1]])".into()).await)); + api018(rejection(db.query("(query [:find ?n :where [?e :n ?n]])"))); + api018(rejection( + db.prepare("(query [:find ?n :where [?e :n ?n]])"), + )); + api018(rejection(db.checkpoint().await)); + api018(rejection(db.export_graph())); + api018(rejection(db.import_graph(blob(&[])).await)); + api018(rejection(db.begin_write())); + api018(rejection(db.register_predicate( + "p?", + js_sys::Function::new_no_args("return true").into(), + ))); + api018(rejection(cursor.next_batch(10))); + api018(rejection(cursor.vars())); + api018(rejection(pq.execute(none.clone()))); + api018(rejection(pq.query(none.clone()))); + api018(rejection(pq.with_max_iterations(1.0))); + api018(rejection( + tx.execute("(query [:find ?n :where [?e :n ?n]])"), + )); + api018(rejection(JsFuture::from(tx.commit()).await)); + // Disposal never throws. + cursor2.close(); + tx.rollback(); + drop(tx); + } + // The OPFS lock is free at once, with the dependents dropped above. + let db = open(&name).await; + assert_eq!( + ns(&db).await, + vec![0, 1, 2], + "the open transaction was discarded" + ); + db.close().unwrap(); + } + + #[wasm_bindgen_test] + async fn close_from_a_user_function_during_a_query_is_busy() { + let name = unique("busy-close"); + let db = open(&name).await; + put(&db, 0..3).await; + let inner = BrowserDb { + inner: db.inner.clone(), + }; + let seen = Rc::new(RefCell::new(Vec::::new())); + let log = seen.clone(); + let f = Closure:: JsValue>::new(move |_v: JsValue| { + let mut log = log.borrow_mut(); + log.push(match inner.close() { + Ok(()) => "closed".into(), + Err(e) => code_of(&e), + }); + // Nested reads keep working. + let nested = inner.query("(query [:find ?n :where [?e :n ?n]])").is_ok() + && inner.export_graph().is_ok(); + log.push(nested.to_string()); + JsValue::TRUE + }) + .into_js_value(); + db.register_predicate("probe?", f).unwrap(); + exec(&db, "(query [:find ?n :where [?e :n ?n] [(probe? ?n)]])").await; + let seen = seen.borrow().clone(); + assert!(!seen.is_empty()); + assert_eq!(seen[0], "API-027", "refused while the query runs"); + assert_eq!(seen[1], "true", "nested calls work"); + // Through a cursor too. + let mut cursor = db + .query("(query [:find ?n :where [?e :n ?n] [(probe? ?n)]])") + .unwrap(); + while cursor.next_batch(1).unwrap().is_some() {} + db.close().expect("close once the query is over"); + } + + #[wasm_bindgen_test] + async fn calls_issued_before_close_change_nothing_after_it() { + // OPFS: an execute whose body has not started rejects and writes + // nothing; a commit that applied resolves and is stored. Both are + // checked against the database reopened before they settle. + let name = unique("before-close"); + let db = open(&name).await; + let late = { + let d = BrowserDb { + inner: db.inner.clone(), + }; + wasm_bindgen_futures::future_to_promise(async move { + d.execute("(transact [[:late :n 7]])".into()) + .await + .map(JsValue::from) + }) + }; + let mut tx = db.begin_write().unwrap(); + tx.execute("(transact [[:c :n 1]])").unwrap(); + let commit = tx.commit(); + db.close().unwrap(); + let reopened = open(&name).await; + let err = rejection(JsFuture::from(late).await); + assert_eq!(code_of(&err), "API-018", "the body started after close()"); + JsFuture::from(commit) + .await + .expect("an applied OPFS commit resolves"); + assert_eq!( + ns(&reopened).await, + vec![1], + "the commit, not the late write" + ); + reopened.close().unwrap(); + + // IndexedDB: a commit whose flush has not started is never flushed. + let name = unique("before-close-idb"); + let db = BrowserDb::open(&name).await.unwrap(); + let mut tx = db.begin_write().unwrap(); + tx.execute("(transact [[:c :n 1]])").unwrap(); + let commit = tx.commit(); + db.close().unwrap(); + let reopened = BrowserDb::open(&name).await.unwrap(); + let err = rejection(JsFuture::from(commit).await); + assert_eq!(code_of(&err), "API-018"); + assert!(ns(&reopened).await.is_empty(), "not stored"); + reopened.close().unwrap(); + } + + // ── AC10: failed commits ─────────────────────────────────────────────── + + #[wasm_bindgen_test] + async fn a_failed_commit_is_stg_048_then_stg_046() { + let cases: Vec<(Faults, &str)> = vec![ + ( + Faults { + write_after: Some((0, WriteFault::Throw(0))), + ..Faults::default() + }, + "fault injection", + ), + ( + Faults { + write_after: Some((1, WriteFault::Short(100))), + ..Faults::default() + }, + "short read/write", + ), + ( + Faults { + flush_after: Some(1), + ..Faults::default() + }, + "flush", + ), + ]; + for (faults, text) in cases { + let name = unique("failed"); + let db = open(&name).await; + put(&db, 0..3).await; + *file(&db).faults.lock().unwrap() = faults; + let err = rejection(db.execute("(transact [[:e9 :n 9]])".into()).await); + let msg = err.as_string().unwrap(); + assert_eq!(code_of(&err), "STG-048", "{msg}"); + assert!(msg.contains(text), "{msg}"); + let err = rejection(db.execute("(transact [[:e10 :n 10]])".into()).await); + assert_eq!(code_of(&err), "STG-046"); + let mut tx = db.begin_write().unwrap(); + tx.execute("(transact [[:e11 :n 11]])").unwrap(); + assert_eq!( + code_of(&rejection(JsFuture::from(tx.commit()).await)), + "STG-046" + ); + // Reads keep working, failed facts included. + assert!(ns(&db).await.contains(&9)); + db.close().expect("close after a failed commit"); + let db = open(&name).await; + let got = ns(&db).await; + assert!( + got == vec![0, 1, 2] || got == vec![0, 1, 2, 9], + "previous state, or the failed commit whole" + ); + db.close().unwrap(); + } + } + + // ── AC5, AC6, AC7: interrupted commits and open outcomes ────────────── + + /// A commit whose meta write tears before the CRC: a native open of + /// these bytes refuses (STG-033); OPFS mode opens at the previous commit, + /// and its export opens under the native rule. + #[wasm_bindgen_test] + async fn a_torn_meta_reopens_at_the_previous_commit() { + // Tear writes of the commit until one leaves the meta invalid. + let mut torn_once = false; + for k in 0..64u64 { + let name_k = unique("torn-k"); + let d = open(&name_k).await; + put(&d, 0..20).await; + *file(&d).faults.lock().unwrap() = Faults { + write_after: Some((k, WriteFault::Throw(16))), + ..Faults::default() + }; + let ok = d.execute("(transact [[:z :n 99]])".into()).await.is_ok(); + d.close().unwrap(); + if ok { + break; + } + let reopened = open(&name_k).await; + let got = ns(&reopened).await; + assert!( + got.len() == 20 || got.len() == 21, + "old or new, never refused" + ); + let bytes = reopened.export_graph().unwrap().to_vec(); + reopened.close().unwrap(); + // The export opens under the native rule (no WAL). + let mut pages = std::collections::HashMap::new(); + for (i, c) in bytes.chunks(4096).enumerate() { + pages.insert(i as u64, c.to_vec()); + } + let native = PersistentFactStorage::open( + BrowserBackend::Buffer(BrowserBufferBackend::load_pages(pages)), + 0, + None, + ); + assert!(native.is_ok(), "the export opens natively"); + // The raw file after a meta tear: one valid slot, next-generation + // pages. Native refuses it; OPFS mode just opened it. + let raw = read_raw(&format!("{name_k}.graph")).await; + let mut pages = std::collections::HashMap::new(); + for (i, c) in raw.chunks(4096).enumerate() { + pages.insert(i as u64, c.to_vec()); + } + if let Err(e) = PersistentFactStorage::open( + BrowserBackend::Buffer(BrowserBufferBackend::load_pages(pages)), + 0, + None, + ) { + assert_eq!(crate::MinigrafError::from(e).code(), "STG-033"); + assert_eq!(got.len(), 20, "the previous commit"); + torn_once = true; + } + } + assert!(torn_once, "some tear left one valid slot and later pages"); + } + + #[wasm_bindgen_test] + async fn open_outcomes_match_native_and_refusals_modify_nothing() { + // A short file that is not a Minigraf database: STG-045, unchanged. + let name = unique("short"); + write_raw(&format!("{name}.graph"), &[0x55; 100]).await; + assert_eq!( + code_of(&rejection(BrowserDb::open_opfs(name.clone()).await)), + "STG-045" + ); + assert_eq!(read_raw(&format!("{name}.graph")).await, vec![0x55; 100]); + // Not a Minigraf file: STG-002, unchanged. + let name = unique("foreign"); + write_raw(&format!("{name}.graph"), &vec![0xAA; 3 * 4096]).await; + assert_eq!( + code_of(&rejection(BrowserDb::open_opfs(name.clone()).await)), + "STG-002" + ); + assert_eq!( + read_raw(&format!("{name}.graph")).await, + vec![0xAA; 3 * 4096] + ); + // A meta counting pages past the end: STG-044, unchanged. + let name = unique("short-tail"); + let db = open(&name).await; + put(&db, 0..300).await; + let bytes = db.export_graph().unwrap().to_vec(); + db.close().unwrap(); + let cut = &bytes[..bytes.len() - 4096]; + write_raw(&format!("{name}.graph"), cut).await; + assert_eq!( + code_of(&rejection(BrowserDb::open_opfs(name.clone()).await)), + "STG-044" + ); + assert_eq!(read_raw(&format!("{name}.graph")).await, cut.to_vec()); + // Both slots damaged: STG-032. + let name = unique("both-damaged"); + let mut damaged = bytes.clone(); + damaged[30] ^= 0xFF; + damaged[4096 + 30] ^= 0xFF; + write_raw(&format!("{name}.graph"), &damaged).await; + assert_eq!( + code_of(&rejection(BrowserDb::open_opfs(name.clone()).await)), + "STG-032" + ); + // A torn initial meta: initialised again. + let name = unique("torn-initial"); + let mut page = vec![0u8; 4096]; + page[..40].copy_from_slice(&crate::storage::meta::MetaPage::empty(1).encode()[..40]); + write_raw(&format!("{name}.graph"), &page).await; + let db = open(&name).await; + assert!(ns(&db).await.is_empty()); + db.close().unwrap(); + } + + // ── AC11: portability ────────────────────────────────────────────────── + + #[wasm_bindgen_test] + async fn export_is_a_complete_file_that_verifies() { + let name = unique("portable"); + let db = portability_db(&name).await; + assert_portability_answers(&db, "OPFS").await; + let bytes = db.export_graph().unwrap().to_vec(); + assert_eq!(&bytes[0..4], b"MGRF"); + image_of(bytes.clone()).expect("the export verifies"); + db.close().unwrap(); + // Back into a new OPFS database, across a reopen. + let copy = unique("portable-copy"); + let db = open(©).await; + db.import_graph(blob(&bytes)).await.unwrap(); + assert_portability_answers(&db, "imported export").await; + db.close().unwrap(); + let db = open(©).await; + assert_portability_answers(&db, "imported export, reopened").await; + db.close().unwrap(); + } + + #[wasm_bindgen_test] + async fn native_v8_and_v7_files_import_and_survive_reopen() { + let v8: &[u8] = include_bytes!("../../tests/fixtures/portability/native_v8.graph"); + let name = unique("native-v8"); + let db = open(&name).await; + db.import_graph(blob(v8)).await.expect("import native v8"); + assert_portability_answers(&db, "native v8").await; + db.close().unwrap(); + let db = open(&name).await; + assert_portability_answers(&db, "native v8, reopened").await; + db.close().unwrap(); + + let v7: &[u8] = include_bytes!("../../tests/fixtures/compat.graph"); + let name = unique("native-v7"); + let db = open(&name).await; + db.import_graph(blob(v7)).await.expect("import native v7"); + let names = exec(&db, "(query [:find ?n :where [?e :name ?n]])").await; + assert_eq!(rows(&names), serde_json::json!([["Alice"]])); + db.close().unwrap(); + let db = open(&name).await; + let names = exec(&db, "(query [:find ?n :where [?e :name ?n]])").await; + assert_eq!(rows(&names), serde_json::json!([["Alice"]])); + db.close().unwrap(); + } + + #[wasm_bindgen_test] + async fn a_smaller_import_leaves_no_trailing_bytes() { + let small = open(&unique("small")).await; + put(&small, 0..2).await; + let small_bytes = small.export_graph().unwrap().to_vec(); + small.close().unwrap(); + let name = unique("large"); + let db = open(&name).await; + for i in 0..20 { + put(&db, i * 100..(i + 1) * 100).await; + } + let large = file(&db).size().unwrap(); + db.import_graph(blob(&small_bytes)).await.unwrap(); + let size = file(&db).size().unwrap(); + assert!(size < large, "the file shrank"); + assert_eq!( + size as usize, + small_bytes.len(), + "exactly the image's length" + ); + assert_eq!(ns(&db).await, vec![0, 1]); + db.close().unwrap(); + } + + // ── AC12: import ─────────────────────────────────────────────────────── + + #[wasm_bindgen_test] + async fn import_validation_rejects_before_replacing_anything() { + let src = open(&unique("valid-src")).await; + put(&src, 0..300).await; + let good = src.export_graph().unwrap().to_vec(); + src.close().unwrap(); + + let name = unique("valid"); + let db = open(&name).await; + put(&db, 0..3).await; + let cursor = db.query("(query [:find ?n :where [?e :n ?n]])").unwrap(); + let pq = db.prepare("(query [:find ?n :where [?e :n ?n]])").unwrap(); + + // Not a whole number of pages: API-017. + let err = rejection(db.import_graph(blob(&good[..4097])).await); + assert_eq!(code_of(&err), "API-017"); + // Page-aligned but truncated: STG-044 (IndexedDB mode accepts it). + let truncated = &good[..good.len() - 4096]; + let err = rejection(db.import_graph(blob(truncated)).await); + assert_eq!(code_of(&err), "STG-044"); + let mem = BrowserDb::open_in_memory().unwrap(); + mem.import_graph(blob(truncated)) + .await + .expect("the buffer modes accept it"); + // A reachable page with a bad CRC: rejected (the buffer modes find + // it only when reading it). + let mut bad = good.clone(); + let page = 2 * 4096; + bad[page + 100] ^= 0xFF; + let err = rejection(db.import_graph(blob(&bad)).await); + assert!(code_of(&err).starts_with("STG-"), "a storage code"); + mem.import_graph(blob(&bad)).await.expect("accepted lazily"); + + // Nothing was replaced; the dependents still work; no leftovers. + assert_eq!(ns(&db).await, vec![0, 1, 2]); + assert!(pq.execute(JsValue::UNDEFINED).is_ok()); + drop(cursor); + let mine: Vec = entries() + .await + .into_iter() + .filter(|e| e.starts_with(&name)) + .collect(); + assert_eq!(mine, vec![format!("{name}.graph")]); + // An empty blob is an empty database. + db.import_graph(blob(&[])).await.expect("empty import"); + assert!(ns(&db).await.is_empty()); + db.close().unwrap(); + } + + #[wasm_bindgen_test] + async fn a_published_import_invalidates_earlier_cursors_and_prepared_queries() { + let name = unique("publish"); + let db = open(&name).await; + put(&db, 0..3).await; + let mut cursor = db.query("(query [:find ?n :where [?e :n ?n]])").unwrap(); + let pq = db.prepare("(query [:find ?n :where [?e :n ?n]])").unwrap(); + let other = BrowserDb::open_in_memory().unwrap(); + put(&other, 10..12).await; + db.import_graph(other.export_graph().unwrap()) + .await + .expect("import"); + assert_eq!(code_of(&rejection(cursor.next_batch(10))), "API-018"); + assert_eq!( + code_of(&rejection(pq.execute(JsValue::UNDEFINED))), + "API-018" + ); + assert_eq!(ns(&db).await, vec![10, 11], "new calls read the import"); + let pq2 = db.prepare("(query [:find ?n :where [?e :n ?n]])").unwrap(); + assert!( + pq2.execute(JsValue::UNDEFINED).is_ok(), + "new dependents work" + ); + db.close().unwrap(); + let db = open(&name).await; + assert_eq!(ns(&db).await, vec![10, 11]); + db.close().unwrap(); + } + + #[wasm_bindgen_test] + async fn calls_admitted_while_importing_are_busy() { + let name = unique("importing"); + let db = open(&name).await; + put(&db, 0..3).await; + let mut cursor = db.query("(query [:find ?n :where [?e :n ?n]])").unwrap(); + let other = BrowserDb::open_in_memory().unwrap(); + put(&other, 10..12).await; + let image = other.export_graph().unwrap(); + let import = { + let d = BrowserDb { + inner: db.inner.clone(), + }; + wasm_bindgen_futures::future_to_promise(async move { + d.import_graph(image).await.map(|()| JsValue::UNDEFINED) + }) + }; + // Let the import's body start and wait on the browser. + JsFuture::from(js_sys::Promise::resolve(&JsValue::NULL)) + .await + .unwrap(); + JsFuture::from(js_sys::Promise::resolve(&JsValue::NULL)) + .await + .unwrap(); + assert!(db.inner.importing.get(), "the import is in progress"); + let busy = |e: JsValue| assert_eq!(code_of(&e), "API-027"); + busy(rejection(db.query("(query [:find ?n :where [?e :n ?n]])"))); + busy(rejection(db.export_graph())); + busy(rejection(db.begin_write())); + busy(rejection(db.close())); + busy(rejection(cursor.next_batch(1))); + busy(rejection(db.execute("(transact [[:x :n 1]])".into()).await)); + assert_eq!( + code_of(&rejection(BrowserDb::open_opfs(name.clone()).await)), + "STG-026", + "locked throughout" + ); + cursor.close(); + JsFuture::from(import).await.expect("the import completes"); + assert_eq!(ns(&db).await, vec![10, 11]); + db.close().unwrap(); + } + + /// An image staged like an import, ready for [`recover_import`] tests. + fn staged_bytes(image: &[u8], generation: u64) -> Vec { + let pfs = image_of(image.to_vec()).unwrap(); + let mut out = Vec::new(); + pfs.write_image(generation, OtherSlot::Zero, |_, page| { + out.extend_from_slice(page); + Ok(()) + }) + .unwrap(); + out + } + + /// Spec AC12: an import interrupted at every write and flush of its + /// staging file, of the database file's copy, and of the copy that an + /// open redoes, reopens as the whole old or the whole new database; a + /// failure before the publication point keeps the old one open, one + /// after it closes the handle. Transient files are gone after an open + /// that completes. + #[wasm_bindgen_test] + async fn an_interrupted_import_reopens_old_or_new() { + let other = BrowserDb::open_in_memory().unwrap(); + put(&other, 100..400).await; + let image = other.export_graph().unwrap().to_vec(); + let new: Vec = (100..400).collect(); + let old: Vec = (0..3).collect(); + let mut published_failures = 0; + let mut kept_old = 0; + for target in ["stage", "main"] { + for flush in [false, true] { + for k in 0..40u64 { + let name = unique("interrupt"); + let db = open(&name).await; + put(&db, 0..3).await; + let faults = Faults { + write_after: (!flush).then_some((k, WriteFault::Throw(16))), + flush_after: flush.then_some(k), + }; + match target { + "stage" => arm(&format!("{name}.graph-import"), faults), + _ => *file(&db).faults.lock().unwrap() = faults, + } + let result = db.import_graph(blob(&image)).await; + let _ = opfs::tests::take_armed(&format!("{name}.graph-import")); + if result.is_ok() { + assert_eq!(ns(&db).await, new); + db.close().unwrap(); + break; + } + if db.inner.closed.get() { + published_failures += 1; + } else { + kept_old += 1; + assert_eq!(ns(&db).await, old, "the old database, still open"); + db.close().unwrap(); + } + drop(db); + let db = open(&name).await; + let got = ns(&db).await; + assert!(got == old || got == new, "old or new, whole"); + db.close().unwrap(); + let mine: Vec = entries() + .await + .into_iter() + .filter(|e| e.starts_with(&name)) + .collect(); + assert_eq!(mine, vec![format!("{name}.graph")], "no leftovers"); + } + } + } + assert!( + published_failures > 0, + "some failures came after publication" + ); + assert!(kept_old > 0, "some came before it"); + + // A second interruption, in the copy an open redoes. + let name = unique("interrupt-twice"); + let db = open(&name).await; + put(&db, 0..3).await; + *file(&db).faults.lock().unwrap() = Faults { + write_after: Some((3, WriteFault::Throw(0))), + ..Faults::default() + }; + assert!(db.import_graph(blob(&image)).await.is_err()); + assert!(db.inner.closed.get(), "failed while copying: closed"); + drop(db); + arm( + &format!("{name}.graph"), + Faults { + write_after: Some((5, WriteFault::Throw(0))), + ..Faults::default() + }, + ); + let err = rejection(BrowserDb::open_opfs(name.clone()).await); + assert_eq!(code_of(&err), "STG-048"); + let db = open(&name).await; + assert_eq!(ns(&db).await, new, "the redone copy completes"); + db.close().unwrap(); + } + + /// A staging file left after its copy finished (a failed cleanup) is + /// removed at the next open, never copied over later commits; one that + /// never reached its publication point is removed too. + #[wasm_bindgen_test] + async fn leftover_staging_files_are_removed_not_replayed() { + let other = BrowserDb::open_in_memory().unwrap(); + put(&other, 100..110).await; + let image = other.export_graph().unwrap().to_vec(); + + let name = unique("consumed"); + let db = open(&name).await; + db.import_graph(blob(&image)).await.unwrap(); + let g = file(&db).newest_generation().unwrap().unwrap(); + put(&db, 0..2).await; + db.close().unwrap(); + write_raw(&format!("{name}.graph-import"), &staged_bytes(&image, g)).await; + let db = open(&name).await; + let mut want: Vec = (100..110).collect(); + want.extend([0, 1]); + want.sort_unstable(); + assert_eq!(ns(&db).await, want, "later commits kept"); + db.close().unwrap(); + + let name = unique("unpublished"); + let db = open(&name).await; + put(&db, 0..2).await; + db.close().unwrap(); + let mut staged = staged_bytes(&image, 99); + staged[..4096].fill(0); // generation 99's slot is page 0: no meta + write_raw(&format!("{name}.graph-import"), &staged).await; + let db = open(&name).await; + assert_eq!( + ns(&db).await, + vec![0, 1], + "an unpublished import is dropped" + ); + db.close().unwrap(); + for n in [&name] { + let mine: Vec = entries() + .await + .into_iter() + .filter(|e| e.starts_with(n.as_str())) + .collect(); + assert_eq!(mine, vec![format!("{n}.graph")]); + } + } + + // ── AC13: bounded reads and memory ───────────────────────────────────── + + /// Spec AC13(a): opening a healthy database reads a fixed number of + /// pages, the same for databases whose fact counts differ tenfold. + #[wasm_bindgen_test] + async fn opfs_open_read_bound() { + let mut opened = Vec::new(); + for facts in [500i64, 5_000] { + let name = unique("read-bound"); + let db = open(&name).await; + for chunk in (0..facts).step_by(500) { + put(&db, chunk..chunk + 500).await; + } + db.close().unwrap(); + let before = reads(); + let db = open(&name).await; + opened.push((reads() - before, file(&db).page_count().unwrap())); + db.close().unwrap(); + } + let (small, large) = (opened[0], opened[1]); + assert!(large.1 > 5 * small.1, "the larger file is much larger"); + assert_eq!(small.0, large.0, "the same pages read at open"); + assert!(small.0 <= 4, "a fixed handful of pages: {}", small.0); + // IndexedDB mode reads every page at open. + let name = unique("read-bound-idb"); + let db = BrowserDb::open(&name).await.unwrap(); + put(&db, 0..500).await; + db.close().unwrap(); + let db = BrowserDb::open(&name).await.unwrap(); + let held = db.inner.borrow().pfs.with_backend(|b| { + b.buffer() + .map_or(0, BrowserBufferBackend::stored_page_count) + }); + assert!(held > 4, "IndexedDB mode holds every page"); + db.close().unwrap(); + } + + /// Spec AC13(b): after touching more pages than the cache holds, with no + /// cursor open, the cache holds at most its capacity, and the backend no + /// pages at all. + #[wasm_bindgen_test] + async fn opfs_cache_residency_bound() { + let name = unique("residency"); + let db = open(&name).await; + for chunk in 0..40 { + put(&db, chunk * 500..(chunk + 1) * 500).await; + } + let pages = file(&db).page_count().unwrap(); + assert!( + pages > 2 * OPFS_PAGE_CACHE as u64, + "more pages than the cache" + ); + exec(&db, "(query [:find (count ?e) :where [?e :n ?n]])").await; + exec(&db, "(query [:find ?n :where [:e19999 :n ?n]])").await; + let inner = db.inner.borrow(); + let cached = inner.pfs.cached_page_count(); + assert!(cached <= OPFS_PAGE_CACHE, "cache within capacity: {cached}"); + assert!( + inner.pfs.with_backend(|b| b.buffer().is_none()), + "no page buffer under OPFS mode" + ); + drop(inner); + db.close().unwrap(); + } + + // ── AC14: IndexedDB in a worker ──────────────────────────────────────── + + #[wasm_bindgen_test] + async fn indexeddb_mode_works_in_a_worker() { + let name = unique("idb-worker"); + let db = BrowserDb::open(&name).await.expect("IndexedDB in a worker"); + put(&db, 0..3).await; + db.checkpoint().await.unwrap(); + db.close().unwrap(); + let db = BrowserDb::open(&name).await.unwrap(); + assert_eq!(ns(&db).await, vec![0, 1, 2]); + db.close().unwrap(); + } +} diff --git a/src/browser/tx_tests.rs b/src/browser/tx_tests.rs index f145922ef0..ced739ba9f 100644 --- a/src/browser/tx_tests.rs +++ b/src/browser/tx_tests.rs @@ -3,11 +3,17 @@ //! while a transaction is open, `[API-018]` after it finishes, and a //! differential check against the native transaction. +use super::tests::{fresh_persistent, reopen_persistent, scratch}; use super::*; use wasm_bindgen_futures::JsFuture; use wasm_bindgen_test::*; +// The worker build (`--cfg minigraf_browser_worker`) runs every test in a +// dedicated worker, where OPFS mode exists (#441). +#[cfg(not(minigraf_browser_worker))] wasm_bindgen_test_configure!(run_in_browser); +#[cfg(minigraf_browser_worker)] +wasm_bindgen_test_configure!(run_in_dedicated_worker); const NAMES: &str = "(query [:find ?n :where [?e :person/name ?n]])"; const SETUP: &str = r#"(transact [[:alice :person/name "Alice"] [:alice :person/age 30] @@ -45,7 +51,7 @@ async fn names(db: &BrowserDb) -> Vec { } async fn setup() -> BrowserDb { - let db = BrowserDb::open_in_memory().expect("open_in_memory"); + let db = scratch().await; db.execute(SETUP.to_string()).await.expect("setup"); db } @@ -206,8 +212,7 @@ async fn fresh_db(name: &str) -> BrowserDb { #[wasm_bindgen_test] async fn commit_is_one_transaction_and_survives_reopen() { - let name = "minigraf-test-tx-commit"; - let db = fresh_db(name).await; + let db = fresh_persistent("minigraf-test-tx-commit").await; db.execute(SETUP.to_string()).await.expect("setup"); let before = db.inner.borrow().fact_storage.current_tx_count(); let mut tx = db.begin_write().expect("begin"); @@ -241,15 +246,13 @@ async fn commit_is_one_transaction_and_survives_reopen() { vec!["[1]", "[1]", "[2]", "[2]"], "Alice and Carol from the setup, Dave and Erin from the one commit" ); - drop(db); - let db = BrowserDb::open(name).await.expect("reopen"); + let db = reopen_persistent(db).await; assert_eq!(names(&db).await.len(), 4, "the commit after reopening"); } #[wasm_bindgen_test] async fn rollback_and_drop_discard_and_survive_reopen() { - let name = "minigraf-test-tx-rollback"; - let db = fresh_db(name).await; + let db = fresh_persistent("minigraf-test-tx-rollback").await; db.execute(SETUP.to_string()).await.expect("setup"); let mut tx = db.begin_write().expect("begin"); @@ -264,8 +267,7 @@ async fn rollback_and_drop_discard_and_survive_reopen() { drop(tx); // as JavaScript's free() does assert_eq!(names(&db).await.len(), 3, "dropped"); - drop(db); - let db = BrowserDb::open(name).await.expect("reopen"); + let db = reopen_persistent(db).await; assert_eq!(names(&db).await.len(), 3, "neither after reopening"); } diff --git a/src/browser/udfs.rs b/src/browser/udfs.rs index 8022eaa1bd..0dcda202cd 100644 --- a/src/browser/udfs.rs +++ b/src/browser/udfs.rs @@ -263,11 +263,17 @@ fn value_to_js(v: &Value) -> JsValue { #[cfg(all(target_arch = "wasm32", feature = "browser", test))] mod tests { use super::*; + use crate::browser::tests::scratch; use std::cell::RefCell; use std::rc::Rc; use wasm_bindgen_test::*; + // The worker build (`--cfg minigraf_browser_worker`) runs every test in a + // dedicated worker, where OPFS mode exists (#441). + #[cfg(not(minigraf_browser_worker))] wasm_bindgen_test_configure!(run_in_browser); + #[cfg(minigraf_browser_worker)] + wasm_bindgen_test_configure!(run_in_dedicated_worker); fn js_fn(args: &str, body: &str) -> JsValue { Function::new_with_args(args, body).into() @@ -308,7 +314,7 @@ mod tests { #[wasm_bindgen_test] async fn a_predicate_filters_rows() { - let db = BrowserDb::open_in_memory().unwrap(); + let db = scratch().await; numbers(&db).await; db.register_predicate( "big?", @@ -328,7 +334,7 @@ mod tests { #[wasm_bindgen_test] async fn values_reach_javascript_as_typed_objects() { - let db = BrowserDb::open_in_memory().unwrap(); + let db = scratch().await; db.execute( r#"(transact [[:a :v "s"] [:a :v 7] [:a :v 1.5] [:a :v true] [:a :v :k] [:a :v nil] [:a :v #{2 "x"}]])"#.into(), ) @@ -373,7 +379,7 @@ mod tests { #[wasm_bindgen_test] async fn a_ref_reaches_javascript_as_its_uuid() { - let db = BrowserDb::open_in_memory().unwrap(); + let db = scratch().await; let kind = Rc::new(RefCell::new(String::new())); let log = kind.clone(); db.register_predicate( @@ -401,7 +407,7 @@ mod tests { #[wasm_bindgen_test] async fn an_aggregate_in_find_and_over() { - let db = BrowserDb::open_in_memory().unwrap(); + let db = scratch().await; numbers(&db).await; let product = js_sys::Object::new(); Reflect::set(&product, &"init".into(), &js_fn("", "return 1n")).unwrap(); @@ -456,7 +462,7 @@ mod tests { #[wasm_bindgen_test] async fn a_plain_finish_result_is_read_as_a_bind_value() { - let db = BrowserDb::open_in_memory().unwrap(); + let db = scratch().await; numbers(&db).await; let agg = js_sys::Object::new(); Reflect::set(&agg, &"init".into(), &js_fn("", "return 0")).unwrap(); @@ -474,7 +480,7 @@ mod tests { #[wasm_bindgen_test] async fn a_throw_fails_the_query_with_qry_014() { - let db = BrowserDb::open_in_memory().unwrap(); + let db = scratch().await; numbers(&db).await; db.register_predicate("boom?", js_fn("v", "throw new Error('bad ' + v.value)")) .unwrap(); @@ -515,7 +521,7 @@ mod tests { #[wasm_bindgen_test] async fn aggregate_failures_fail_the_query() { - let db = BrowserDb::open_in_memory().unwrap(); + let db = scratch().await; numbers(&db).await; let ok_init = "return 0"; let ok_step = "return acc"; @@ -591,7 +597,7 @@ mod tests { #[wasm_bindgen_test] async fn a_function_may_query_the_same_database() { - let db = BrowserDb::open_in_memory().unwrap(); + let db = scratch().await; numbers(&db).await; let inner_db = clone_db(&db); // A nested query that runs a UDF too: true only for ?n = 1. @@ -617,7 +623,7 @@ mod tests { #[wasm_bindgen_test] async fn a_function_may_register_a_function() { - let db = BrowserDb::open_in_memory().unwrap(); + let db = scratch().await; numbers(&db).await; let inner_db = clone_db(&db); let registered = Rc::new(RefCell::new(false)); @@ -657,7 +663,7 @@ mod tests { /// promise's body starts from a microtask. #[wasm_bindgen_test] async fn async_methods_from_a_function_run_after_the_query() { - let db = BrowserDb::open_in_memory().unwrap(); + let db = scratch().await; numbers(&db).await; let promises = Rc::new(RefCell::new(Vec::::new())); let started = promises.clone(); @@ -696,7 +702,7 @@ mod tests { #[wasm_bindgen_test] async fn import_graph_from_a_function_runs_after_the_query() { - let db = BrowserDb::open_in_memory().unwrap(); + let db = scratch().await; numbers(&db).await; let other = BrowserDb::open_in_memory().unwrap(); other diff --git a/src/storage/persistent_facts.rs b/src/storage/persistent_facts.rs index fb3ac15eb5..510c156eac 100644 --- a/src/storage/persistent_facts.rs +++ b/src/storage/persistent_facts.rs @@ -62,6 +62,19 @@ pub struct PersistentFactStorage { #[cfg(any(not(target_arch = "wasm32"), feature = "browser"))] const VERIFY_CACHE_PAGES: usize = 64; +/// The meta slot an image ([`PersistentFactStorage::write_image`]) does not +/// put its meta page in. +#[cfg(any(test, all(target_arch = "wasm32", feature = "browser")))] +#[derive(Clone, Copy)] +pub(crate) enum OtherSlot { + /// The file's own page if it is a valid meta of an older generation + /// (an export: page 0 then usually starts with the magic), else zeros. + KeepOlder, + /// Zeros: the image's only valid meta is the one it is written with (an + /// import's staging file, whose meta page is its publication point). + Zero, +} + /// How open treats a file with exactly one valid meta slot (spec §4.1.1). #[derive(Clone, Copy)] enum Recovery { @@ -771,14 +784,15 @@ impl PersistentFactStorage { /// The committed state as a file of its own, one page at a time in /// page order (#441): every page the active meta counts, its meta page /// stamped `generation` (at least the active one) in that generation's - /// slot, and zeros in the other slot and in every free page. Such a file - /// holds nothing a later, interrupted commit wrote, so a native open with - /// no WAL never finds evidence of one (STG-033) in it. A larger - /// `generation` makes the image newer than any meta it replaces. + /// slot, the other slot as `other` says, and zeros in every free page. + /// Such a file holds nothing a later, interrupted commit wrote, so a + /// native open with no WAL never finds evidence of one (STG-033) in it. + /// A larger `generation` makes the image newer than any meta it replaces. #[cfg(any(test, all(target_arch = "wasm32", feature = "browser")))] pub(crate) fn write_image( &self, generation: u64, + other: OtherSlot, mut emit: impl FnMut(u64, &[u8]) -> Result<()>, ) -> Result<()> { let m = self.meta; @@ -802,7 +816,12 @@ impl PersistentFactStorage { for id in 0..m.page_count { if id == slot_page(generation) { emit(id, &meta.encode())?; - } else if id < 2 || free.contains(&id) { + } else if id < 2 { + let page = self.lock()?.read_page(id)?; + let keep = matches!(other, OtherSlot::KeepOlder) + && matches!(MetaPage::decode(&page), SlotState::Valid(o) if o.generation < generation); + emit(id, if keep { &page } else { &zeros })?; + } else if free.contains(&id) { emit(id, &zeros)?; } else { let page = self.lock()?.read_page(id)?; @@ -3898,4 +3917,377 @@ mod tests { let facts = assert_size_per_fact(80_000); assert!(facts > 1_000_000); } + + // ── WAL-less recovery (#441, spec D1) ────────────────────────────────── + + /// Every `:test/n` value, sorted: the full answer to one query. + fn values(pfs: &PersistentFactStorage) -> Vec { + let mut v: Vec = pfs + .storage() + .get_facts_by_attribute(&":test/n".to_string()) + .unwrap() + .into_iter() + .filter_map(|f| match f.value { + Value::Integer(i) => Some(i), + _ => None, + }) + .collect(); + v.sort_unstable(); + v + } + + fn open_wl(mem: &MemoryBackend) -> Result> { + PersistentFactStorage::open_wal_less(mem.clone(), 16) + } + + /// `pfs`'s [`write_image`](PersistentFactStorage::write_image) as a new + /// store. + fn image_of(pfs: &PersistentFactStorage) -> MemoryBackend { + let mut img = MemoryBackend::new(); + pfs.write_image(pfs.generation(), OtherSlot::KeepOlder, |id, page| { + img.write_page(id, page) + }) + .unwrap(); + img + } + + /// The image of a store opens under the native rule (no WAL) with the + /// same answers, and verifies clean (spec AC7). + fn assert_image_opens_natively(pfs: &PersistentFactStorage) { + let img = image_of(pfs); + let native = open_mem(&img, None).expect("image opens natively without a WAL"); + assert_eq!(values(&native), values(pfs), "image answers"); + assert!( + native.verify().unwrap().problems.is_empty(), + "image verifies" + ); + } + + /// A store at the state `batches` saves leave, with `per` facts each. + fn saved(batches: u128, per: u128) -> MemoryBackend { + let mem = MemoryBackend::new(); + let mut pfs = open_wl(&mem).unwrap(); + for b in 0..batches { + put_batch(&mut pfs, b * per..(b + 1) * per); + pfs.save().unwrap(); + } + mem + } + + /// Spec AC5(a): a crash at every write and sync of a commit, torn writes + /// of every page (meta pages among them) and a failed final sync, on the + /// first commit of a new database and on a later one. The reopen is + /// never refused, answers exactly the previous committed state or the + /// new one, holds the new one after a reported success, accounts for + /// every page, takes further commits, and exports an image that opens + /// natively with no WAL. + #[test] + fn wal_less_crash_at_every_point_of_a_commit_opens_old_or_new() { + const PER: u128 = 150; + let mut kinds: Vec<(bool, Option)> = Vec::new(); + for torn in [None, Some(0usize), Some(16), Some(512), Some(4095)] { + kinds.push((true, torn)); + } + kinds.push((false, None)); + let mut points = 0; + for old_batches in [0u128, 3] { + let old: Vec = (0..(old_batches * PER) as i64).collect(); + let new: Vec = (0..((old_batches + 1) * PER) as i64).collect(); + for &(write, torn) in &kinds { + for k in 0u64.. { + let mem = saved(old_batches, PER); + let (backend, config) = FaultInjectingBackend::with_config(mem.clone()); + let mut pfs = PersistentFactStorage::open_wal_less(backend, 16).unwrap(); + put_batch(&mut pfs, old_batches * PER..(old_batches + 1) * PER); + { + let mut cfg = config.lock().unwrap(); + cfg.fail_write_after = write.then_some(k); + cfg.fail_sync_after = (!write).then_some(k); + cfg.torn_write_bytes = torn; + } + let ok = pfs.save().is_ok(); + pfs.dirty = false; // a kill: nothing more is written + drop(pfs); + + let mut reopened = open_wl(&mem).expect("a WAL-less reopen is never refused"); + let got = values(&reopened); + assert!(got == old || got == new, "old or new state, never a mix"); + if ok { + assert!(got == new, "a reported success is kept"); + } + assert_space_accounted(&reopened.meta(), &mem); + assert_image_opens_natively(&reopened); + + // The reopened database takes the next commit, over + // whatever the interrupted one left. + let next = got.len() as u128; + put_batch(&mut reopened, 10_000 + next..10_000 + next + 10); + reopened.save().expect("commit after recovery"); + drop(reopened); + let again = open_wl(&mem).expect("reopen after that commit"); + assert_eq!(values(&again).len(), got.len() + 10, "the later commit"); + assert_space_accounted(&again.meta(), &mem); + + if ok { + break; + } + points += 1; + } + } + } + assert!(points > 50, "expected many crash points"); + } + + /// Spec AC5(a): interrupting the creation of a new database (the first + /// meta writes) reopens as an empty database. + #[test] + fn wal_less_crash_while_creating_opens_empty() { + for torn in [None, Some(0usize), Some(16), Some(512)] { + for k in 0u64..3 { + let mem = MemoryBackend::new(); + let (backend, config) = FaultInjectingBackend::with_config(mem.clone()); + { + let mut cfg = config.lock().unwrap(); + cfg.fail_write_after = Some(k); + cfg.torn_write_bytes = torn; + } + let created = PersistentFactStorage::open_wal_less(backend, 16); + drop(created); + let pfs = open_wl(&mem).expect("reopen after a torn creation"); + assert!(values(&pfs).is_empty(), "nothing committed"); + } + } + } + + /// How many page writes one save of `PER` new facts makes on `mem`. + fn writes_of_a_save(mem: &MemoryBackend, ids: std::ops::Range) -> u64 { + for k in 0u64.. { + let copy = deep_copy(mem); + let (backend, config) = FaultInjectingBackend::with_config(copy); + let mut pfs = PersistentFactStorage::open_wal_less(backend, 16).unwrap(); + put_batch(&mut pfs, ids.clone()); + config.lock().unwrap().fail_write_after = Some(k); + let ok = pfs.save().is_ok(); + pfs.dirty = false; + if ok { + return k; + } + } + unreachable!() + } + + /// Run a save of `ids` on `mem` whose `k`-th write fails after writing + /// its first `torn` bytes. + fn interrupted_save(mem: &MemoryBackend, ids: std::ops::Range, k: u64, torn: usize) { + let (backend, config) = FaultInjectingBackend::with_config(mem.clone()); + let mut pfs = PersistentFactStorage::open_wal_less(backend, 16).unwrap(); + put_batch(&mut pfs, ids); + { + let mut cfg = config.lock().unwrap(); + cfg.fail_write_after = Some(k); + cfg.torn_write_bytes = Some(torn); + } + assert!(pfs.save().is_err(), "the save is interrupted"); + pfs.dirty = false; + } + + /// Spec AC5(a) fixed cases. The commit point is the bytes, not the + /// write's result: a torn meta write that still decodes as the new meta + /// (the meaningful bytes and CRC are at the page's start) is the new + /// state; a tear that leaves it invalid is the previous one, where a + /// native open refuses (STG-033). And a commit that completed without + /// being acknowledged is the previous committed state for the next one. + #[test] + fn wal_less_fixed_cases() { + const PER: u128 = 100; + let a: Vec = (0..PER as i64).collect(); + let b: Vec = (0..2 * PER as i64).collect(); + let base = saved(1, PER); + let n = writes_of_a_save(&base, PER..2 * PER); + let meta_write = n - 1; + + // A torn meta write whose first 512 bytes hold the whole new meta. + let mem = deep_copy(&base); + interrupted_save(&mem, PER..2 * PER, meta_write, 512); + let pfs = open_wl(&mem).unwrap(); + assert_eq!(values(&pfs), b, "torn but valid meta: the new state"); + assert_eq!(open_mem(&mem, None).unwrap().generation(), pfs.generation()); + + // A tear that leaves the new meta invalid. + let mem = deep_copy(&base); + interrupted_save(&mem, PER..2 * PER, meta_write, 16); + let before = snapshot(&mem); + assert_eq!( + code(open_mem(&mem, None).err().unwrap()), + "STG-033", + "a native open refuses these bytes" + ); + assert!(snapshot(&mem) == before, "the refused open wrote nothing"); + let pfs = open_wl(&mem).unwrap(); + assert_eq!(values(&pfs), a, "invalid meta: the previous state"); + assert_image_opens_natively(&pfs); + + // B completes (never acknowledged); C fails before its meta, or + // tears it: B's state, not A's. + let c = 2 * PER..3 * PER; + let n_c = writes_of_a_save(&saved(2, PER), c.clone()); + for (k, torn) in [(0, 0), (n_c / 2, 0), (n_c - 1, 0), (n_c - 1, 16)] { + let mem = saved(2, PER); + interrupted_save(&mem, c.clone(), k, torn); + let pfs = open_wl(&mem).unwrap(); + assert_eq!(values(&pfs), b, "B's state survives C's failure"); + } + } + + /// Spec AC6: apart from the rule above, a WAL-less open has the native + /// outcome for the same bytes, and a refused open modifies nothing. + #[test] + fn wal_less_open_outcomes_match_native() { + let same = |mem: &MemoryBackend| { + let before = snapshot(mem); + let native = open_mem(mem, None).map(|p| (p.generation(), values(&p))); + let wal_less = open_wl(mem).map(|p| (p.generation(), values(&p))); + match (native, wal_less) { + (Ok(n), Ok(w)) => assert!(n == w, "same state"), + (Err(n), Err(w)) => { + assert_eq!(code(n), code(w), "same refusal"); + assert!(snapshot(mem) == before, "a refused open writes nothing"); + } + _ => panic!("one open refused and the other did not"), + } + }; + // Two valid slots: the higher generation. + let mem = three_generations(); + same(&mem); + assert_eq!(open_wl(&mem).unwrap().generation(), 3); + // Both slots damaged: STG-032. + let mut mem = three_generations(); + damage(&mut mem, 0); + damage(&mut mem, 1); + same(&mem); + assert_eq!(code(open_wl(&mem).err().unwrap()), "STG-032"); + // Not a Minigraf file: STG-002. + let mut mem = MemoryBackend::new(); + for id in 0..3 { + mem.write_page(id, &vec![0xAA; PAGE_SIZE]).unwrap(); + } + same(&mem); + assert_eq!(code(open_wl(&mem).err().unwrap()), "STG-002"); + // An unsupported feature bit: STG-034. + let mut mem = three_generations(); + let mut m = match MetaPage::decode(&mem.read_page(slot_page(3)).unwrap()) { + SlotState::Valid(m) => m, + _ => panic!("gen 3 valid"), + }; + m.required_features = 1 << 40; + mem.write_page(slot_page(3), &m.encode()).unwrap(); + same(&mem); + assert_eq!(code(open_wl(&mem).err().unwrap()), "STG-034"); + // A meta page_count past the end of the file: STG-044. + let mut mem = three_generations(); + forge_page_count(&mut mem, 1 << 40); + same(&mem); + assert_eq!(code(open_wl(&mem).err().unwrap()), "STG-044"); + // A torn initial meta: initialised again. + let mut mem = MemoryBackend::new(); + let mut page = vec![0u8; PAGE_SIZE]; + page[..40].copy_from_slice(&MetaPage::empty(1).encode()[..40]); + mem.write_page(0, &page).unwrap(); + assert_eq!(open_wl(&mem).unwrap().generation(), 1); + // A damaged page: the open succeeds, the read fails, in both. + let mut mem = corruptible(); + let m = open_mem(&mem, None).unwrap().meta(); + let leaf = *leaves_of(m.eavt_root, &mem).first().expect("a leaf"); + flip(&mut mem, leaf); + same(&mem); + let pfs = open_wl(&mem).unwrap(); + let read = pfs + .storage() + .get_facts_by_attribute(&":test/n".to_string()) + .and_then(|_| pfs.verify().map(|f| f.problems)); + assert!( + read.map(|p| !p.is_empty()).unwrap_or(true), + "the damage is found" + ); + } + + /// Spec AC6: a short file that is not a new Minigraf database is STG-045 + /// in both modes, and is left as it was. + #[test] + fn wal_less_short_foreign_file_is_stg_045_and_unmodified() { + use crate::storage::backend::FileBackend; + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("short.graph"); + std::fs::write(&path, [0x55u8; 100]).unwrap(); + let native = PersistentFactStorage::open(FileBackend::open(&path).unwrap(), 16, None); + assert_eq!(code(native.err().unwrap()), "STG-045"); + let wal_less = PersistentFactStorage::open_wal_less(FileBackend::open(&path).unwrap(), 16); + assert_eq!(code(wal_less.err().unwrap()), "STG-045"); + assert_eq!( + std::fs::read(&path).unwrap(), + vec![0x55u8; 100], + "unmodified" + ); + } + + /// Spec AC7 natively: the image of a WAL-less database reopened after an + /// interrupted commit, written as a file, opens with `Minigraf::open` + /// (no WAL beside it) and verifies clean. + #[test] + fn wal_less_image_after_recovery_opens_with_minigraf_open() { + const PER: u128 = 100; + let base = saved(1, PER); + let n = writes_of_a_save(&base, PER..2 * PER); + let mem = deep_copy(&base); + interrupted_save(&mem, PER..2 * PER, n - 1, 16); + let pfs = open_wl(&mem).unwrap(); + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("export.graph"); + let mut bytes = Vec::new(); + pfs.write_image(pfs.generation(), OtherSlot::KeepOlder, |_, page| { + bytes.extend_from_slice(page); + Ok(()) + }) + .unwrap(); + std::fs::write(&path, &bytes).unwrap(); + let db = crate::Minigraf::open(&path).expect("opens natively"); + assert!(db.verify().unwrap().is_ok(), "verifies clean"); + let rows = match db + .execute("(query [:find (count ?e) :where [?e :test/n _]])") + .unwrap() + { + crate::QueryResult::QueryResults { results, .. } => results, + _ => panic!("a query result"), + }; + assert_eq!(rows.len(), 1); + assert!( + matches!(rows[0].first(), Some(Value::Integer(100))), + "the previous committed state" + ); + } + + /// An image stamped with a later generation (as an OPFS import stages + /// one) is the same database: it opens, answers and verifies the same. + #[test] + fn image_with_a_later_generation_is_the_same_database() { + let mem = saved(3, 50); + let pfs = open_wl(&mem).unwrap(); + for later in [ + pfs.generation(), + pfs.generation() + 1, + pfs.generation() + 10, + ] { + let mut img = MemoryBackend::new(); + pfs.write_image(later, OtherSlot::Zero, |id, page| img.write_page(id, page)) + .unwrap(); + let opened = open_mem(&img, None).expect("opens"); + assert_eq!(opened.generation(), later); + assert_eq!(values(&opened), values(&pfs)); + assert!(opened.verify().unwrap().problems.is_empty()); + } + assert!( + pfs.write_image(pfs.generation() - 1, OtherSlot::Zero, |_, _| Ok(())) + .is_err() + ); + } } diff --git a/src/storage/persistent_facts/browser_access.rs b/src/storage/persistent_facts/browser_access.rs index a5b3242d2e..af21c7db19 100644 --- a/src/storage/persistent_facts/browser_access.rs +++ b/src/storage/persistent_facts/browser_access.rs @@ -60,6 +60,12 @@ impl PersistentFactStorage { Ok(ids) } + /// Pages in the page cache now (#441). + #[cfg(test)] + pub(crate) fn cached_page_count(&self) -> usize { + self.page_cache.cached_page_count() + } + /// The active meta's `page_count`: the length of the file it describes. pub(crate) fn committed_page_count(&self) -> u64 { self.meta.page_count diff --git a/tests/fixtures/portability/README.md b/tests/fixtures/portability/README.md new file mode 100644 index 0000000000..8e2663a650 --- /dev/null +++ b/tests/fixtures/portability/README.md @@ -0,0 +1,12 @@ +# Portability fixtures (#441) + +The same database, built from `commands.txt`, must answer every query in +`queries.txt` (tab, then the JSON rows, sorted) in every mode and on every +platform. + +- `native_v8.graph`: written by the native build (checkpointed, no WAL). + Regenerate with `cargo test --test opfs_portability_test -- --ignored`. + The browser tests import it into OPFS mode. +- `opfs_export.graph`: `exportGraph()` of an OPFS-mode database built in a + browser. `tests/opfs_portability_test.rs` opens it with `Minigraf::open`, + checks `verify()` and the answers. diff --git a/tests/fixtures/portability/commands.txt b/tests/fixtures/portability/commands.txt new file mode 100644 index 0000000000..9e491c1423 --- /dev/null +++ b/tests/fixtures/portability/commands.txt @@ -0,0 +1,5 @@ +(transact [[:doc :body "A value longer than sixty-four bytes, so that it is kept once in a value page."]]) +(transact [[:alice :tag :a] [:alice :tag :b] [:alice :name "Alice"]]) +(retract [[:alice :tag :a]]) +(transact {:valid-from "2020-01-01" :valid-to "2021-01-01"} [[:alice :employer :old]]) +(transact {:valid-from "2021-01-01"} [[:alice :employer :new]]) diff --git a/tests/fixtures/portability/native_v8.graph b/tests/fixtures/portability/native_v8.graph new file mode 100644 index 0000000000000000000000000000000000000000..83b6a3616f51e8c80c709c571ca4e32d453985bc GIT binary patch literal 32768 zcmeI)O-~a+7zglqW@fjGl8TWLQV;IQ#Hf6Ll^8H0i3zAkxM{M51sh7Y&;{(lI|sPv z$>7CCDdT&UAm<%QLe(1N}o6 z9md!|-*9iV_W3U9e7y0^1m6&V00bZa0SG_<0uX=z1Rwwb2sDAfKwbR*_422uChm{N z1mZ2VnJ_C!MUu)b<#w`fj!$;0i8;>{w3rN12tWV=5P$##AOHafKmY;|fB*#cT;M7r z))UYd6Zf0+@u9&z?;9OJ00Izz00bZa0SG_<0uX=z1R$`JK(9OJ=Vt>q9~LHp88^D= z7u-_rR#ZMS9?s6V*>V(=PP?Tru8G{7{*>IwpcuJfVKi`a1=mj+DEbpYm#=5|*YCZJ z%>2BG|JemaimbnN#;9*_df{dF`myBa>hmkVGW6ip$y3KSh|xi=xUEFxNBah2&Viq` zD!Wxy>UKrDW%cHBqk(RqzF66|N>#pRD7-QLfn{j!`MIJVv#?v8%C}@s_k#@r5P$## zAOHafKmY;|fB*y_0D&C^I$0e5k1S6=GSBn>xA>oqT8eKs0ze&hJcNKTfF185Dj)y> z2tWV=5P$##AOHafKmY=}3v9;!<>42T=KcQq`2U|~{}pYS|JSqsif=RjFVj+&*G={7 z5dahaZ>i{rp8?Qi4Nn2cw4;mh9Kh~(8MP3A00bZa0SG_<0uX=z1R${A1>*RhvDMov z@67sr)%btE)8P6c009U<00Izz00bZa0SG_<0!<{~vkbd`es$5T+h2|U57UXIbb4iZ zh(CFFXQaFT^%7S^ZSQ+NK6`WJ`q{;>{@uxGXmnHDB@u|e67nhRY zx>=Ik%Q7LAWqE(vAiKq@woEDM#VwH|y+Sb0grz8zEJmOks%1qjD{EQHWZ4nBam9ls f`aj@OAOHafKmY;|fB*y_009U9Oh7)e559Mt5A!mV0b;`8->8!R2vfA_52?fB*sr zAbGpoS8xL^x?mTz=oEgq@cjCqmT$zXf z0tg_000IagfB*srAb5I_I{1Q0*~0R#|0 z0D<2K3`9rb(%~d3Rmx+@L{#4&m!sPFv3g_WzRKZ=Xt+^NYAd2zC2grkbdp#ZXRE3&{E6bcNY_KyMQ>pc43x%45|dzRwGKm_X?`ro z%HDl%L_r?8xvpx)EW(~_^Mfs!WNZi^fB*srAb1n`~UHc!Mkof z|6l9>%SC}!KN|s%i^FsXfiZv?e?=M)KmY**5I_I{1Q0*~0R#~EwLq)?zwTk~di!(y z>HPm6&HiiIHveyC|F!zb{J)>~P5qRqelr5#`u}YWi_f^{-4T z0tg_000IagfB*srAb`OCE|B*Bg?M!Q@(VZb@1Fnv-(|Re1Q0*~0R#|0009ILKmY** zI!PcF1#$Aq?hEd{{capU%$Li~?kz zIjVG4d6m+r(i!E|z^+5I_I{ z1Q0*~0R#}}IDuaAxa01L8$tj91Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009IL zKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~0R#|0009ILKmY**5I_I{1Q0*~ z0R#|0009ILK;T~#I3WV@YUtk6Irhu_c>ljGXKn1VvD?O+jXgFJ5I_I{1Q0*~0R#|0 N009ILK;YjM_y#O`DdhkF literal 0 HcmV?d00001 diff --git a/tests/fixtures/portability/queries.txt b/tests/fixtures/portability/queries.txt new file mode 100644 index 0000000000..ac7670d00d --- /dev/null +++ b/tests/fixtures/portability/queries.txt @@ -0,0 +1,6 @@ +(query [:find ?b :where [:doc :body ?b]]) [["A value longer than sixty-four bytes, so that it is kept once in a value page."]] +(query [:find ?t :where [:alice :tag ?t]]) [[":b"]] +(query [:find ?t :as-of 2 :where [:alice :tag ?t]]) [[":a"],[":b"]] +(query [:find ?o :valid-at "2020-06-01" :where [:alice :employer ?o]]) [[":old"]] +(query [:find ?o :valid-at "2022-06-01" :where [:alice :employer ?o]]) [[":new"]] +(query [:find ?n :where [?e :name ?n] [?e :tag :b]]) [["Alice"]] diff --git a/tests/opfs_portability_test.rs b/tests/opfs_portability_test.rs new file mode 100644 index 0000000000..65b1156967 --- /dev/null +++ b/tests/opfs_portability_test.rs @@ -0,0 +1,101 @@ +//! Portability between native files and the browser's OPFS mode (#441, +//! spec AC11). The database `tests/fixtures/portability/commands.txt` +//! builds answers `queries.txt` the same everywhere: +//! +//! - `native_v8.graph`, a checkpointed native file the browser tests import +//! into OPFS mode, answers here as written (regenerated by the ignored +//! test below); +//! - `opfs_export.graph`, the `exportGraph()` of an OPFS database built in +//! a browser, opens here with `Minigraf::open` (no WAL beside it), passes +//! `verify()` and gives the same answers. +#![cfg(not(target_arch = "wasm32"))] + +use minigraf::{Minigraf, QueryResult, Value}; +use std::path::Path; + +const COMMANDS: &str = include_str!("fixtures/portability/commands.txt"); +const QUERIES: &str = include_str!("fixtures/portability/queries.txt"); + +/// A value as `BrowserDb` encodes it in JSON. +fn json(v: &Value) -> serde_json::Value { + match v { + Value::String(s) | Value::Keyword(s) => serde_json::Value::String(s.clone()), + Value::Integer(i) => (*i).into(), + Value::Boolean(b) => (*b).into(), + Value::Null => serde_json::Value::Null, + Value::Ref(u) => u.to_string().into(), + _ => serde_json::Value::String("unsupported value".into()), + } +} + +fn assert_answers(db: &Minigraf, source: &str) { + for line in QUERIES.lines().filter(|l| !l.is_empty()) { + let (query, expected) = line.split_once('\t').expect("queryrows"); + let expected: serde_json::Value = serde_json::from_str(expected).expect("rows"); + let QueryResult::QueryResults { results, .. } = db.execute(query).expect("query runs") + else { + panic!("{source}: not a query result"); + }; + let mut rows: Vec = results + .iter() + .map(|r| r.iter().map(json).collect()) + .collect(); + rows.sort_by_key(ToString::to_string); + assert_eq!( + serde_json::Value::Array(rows), + expected, + "{source}: answer to {query}" + ); + } +} + +fn open_copy(fixture: &Path) -> (tempfile::TempDir, Minigraf) { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("copy.graph"); + std::fs::copy(fixture, &path).expect("copy fixture"); + let db = Minigraf::open(&path).expect("opens without a WAL"); + (dir, db) +} + +fn fixture(name: &str) -> std::path::PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/portability") + .join(name) +} + +#[test] +fn native_fixture_answers_the_portability_queries() { + let (_dir, db) = open_copy(&fixture("native_v8.graph")); + assert!(db.verify().expect("verify runs").is_ok(), "verifies clean"); + assert_answers(&db, "native_v8.graph"); +} + +#[test] +fn opfs_export_opens_natively_verifies_and_answers() { + let bytes = std::fs::read(fixture("opfs_export.graph")).expect("fixture"); + assert_eq!(&bytes[0..4], b"MGRF", "starts with the magic"); + assert_eq!(bytes.len() % 4096, 0, "whole pages"); + let (_dir, db) = open_copy(&fixture("opfs_export.graph")); + assert!(db.verify().expect("verify runs").is_ok(), "verifies clean"); + assert_answers(&db, "opfs_export.graph"); +} + +/// Regenerate `native_v8.graph` from `commands.txt`. +#[test] +#[ignore] +fn regenerate_native_fixture() { + let dir = tempfile::tempdir().expect("tempdir"); + let path = dir.path().join("native.graph"); + { + let db = Minigraf::open(&path).expect("create"); + for command in COMMANDS.lines().filter(|l| !l.is_empty()) { + db.execute(command).expect("command"); + } + db.checkpoint().expect("checkpoint"); + } + assert!( + !dir.path().join("native.graph.wal").exists(), + "checkpointed, no WAL" + ); + std::fs::copy(&path, fixture("native_v8.graph")).expect("write fixture"); +} From 1f1290aaef612ffb8fabc34d1c90e85a1a692846 Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Sat, 14 Nov 2026 09:28:20 +0530 Subject: [PATCH 03/20] ci(browser): run OPFS tests in Chrome and Firefox workers (#441) Document the OPFS error codes alongside the registry and retain main-thread browser coverage. Refs #441 Co-Authored-By: GPT-6 --- .github/workflows/wasm.yml | 21 ++++++++++++++++++ docs/ERROR_REFERENCE.md | 44 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+) diff --git a/.github/workflows/wasm.yml b/.github/workflows/wasm.yml index 003f5cf519..b1b89dd2fc 100644 --- a/.github/workflows/wasm.yml +++ b/.github/workflows/wasm.yml @@ -38,6 +38,27 @@ jobs: - name: wasm-bindgen tests (lib, incl. BrowserDb and IndexedDB) run: wasm-pack test --headless --${{ matrix.browser }} --features browser + browser-worker: + name: Browser worker tests (headless ${{ matrix.browser }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + browser: [firefox, chrome] + env: + # wasm-bindgen configures the whole lib test binary for a dedicated + # worker; the browser job above retains main-thread coverage. + RUSTFLAGS: --cfg minigraf_browser_worker + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + targets: wasm32-unknown-unknown + - name: Install wasm-pack + run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh + - name: wasm-bindgen worker tests (OPFS and IndexedDB) + run: wasm-pack test --headless --${{ matrix.browser }} --features browser --lib + wasi: name: WASI tests (Wasmtime) runs-on: ubuntu-latest diff --git a/docs/ERROR_REFERENCE.md b/docs/ERROR_REFERENCE.md index 875caa8224..96b8743fbc 100644 --- a/docs/ERROR_REFERENCE.md +++ b/docs/ERROR_REFERENCE.md @@ -208,6 +208,7 @@ with no `CodedError` anywhere in its chain. | STG-045 | File shorter than one page and not a new database | Storage | | STG-046 | Earlier checkpoint failed on this handle | Storage | | STG-047 | WAL is from a later checkpoint | Storage | +| STG-048 | Browser storage operation failed | Storage | | WAL-001 | Invalid WAL magic number | WAL | | WAL-002 | Unsupported WAL version | WAL | | WAL-003 | Value size exceeds maximum | WAL | @@ -240,6 +241,8 @@ with no `CodedError` anywhere in its chain. | API-023 | Bind slot not permitted in attribute position | Database API | | API-024 | Bind slot type mismatch | Database API | | API-025 | Query option override not allowed | Database API | +| API-026 | OPFS mode is not available here | Database API | +| API-027 | Database is busy | Database API | | INT-000 | Unclassified internal error | Internal | | INT-001 | Write transaction already in progress on this thread | Internal | | INT-002 | Invalid entity (API layer) | Internal | @@ -3167,6 +3170,21 @@ Nothing committed is lost. Every committed transaction is still in the WAL, and --- +### STG-048 Browser storage operation failed + +**Error text**: `browser storage operation failed: {} ({})` + +**Cause**: An OPFS operation failed: opening an entry, acquiring its sync access handle, reading, writing, flushing, checking size, truncating, or staging an import. The message names the operation and the browser exception (for example `QuotaExceededError`), or a short read/write. Short transfers are never accepted as complete pages. Lock contention uses STG-026; an unavailable OPFS context uses API-026. + +**Resolution**: +- Check the operation and exception name, available origin quota and browser permissions. +- After a commit may have written, later writes fail with STG-046. Call `close()` and reopen. Reads may still include the unconfirmed transaction; a rejected commit does not prove it is absent on disk. +- An import failure at publication closes the database; reopen to recover the complete old or new image. + +**Scenario**: Origin quota is exhausted while a worker writes a data page; `execute()` rejects with STG-048 naming the write and `QuotaExceededError`. + +--- + ## WAL — Write-Ahead Log Errors WAL errors relate to the sidecar `.wal` file written alongside the `.graph` file. @@ -3667,6 +3685,32 @@ ValueSet::new(vec![inner]) // API-021 --- +### API-026 OPFS mode is not available here + +**Error text**: `OPFS mode is not available here: {}` + +**Cause**: `BrowserDb.openOpfs(name)` requires a secure context and a dedicated Web Worker with OPFS and sync access handles. It is unavailable on the main thread, in shared/service workers, when the APIs are absent, or when the browser denies access to the origin directory (for example `SecurityError`). It never falls back to IndexedDB. + +**Resolution**: +- Run the call in a dedicated worker served over HTTPS or localhost. +- Check browser support and origin access policy. Choose `BrowserDb.open(name)` explicitly if IndexedDB suits the application. + +**Scenario**: A page calls `BrowserDb.openOpfs('memory')` on its main thread. + +### API-027 Database is busy + +**Error text**: `the database is busy: {}` + +**Cause**: A call starts work while an OPFS import is in progress, or `BrowserDb.close()` is called reentrantly while a database operation is running. Busy calls change nothing. Closed objects (API-018) take precedence over busy; busy takes precedence over an open transaction (INT-001). Disposal operations remain permitted. + +**Resolution**: +- Await `importGraph()` before making other calls, including `close()` or another import. +- Call `close()` after the current operation returns, rather than inside its UDF. + +**Scenario**: A worker calls `query()` while `importGraph()` is awaiting staging I/O. + +--- + ## INT — Internal Errors Internal errors are invariant violations or edge cases that are not expected From 6729e5204931294126efe6a8e767688c383ed14a Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Sat, 14 Nov 2026 09:41:38 +0530 Subject: [PATCH 04/20] docs(browser): document OPFS lifecycle and recovery guarantees (#441) Add reproducible bounded-memory probes and distinguish outstanding browser measurements from completed native and WASI checks. Refs #441 Co-Authored-By: GPT-6 --- CHANGELOG.md | 3 + CLAUDE.md | 8 +- PHILOSOPHY.md | 4 +- README.md | 4 +- ROADMAP.md | 3 +- docs/DURABILITY.md | 73 ++++++- docs/ERROR_REFERENCE.md | 24 +++ docs/TEST_COVERAGE.md | 46 ++++- docs/benchmarks/441-opfs/README.md | 70 +++++++ docs/benchmarks/441-opfs/build-fixtures.sh | 18 ++ docs/benchmarks/441-opfs/fixture.rs | 43 ++++ docs/benchmarks/441-opfs/measure-worker.js | 54 +++++ docs/benchmarks/441-opfs/measure.mjs | 64 ++++++ .../2026-10-05-v8-storage-format-design.md | 7 + .../2026-11-14-441-browser-opfs-design.md | 194 ++++++++++++++++++ 15 files changed, 600 insertions(+), 15 deletions(-) create mode 100644 docs/benchmarks/441-opfs/README.md create mode 100644 docs/benchmarks/441-opfs/build-fixtures.sh create mode 100644 docs/benchmarks/441-opfs/fixture.rs create mode 100644 docs/benchmarks/441-opfs/measure-worker.js create mode 100644 docs/benchmarks/441-opfs/measure.mjs create mode 100644 docs/superpowers/specs/2026-11-14-441-browser-opfs-design.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 8346b3849b..837265bda2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **OPFS-backed `BrowserDb` persistence and explicit close** (#441). `await BrowserDb.openOpfs(name)` opens one `.graph` file through an exclusive sync access handle in a dedicated worker on a secure origin. It uses the default 256-page cache, reads pages on demand and commits by copy-on-write with data/meta flushes, without a WAL or a whole-file buffer. IndexedDB `open(name)` now also works in workers. The two modes have independent namespaces and no automatic fallback. OPFS starts at Tier 2 within the Tier 1 browser binding. All modes gain synchronous, idempotent `close()`, which saves nothing, discards an open transaction, invalidates dependents and immediately releases an OPFS lock. OPFS import validates the whole image, stages and publishes the complete old or new database across interruption, and invalidates earlier cursors/prepared queries after publication. OPFS recovery accepts the sole valid meta; unlike native recovery this can silently fall back after later meta rot. A failed commit is unconfirmed (it may be stored), leaves reads working, and refuses later writes with STG-046 until reopen. New registered errors API-026 (unavailable OPFS context), API-027 (busy database) and STG-048 (browser operation and exception name); lock contention is STG-026. No file-format change or new crate dependency. See [durability](docs/DURABILITY.md#browser-storage-modes) and [measurement status](docs/benchmarks/441-opfs/README.md). + + - **`OpenOptions::allow_overrides`** (#869, part of #862): the embedder lists, as `Overridable` variants, the query keywords that may override the database's settings. The default, `Overridable::DEFAULTS` (`MaxDerivedFacts`, `MaxResults`), allows every keyword that could do so before, so no query changes. A query using a keyword the list leaves out is refused with the new **API-025** before it runs, from `execute`, `query`, `prepare` and `WriteTransaction::execute`; the list replaces the defaults rather than adding to them, and is not stored in the file. `BrowserDb` takes no options and uses the defaults. `Overridable` is `#[non_exhaustive]`. - **`:query-memory-limit`, limit bind slots and `PreparedQuery` limit setters** (#870, part of #862). `:query-memory-limit N` replaces `OpenOptions::query_memory_limit` for one query, up or down; `0` is allowed and lets only a query that holds no state run (QRY-012 otherwise). It is gated by the new `Overridable::QueryMemoryLimit`, which is **not** in `Overridable::DEFAULTS`, so a database refuses it with API-025 unless the embedder lists it; `BrowserDb` always refuses it. `:max-derived-facts`, `:max-results` and `:query-memory-limit` take a `$slot` filled with `BindValue::Val(Value::Integer(n))` (INT-014/INT-015 out of range as for a literal, API-024 for another type, API-022 when missing). `PreparedQuery::with_max_derived_facts` and `with_query_memory_limit` replace the database values for one statement; they are not gated, and an allowed keyword still overrides them. See [docs/CONFIGURATION.md](docs/CONFIGURATION.md). The v3 binary-size gate rises from 1,234 to 1,240 KiB (+672 B measured here; the rest is room for #861). diff --git a/CLAUDE.md b/CLAUDE.md index 1c8f469166..d3447986b2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -195,9 +195,9 @@ cargo run < demos/demo_negation.txt 7. **`src/wal.rs`** — Fact-level sidecar WAL, CRC32-protected entries, crash recovery; `append_entry` takes the `SyncMode` of each append (#871), and in tests the writer tracks its synced length (lost-sync fault, `Minigraf::power_loss`). Like `storage::verify` and `storage::dir_sync`, not compiled for wasm32 (no file backend there) -8. **`src/error.rs`** — Structured error codes: `MinigrafError`, `ErrorCategory`, `ErrorCode` registry (PRS/QRY/STG/WAL/API/INT codes); must match `docs/ERROR_REFERENCE.md`. `MinigrafError::invalid_argument` (API-017) and `MinigrafError::closed` (API-018) exist for language bindings +8. **`src/error.rs`** — Structured error codes: `MinigrafError`, `ErrorCategory`, `ErrorCode` registry (PRS/QRY/STG/WAL/API/INT codes); must match `docs/ERROR_REFERENCE.md`. `MinigrafError::invalid_argument` (API-017) and `MinigrafError::closed` (API-018) exist for language bindings; browser OPFS adds API-026 (unavailable context), API-027 (busy) and STG-048 (failed browser storage operation) -9. **`src/browser/`** — Browser WASM backend (`browser` feature): `buffer.rs` (`BrowserBufferBackend`, in-memory pages with dirty tracking; free pages a pinned generation may read are withheld from release, `pins::split_released`), `indexeddb.rs` (IndexedDB persistence); `BrowserDb.query` returns a `BrowserCursor` (JSON row batches); `BrowserDb.prepare` returns a `BrowserPreparedQuery` (`execute`/`query` with a bindings object; `withMaxDerivedFacts`/`withQueryMemoryLimit`/`withMaxIterations` return a new one, #872; read by `binds.rs`: plain JS values or `{type, value}` forms, #181; a set as `{type: 'set', value: [..]}` or a JS `Set`, #316); `BrowserDb.beginWrite` returns a `BrowserWriteTransaction` (#824: sync `execute` staging through `db::Staged`, `commit()` a `Promise` resolved after the IndexedDB flush, `rollback()`; `BrowserDbInner::tx_open` makes every other read/write call `INT-001`; finished = `API-018`; tests in `tx_tests.rs`); `udfs.rs`: `registerPredicate`/`registerAggregate` (#180; values as `{type, value}` objects, a throw is `QRY-014`; JS values held in a thread-local table, the registry keeps ids). Tests here must be `#[wasm_bindgen_test]` (the module only builds for wasm32); `.github/workflows/wasm.yml` runs them in headless browsers, runs the WASI tests under Wasmtime, and lints both wasm targets. New file-backed integration test files need `#![cfg(not(target_arch = "wasm32"))]` +9. **`src/browser/`** — Browser WASM backend (`browser` feature): `backend.rs` (buffer/OPFS dispatch), `opfs.rs` (`OpfsBackend` over an exclusive dedicated-worker sync access handle, one `.graph` file and a 256-page cache; no WAL; sole-valid-meta recovery; crash-atomic import stages `.graph-import`), `buffer.rs` (`BrowserBufferBackend`, in-memory pages with dirty tracking; free pages a pinned generation may read are withheld from release, `pins::split_released`), `indexeddb.rs` (IndexedDB persistence on the main thread or in workers); `openOpfs(name)` selects OPFS explicitly (Tier 2 in v3.0.0); all modes have synchronous `close()`, invalidating dependents and releasing the OPFS lock, with no save; `BrowserDb.query` returns a `BrowserCursor` (JSON row batches); `BrowserDb.prepare` returns a `BrowserPreparedQuery` (`execute`/`query` with a bindings object; `withMaxDerivedFacts`/`withQueryMemoryLimit`/`withMaxIterations` return a new one, #872; read by `binds.rs`: plain JS values or `{type, value}` forms, #181; a set as `{type: 'set', value: [..]}` or a JS `Set`, #316); `BrowserDb.beginWrite` returns a `BrowserWriteTransaction` (#824: sync `execute` staging through `db::Staged`, `commit()` a `Promise` resolved after the IndexedDB flush, `rollback()`; `BrowserDbInner::tx_open` makes every other read/write call `INT-001`; finished = `API-018`; tests in `tx_tests.rs`); `udfs.rs`: `registerPredicate`/`registerAggregate` (#180; values as `{type, value}` objects, a throw is `QRY-014`; JS values held in a thread-local table, the registry keeps ids). Tests here must be `#[wasm_bindgen_test]` (the module only builds for wasm32); `.github/workflows/wasm.yml` runs them in headless browsers, runs the WASI tests under Wasmtime, and lints both wasm targets. New file-backed integration test files need `#![cfg(not(target_arch = "wasm32"))]` ### Data Model @@ -250,7 +250,7 @@ Auto-migrates v7 → v8 on open (spec §9, with a backup meta page). v1–v6 are ## Test Coverage -**2159 tests** (2138 active, 21 ignored; unit + integration + doc). Engine answers once checked by the executor are frozen in `tests/answers/` (#551). +**2169 tests** (2147 active, 22 ignored; unit + integration + doc). Engine answers once checked by the executor are frozen in `tests/answers/` (#551). See `docs/TEST_COVERAGE.md` for the full per-file breakdown. The #760 deterministic storage suite adds three native unit tests in a separate `--cfg minigraf_shuttle` build, outside the default count. Run only `graph::storage::deterministic::` with that cfg; ordinary unit tests do not enter its scheduler. `deterministic-schedules.yml` runs relevant v3 PRs on Linux/macOS/Windows plus Linux MSRV, and larger nightly/manual searches with replay artifacts. The identical main workflow activates the v3 schedule after both PRs merge. See `docs/testing/deterministic-schedules.md` for scope, controls and findings. @@ -273,7 +273,7 @@ Current plan (see `ROADMAP.md` and the milestones): - **v3.0.0** (format v8 and data integrity, plus `:limit` #306, optional `:distinct` output #493, `lag`/`lead` #182, sliding row frames #183, query profiler #185, set membership predicates #316 and prepared queries in every binding #181, UDF registration in every binding #180; [v3.0.0 milestone](https://github.com/project-minigraf/minigraf/milestone/1)) is developed on the long-lived `v3` branch. All v3 PRs target `v3`; `main` stays untouched. CI runs on PRs to `v3` the same as on PRs to `main`. Do not merge `main` into `v3` routinely: `main` only gets backports of fixes already on `v3`. Shared CI and workflow changes land on both branches as separate PRs. Just before the v3.0.0 cut, merge `main` into `v3` once (merge commit; source conflicts resolved to `v3`, `main`'s v2.0.x CHANGELOG sections brought over) so the cut is a fast-forward. At the first v3 release, the current `main` is copied to `v2` and `v3` becomes the new `main`. - **Input order (#641)** targets v3.0.0 in three slices: committed scalar Sort omission, one-group count/count-distinct and partition-at-a-time windows are implemented in the dependent PRs for `v3`. The reviewed design, fallback scope and acceptance gates are in `docs/superpowers/specs/2026-10-28-641-input-order-design.md`. - **Scalar key decoder proofs (#693)** target v3.0.0 before the release cut. The pure decoder refactor preserves key bytes and INT-049 behavior; Kani covers round trips and bounded malformed inputs. The approved design and CI limits are in `docs/superpowers/specs/2026-10-29-693-decoder-proofs.md`. -- **v3.next** (the release after v3.0.0) holds new features (rule persistence #241; OPFS `BrowserDb` #441). +- **v3.next** (the release after v3.0.0) holds rule persistence (#241). OPFS `BrowserDb` (#441) targets v3.0.0, starts at Tier 2, and is implemented on its feature branch with browser evidence still pending. - Known v2.x data issue: same-transaction multi-valued facts (#371). Fix ships in v3.0.0. All v2.x known issues are listed in pinned issue #421 (`known-issue` label); `data-integrity`/`corruption`/`durability` issues fixed in a v2.x patch stay open until that patch is published; issues fixed on the unreleased `v3` branch close on merge (closing keywords do not fire on `v3`, so close them with `gh issue close`) and stay listed in #421 (CONTRIBUTING.md). - Support policy and tiers: PHILOSOPHY.md §7 and §10. v2.x gets data-integrity and security fixes for 12 months after v3.0.0. Tier 1 = Rust + Python + browser WASM (from v3.0.0); other bindings are Tier 2. diff --git a/PHILOSOPHY.md b/PHILOSOPHY.md index 935cd01bf5..c0da75c2ee 100644 --- a/PHILOSOPHY.md +++ b/PHILOSOPHY.md @@ -97,7 +97,7 @@ SQLite's success comes from a clear philosophy: be a library, not a server. Be s - Easy to backup: copy one file - Easy to share: email, USB drive, version control (for small DBs) - Easy to delete: remove one file -- WASM: Store in browser's IndexedDB as single blob +- WASM: IndexedDB stores page records; OPFS mode (v3.0.0, dedicated worker) stores one portable `.graph` file, with a transient staging file only during import/recovery **Anti-pattern**: Multiple files, directories, or complex file structures that are hard to manage. @@ -267,7 +267,7 @@ Data-integrity bugs stay visible until they are fixed in a published release; se A binding moves to Tier 1 when real users need it: issues from outside users, or known dependents. The project's own tools count as dependents when they ship to users and rely on the binding: Python is Tier 1 because `temporal_reasoning` stores its memory through it, and browser WASM is Tier 1 because the playground, the graph visualizer and the docs site run on it. A promotion takes effect at a major or minor release, never in a patch. No binding is removed by being Tier 2. -A new storage backend inside a Tier 1 binding starts at Tier 2 until it has shipped in a release; the OPFS backend for browser WASM (#441, v3.next) is the current example. The IndexedDB backend stays Tier 1. +A new storage backend inside a Tier 1 binding starts at Tier 2 until it has shipped in a release; the OPFS backend for browser WASM (#441, v3.0.0) is the current example. The IndexedDB backend stays Tier 1. | Tier | Platform and filesystem | |---|---| diff --git a/README.md b/README.md index 3b4d72e6db..e3cf225198 100644 --- a/README.md +++ b/README.md @@ -281,7 +281,7 @@ See the [Mobile Integration](https://project-minigraf.github.io/minigraf-docs/la ### For WASM / Browser -Published as [`@minigraf/browser`](https://www.npmjs.com/package/@minigraf/browser) on npm (IndexedDB-backed, `wasm-pack`). `BrowserDb.beginWrite()` opens a write transaction (from v3.0.0): `execute` stages and reads back, `await tx.commit()` stores it in IndexedDB, `rollback()` discards it, and the database's other calls throw `[INT-001]` until it ends. The WASI package is the REPL: each command commits on its own, with no explicit transactions. WASI build (`wasm32-wasip1`) available as [`@minigraf/wasi`](https://www.npmjs.com/package/@minigraf/wasi) on npm and as a GitHub Releases artifact (Wasmtime / Wasmer). See the [Use Cases](https://project-minigraf.github.io/minigraf-docs/latest/use-cases/). The [playground](https://minigraf-playground.vercel.app/) and the [time travel visualizer](https://project-minigraf.github.io/minigraf-visualizer/) are both built on `@minigraf/browser`. +Published as [`@minigraf/browser`](https://www.npmjs.com/package/@minigraf/browser) on npm (`wasm-pack`). `BrowserDb.open(name)` uses IndexedDB on the main thread or in a worker. From v3.0.0, `await BrowserDb.openOpfs(name)` uses one `.graph` file in OPFS, in a dedicated worker on HTTPS or localhost, with a 256-page cache instead of a whole-file memory buffer. OPFS starts at Tier 2 inside the Tier 1 browser binding; the modes have separate namespaces and never fall back to each other. `close()` is synchronous and releases the database and its lock in every mode. See the [complete module-worker example](https://github.com/project-minigraf/minigraf-wasm/tree/v3#opfs-worker) and [browser durability contract](docs/DURABILITY.md#browser-storage-modes). `BrowserDb.beginWrite()` opens a write transaction (from v3.0.0): `execute` stages and reads back, `await tx.commit()` stores it in IndexedDB or flushes it to OPFS, `rollback()` discards it, and the database's other calls throw `[INT-001]` until it ends. The WASI package is the REPL: each command commits on its own, with no explicit transactions. WASI build (`wasm32-wasip1`) available as [`@minigraf/wasi`](https://www.npmjs.com/package/@minigraf/wasi) on npm and as a GitHub Releases artifact (Wasmtime / Wasmer). See the [Use Cases](https://project-minigraf.github.io/minigraf-docs/latest/use-cases/). The [playground](https://minigraf-playground.vercel.app/) and the [time travel visualizer](https://project-minigraf.github.io/minigraf-visualizer/) are both built on `@minigraf/browser`. ### For Python / Node.js / Java / C @@ -292,7 +292,7 @@ Language bindings ship as `minigraf` on PyPI, `minigraf` on npm (Node.js native Minigraf runs as: - ✅ An embedded library - ✅ A standalone binary (interactive REPL) -- ✅ Browser WASM — `@minigraf/browser` (IndexedDB-backed, `wasm-pack`) +- ✅ Browser WASM — `@minigraf/browser` (IndexedDB or OPFS in a dedicated worker, `wasm-pack`; OPFS starts at Tier 2 in v3.0.0) - ✅ Server-side WASM — `wasm32-wasip1` / WASI (Wasmtime, Wasmer, Cloudflare Workers) - ✅ Android, iOS, Python, Node.js, Java, C — via UniFFI / napi-rs / cbindgen diff --git a/ROADMAP.md b/ROADMAP.md index 4277166ddc..2c5f1982b0 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -112,6 +112,8 @@ Scope — query API: Scope — features: +- OPFS-backed persistent mode for `BrowserDb`: a sync access handle in a dedicated Worker, no in-memory buffer; IndexedDB stays the main-thread backend. Starts at Tier 2 inside the Tier 1 browser binding until it has shipped (#441; implemented on the feature branch, browser evidence pending) + - Query profiler (#185) - Set values `#{...}`, the set predicates `in?`/`not-in?`/`any-in?`/`all-in?`/`none-in?` and `collect-set` (#316; done on `v3`: core, `BrowserDb` set binds, and set values in Python, Java, Android, Swift, Node.js, C and the browser/WASI packages on their `v3` branches) - Prepared queries in every binding (#181; done on `v3`: core, `BrowserDb.prepare`, and Python, Java, Android, Swift, Node.js and C on their `v3` branches) @@ -147,7 +149,6 @@ Scope — release-gate testing and documentation: Additive features, kept out of v3.0.0 so the #371 fix is not held up by them: - Rule persistence — design first (#241) -- OPFS-backed persistent mode for `BrowserDb`: a sync access handle in a dedicated Worker, no in-memory buffer; IndexedDB stays the main-thread backend. Starts at Tier 2 inside the Tier 1 browser binding until it has shipped (#441) - Temporal graph-traversal research (#273; may result in documentation or a blog post rather than code) --- diff --git a/docs/DURABILITY.md b/docs/DURABILITY.md index 5410fcb78e..f53900254f 100644 --- a/docs/DURABILITY.md +++ b/docs/DURABILITY.md @@ -56,10 +56,81 @@ For example, a database opened `Full` can run a bulk load as `Normal` transactio **What durability assumes.** A sync is only as good as the storage under it. Minigraf assumes that when `fdatasync`/`fsync` (`FlushFileBuffers` on Windows) returns, the data is on stable storage. Disks or controllers with a volatile write cache that ignore flushes, virtual machines with an "unsafe" disk cache mode, and some network or FUSE filesystems break that assumption, and a power loss there can lose committed transactions or tear a page. A torn or rotted page is detected by its checksum on the next read (`STG-029`); it is not silently read. -**Browser (`BrowserDb`).** There is no WAL. Each `transact` and `retract` is a checkpoint whose pages, meta page and freed page keys are written in one IndexedDB `readwrite` transaction, and the returned promise resolves after that transaction completes. Closing the tab before then loses that write, whole. How durable a completed IndexedDB transaction is on a power loss is up to the browser. +**Browser IndexedDB (`BrowserDb.open`).** There is no WAL. Each `transact` and `retract` is a checkpoint whose pages, meta page and freed page keys are written in one IndexedDB `readwrite` transaction, and the returned promise resolves after that transaction completes. Closing the tab before then loses that write, whole. How durable a completed IndexedDB transaction is on a power loss is up to the browser. **WASI.** The `wasm32-wasip1` build has no file backend: its databases are in memory and end with the process. +## Browser storage modes + +From v3.0.0 `BrowserDb.openOpfs(name)` is an explicit second persistent mode, +starting at Tier 2 inside the Tier 1 browser binding. It requires a dedicated +worker in a secure context (HTTPS or localhost). `BrowserDb.open(name)` keeps +its IndexedDB mode, now also available in workers; `openInMemory()` is transient. +The same name in the two persistent modes identifies separate databases. There +is no automatic fallback or migration. Move data explicitly with `exportGraph()` +and `importGraph()`; exported v8 bytes open natively without a WAL, and OPFS can +import supported v7 files through migration. + +OPFS maps the name exactly to `.graph` in the origin's root directory. +Empty names, `.`, `..`, slash/backslash and reserved import names are API-017. +The exclusive sync access handle is the lock: another open in the same worker, +another worker or tab fails with STG-026. `close()` releases it immediately; +worker termination also releases it. A refused open releases any handle it took. +Unsupported contexts or denied origin directory access are API-026, with no +fallback. Other storage failures are STG-048, naming the operation and browser +exception (or short transfer). + +Every OPFS transaction commits directly through v8 copy-on-write: write data +pages, `flush()`, write the alternate meta, `flush()`. Its Promise resolves after +both flushes; there is no WAL or separate asynchronous persistence step. On +interruption, reopen selects the complete previous or new commit, retaining +acknowledged commits subject to the browser's storage guarantees. Worker kills +exercise process interruption, not power loss. Browser `flush()` only attempts +to transfer cached changes to underlying storage; it is not a power-loss promise. + +With exactly one valid meta page, OPFS opens at it regardless of later-generation +pages or damage in the other slot. This recovers an interrupted WAL-less commit. +It also gives up native STG-033's protection against silently losing a completed +checkpoint when its newest meta later rots. Native recovery is unchanged. +With two valid slots the higher generation wins; other integrity checks remain. + +A rejected commit is **unconfirmed**, not necessarily absent: a short/failed meta +write may still have installed a complete valid meta. Later writes/checkpoints +fail with STG-046 until close/reopen. Reads may include the attempted transaction. +`close()` saves nothing and does not retry the failed commit. Reopen and inspect +before retrying the transaction. + +`close()` is synchronous and idempotent in every mode. It discards open staged +transactions and invalidates earlier dependents (API-018), even while they remain +referenced. Reentrant close inside a running query's UDF or close during OPFS +import is API-027 and leaves the database open. Disposal operations remain valid. +Deferred execute/checkpoint/import bodies that start after close are API-018. +An already applied OPFS/in-memory transaction commit resolves after close; an +IndexedDB commit whose flush has not been submitted rejects with API-018 and +never submits it. An already submitted IndexedDB flush settles by its own result. + +OPFS import first validates and verifies its complete input, then stages it in +`.graph-import`. Publication makes the new image recoverable before copying +it over the main file; interruption and reopen finish with the complete old or +new image. A pre-publication failure preserves the old database and dependents; +a publication failure closes the handle because the outcome is unknown. Success +invalidates earlier cursors/prepared queries. Cleanup failure after publication +does not reject the import; a later open finishes it. The main file stays locked +during staging, publication and recovery. Calls admitted while importing are +API-027; API-018 takes precedence, and busy takes precedence over INT-001. + +OPFS reads healthy v8 pages on demand through a 256-page cache. The storage bound +excludes query working state, results/export blobs, import input, v7 migration, +transaction facts, pages and quarantine metadata held for long-open cursors, +and recovery scans. [The measurement record](benchmarks/441-opfs/README.md) +separates inherited browser test results from outstanding engine and memory runs. + +Both persistent modes use browser-managed origin quota and eviction. Applications +may request `navigator.storage.persist()` themselves; Minigraf never requests it. +Quota errors during OPFS commit are STG-048 and follow the failed-handle rule. +Private-browsing behavior is engine/policy dependent; checks for this change +remain unrun in the current sandbox, as recorded in the measurement record. + ## Opening after a crash Opening a file-backed database: diff --git a/docs/ERROR_REFERENCE.md b/docs/ERROR_REFERENCE.md index 96b8743fbc..97106a7356 100644 --- a/docs/ERROR_REFERENCE.md +++ b/docs/ERROR_REFERENCE.md @@ -2884,6 +2884,11 @@ Read-only handles (`OpenOptions::read_only`) share a file with each other, but n **Cause**: Another process holds the kernel file lock on this `.graph` file. Minigraf is single-writer: one process at a time. This is reported only after a bounded retry (up to ~375ms) rules out a transient `fork`-related false positive — see the v2.0.0 CHANGELOG entry on the bounded retry. +In browser OPFS mode, the same code means another tab or worker (or another +handle in this worker) holds the file's exclusive sync access handle. The browser +reports `NoModificationAllowedError`; `close()` or worker termination releases it. +There is no retry on acquisition and no fallback to IndexedDB. + **Resolution**: - Wait for the other process to exit; the kernel releases the lock automatically, including when the process is killed. - There is no lock file to delete. If you find a stale `.graph.lock`, it is a leftover from a version before 2.0 and has no effect — it is not read or deleted, and should not be removed by hand in case an old process still depends on it. @@ -2972,6 +2977,11 @@ A read-only handle (`OpenOptions::read_only`) in another process also refuses a **Cause**: One meta page is valid and the other is damaged, and there is evidence that the damaged one held a newer, completed checkpoint: the WAL records a newer base generation, or there is no WAL and data pages from the newer generation exist. Opening at the older generation would lose the facts of the newest checkpoint, whose WAL is already deleted. +OPFS mode has no WAL and deliberately opens at the sole valid meta instead +of raising this error, even with later-generation data pages. That recovers an +interrupted commit, but can silently lose the latest acknowledged checkpoint if +its meta later rots. Native recovery keeps this refusal unchanged. + **Resolution**: - Restore from backup. The file is not modified by the failed open. @@ -3148,6 +3158,12 @@ let mut out = LogWriter::create("memory.v2.graph", OpenOptions::new())?; Nothing committed is lost. Every committed transaction is still in the WAL, and the `.graph` file holds either the previous checkpoint or the failed one complete. Queries through the handle keep working. The close-time checkpoint is skipped. +In OPFS mode a failed write, short write or flush has the same failed-handle +rule: later writes/checkpoints fail with STG-046 while reads and `close()` keep +working. There is no WAL. Close and reopen to select the complete previous or +new generation; the rejected transaction may be wholly present. Closing does +not retry the failed commit. + **Resolution**: - Fix the cause (free disk space, check the device and the kernel log), then drop every clone of the handle and open the database again. Opening picks the newest complete checkpoint and replays the WAL. - Do not copy the `.graph` file without its `.wal` while the handle is in this state. @@ -3578,6 +3594,14 @@ let mut log = src.fact_log(&FactFilter::new())?; **Cause**: A language binding object was used after it was closed or finished, for example `append` or `finish` on a log writer after `finish` or `close`, or `execute` or `commit` on a write transaction after it committed (or failed to), rolled back, or its database was closed (#824). A finished transaction's `rollback` does nothing instead. Rust consumes these objects, so the Rust API never returns this code. Bindings cannot consume an object a caller still holds, so they return this error instead. A cursor or fact log reports the end (no more batches) after `close`, not this error. The `{}` names the object. +`BrowserDb.close()` in every mode also closes its cursors, prepared queries and +transactions logically, even while those objects are still referenced. A +successful OPFS `importGraph()` invalidates earlier cursors and prepared queries. +A failure at import publication closes the OPFS database itself. Disposal +(`free()`, cursor `close()`, transaction `rollback()`, repeated database `close()`) +remains permitted. API-018 takes precedence over busy (API-027) and an open +transaction (INT-001). + **Resolution**: - Create a new object instead of reusing a finished one. - In Python, use the writer as a context manager so it is finished once, on exit. diff --git a/docs/TEST_COVERAGE.md b/docs/TEST_COVERAGE.md index 858f63de05..1c174f343e 100644 --- a/docs/TEST_COVERAGE.md +++ b/docs/TEST_COVERAGE.md @@ -4,12 +4,38 @@ | | Default features | With `--features test-hooks` | |---|---|---| -| **Active** | 2,138 | 2,176 | -| **Ignored** | 21 | 30 | -| **Total** | 2,159 | 2,206 | +| **Active** | 2,147 | 2,185 | +| **Ignored** | 22 | 31 | +| **Total** | 2,169 | 2,216 | Counts above are the current Cargo test inventory (active versus ignored), -including #777, #789, #781, #794, #795, #837, #865, #869, #870, #861, #863, #871, #872 and #824 (Linux). Verification results below identify which suites ran. +including #441, #777, #789, #781, #794, #795, #837, #865, #869, #870, #861, #863, #871, #872 and #824 (Linux). Verification results below identify which suites ran. + +**#441 verification (2026-11-14)**: the full native default suite passes +(2,147 active, 22 ignored), including the grammar verifier and seven new +WAL-less/image unit tests. The WASI suite passes (1,222 active, seven ignored). +Native all-feature lib and browser/WASI lib strict clippy pass, as does formatting. +The preceding implementation session reported 101 Firefox worker and 81 +main-thread tests passing; this continuation compiled the worker suite but could +not run it because the sandbox refuses localhost sockets (`EPERM`). No fresh +Chrome/Firefox/WebKit, private-browsing or memory measurement pass is claimed. +The packaged consumer adds cross-worker locking, acknowledged-write recovery, +six sequential-commit termination timings and four import-interruption timings; +those new harnesses have syntax checks only here. See +[measurement status and reproduction](benchmarks/441-opfs/README.md). + +The native additions are `wal_less_crash_at_every_point_of_a_commit_opens_old_or_new`, +`wal_less_crash_while_creating_opens_empty`, `wal_less_fixed_cases`, +`wal_less_open_outcomes_match_native`, +`wal_less_short_foreign_file_is_stg_045_and_unmodified`, +`wal_less_image_after_recovery_opens_with_minigraf_open` and +`image_with_a_later_generation_is_the_same_database` in +`storage::persistent_facts`. They cover write/sync/tear boundaries, uncertain +meta publication, completed-unacknowledged predecessors, native STG-033 parity, +refused-open immutability and portable export after recovery. +`tests/opfs_portability_test.rs` adds two active fixture checks and one ignored +native fixture generator; long values, retraction, transaction/valid-time queries +and `verify()` agree on checked-in native and browser images. **#863 nightly candidates**: four engine unit tests check the `test-hooks` path events and memory peaks (both builds). With `test-hooks` only, @@ -676,7 +702,17 @@ These are not `#[test]`s; run them with `cargo kani` or `.github/workflows/kani. None of these are counted in the summary. The wasm32 targets run in CI (`.github/workflows/wasm.yml`). -- **In a browser** (`wasm-pack test --headless`, Firefox and Chrome): 80 `wasm_bindgen_test` tests (Firefox, 2026-11-11, after #824). The final #641 snapshot passed in headless Firefox on the bench; Chrome remains a CI run. +- **OPFS worker mode** (#441; `src/browser/opfs_tests.rs`, 21 worker tests plus one main-thread test): + - AC1/AC4: `acknowledged_writes_survive_close_and_reopen`, `the_database_is_one_file_and_no_wal`. + - AC2/AC8: `invalid_names_are_api_017_before_anything_is_created`, `a_second_open_is_stg_026_until_close`, `open_opfs_on_the_main_thread_is_api_026_and_creates_nothing`. + - AC9: `after_close_every_call_is_api_018_but_disposal`, `close_from_a_user_function_during_a_query_is_busy`, `calls_issued_before_close_change_nothing_after_it`. + - AC5/AC6/AC10: `a_failed_commit_is_stg_048_then_stg_046`, `a_torn_meta_reopens_at_the_previous_commit`, `open_outcomes_match_native_and_refusals_modify_nothing`. + - AC7/AC11: `export_is_a_complete_file_that_verifies`, `native_v8_and_v7_files_import_and_survive_reopen`, `a_smaller_import_leaves_no_trailing_bytes`. + - AC12: `import_validation_rejects_before_replacing_anything`, `a_published_import_invalidates_earlier_cursors_and_prepared_queries`, `calls_admitted_while_importing_are_busy`, `an_interrupted_import_reopens_old_or_new`, `leftover_staging_files_are_removed_not_replayed`. + - AC13/AC14: `opfs_open_read_bound`, `opfs_cache_residency_bound`, `indexeddb_mode_works_in_a_worker`. + - AC3 parity: the worker build runs the existing transaction, prepared-query and UDF suites against OPFS as well as IndexedDB. The main-thread build preserves the existing IndexedDB tests. Worker CI sets `RUSTFLAGS=--cfg minigraf_browser_worker` for Firefox and Chrome. + +- **In a browser** (`wasm-pack test --headless`, Firefox and Chrome): 81 `wasm_bindgen_test` tests (inherited Firefox main-thread result, 2026-11-14, after #441). The final #641 snapshot passed in headless Firefox on the bench; Chrome remains a CI run. - Three committed-memory-backend input-order fixtures: scalar Sort omission, streaming counts with Boolean/float representatives and mixed numeric group order, and partition windows preserving float edges and numeric group classes (#641). - Twenty-six for `BrowserDb`, including IndexedDB persistence, the default `allow_overrides` list (#869), the cursor, and the flush returning every dirty page and failing on one it cannot read (#470). - Eleven for `BrowserBufferBackend` (which used `#[test]` and never ran before). diff --git a/docs/benchmarks/441-opfs/README.md b/docs/benchmarks/441-opfs/README.md new file mode 100644 index 0000000000..1acc56d0e8 --- /dev/null +++ b/docs/benchmarks/441-opfs/README.md @@ -0,0 +1,70 @@ +# BrowserDb OPFS bounded-memory evidence (#441) + +The measurement is **not completed in this environment**. The browser runner +cannot bind its localhost server (`Operation not permitted`, `EPERM`); an +independent Node `listen(0, '127.0.0.1')` probe also fails with `EPERM`. +No open-time, page-read or memory numbers are claimed. AC13(c)'s required +4 MiB/40 MiB comparison remains unverified. + +## Reproduction + +Run on the designated benchmark host, with Playwright 1.55.0 and its chosen +engine installed. Build `@minigraf/browser` from the companion wasm repository +using `wasm-pack build --target web --features browser` (release profile, +Cargo default `opt-level = 3` in the packaging shim; record any wasm-opt setting and version). +The offline comparison in this session explicitly disabled wasm-opt for both +revisions; memory measurement should identify the package used in its output. + +```bash +bash docs/benchmarks/441-opfs/build-fixtures.sh /tmp/opfs-fixtures +cd docs/benchmarks/441-opfs +npm install --no-save --package-lock=false playwright@1.55.0 +npx playwright install chromium firefox webkit +node measure.mjs /path/to/minigraf-wasm/pkg /tmp/opfs-fixtures chromium > chromium.json +node measure.mjs /path/to/minigraf-wasm/pkg /tmp/opfs-fixtures firefox > firefox.json +node measure.mjs /path/to/minigraf-wasm/pkg /tmp/opfs-fixtures webkit > webkit.json +``` + +`fixture.rs` builds and verifies two native v8 files of at least 4 MiB and +40 MiB with the same schema. Its data generation and fixture import occur +outside the measured wasm instance. Each mode/size is seeded in a worker which +is destroyed, then opened in another worker with a fresh wasm instance. The +query is always `(query [:find ?a :where [:probe :answer ?a]])`, whose single short +answer must be `42`. + +`measure-worker.js` wraps `FileSystemSyncAccessHandle.prototype.read` before open +and records open milliseconds, read calls before and after the query, bytes read +expressed in 4 KiB pages, and `memory.buffer.byteLength` after the query. +IndexedDB's page count is recorded from the `getAll()` request's result length; +the output separately labels total fixture pages as `indexedDbImagePages` +(including free pages). Read-call counts specifically refer to OPFS. + +WebAssembly memory size is an allocated **high-water mark**, not live memory. +The script fails if OPFS's larger-file high-water mark exceeds the smaller-file +mark by more than 4 MiB. Results are needed before claiming that bound. + +The storage bound excludes query working state, results and export blobs, +import input, v7 migration, a transaction's facts, pages and quarantine metadata +held for long-open cursors, and recovery scans. Browser-side IndexedDB memory, +OS cache state and cold-start latency are not measured. + +## Browser execution record + +The preceding implementation session reported 101 Firefox worker and 81 +Firefox main-thread tests passing. Those are inherited results from the handoff, +not fresh runs here. This session compiled the worker tests but could not start +the local test server. The built package's new cross-worker, termination and +import-interruption harnesses have syntax checks only; no engine pass is claimed. +CI now specifies Chrome/Firefox core worker lanes and Chromium/Firefox/WebKit +package-consumer lanes. + +WebKit limitation here: no Playwright WebKit binary is installed, and the +sandbox's localhost bind restriction blocks the package harness before launch. +A reproducible WebKit command is the consumer harness with `BROWSER=webkit`; +its result here is **not run**, not passed. A manual result remains outstanding. + +Private browsing: Chromium, Firefox and WebKit checks are **not run** here. +There is no evidence of how these engines expose OPFS in private windows in this +session. The same consumer page can be served on localhost and opened in a real +private window to record the actual `openOpfs` outcome; automated isolated browser +contexts are not represented as proof of UI private-browsing behavior. diff --git a/docs/benchmarks/441-opfs/build-fixtures.sh b/docs/benchmarks/441-opfs/build-fixtures.sh new file mode 100644 index 0000000000..7edd3aaed0 --- /dev/null +++ b/docs/benchmarks/441-opfs/build-fixtures.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Run on the designated benchmark host. No fixture generation is timed. +set -euo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CORE="$(cd "$HERE/../../.." && pwd)" +OUTPUT="${1:?usage: build-fixtures.sh output-directory}" +mkdir -p "$OUTPUT" +OUTPUT="$(cd "$OUTPUT" && pwd)" +SCRATCH="$(mktemp -d)" +trap 'rm -rf "$SCRATCH"' EXIT +mkdir "$SCRATCH/src" +cp "$HERE/fixture.rs" "$SCRATCH/src/main.rs" +python3 - "$CORE" "$SCRATCH/Cargo.toml" <<'PY' +import json, sys +from pathlib import Path +Path(sys.argv[2]).write_text('[package]\nname = "opfs-fixtures"\nversion = "0.0.0"\nedition = "2024"\n[dependencies]\nminigraf = { path = ' + json.dumps(sys.argv[1]) + ' }\n[workspace]\n') +PY +cargo run --release --manifest-path "$SCRATCH/Cargo.toml" -- "$OUTPUT" diff --git a/docs/benchmarks/441-opfs/fixture.rs b/docs/benchmarks/441-opfs/fixture.rs new file mode 100644 index 0000000000..305f687739 --- /dev/null +++ b/docs/benchmarks/441-opfs/fixture.rs @@ -0,0 +1,43 @@ +//! Builds native v8 fixtures outside every measured WebAssembly instance. +use minigraf::{Minigraf, OpenOptions}; +use std::{env, fs, path::Path}; + +fn main() -> Result<(), Box> { + let directory = env::args().nth(1).ok_or("expected output directory")?; + fs::create_dir_all(&directory)?; + for mib in [4_u64, 40] { + let path = Path::new(&directory).join(format!("{mib}mib.graph")); + if path.exists() { + return Err("fixture already exists".into()); + } + let db = Minigraf::open_with_options(&path, OpenOptions::new())?; + db.execute("(transact [[:probe :answer 42]])")?; + let mut count = 0; + loop { + let mut statement = String::from("(transact ["); + for i in count..count + 128 { + statement.push_str(&format!( + "[:item-{i} :payload \"{i:08}{}\"] [:item-{i} :id {i}] ", + "x".repeat(1000) + )); + } + statement.push_str("])"); + db.execute(&statement)?; + db.checkpoint()?; + count += 128; + if fs::metadata(&path)?.len() >= mib * 1024 * 1024 { + break; + } + } + if !db.verify()?.is_ok() { + return Err("fixture failed integrity verification".into()); + } + println!( + "{}: {} bytes, {} payload entities", + path.display(), + fs::metadata(&path)?.len(), + count + ); + } + Ok(()) +} diff --git a/docs/benchmarks/441-opfs/measure-worker.js b/docs/benchmarks/441-opfs/measure-worker.js new file mode 100644 index 0000000000..60d31dd50f --- /dev/null +++ b/docs/benchmarks/441-opfs/measure-worker.js @@ -0,0 +1,54 @@ +// One worker = one fresh wasm instance. Seeding workers are always destroyed. +import init, { BrowserDb } from '/pkg/minigraf_wasm.js' + +const wasm = await init() +self.onmessage = async ({ data: { operation, name, mode, fixture } }) => { + let db + try { + if (operation === 'seed') { + const bytes = new Uint8Array(await (await fetch(`/fixtures/${fixture}`)).arrayBuffer()) + if (mode === 'opfs') { + const root = await navigator.storage.getDirectory() + const file = await root.getFileHandle(`${name}.graph`, { create: true }) + const handle = await file.createSyncAccessHandle() + try { + if (handle.write(bytes, { at: 0 }) !== bytes.length) throw new Error('short fixture write') + handle.truncate(bytes.length) + handle.flush() + } finally { handle.close() } + } else { + db = await BrowserDb.open(name) + await db.importGraph(bytes) + } + self.postMessage({ ok: true }) + } else { + let reads = 0, bytesRead = 0, indexedDbPages = 0 + const originalGetAll = IDBObjectStore.prototype.getAll + IDBObjectStore.prototype.getAll = function (...args) { + const request = originalGetAll.apply(this, args) + request.addEventListener('success', () => { indexedDbPages += request.result.length }) + return request + } + const originalRead = FileSystemSyncAccessHandle.prototype.read + FileSystemSyncAccessHandle.prototype.read = function (buffer, options) { + reads++ + const transferred = originalRead.call(this, buffer, options) + bytesRead += transferred + return transferred + } + const start = performance.now() + db = mode === 'opfs' ? await BrowserDb.openOpfs(name) : await BrowserDb.open(name) + const openMs = performance.now() - start + const openReads = reads + const answer = JSON.parse(await db.execute('(query [:find ?a :where [:probe :answer ?a]])')).results + if (JSON.stringify(answer) !== '[[42]]') throw new Error('point query answer changed') + // memory.buffer.byteLength is the allocated linear-memory high-water + // mark. It is not live memory and does not include browser-side IDB state. + self.postMessage({ mode, fixture, openMs, openReads, readCalls: reads, + pagesRead: mode === 'opfs' ? bytesRead / 4096 : indexedDbPages, memoryHighWaterBytes: wasm.memory.buffer.byteLength }) + } + } catch (error) { + self.postMessage({ error: String(error) }) + } finally { db?.close() } +} +self.postMessage({ ready: true }) diff --git a/docs/benchmarks/441-opfs/measure.mjs b/docs/benchmarks/441-opfs/measure.mjs new file mode 100644 index 0000000000..d2bd0c6677 --- /dev/null +++ b/docs/benchmarks/441-opfs/measure.mjs @@ -0,0 +1,64 @@ +// Usage: node measure.mjs [chromium|firefox|webkit] +// Install Playwright 1.55.0 and the selected engine in this script's directory. +import { createServer } from 'node:http' +import { readFile, stat } from 'node:fs/promises' +import { resolve, join } from 'node:path' +import { fileURLToPath } from 'node:url' +import { chromium, firefox, webkit } from 'playwright' + +const [packageDir, fixtureDir, engine = 'chromium'] = process.argv.slice(2) +if (!packageDir || !fixtureDir) throw new Error('expected package directory and fixture directory') +const here = fileURLToPath(new URL('.', import.meta.url)) +const fixtures = ['4mib.graph', '40mib.graph'] +const sizes = Object.fromEntries(await Promise.all(fixtures.map(async (name) => [name, (await stat(join(fixtureDir, name))).size]))) +if (sizes[fixtures[0]] < 4 * 1024 ** 2 || sizes[fixtures[1]] < 40 * 1024 ** 2) throw new Error('undersized fixture') +const server = createServer(async (req, res) => { + const path = new URL(req.url, 'http://localhost').pathname + try { + if (path === '/') { res.end('OPFS memory measurement'); return } + const base = path.startsWith('/pkg/') ? resolve(packageDir) : path.startsWith('/fixtures/') ? resolve(fixtureDir) : here + const relative = path.replace(/^\/(pkg|fixtures)\//, '').replace(/^\//, '') + const file = resolve(base, relative) + if (!file.startsWith(`${base}/`)) { res.writeHead(403).end(); return } + const body = await readFile(file) + res.setHeader('content-type', file.endsWith('.wasm') ? 'application/wasm' : 'text/javascript') + res.end(body) + } catch { res.writeHead(404).end() } +}) +await new Promise((resolve, reject) => { server.once('error', reject); server.listen(0, '127.0.0.1', resolve) }) +let browser +try { + browser = await ({ chromium, firefox, webkit })[engine].launch() + const page = await browser.newPage() + await page.goto(`http://127.0.0.1:${server.address().port}`) + const measurements = [] + for (const mode of ['opfs', 'indexeddb']) for (const fixture of fixtures) { + const name = `memory-${crypto.randomUUID()}` + const invoke = (operation) => page.evaluate(({ operation, mode, fixture, name }) => new Promise((resolve, reject) => { + const worker = new Worker('/measure-worker.js', { type: 'module' }) + const timeout = setTimeout(() => { worker.terminate(); reject(new Error('worker timed out')) }, 180000) + worker.onerror = (e) => { clearTimeout(timeout); worker.terminate(); reject(new Error(e.message)) } + worker.onmessage = ({ data }) => { + if (data.ready) worker.postMessage({ operation, mode, fixture, name }) + else { + clearTimeout(timeout) + worker.terminate() + data.error ? reject(new Error(data.error)) : resolve(data) + } + } + }), { operation, mode, fixture, name }) + await invoke('seed') + const measurement = await invoke('measure') + measurement.databaseBytes = sizes[fixture] + measurement.indexedDbImagePages = mode === 'indexeddb' ? sizes[fixture] / 4096 : null + measurements.push(measurement) + } + const opfs = measurements.filter((row) => row.mode === 'opfs') + const delta = opfs[1].memoryHighWaterBytes - opfs[0].memoryHighWaterBytes + console.log(JSON.stringify({ engine, version: browser.version(), measurements, opfsMemoryDeltaBytes: delta, + within4MiB: delta <= 4 * 1024 ** 2 }, null, 2)) + if (delta > 4 * 1024 ** 2) process.exitCode = 1 +} finally { + await browser?.close() + server.close() +} diff --git a/docs/superpowers/specs/2026-10-05-v8-storage-format-design.md b/docs/superpowers/specs/2026-10-05-v8-storage-format-design.md index 060374267b..bdaa6d417f 100644 --- a/docs/superpowers/specs/2026-10-05-v8-storage-format-design.md +++ b/docs/superpowers/specs/2026-10-05-v8-storage-format-design.md @@ -165,6 +165,13 @@ without a layout change. The 84-byte `FileHeader` and its `index_checksum` and #### 4.1.1 Choosing the meta on open +**2026-11-14 — OPFS exception (#441).** The native rules below are unchanged. +Browser OPFS mode has no WAL and opens at the sole valid slot regardless of +later-generation pages or the other slot's damage. This permits recovery after +an interrupted commit, at the cost of silently falling back if a completed +newest meta later rots. With two valid slots, the higher generation still wins. +See [the OPFS design](2026-11-14-441-browser-opfs-design.md), D1. + Each slot is classified as one of: - **empty:** no `"MGRF"`/`"META"` magic. This is slot B before generation 2. diff --git a/docs/superpowers/specs/2026-11-14-441-browser-opfs-design.md b/docs/superpowers/specs/2026-11-14-441-browser-opfs-design.md new file mode 100644 index 0000000000..421a36c0e9 --- /dev/null +++ b/docs/superpowers/specs/2026-11-14-441-browser-opfs-design.md @@ -0,0 +1,194 @@ +# 005-browser-opfs — BrowserDb OPFS-backed persistent mode (#441) + +**Mission Type:** mixed (Rust/WASM code, browser tests, documentation) + +**Base branches:** core `v3` (read at `origin/v3` 108b99394; re-checked at 9fd321e7 with no change under `src/browser`, `src/storage` or `src/error.rs`; the Generator records the commit the worktree is cut from); minigraf-wasm `v3`; minigraf-docs `main`. Nothing lands on core `main`. + +**Deliverables:** three PRs — core (`feat/441-browser-opfs`, worktree under `.worktrees/`), minigraf-wasm, minigraf-docs — merged in that order. #441 is closed by hand after the last merge. PRs say `Refs #441`, no closing keywords. + +## 1. User Intent + +> #441 + +GitHub issue project-minigraf/minigraf#441, "BrowserDb: OPFS-backed persistent mode (sync access handle, no in-memory buffer)", milestone v3.0.0. Its body and comments are part of the intent. Answers given during planning (2026-11-14): + +> [Recovery without a WAL after an interrupted commit] Reopen at last commit. + +> [IndexedDB failed-flush fix] Separate issue. *(filed as #894)* + +> [Releasing the database / lock] close() on every mode. + +> [Interrupted importGraph in OPFS mode] Old or new, never a mix. + +> [Cursors and prepared queries created before an OPFS import] Invalidate them. + +## 2. Goal + +Give `BrowserDb` a second persistent mode that keeps the database as one real `.graph` file in the Origin Private File System, read and written through a `FileSystemSyncAccessHandle` inside a dedicated Web Worker. Unlike IndexedDB mode it holds no in-memory copy of the pages: the storage layer keeps at most a page cache's worth of pages, and opening a healthy database does not read all of it (AC13 states what this bound leaves out). Commits use the v8 copy-on-write protocol directly on the file, with no WAL and no separate flush step. IndexedDB mode stays as it is (and starts working inside workers); OPFS mode ships in v3.0.0 as a Tier 2 backend inside the Tier 1 browser binding. + +## 3. Terms + +- **IndexedDB mode**: `BrowserDb.open(name)`; every page in memory (`BrowserBufferBackend`), written through to IndexedDB after each commit. +- **OPFS mode**: the new mode, `BrowserDb.openOpfs(name)`. +- **Commit**: one `PersistentFactStorage::save()`: data pages, sync, meta page, sync. +- **Commit point**: a commit has reached its commit point when the file holds a valid meta image of its generation (one that open would select). This is a fact about the bytes, not about what the write call reported: a meta write that returned an error or a short count may still have left the complete new image (the meaningful bytes and CRC of a meta page are at its start, `src/storage/meta.rs:106-168`; the fault backend's 512-byte tear does exactly this, `persistent_facts.rs:2163-2203`). A crash before the commit point reopens at the previous committed state; after it, at the new one, even if the final sync or the caller's promise never completed. +- **Previous committed state**: the state of the last commit that reached its commit point before the attempt in question, whether or not its caller had observed the acknowledgment yet. Acknowledgments say what must be kept; they do not cap what recovery may return. +- **Uncertain commit**: a commit whose call was interrupted or rejected after it may have written. After reopen it is wholly present (it reached its commit point) or wholly absent; a rejection does not prove absence. +- **Acknowledged**: the promise of `execute`, `tx.commit()`, `checkpoint()` or `importGraph()` has resolved and the code that awaited it has observed that (in the termination tests: the page that owns the worker has received the worker's message for that commit). +- **Importing**: an OPFS-mode database is importing from the start of an `importGraph()` body until that body ends (resolve or reject), cleanup included. +- **Disposal operations**: generated `free()`, a cursor's `close()`, a transaction's `rollback()`, and `BrowserDb.close()` on an already closed database. They are always permitted and never throw the errors below. +- **Slot**: one of the two meta pages (pages 0 and 1). **Valid / damaged / empty** as in the v8 format spec §4.1.1. +- **Dependents**: `BrowserCursor`, `BrowserPreparedQuery` and `BrowserWriteTransaction` objects obtained from a `BrowserDb`. + +## 4. Context & Investigation Findings + +### Existing code + +- `src/browser/mod.rs:35-48`: `BrowserDbInner` is typed to `PersistentFactStorage` plus `Option`; every write path (`apply_write` :435, `commit_stamped` :495, `tx_commit` :640, `checkpoint` :231, `import_graph` :288, `open` :107) is "sync save into the buffer, then async IndexedDB flush". OPFS mode changes the facade, not only adds a backend. +- `src/storage/mod.rs:80-127`: `StorageBackend` (sync, `Send + Sync`), including `holds_every_page()` (file-length checks STG-044/INT-049, `persistent_facts.rs:810-830`) and `partial_page()` (short-file check STG-045, `:203-212`). A file-like backend must answer both truthfully. +- `src/storage/persistent_facts.rs:79-113`: the storage is generic over the backend and all backends share `select_meta` (:198-283). `save()` :433-520; a failed save sets a state that refuses later saves with STG-046 (:434, :442). +- `src/storage/persistent_facts.rs:855-893` `check_single_valid`: with one valid slot at generation `g` and no WAL, a valid page stamped `g+1` means STG-033. Native test `no_wal_with_next_generation_pages_is_stg_033` (:1612). This is the rule OPFS mode cannot keep (§7 D1). +- `src/browser/mod.rs:86,113`: browser storage opens with page cache capacity 0, right only for the buffer. +- `src/browser/mod.rs:500-507`: facts are loaded into `FactStorage` before `save()`, so after a failed commit they stay readable. +- `src/browser/mod.rs:716-719`: a prepared query holds the `Rc>`; `:670-674` a cursor holds a `crate::Cursor`, which keeps storage alive through its snapshot and readers. Both extend the database's lifetime, by different routes, and there is no `close()`. `:327`: in IndexedDB mode, cursors opened before an import keep the old buffer. +- `src/browser/udfs.rs:22-27, 657, 696`: `execute`, `checkpoint` and `importGraph` are Rust `async` methods, so their bodies run from a microtask after the caller's synchronous code (and any outer query) has returned; tests `async_methods_from_a_function_run_after_the_query` and `import_graph_from_a_function_runs_after_the_query` pin this, and minigraf-wasm's README documents it. `tx.commit()` differs: it applies the transaction synchronously and only the flush is deferred (`mod.rs:552-574`). +- `src/storage/persistent_facts.rs:792-799`: dropping the storage may save dirty state and never calls `StorageBackend::close()`; `:2182-2203` the native crash test accepts either whole generation after a failed save and requires the new one after a successful save — the oracle AC5 follows. +- `src/browser/mod.rs:288-313`: IndexedDB-mode import validates only page alignment and the selected meta; other pages are checked lazily on read, and the buffer skips file-length checks. `src/error.rs:2206-2226`: an uncoded error reaches JS as `INT-000`. +- `src/browser/indexeddb.rs:75`: `web_sys::window()` makes IndexedDB mode fail in workers. +- `src/error.rs:1348`: STG-026, the kernel-lock contention code. `:1469` STG-046. `:1625` API-018 (`{} is closed`). `src/browser/mod.rs:864`: every engine error reaches JS as `[CODE] text` (#673); environment errors in `indexeddb.rs` are still uncoded strings. +- `src/storage/persistent_facts.rs:29-33`: large-transaction streaming is not built for wasm; `:311-325` v7 migration reads all facts into memory. Neither changes here. +- Tests today: `.github/workflows/wasm.yml:22-39` runs the lib's `wasm_bindgen_test`s in headless Firefox and Chrome on the main thread (`run_in_browser`); nothing runs in a worker. minigraf-wasm `consumer-test/browser/driver.mjs` drives the built package in Playwright Chromium; `Cargo.toml:16` there pins core by git rev. +- Dependencies: wasm-bindgen 0.2.126 / web-sys 0.3.103 (minigraf-wasm lockfile). The sync-access-handle bindings without options are stable in web-sys; `create_sync_access_handle_with_options` is unstable and must not be used. wasm-bindgen 0.2.126 already implements `Send`/`Sync` for `JsValue` when atomics are off, so the issue's unsafe wrapper may be unnecessary. +- Native size gate: `binary-size.yml:17`, 1,280,000 bytes (1,250 KiB), native `dist` binary only. Browser code is wasm-only. + +### External facts + +- [File System Standard](https://fs.spec.whatwg.org/#api-filesystemsyncaccesshandle): sync access handles exist only in dedicated workers in secure contexts; the default mode is exclusive and a second acquisition fails with `NoModificationAllowedError`; `read`/`write` may be short; `flush()` attempts to transfer cached changes to the underlying storage, which is not a power-loss guarantee, and nothing is said about torn writes. The standard does not define `FileSystemFileHandle.move()`; stable web-sys has no binding for it, and only Chrome documents it. +- [Storage Standard](https://storage.spec.whatwg.org/#persistence): OPFS shares the origin's best-effort bucket with IndexedDB; persistence is a permission the application requests. + +### Stale statements to correct + +`ROADMAP.md:150` (#441 under v3.next), `PHILOSOPHY.md:270` ("#441, v3.next"), docs site `content/use-cases.md:468` ("v3.1.0"), the issue's 2026-10-14 comment. The milestone is v3.0.0. The issue's "wiki" doc targets mean the docs site. + +### Deviations from the issue text + +1. Recovery is not "the native recovery path" unchanged: D1. +2. `close()` is added (the issue names no lifecycle API): D3. +3. The IndexedDB failed-flush fix is not in this mission: #894. +4. The unsafe `Send`/`Sync` wrapper is not required: D8. +5. "`getFile()` is blocked while a sync handle is open" is not asserted; export and import go through the held handle regardless. +6. Import invalidates earlier cursors and prepared queries in OPFS mode: D5. +7. The promise-based main-thread wrapper is left out: §9. + +## 5. Acceptance Criteria + +Test names are suggestions unless marked **(named)**. Every behavioural AC is covered by at least one automated test whose name states the behaviour, listed in `docs/TEST_COVERAGE.md`; AC13(c), AC15's WebKit and private-browsing records and AC17 are checked as evidence and documentation. + +**AC1 Entry point.** In a dedicated worker with OPFS, `await BrowserDb.openOpfs(name)` opens or creates the database `name`; data written and acknowledged is there after `close()` and a new `openOpfs(name)`. `open(name)` (IndexedDB) and `openInMemory()` are unchanged. The same name in IndexedDB mode and OPFS mode are two separate databases; there is no automatic fallback or migration between them. + +**AC2 Unsupported context and names.** `openOpfs` on the main thread, or where OPFS or sync access handles are unavailable, rejects with one new API-category code ("OPFS mode is not available here", naming what is missing); nothing is created and no wasm trap occurs. An invalid name (empty, containing `/` or `\`, `.` or `..`, or a name the import scheme reserves) rejects with `[API-017]` before any file is created. + +**AC3 API parity.** Every `BrowserDb` method keeps its name, sync-or-Promise shape, result encoding and scheduling in OPFS mode: `execute`, `checkpoint` and `importGraph` bodies still run from a microtask (so called from a user function they run after the query, and an import then succeeds, as the two `udfs.rs` tests pin); `query`, `prepare`, `exportGraph`, `beginWrite` and the transaction's `execute` stay synchronous; `tx.commit()` applies synchronously and returns a Promise. The behaviours the IndexedDB-mode tests pin and that are not about the buffer or IndexedDB records pass against OPFS mode in a dedicated worker: persistence across reopen, cursors fixed at open, prepared binds and `with*` setters, temporal queries, rules, UDFs incl. re-entry, default override list, transaction staging/read-your-own-writes/one `tx_count` per commit/rollback/drop/`INT-001` guards/`API-018` after finish, and the native differential check of `src/browser/tx_tests.rs`. + +**AC4 Acknowledged commits survive.** After a write is acknowledged, terminating the worker without `close()` and reopening shows that commit (its history: later valid retractions still apply). A rolled-back or dropped transaction leaves nothing after reopen. OPFS mode creates no WAL and, outside an import in progress or awaiting cleanup, no file other than the database file. + +**AC5 Interrupted commit (D1).** For an OPFS-mode database, a crash at any point of a commit leaves a file that `openOpfs` opens. The reopened state is the previous committed state, or that plus the interrupted commit whole (when it reached its commit point). It always contains every acknowledged commit. Never a partial transaction, never a refused open. Verified at two levels: +- (a) natively, with the existing fault-injection approach, on a backend configured as OPFS mode is (every page stored, no WAL, WAL-less recovery rule): for every write/sync boundary of a first commit on a new database and of a later commit, including torn writes of the meta page and of a data page and a failed final sync, reopen succeeds and full query answers equal the previous committed state or the new state. Fixed cases: `save()` returned success ⇒ new state; a torn meta write whose bytes decode as the valid new meta ⇒ new state; a tear that leaves that meta invalid ⇒ previous committed state; commit B completes (unacknowledged), commit C then fails before its meta is written ⇒ B's state, not A's; +- (b) in a browser: a worker is terminated at arbitrary times during a stream of commits with sequential identifiers, while the owning page records each acknowledgment it receives. After reopen the database holds a gap-free prefix of the commits that includes every recorded one (it may hold more: a commit whose acknowledgment was lost), and no partial commit. Runs in CI on Chromium and Firefox from minigraf-wasm's packaged consumer test. + +**AC6 Native rule unchanged.** A native open without a WAL of a file with one valid slot and next-generation pages still fails with STG-033 (`no_wal_with_next_generation_pages_is_stg_033` keeps passing), as do all other native recovery tests. In OPFS mode, every other open outcome matches native for the same bytes: two valid slots (higher generation wins), both slots damaged (STG-032), not a Minigraf file (STG-002), page CRC damage, unsupported feature bits, meta `page_count` past the file end (STG-044), a short non-Minigraf file (STG-045, file not modified), torn initial meta (initialised again). A refused open does not modify the file and releases the handle. + +**AC7 Export after recovery.** The bytes `exportGraph()` returns from an OPFS database always open natively with `Minigraf::open()` (no WAL beside them), including when the database was last opened after an interrupted commit and has not committed since. + +**AC8 Lock.** While a database is open in OPFS mode, a second `openOpfs` of the same name — from another worker or the same worker — rejects with `[STG-026]`. Different names open side by side. The lock is free again after `close()`, after a failed `openOpfs` (any reason past handle acquisition), and after the worker is terminated. + +**AC9 close() (D3).** `BrowserDb.close()` exists in all three modes, is synchronous and idempotent. +- It does no durable work and saves nothing: in the persistent modes every acknowledged write is already stored (an in-memory database is simply discarded), an open write transaction is discarded, and facts of a failed commit are not retried. It therefore reports no storage error; unless refused (next item) the database always ends up closed, with its OPFS handle and lock released, including after a failed commit (AC10). +- `close()` is refused, with the new "database is busy" API-category code and the database left open (no panic), in exactly two situations: it is called from a user function while a query of this database is running, or the database is importing. Other calls from user functions keep today's behaviour (nested `query`, `prepare`, `exportGraph` and `register*` work; `beginWrite` fails as today). +- After `close()`, every method of the database and of its dependents fails with `[API-018]`, except the disposal operations. This holds while dependents are still referenced: AC8's reopen works at once. +- Operations issued before `close()`: (i) an `execute`/`checkpoint`/`importGraph` whose body has not started rejects with `[API-018]` and changes nothing; (ii) a `tx.commit()` whose synchronous commit phase succeeded (a commit that already failed keeps its own rejection) — OPFS mode and in-memory: its promise resolves (in OPFS mode the commit is already stored); IndexedDB mode with the flush not yet submitted: the flush is never submitted and the promise rejects with `[API-018]` (the commit is not stored); (iii) an IndexedDB flush already submitted settles by its own outcome. Nothing issued before `close()` writes into, or publishes state to, a database opened afterwards; tests reopen the same name before the old promise settles and assert both the promise's outcome and the reopened data. +- Freeing a `BrowserDb` and all its dependents without `close()` also releases the OPFS handle and saves nothing as a side effect. + +**AC10 Failed commit.** When a commit fails after it may have written (an injected write or flush failure, a short write, quota exhaustion), the call rejects with the "browser storage operation failed" code (D9), whose text carries the operation and the browser exception's name (or "short read/write" when there is no exception) — never `INT-000`. Later writes on that handle reject with `[STG-046]`; reads keep working; `close()` works and frees the lock; a reopen opens cleanly at the previous committed state or, if the failed commit reached its commit point, with it whole. No short read or write is ever treated as a complete page. + +**AC11 Portability.** (a) `exportGraph()` from OPFS mode returns a complete v8 `.graph` file: opened natively it passes `verify()` and answers a fixed set of queries (long values, retractions, valid-time and `:as-of`) the same as the browser did. (b) A checkpointed native v8 file and a supported v7 file import into OPFS mode, answer those queries, and still do after `close()` and reopen. (c) Importing a smaller database over a larger one leaves a file with no trailing content of the old one. The native half may use fixture bytes checked in from the browser half, with a test on each side. + +**AC12 Import (D4, D5).** In OPFS mode an import has a **publication point**: before it the database is the old one, from it on the new one. +- *Validation, before anything is replaced.* Rejected: a length that is not a page multiple (`[API-017]`; IndexedDB mode keeps its uncoded message); a v8 image with no selectable meta, unsupported feature bits or version, a meta `page_count` beyond the image (`STG-044`), or for which the native verifier's checks (`src/storage/verify.rs`: page checks, index agreement, dictionary consistency, space accounting) report any finding; a v7 image that does not migrate completely. An image with several problems may be rejected with any one applicable registered code. Engine errors that both modes raise keep their codes; OPFS mode rejects more than IndexedDB mode (a page-aligned truncated image and a reachable page with a bad CRC are both tested). An empty blob gives an empty database, as in IndexedDB mode. +- *Failure before publication* (validation, staging I/O, quota): the import rejects with a coded error; the old database, its lock and its earlier dependents keep working; leftovers are removed then or at the next open. +- *Publication* is complete only when the new database is durable; a failure before that is the next item. From publication on, cursors and prepared queries created before the import fail with `[API-018]`. Cleanup follows publication; a cleanup failure does not fail the import (a later open finishes it). When the import has ended, new calls read the imported data. +- *Failure at the publication step itself* (outcome unknown): the import rejects with a coded error and the database becomes closed (`[API-018]` everywhere); reopening gives the complete old or the complete new database. +- *While importing:* admission is checked when a call starts work — at once for synchronous methods, when the body starts for `execute`/`checkpoint`/`importGraph` (which keep their deferred scheduling). A call admitted while the database is importing fails with the "database is busy" code and changes nothing: database methods including `close()` and a second import, and methods of dependents. A call issued during the import whose body starts after it has ended runs normally, against the database as it then is. Precedence when several apply: `[API-018]` (closed database, or a dependent invalidated by publication) over busy over `[INT-001]` (transaction open). Disposal operations are always permitted. An `openOpfs` of the same name from anywhere rejects with `[STG-026]` at every moment of the import and of any recovery or cleanup at open. +- *Interruption* at any point (fault injection at each staging/publication/cleanup boundary, plus worker termination in a browser), including a second interruption during the next open's recovery, reopens as the complete old or the complete new database. Transient files are kept until the chosen database is safely recoverable, are gone after an open that completes cleanup, and their names cannot be opened as databases. +- IndexedDB-mode and in-memory import behaviour is unchanged. + +**AC13 Bounded reads and memory.** (a) **(named `opfs_open_read_bound`)** `openOpfs` of a healthy v8 database reads at most a fixed number of pages, the same bound for two databases whose fact counts differ tenfold; IndexedDB mode reads all. (b) **(named `opfs_cache_residency_bound`)** after a workload that touches more pages than the cache holds, with no cursor open, the page cache holds at most its capacity (256 entries, the default; `BrowserDb` takes no options) and the storage layer keeps no other page store. (c) A recorded measurement, `docs/benchmarks/441-opfs/README.md` with the script that produced it and the build profile: databases of at least 4 MiB and 40 MiB with the same schema, built outside the measured instance; each opened in a fresh wasm instance, one fixed point query (one entity, short values) run, then WebAssembly memory size, open time and pages read recorded for OPFS mode and IndexedDB mode. Required: OPFS memory for the larger database is within 4 MiB of the smaller one's (a larger difference fails this AC); the actual numbers are published, and wasm memory size is reported as a high-water mark, not as live memory. The docs state what the bound excludes: query working state, results and export blobs, import input, v7 migration, a transaction's facts, pages and quarantine metadata held for long-open cursors, and recovery scans. + +**AC14 IndexedDB in a worker.** `BrowserDb.open(name)`, write, query, `checkpoint` and reopen work in a dedicated worker as on the main thread; existing main-thread IndexedDB tests still pass. + +**AC15 Browser evidence.** In CI: the OPFS worker tests (AC1–AC4, AC8–AC12 where browser-level, AC14) run in headless Chrome and Firefox from core (`wasm.yml`); the built `@minigraf/browser` package is exercised in a real dedicated worker (open, write, reopen, lock, main-thread rejection, termination test AC5b) from minigraf-wasm's consumer test on Chromium and Firefox. WebKit: the same package test runs in CI, or the evidence file records the exact harness limitation and a reproducible manual run with its result; it is never reported as passed without a run. Private-browsing behaviour per engine is recorded in the docs from an actual check, not assumed. + +**AC16 No regressions, no new dependencies.** Format stays v8. No new crate; only web-sys features are added; no `web_sys_unstable_apis`. Native, WASI and browser test suites, both wasm clippy jobs, coverage gates, the grammar verifier and the native size gate (1,250 KiB) pass. A build with wasm atomics enabled either compiles with sound handle confinement or fails to compile; it never shares the JS handle across threads through an unconditional unsafe impl. The `.wasm` package size before and after is reported in the PR. + +**AC17 Documentation.** Updated together and consistent: +- core: `CHANGELOG.md`; `README.md` browser section; `ROADMAP.md` (#441 moves into v3.0.0); `PHILOSOPHY.md:270` (version; still "starts at Tier 2") and §"Single file" line on WASM if it now misstates storage; `docs/DURABILITY.md` (OPFS: durable as far as the browser's `flush()` goes; the WAL-less recovery rule and what it gives up; eviction and `navigator.storage.persist()`, which Minigraf never calls itself; failed-commit behaviour; lock and `close()`); `docs/ERROR_REFERENCE.md` + `src/error.rs` (new codes; STG-026, STG-033, STG-046 and API-018 entries mention OPFS mode); `docs/TEST_COVERAGE.md`; `CLAUDE.md` (module map, test counts, status); a dated note in `docs/superpowers/specs/2026-10-05-v8-storage-format-design.md` §4.1.1; the approved spec copied to `docs/superpowers/specs/2026-11-14-441-browser-opfs-design.md`. +- minigraf-wasm: core pin, `README.md` (OPFS mode, a complete module-worker example with `postMessage`, `close()`, choosing a mode, moving a database between modes by export/import), consumer tests, `readme_sync.py` kept passing. +- minigraf-docs: `content/use-cases.md`, `content/architecture.md`, `content/comparison.md` and the durability page, with `` markers; link check passes. + +## 6. Edge Cases & Boundaries + +- **Short reads/writes and offsets:** a read or write that transfers fewer bytes than a page is an error or is completed, never a silently short page; page offsets are exact JS numbers or the operation fails before I/O. +- **Partial trailing page:** a file whose length is not a page multiple follows the native rules (`partial_page`), not a blanket rejection. +- **Read failures:** a page that cannot be read is an error; never zero-filled. +- **Empty transaction / no-op checkpoint:** same `tx_count` and write behaviour as IndexedDB mode. +- **Failed commit visibility:** the failed transaction's facts may stay readable on that handle until it is closed (as in IndexedDB mode today), and may or may not be stored (AC10); docs say a rejection means "not confirmed", not "absent". +- **Overlapping calls:** OPFS storage I/O is synchronous while an operation's body runs, so the bodies of un-awaited `execute` calls run one after another in call order, each complete before the next starts; none is acknowledged over a failed predecessor (STG-046). The order of synchronous calls relative to deferred bodies is the same as in IndexedDB mode (a `query` issued right after an un-awaited `execute` runs before that write; a `beginWrite` makes the deferred body reject with `INT-001`). +- **Long-open cursors:** keep their snapshot across commits (page quarantine, as native); they cost file space and quarantine metadata, not correctness. +- **Lock error text:** STG-026's message speaks of a process; the code is reused as is, and its reference entry explains the browser case (another tab or worker; released on `close()` or when the worker ends). +- **Storage pressure:** quota exhaustion and eviction are browser-managed; quota errors surface as AC10 failures, never as lock or corruption codes. +- **v7 import:** supported; may use memory in proportion to the database (existing migration). +- **Names:** distinct names never map to one file (no normalisation); the mapping from name to OPFS entry is documented. + +## 7. Key Decisions + +- **D1 WAL-less recovery rule (user).** An OPFS-mode database that has exactly one valid slot opens at it, whatever the other slot or later-generation pages look like. Why: with no WAL, an interrupted commit can leave that state (a kill during the first commit of a new database, or a torn meta write later), and the browser has no repair tool, so refusing would lose the database. Given up: if the newest meta page of an acknowledged commit later rots, open falls back to the previous commit silently (native refuses with STG-033). The rule applies to OPFS mode only; native opens are unchanged (AC6). Codex recommended keeping the refusal; the user chose reopening. +- **D2 Entry point `BrowserDb.openOpfs(name)`.** Explicit and additive; no options object; no fallback to IndexedDB (a silent fallback would show an empty database from the other namespace). +- **D3 `close()` on every mode (user).** It does no durable work, so it cannot fail on storage; it is refused only in the two situations AC9 names. +- **D4 Crash-atomic import (user).** A transient second OPFS file during an import is allowed; the single-file property holds outside that window. +- **D5 Import invalidates earlier cursors and prepared queries in OPFS mode (user).** Codex recommended refusing the import instead. +- **D6 No WAL, checkpoint per transaction,** as the issue says. +- **D7 Promise shapes kept.** `execute`, `checkpoint`, `importGraph` and `tx.commit()` stay Promise-returning in OPFS mode: one class and one TypeScript surface; they resolve after the synchronous durable write. +- **D8 Thread confinement without bespoke unsafe where the dependency already provides it** (AC16). +- **D9 Errors.** Lock contention (`NoModificationAllowedError` on acquiring the handle) is exactly STG-026. Invalid name and malformed import length are API-017. Three new registered codes, numbers assigned at implementation: (1) API, "OPFS mode is not available here": no dedicated worker, no OPFS or sync-access-handle API, insecure context, or the browser denies access to the origin's directory (for example `SecurityError` from `getDirectory()`); (2) API, "database is busy" (AC9, AC12); (3) storage, "browser storage operation failed": every other failed OPFS operation — creating or opening an entry, acquiring a handle for a reason other than the lock, read, write, flush, size, truncate, and staging steps of an import — with the operation and the exception name or a short-transfer note; quota exhaustion is here, never under (1) or STG-026. Post-publication cleanup failures are not reported (AC12). STG-003 is not reused. No uncoded or `INT-000` rejection from the new code paths; IndexedDB mode's existing uncoded messages are unchanged. Codes are never removed or recycled. +- **D10 Page cache:** the default 256 pages in OPFS mode. +- **D11 Tier:** OPFS mode is Tier 2 in v3.0.0. +- **D12 IndexedDB failed-flush gap** is #894 (v3.0.0, data-integrity), not this mission. This mission must not make it worse. + +## 8. Out of Scope + +- #894 (IndexedDB failed flush, abort handling); any other IndexedDB behaviour change beyond working in a worker and `close()`. +- A WAL in the browser; format changes; changes to native recovery. +- Multi-tab or multi-worker shared access, read-only OPFS handles, the Chrome-only handle modes, SharedWorker/ServiceWorker, threaded wasm. +- A packaged promise-based main-thread wrapper (file a follow-up issue in minigraf-wasm at the end of the mission; the README example covers hand-rolled `postMessage`). +- Automatic IndexedDB→OPFS migration; a database-delete or list API; `BrowserDb` open options; calling `navigator.storage.persist()`. +- Streaming import/export APIs; browser large-transaction streaming; bounded-memory v7 migration. +- New nightly benchmark lanes for OPFS (belongs to the #844 re-audit). +- Claims of power-loss durability from worker-termination tests. + +## 9. Possible Directions & Ideas (non-binding) + +- The file backend (`src/storage/backend/file.rs`) is the model for length, partial-page and lock behaviour; `src/storage/backend/fault_inject.rs` and the recovery tests around `persistent_facts.rs:1537-1645` are the model for AC5a/AC6. +- `wasm_bindgen_test_configure!(run_in_dedicated_worker)` is per test binary, so worker tests probably need their own test target; lock and termination tests need a page that owns workers, which the minigraf-wasm Playwright harness already has (`consumer-test/browser/driver.mjs`). +- For AC7, either repairing the leftover state at open or normalising on export would do; so would any commit protocol that never leaves it. +- For D4, `FileSystemFileHandle.move()` is only a candidate: it is not in the standard or in stable web-sys, and whatever scheme is used must show AC12's old-or-new and lock outcomes on Chromium and Firefox (and WebKit per AC15) before completion is claimed. Native `verify()` (`persistent_facts.rs:519`, `src/storage/verify.rs`) is `cfg(not(wasm32))` today. +- wasm linear memory never shrinks, which is why AC13(c) asks for a fresh instance per measurement. +- Related designs: `2026-10-05-v8-storage-format-design.md`, `2026-10-07-browser-page-reuse-design.md`, `2026-10-21-streaming-transactions-design.md` §WAL guard, `2026-11-10-824-binding-transactions-design.md`. + +## 10. What the user should test manually + +- The README worker example in a real page on Chrome, Firefox and Safari. +- Private-browsing behaviour on each engine, against what the docs record. + +## Open Questions + +None. Converged with Codex (gpt-6-astra) in four rounds; Codex-04 is APPROVED, and its two wording notes are applied here. From fd03fe68d92a92cff776ef322782afc0f6a9ace9 Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Sat, 14 Nov 2026 10:09:40 +0530 Subject: [PATCH 05/20] test(browser): record OPFS build sizes and verification results (#441) Publish matching-profile wasm sizes, the native size gate, coverage results and a bounded-memory harness with a validated worker route. Refs #441 Co-Authored-By: GPT-6 --- docs/TEST_COVERAGE.md | 13 ++++++++++++- docs/benchmarks/441-opfs/README.md | 2 ++ docs/benchmarks/441-opfs/build-sizes.md | 26 +++++++++++++++++++++++++ docs/benchmarks/441-opfs/measure.mjs | 6 ++++-- 4 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 docs/benchmarks/441-opfs/build-sizes.md diff --git a/docs/TEST_COVERAGE.md b/docs/TEST_COVERAGE.md index 1c174f343e..4e67e4fbbc 100644 --- a/docs/TEST_COVERAGE.md +++ b/docs/TEST_COVERAGE.md @@ -13,8 +13,19 @@ including #441, #777, #789, #781, #794, #795, #837, #865, #869, #870, #861, #863 **#441 verification (2026-11-14)**: the full native default suite passes (2,147 active, 22 ignored), including the grammar verifier and seven new -WAL-less/image unit tests. The WASI suite passes (1,222 active, seven ignored). +WAL-less/image unit tests. The WASI suite passes (1,222 active, seven ignored). The CI-equivalent +nightly branch-coverage run passes 2,125 tests with 18 ignored (doctests excluded), +95.71% line / 86.64% branch coverage, and every module floor. Its first attempt +omitted the lock helper override and failed two helper lookups; rebuilding the +helper and supplying `MINIGRAF_PID_NS_HELPER` resolved that harness setup error. Native all-feature lib and browser/WASI lib strict clippy pass, as does formatting. +The broader `--all-targets` lint fails on existing test-code lints on both the base +and this branch (including 21 library-test diagnostics); it is not reported as +passing. Rust 1.99.1's distribution binary is 1,269,568 bytes, below the unchanged +1,280,000-byte gate; [build sizes](benchmarks/441-opfs/build-sizes.md) record the +older compiler's oversized base/control and the wasm increase. A nightly build +with wasm atomics enabled fails with E0277 because the JS handle is not +`Send + Sync`, preserving worker confinement. The preceding implementation session reported 101 Firefox worker and 81 main-thread tests passing; this continuation compiled the worker suite but could not run it because the sandbox refuses localhost sockets (`EPERM`). No fresh diff --git a/docs/benchmarks/441-opfs/README.md b/docs/benchmarks/441-opfs/README.md index 1acc56d0e8..ed0a75ff27 100644 --- a/docs/benchmarks/441-opfs/README.md +++ b/docs/benchmarks/441-opfs/README.md @@ -48,6 +48,8 @@ import input, v7 migration, a transaction's facts, pages and quarantine metadata held for long-open cursors, and recovery scans. Browser-side IndexedDB memory, OS cache state and cold-start latency are not measured. +Build-size results and exact toolchains are recorded in [build-sizes.md](build-sizes.md). + ## Browser execution record The preceding implementation session reported 101 Firefox worker and 81 diff --git a/docs/benchmarks/441-opfs/build-sizes.md b/docs/benchmarks/441-opfs/build-sizes.md new file mode 100644 index 0000000000..191a4cd3d8 --- /dev/null +++ b/docs/benchmarks/441-opfs/build-sizes.md @@ -0,0 +1,26 @@ +# Build sizes (#441) + +Measured 2026-11-14. No dependency was added. Before is core `9fd321e75`; +after is `6fa1582f8` (subsequent documentation commits do not change code). + +| Artifact | Toolchain / profile | Before bytes | After bytes | Difference | +|---|---|---:|---:|---:| +| `minigraf_wasm_bg.wasm` in the packaging shim | Rust 1.94.0, release, Cargo default opt-level 3, wasm-opt disabled in both builds | 1,754,394 | 1,822,855 | +68,461 (+3.90%) | +| Native CLI | Rust 1.94.0, `--profile dist` | 1,306,088 | 1,307,088 | +1,000 | +| Native CLI | Rust 1.99.1, `--profile dist` | Not measured | 1,269,568 | Not compared | + +The 1.99.1 distribution build **passes** the unchanged 1,280,000-byte +(1,250 KiB) gate, with 10,432 bytes left. The 1.94.0 build **fails** that gate +on both revisions. The newer installed compiler was used to reproduce the +repository's documented size environment; the gate was not raised. + +Both wasm builds use the same minimal `minigraf-wasm` re-export shim, a local +path dependency on each core revision, wasm-bindgen 0.2.130 / web-sys 0.3.107, +and `wasm-pack build --mode no-install --target web --no-opt --features browser`. +These are the raw wasm sizes, not gzipped transfer sizes or linear memory. +The companion repository's pinned lockfile still uses its existing dependency +versions; a release package built with those versions may differ. + +The after package was also stamped as `@minigraf/browser@0.0.0-ci` and packed +successfully (665,211 bytes) with JS glue, TypeScript definitions, README and both licenses. +No registry publish or remote branch operation was performed. diff --git a/docs/benchmarks/441-opfs/measure.mjs b/docs/benchmarks/441-opfs/measure.mjs index d2bd0c6677..5cb3ab7f4f 100644 --- a/docs/benchmarks/441-opfs/measure.mjs +++ b/docs/benchmarks/441-opfs/measure.mjs @@ -1,6 +1,7 @@ // Usage: node measure.mjs [chromium|firefox|webkit] // Install Playwright 1.55.0 and the selected engine in this script's directory. import { createServer } from 'node:http' +import { createHash } from 'node:crypto' import { readFile, stat } from 'node:fs/promises' import { resolve, join } from 'node:path' import { fileURLToPath } from 'node:url' @@ -8,7 +9,8 @@ import { chromium, firefox, webkit } from 'playwright' const [packageDir, fixtureDir, engine = 'chromium'] = process.argv.slice(2) if (!packageDir || !fixtureDir) throw new Error('expected package directory and fixture directory') -const here = fileURLToPath(new URL('.', import.meta.url)) +const here = resolve(fileURLToPath(new URL('.', import.meta.url))) +const wasmSha256 = createHash('sha256').update(await readFile(join(packageDir, 'minigraf_wasm_bg.wasm'))).digest('hex') const fixtures = ['4mib.graph', '40mib.graph'] const sizes = Object.fromEntries(await Promise.all(fixtures.map(async (name) => [name, (await stat(join(fixtureDir, name))).size]))) if (sizes[fixtures[0]] < 4 * 1024 ** 2 || sizes[fixtures[1]] < 40 * 1024 ** 2) throw new Error('undersized fixture') @@ -55,7 +57,7 @@ try { } const opfs = measurements.filter((row) => row.mode === 'opfs') const delta = opfs[1].memoryHighWaterBytes - opfs[0].memoryHighWaterBytes - console.log(JSON.stringify({ engine, version: browser.version(), measurements, opfsMemoryDeltaBytes: delta, + console.log(JSON.stringify({ engine, version: browser.version(), wasmSha256, buildLabel: process.env.BUILD_LABEL ?? 'not specified', measurements, opfsMemoryDeltaBytes: delta, within4MiB: delta <= 4 * 1024 ** 2 }, null, 2)) if (delta > 4 * 1024 ** 2) process.exitCode = 1 } finally { From 291b1769be58e49d58807340ff97436e02fb13fc Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Sat, 14 Nov 2026 10:20:16 +0530 Subject: [PATCH 06/20] test(browser): publish OPFS browser and memory measurements (#441) Record successful packaged consumer runs on Chromium and Firefox, bounded-memory results for 4 MiB and 40 MiB databases, and observed private browsing behavior. Identify the missing WebKit engine and Chrome driver mismatch explicitly. Refs #441 Co-Authored-By: GPT-6 --- docs/TEST_COVERAGE.md | 20 ++-- docs/benchmarks/441-opfs/README.md | 141 +++++++++++++++--------- docs/benchmarks/441-opfs/build-sizes.md | 11 ++ 3 files changed, 112 insertions(+), 60 deletions(-) diff --git a/docs/TEST_COVERAGE.md b/docs/TEST_COVERAGE.md index 4e67e4fbbc..043922cde7 100644 --- a/docs/TEST_COVERAGE.md +++ b/docs/TEST_COVERAGE.md @@ -26,14 +26,18 @@ passing. Rust 1.99.1's distribution binary is 1,269,568 bytes, below the unchang older compiler's oversized base/control and the wasm increase. A nightly build with wasm atomics enabled fails with E0277 because the JS handle is not `Send + Sync`, preserving worker confinement. -The preceding implementation session reported 101 Firefox worker and 81 -main-thread tests passing; this continuation compiled the worker suite but could -not run it because the sandbox refuses localhost sockets (`EPERM`). No fresh -Chrome/Firefox/WebKit, private-browsing or memory measurement pass is claimed. -The packaged consumer adds cross-worker locking, acknowledged-write recovery, -six sequential-commit termination timings and four import-interruption timings; -those new harnesses have syntax checks only here. See -[measurement status and reproduction](benchmarks/441-opfs/README.md). +Fresh Firefox 159.0.1 runs pass **101 worker** and **81 main-thread** tests. +The packaged consumer passes **52 checks per engine** on Chromium 140.0.7339.16 +(Playwright) and Firefox 159.0.1 (installed Firefox via WebDriver, because the +Playwright Firefox executable is absent). These include cross-worker locking, +acknowledged-write recovery, six sequential-commit termination timings, four +import-interruption timings, worker IndexedDB and the README worker example. +The 4,239,360-byte and 42,135,552-byte fixtures both use 1,245,184 bytes of OPFS +wasm memory high-water, on both engines: **0-byte growth**, passing AC13(c). +Chromium incognito opens OPFS and queries `[[42]]`; Firefox private mode rejects +with API-026 / SecurityError. Chrome core tests lack a matching driver/browser; +WebKit and WebKit private tests lack an installed engine and remain unrun. +See [measurement evidence and reproduction](benchmarks/441-opfs/README.md). The native additions are `wal_less_crash_at_every_point_of_a_commit_opens_old_or_new`, `wal_less_crash_while_creating_opens_empty`, `wal_less_fixed_cases`, diff --git a/docs/benchmarks/441-opfs/README.md b/docs/benchmarks/441-opfs/README.md index ed0a75ff27..673fef98bb 100644 --- a/docs/benchmarks/441-opfs/README.md +++ b/docs/benchmarks/441-opfs/README.md @@ -1,72 +1,109 @@ # BrowserDb OPFS bounded-memory evidence (#441) -The measurement is **not completed in this environment**. The browser runner -cannot bind its localhost server (`Operation not permitted`, `EPERM`); an -independent Node `listen(0, '127.0.0.1')` probe also fails with `EPERM`. -No open-time, page-read or memory numbers are claimed. AC13(c)'s required -4 MiB/40 MiB comparison remains unverified. +Measured locally on 2026-11-14 with fresh wasm instances. **AC13(c) passes**: +OPFS linear-memory high-water is 1,245,184 bytes for both fixtures, a zero-byte +increase (limit: 4 MiB). These are single-run observations under concurrent +native-test load, not latency benchmarks or a power-loss durability claim. + +## Build and fixtures + +The built companion package uses Rust 1.94.0, release profile with the packaging +shim's default `opt-level = 3`, wasm-bindgen 0.2.126 / web-sys 0.3.103, and +wasm-opt disabled. It was built from the minigraf-wasm feature worktree with a +temporary local core path; no path pin is committed. Core code is identical to +`6fa1582f8` (later commits contain documentation and evidence only). + +- Raw wasm: **1,812,953 bytes**; SHA-256 + `32ba454e40152bd017d5a994529d4294cbf1d43a67ee3b7608ecd7e8189dfbc1`. +- `4mib.graph`: **4,239,360 bytes**, 2,688 payload entities, 1,035 file pages. +- `40mib.graph`: **42,135,552 bytes**, 27,392 payload entities, 10,287 file pages. + +`fixture.rs` builds and verifies native v8 files with the same schema. Generation +and seeding happen outside the measured wasm instance. Each mode/size is seeded +in a worker which is destroyed, then opened in another worker with a fresh wasm +instance. The fixed query is `(query [:find ?a :where [:probe :answer ?a]])`, +whose single short answer must be `42`. + +## Results + +“Open reads” and “pages incl. query” count OPFS read calls/pages; IndexedDB's page +count is the number of records returned by `getAll()`, excluding unused file +pages. IndexedDB does not call the OPFS read API. + +| Engine | Mode | Fixture | Open ms | Open reads | Pages incl. query | Wasm high-water bytes | +|---|---|---|---:|---:|---:|---:| +| Chromium 140.0.7339.16 | OPFS | 4 MiB | 21.6 | 2 | 7 | 1,245,184 | +| Chromium 140.0.7339.16 | OPFS | 40 MiB | 26.3 | 2 | 10 | 1,245,184 | +| Chromium 140.0.7339.16 | IndexedDB | 4 MiB | 136.1 | n/a | 989 | 5,373,952 | +| Chromium 140.0.7339.16 | IndexedDB | 40 MiB | 1,381.6 | n/a | 10,071 | 42,991,616 | +| Firefox 159.0.1 | OPFS | 4 MiB | 30 | 2 | 7 | 1,245,184 | +| Firefox 159.0.1 | OPFS | 40 MiB | 26 | 2 | 10 | 1,245,184 | +| Firefox 159.0.1 | IndexedDB | 4 MiB | 96 | n/a | 989 | 5,373,952 | +| Firefox 159.0.1 | IndexedDB | 40 MiB | 1,474 | n/a | 10,071 | 42,991,616 | + +Both engines pass the 4 MiB delta limit with **0 bytes** of OPFS growth. +WebAssembly memory size is an allocated **high-water mark**, not live memory. +The storage bound excludes query working state, results and export blobs, +import input, v7 migration, a transaction's facts, pages and quarantine metadata +held for long-open cursors, and recovery scans. Browser-side IndexedDB memory, +OS cache state and cold-start latency are not measured. ## Reproduction -Run on the designated benchmark host, with Playwright 1.55.0 and its chosen -engine installed. Build `@minigraf/browser` from the companion wasm repository -using `wasm-pack build --target web --features browser` (release profile, -Cargo default `opt-level = 3` in the packaging shim; record any wasm-opt setting and version). -The offline comparison in this session explicitly disabled wasm-opt for both -revisions; memory measurement should identify the package used in its output. +Use an already installed Playwright 1.55.0 engine; installing browser binaries is +a separate setup step. Build the companion with +`wasm-pack build --target web --no-opt --features browser`. ```bash bash docs/benchmarks/441-opfs/build-fixtures.sh /tmp/opfs-fixtures cd docs/benchmarks/441-opfs npm install --no-save --package-lock=false playwright@1.55.0 -npx playwright install chromium firefox webkit -node measure.mjs /path/to/minigraf-wasm/pkg /tmp/opfs-fixtures chromium > chromium.json +BUILD_LABEL='Rust 1.94.0; release opt-level=3; wasm-opt disabled' \ + node measure.mjs /path/to/minigraf-wasm/pkg /tmp/opfs-fixtures chromium > chromium.json node measure.mjs /path/to/minigraf-wasm/pkg /tmp/opfs-fixtures firefox > firefox.json -node measure.mjs /path/to/minigraf-wasm/pkg /tmp/opfs-fixtures webkit > webkit.json ``` -`fixture.rs` builds and verifies two native v8 files of at least 4 MiB and -40 MiB with the same schema. Its data generation and fixture import occur -outside the measured wasm instance. Each mode/size is seeded in a worker which -is destroyed, then opened in another worker with a fresh wasm instance. The -query is always `(query [:find ?a :where [:probe :answer ?a]])`, whose single short -answer must be `42`. - `measure-worker.js` wraps `FileSystemSyncAccessHandle.prototype.read` before open and records open milliseconds, read calls before and after the query, bytes read -expressed in 4 KiB pages, and `memory.buffer.byteLength` after the query. -IndexedDB's page count is recorded from the `getAll()` request's result length; -the output separately labels total fixture pages as `indexedDbImagePages` -(including free pages). Read-call counts specifically refer to OPFS. - -WebAssembly memory size is an allocated **high-water mark**, not live memory. -The script fails if OPFS's larger-file high-water mark exceeds the smaller-file -mark by more than 4 MiB. Results are needed before claiming that bound. - -The storage bound excludes query working state, results and export blobs, -import input, v7 migration, a transaction's facts, pages and quarantine metadata -held for long-open cursors, and recovery scans. Browser-side IndexedDB memory, -OS cache state and cold-start latency are not measured. +expressed in 4 KiB pages, and `memory.buffer.byteLength` after the query. The +script fails if OPFS's larger-file high-water exceeds the smaller-file mark by +more than 4 MiB. The Chromium run used byte-identical copies of these scripts +under `/tmp` for Playwright module resolution. -Build-size results and exact toolchains are recorded in [build-sizes.md](build-sizes.md). +Playwright Firefox is absent locally. The recorded Firefox run instead served +the **same measurement worker** to the installed Firefox via geckodriver. Its +fallback driver and raw logs are local review artifacts in +`TandemKit/005-browser-opfs/Evidence/` in the main checkout, outside version +control. This is a real Firefox run, not a Playwright Firefox pass. -## Browser execution record +Build-size comparisons and exact toolchains are in [build-sizes.md](build-sizes.md). -The preceding implementation session reported 101 Firefox worker and 81 -Firefox main-thread tests passing. Those are inherited results from the handoff, -not fresh runs here. This session compiled the worker tests but could not start -the local test server. The built package's new cross-worker, termination and -import-interruption harnesses have syntax checks only; no engine pass is claimed. -CI now specifies Chrome/Firefox core worker lanes and Chromium/Firefox/WebKit -package-consumer lanes. +## Browser execution and private browsing -WebKit limitation here: no Playwright WebKit binary is installed, and the -sandbox's localhost bind restriction blocks the package harness before launch. -A reproducible WebKit command is the consumer harness with `BROWSER=webkit`; -its result here is **not run**, not passed. A manual result remains outstanding. +- Firefox 159.0.1 core: **81 main-thread and 101 worker tests pass**. +- Packaged consumer: **Chromium 140.0.7339.16 via Playwright** and + **Firefox 159.0.1 via WebDriver** each pass 52 checks, including six + sequential-commit termination timings, four import-interruption timings, + cross-worker locking, IndexedDB in a worker and the README worker example. +- Chrome core wasm-bindgen tests: not run. Cached ChromeDriver is version 156; + installed Playwright Chromium is version 140 (Brave is 157). No matching + Chrome/driver pair exists locally; no browser binaries were downloaded. +- Playwright Firefox launch: unavailable executable + `~/.cache/ms-playwright/firefox-1490/firefox/firefox`. The WebDriver result + above is the explicitly recorded fallback. +- WebKit: **not run**. Playwright 1.55.0 cannot find + `~/.cache/ms-playwright/webkit_ubuntu20.04_x64_special-2092/pw_run.sh`. + No WebKit binary is installed and no manual Safari result exists. Reproduce + with an installed engine using `BROWSER=webkit node driver.mjs 0.0.0-ci` + from an installed consumer app; the local attempt failed before any test. +- Chromium incognito: an actual Playwright `browser.newContext()` opened + OPFS, acknowledged a write and queried `[[42]]`. This records the automated + incognito context, not a separate manual browser-window run. +- Firefox private: geckodriver launched with + `browser.privatebrowsing.autostart = true`; `openOpfs` rejected with + `[API-026] OPFS mode is not available here: the browser denies access to this origin's storage (SecurityError)`. +- WebKit private: unchecked because the engine is absent. -Private browsing: Chromium, Firefox and WebKit checks are **not run** here. -There is no evidence of how these engines expose OPFS in private windows in this -session. The same consumer page can be served on localhost and opened in a real -private window to record the actual `openOpfs` outcome; automated isolated browser -contexts are not represented as proof of UI private-browsing behavior. +CI specifies Chrome/Firefox core main-thread and worker lanes and +Chromium/Firefox/WebKit package-consumer lanes; this local record does not +claim those remote CI jobs have run. diff --git a/docs/benchmarks/441-opfs/build-sizes.md b/docs/benchmarks/441-opfs/build-sizes.md index 191a4cd3d8..24f3cc2caa 100644 --- a/docs/benchmarks/441-opfs/build-sizes.md +++ b/docs/benchmarks/441-opfs/build-sizes.md @@ -24,3 +24,14 @@ versions; a release package built with those versions may differ. The after package was also stamped as `@minigraf/browser@0.0.0-ci` and packed successfully (665,211 bytes) with JS glue, TypeScript definitions, README and both licenses. No registry publish or remote branch operation was performed. + +## Packaged consumer run with the companion lockfile + +The resumed local run built the actual companion worktree with its pinned +wasm-bindgen 0.2.126 / web-sys 0.3.103, Rust 1.94.0, release opt-level 3 and +wasm-opt disabled. Its raw wasm is **1,812,953 bytes** and the stamped +`@minigraf/browser@0.0.0-ci` tarball is **655,336 bytes**. This package passed +52 consumer checks in each of Chromium 140.0.7339.16 (Playwright) and Firefox +159.0.1 (WebDriver). The memory record identifies its exact SHA-256. +These sizes use different dependency versions from the matching-profile +before/after comparison above; they are not a replacement size delta. From 51681ecda94d45f63d4ff3e87d866f7f5a2f0d56 Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Sat, 14 Nov 2026 11:30:59 +0530 Subject: [PATCH 07/20] fix(browser): validate imports before recovery and complete recovered exports (#441) Reject nonempty images without a selectable meta before open-time recovery can initialize them. Enforce the supplied page count before migration or image emission, and fill only absent meta slots during export. Include native creation-interruption regressions and a full v7 portability fixture. Refs #441 Co-Authored-By: Codex (GPT-6.1-sol) --- src/browser/mod.rs | 27 ++++++- src/storage/persistent_facts.rs | 69 +++++++++++++++++- tests/fixtures/portability/README.md | 6 ++ tests/fixtures/portability/native_v7.graph | Bin 0 -> 8192 bytes tests/opfs_portability_test.rs | 79 ++++++++++++++------- 5 files changed, 152 insertions(+), 29 deletions(-) create mode 100644 tests/fixtures/portability/native_v7.graph diff --git a/src/browser/mod.rs b/src/browser/mod.rs index 26b5177a4e..bdc78fb0f3 100644 --- a/src/browser/mod.rs +++ b/src/browser/mod.rs @@ -1304,6 +1304,31 @@ impl Drop for Importing<'_> { /// migrated, a meta counting pages the image lacks is `STG-044` as for a /// file, and any finding of the integrity walk (`verify()`) rejects it. fn image_of(bytes: Vec) -> anyhow::Result> { + // Import admits only an actual empty blob, a selectable v8 meta, or + // a supported v7 header. Open/create may repair a torn initial meta; + // doing that to a nonempty import would silently replace user data. + if !bytes.is_empty() { + use crate::storage::meta::{MetaPage, SlotState}; + let have = (bytes.len() / crate::storage::PAGE_SIZE) as u64; + let selected = bytes + .chunks(crate::storage::PAGE_SIZE) + .take(2) + .filter_map(|page| match MetaPage::decode(page) { + SlotState::Valid(meta) => Some(meta), + _ => None, + }) + .max_by_key(|meta| meta.generation); + if let Some(meta) = selected { + meta.check_features()?; + if meta.page_count > have { + crate::error::bail_coded!(ErrorCode::Stg044, meta.page_count, have); + } + } else if crate::storage::LegacyHeaderV7::detect(&bytes[..crate::storage::PAGE_SIZE])? + .is_none() + { + crate::error::bail_coded!(ErrorCode::Stg032); + } + } let pages: std::collections::HashMap> = bytes .chunks(crate::storage::PAGE_SIZE) .enumerate() @@ -1317,7 +1342,7 @@ fn image_of(bytes: Vec) -> anyhow::Result have.max(2) { + if need > have { crate::error::bail_coded!(ErrorCode::Stg044, need, have); } if let Some(problem) = pfs.verify()?.problems.into_iter().next() { diff --git a/src/storage/persistent_facts.rs b/src/storage/persistent_facts.rs index 510c156eac..c56768f058 100644 --- a/src/storage/persistent_facts.rs +++ b/src/storage/persistent_facts.rs @@ -817,7 +817,15 @@ impl PersistentFactStorage { if id == slot_page(generation) { emit(id, &meta.encode())?; } else if id < 2 { - let page = self.lock()?.read_page(id)?; + // Initial creation can stop after page 0, before the blank + // second slot. Only an absent slot is synthesized; a failed + // read of a slot that exists must still propagate. + let backend = self.lock()?; + let page = if id < backend.page_count()? { + backend.read_page(id)? + } else { + zeros.clone() + }; let keep = matches!(other, OtherSlot::KeepOlder) && matches!(MetaPage::decode(&page), SlotState::Valid(o) if o.generation < generation); emit(id, if keep { &page } else { &zeros })?; @@ -2061,6 +2069,26 @@ mod tests { mem } + /// Rebuild the portable v7 input with the same legacy writer as migration tests. + #[test] + #[ignore] + fn regenerate_portability_v7_fixture() -> Result<()> { + use crate::query::datalog::{executor::DatalogExecutor, parser::parse_datalog_command}; + let storage = FactStorage::new(); + let executor = DatalogExecutor::new(storage.clone()); + for command in include_str!("../../tests/fixtures/portability/commands.txt").lines() { + executor.execute(parse_datalog_command(command)?)?; + } + let mem = v7_file(&storage.get_all_facts()?, storage.current_tx_count()); + let mut bytes = Vec::new(); + for id in 0..mem.page_count()? { + bytes.extend(mem.read_page(id)?); + } + assert_eq!(u32::from_le_bytes(bytes[4..8].try_into()?), 7); + std::fs::write("tests/fixtures/portability/native_v7.graph", bytes)?; + Ok(()) + } + fn sample_facts(n: u128) -> Vec { let pfs = PersistentFactStorage::new(MemoryBackend::new(), 0).unwrap(); for i in 0..n { @@ -4055,8 +4083,47 @@ mod tests { drop(created); let pfs = open_wl(&mem).expect("reopen after a torn creation"); assert!(values(&pfs).is_empty(), "nothing committed"); + assert_image_opens_natively(&pfs); } } + for k in 0..2 { + let mem = MemoryBackend::new(); + let (backend, config) = FaultInjectingBackend::with_config(mem.clone()); + config.lock().unwrap().fail_sync_after = Some(k); + drop(PersistentFactStorage::open_wal_less(backend, 16)); + let pfs = open_wl(&mem).expect("reopen after failed initial flush"); + assert!(values(&pfs).is_empty()); + assert_image_opens_natively(&pfs); + } + } + + #[test] + fn initial_creation_exports_open_with_minigraf_and_verify() -> Result<()> { + let meta = MetaPage::empty(1).encode(); + for prefix in [0, 16, 24, 40, 512, PAGE_SIZE, 2 * PAGE_SIZE] { + let dir = tempfile::tempdir()?; + let path = dir.path().join("initial.graph"); + let mut original = vec![0; 2 * PAGE_SIZE]; + original[..PAGE_SIZE].copy_from_slice(&meta); + std::fs::write(&path, &original[..prefix])?; + let pfs = PersistentFactStorage::open_wal_less( + crate::storage::backend::FileBackend::open(&path)?, + 16, + )?; + let mut bytes = Vec::new(); + pfs.write_image(pfs.generation(), OtherSlot::KeepOlder, |_, page| { + bytes.extend_from_slice(page); + Ok(()) + })?; + let export = dir.path().join("export.graph"); + std::fs::write(&export, bytes)?; + let native = crate::Minigraf::open(&export)?; + assert!( + native.verify()?.is_ok(), + "initial-interruption export verifies natively" + ); + } + Ok(()) } /// How many page writes one save of `PER` new facts makes on `mem`. diff --git a/tests/fixtures/portability/README.md b/tests/fixtures/portability/README.md index 8e2663a650..0591da3e8a 100644 --- a/tests/fixtures/portability/README.md +++ b/tests/fixtures/portability/README.md @@ -10,3 +10,9 @@ platform. - `opfs_export.graph`: `exportGraph()` of an OPFS-mode database built in a browser. `tests/opfs_portability_test.rs` opens it with `Minigraf::open`, checks `verify()` and the answers. + +- `native_v7.graph`: real format v7, built from the same `commands.txt` with + the legacy writer used by migration tests. Regenerate with + `cargo test --lib regenerate_portability_v7_fixture -- --ignored`. + Native migration and OPFS import answer every query in `queries.txt`, + including long values, retraction, valid time and transaction time. diff --git a/tests/fixtures/portability/native_v7.graph b/tests/fixtures/portability/native_v7.graph new file mode 100644 index 0000000000000000000000000000000000000000..47a3c00c882e6788bce84a762f5074cdbbb1ce64 GIT binary patch literal 8192 zcmeZu4{~E?U|?VZVh{kaL4XxP5k-RZGD1}fHCmk^O4F#K(GVC7fzc2c4S~@R7!85Z z5Eu=C(GVC7fzc2c4S~@R7!3isg#Z)0$Irmf$Y0Ok&u_#a3&hX)%ow)v88I~QnKHQZ zS<=ltqZW;Zz-S1JhQMeDjE2By2#kinXb6mkz-S1JhQMeD4C)XNcmI$} zj&l-Tm7G?oxdl1-m8nInELM4` zfA%^Wq8(Eci&aTtIxCY^B1AQF*Y_ixyE72hFko22YL%Cmo65lIn3I{D3em-ct_!z4 zNhIrlx(tgQYQ21X%V!5MZC*Ml%*Oq0FRN8jeo7^SpQA!qVoqtQLQZ~OdTNnENk(Fx zLUCqANu_RDerb_HQe{bMv5rDF8}}l literal 0 HcmV?d00001 diff --git a/tests/opfs_portability_test.rs b/tests/opfs_portability_test.rs index 65b1156967..fb30ff77f0 100644 --- a/tests/opfs_portability_test.rs +++ b/tests/opfs_portability_test.rs @@ -10,6 +10,7 @@ //! `verify()` and gives the same answers. #![cfg(not(target_arch = "wasm32"))] +use anyhow::{Result, bail}; use minigraf::{Minigraf, QueryResult, Value}; use std::path::Path; @@ -28,13 +29,14 @@ fn json(v: &Value) -> serde_json::Value { } } -fn assert_answers(db: &Minigraf, source: &str) { +fn assert_answers(db: &Minigraf, source: &str) -> Result<()> { for line in QUERIES.lines().filter(|l| !l.is_empty()) { - let (query, expected) = line.split_once('\t').expect("queryrows"); - let expected: serde_json::Value = serde_json::from_str(expected).expect("rows"); - let QueryResult::QueryResults { results, .. } = db.execute(query).expect("query runs") - else { - panic!("{source}: not a query result"); + let (query, expected) = line + .split_once('\t') + .ok_or_else(|| anyhow::anyhow!("queryrows"))?; + let expected: serde_json::Value = serde_json::from_str(expected)?; + let QueryResult::QueryResults { results, .. } = db.execute(query)? else { + bail!("{source}: not a query result"); }; let mut rows: Vec = results .iter() @@ -47,14 +49,15 @@ fn assert_answers(db: &Minigraf, source: &str) { "{source}: answer to {query}" ); } + Ok(()) } -fn open_copy(fixture: &Path) -> (tempfile::TempDir, Minigraf) { - let dir = tempfile::tempdir().expect("tempdir"); +fn open_copy(fixture: &Path) -> Result<(tempfile::TempDir, Minigraf)> { + let dir = tempfile::tempdir()?; let path = dir.path().join("copy.graph"); - std::fs::copy(fixture, &path).expect("copy fixture"); - let db = Minigraf::open(&path).expect("opens without a WAL"); - (dir, db) + std::fs::copy(fixture, &path)?; + let db = Minigraf::open(&path)?; + Ok((dir, db)) } fn fixture(name: &str) -> std::path::PathBuf { @@ -64,38 +67,60 @@ fn fixture(name: &str) -> std::path::PathBuf { } #[test] -fn native_fixture_answers_the_portability_queries() { - let (_dir, db) = open_copy(&fixture("native_v8.graph")); - assert!(db.verify().expect("verify runs").is_ok(), "verifies clean"); - assert_answers(&db, "native_v8.graph"); +fn native_fixture_answers_the_portability_queries() -> Result<()> { + let (_dir, db) = open_copy(&fixture("native_v8.graph"))?; + assert!(db.verify()?.is_ok(), "verifies clean"); + assert_answers(&db, "native_v8.graph") } #[test] -fn opfs_export_opens_natively_verifies_and_answers() { - let bytes = std::fs::read(fixture("opfs_export.graph")).expect("fixture"); - assert_eq!(&bytes[0..4], b"MGRF", "starts with the magic"); +fn v7_fixture_is_legacy_and_answers_the_full_portability_queries() -> Result<()> { + let bytes = std::fs::read(fixture("native_v7.graph"))?; + assert_eq!( + u32::from_le_bytes( + bytes + .get(4..8) + .ok_or_else(|| anyhow::anyhow!("version"))? + .try_into()? + ), + 7 + ); + let (_dir, db) = open_copy(&fixture("native_v7.graph"))?; + assert!(db.verify()?.is_ok()); + assert_answers(&db, "native_v7.graph") +} + +#[test] +fn opfs_export_opens_natively_verifies_and_answers() -> Result<()> { + let bytes = std::fs::read(fixture("opfs_export.graph"))?; + assert_eq!( + bytes.get(0..4), + Some(b"MGRF".as_slice()), + "starts with the magic" + ); assert_eq!(bytes.len() % 4096, 0, "whole pages"); - let (_dir, db) = open_copy(&fixture("opfs_export.graph")); - assert!(db.verify().expect("verify runs").is_ok(), "verifies clean"); - assert_answers(&db, "opfs_export.graph"); + let (_dir, db) = open_copy(&fixture("opfs_export.graph"))?; + assert!(db.verify()?.is_ok(), "verifies clean"); + assert_answers(&db, "opfs_export.graph") } /// Regenerate `native_v8.graph` from `commands.txt`. #[test] #[ignore] -fn regenerate_native_fixture() { - let dir = tempfile::tempdir().expect("tempdir"); +fn regenerate_native_fixture() -> Result<()> { + let dir = tempfile::tempdir()?; let path = dir.path().join("native.graph"); { - let db = Minigraf::open(&path).expect("create"); + let db = Minigraf::open(&path)?; for command in COMMANDS.lines().filter(|l| !l.is_empty()) { - db.execute(command).expect("command"); + db.execute(command)?; } - db.checkpoint().expect("checkpoint"); + db.checkpoint()?; } assert!( !dir.path().join("native.graph.wal").exists(), "checkpointed, no WAL" ); - std::fs::copy(&path, fixture("native_v8.graph")).expect("write fixture"); + std::fs::copy(&path, fixture("native_v8.graph"))?; + Ok(()) } From f13abadfc4123529e6f9e7722ddeeda226c2e3b4 Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Sat, 14 Nov 2026 11:38:01 +0530 Subject: [PATCH 08/20] test(browser): cover lifecycle and import fault boundaries (#441) Exercise rejected import admission, creation exports, all close outcomes, real API errors, v7 portability and staging failures in Firefox workers. Sweep repeated recovery-copy interruptions and retain browser lock and dependent assertions at each outcome. Allow the worker job time for the complete boundary matrix. Refs #441 Co-Authored-By: Codex (GPT-6.1-sol) --- .github/workflows/wasm.yml | 2 + src/browser/opfs_tests.rs | 680 ++++++++++++++++++++++++++++++++++++- 2 files changed, 677 insertions(+), 5 deletions(-) diff --git a/.github/workflows/wasm.yml b/.github/workflows/wasm.yml index b1b89dd2fc..f82e14aaf7 100644 --- a/.github/workflows/wasm.yml +++ b/.github/workflows/wasm.yml @@ -49,6 +49,8 @@ jobs: # wasm-bindgen configures the whole lib test binary for a dedicated # worker; the browser job above retains main-thread coverage. RUSTFLAGS: --cfg minigraf_browser_worker + # Import interruption sweeps acquire real browser locks at each point. + WASM_BINDGEN_TEST_TIMEOUT: 180 steps: - uses: actions/checkout@v4 - uses: dtolnay/rust-toolchain@stable diff --git a/src/browser/opfs_tests.rs b/src/browser/opfs_tests.rs index d6f8920494..1ed89eeae9 100644 --- a/src/browser/opfs_tests.rs +++ b/src/browser/opfs_tests.rs @@ -199,6 +199,170 @@ mod worker { db } + // Override one browser boundary and restore its exact descriptor on drop. + // The database still uses real OPFS files and browser handles. + struct JsPatch(js_sys::Function); + impl JsPatch { + fn new(target: &JsValue, key: &str, replacement: &JsValue) -> Self { + let restore = js_sys::Function::new_with_args( + "obj,key,value", + r#" + const old = Object.getOwnPropertyDescriptor(obj, key); + Object.defineProperty(obj, key, {value, configurable:true, writable:true}); + return () => old ? Object.defineProperty(obj,key,old) : delete obj[key]; + "#, + ) + .call3(&JsValue::NULL, target, &key.into(), replacement) + .unwrap(); + Self(restore.unchecked_into()) + } + fn wrap(target: &JsValue, key: &str, body: &str) -> Self { + let original = js_sys::Reflect::get(target, &key.into()).unwrap(); + let replacement = js_sys::Function::new_with_args( + "original", + &format!("return function(...args){{{body}}}"), + ) + .call1(&JsValue::NULL, &original) + .unwrap(); + Self::new(target, key, &replacement) + } + fn global(key: &str, value: &JsValue) -> Self { + Self::new(&js_sys::global(), key, value) + } + fn method(target: &JsValue, key: &str, body: &str) -> Self { + Self::new( + target, + key, + &js_sys::Function::new_with_args("...args", body), + ) + } + } + impl Drop for JsPatch { + fn drop(&mut self) { + self.0.call0(&JsValue::NULL).unwrap(); + } + } + fn prototype(class: &str) -> JsValue { + let class = js_sys::Reflect::get(&js_sys::global(), &class.into()).unwrap(); + js_sys::Reflect::get(&class, &"prototype".into()).unwrap() + } + + #[wasm_bindgen_test] + async fn unavailable_contexts_reject_before_creating_files() { + let before = entries().await; + for case in [ + "insecure", + "missing-storage", + "missing-directory", + "missing-sync", + "denied", + ] { + let nav = js_sys::Reflect::get(&js_sys::global(), &"navigator".into()).unwrap(); + let storage = js_sys::Reflect::get(&nav, &"storage".into()).unwrap(); + let patch = match case { + "insecure" => JsPatch::global("isSecureContext", &JsValue::FALSE), + "missing-storage" => JsPatch::new(&nav, "storage", &JsValue::UNDEFINED), + "missing-directory" => JsPatch::new(&storage, "getDirectory", &JsValue::UNDEFINED), + "missing-sync" => JsPatch::new( + &prototype("FileSystemFileHandle"), + "createSyncAccessHandle", + &JsValue::UNDEFINED, + ), + _ => JsPatch::method( + &storage, + "getDirectory", + "return Promise.reject(new DOMException('denied','SecurityError'))", + ), + }; + let err = rejection(BrowserDb::open_opfs(unique(case)).await); + assert_eq!(code_of(&err), "API-026", "{case}"); + if case == "denied" { + assert!(err.as_string().unwrap().contains("SecurityError")); + } + drop(patch); + assert_eq!(entries().await, before, "{case}: no entries created"); + } + } + + #[wasm_bindgen_test] + async fn short_and_thrown_reads_are_errors_including_existing_meta_slots() { + let name = unique("read-fault"); + let db = open(&name).await; + put(&db, 0..3).await; + for (body, detail) in [ + ("return args[0].length - 1", "short read/write"), + ( + "throw new DOMException('unreadable','NotReadableError')", + "NotReadableError", + ), + ] { + let patch = JsPatch::method(file(&db).handle_for_tests(), "read", body); + let err = rejection(db.export_graph()); + assert_eq!(code_of(&err), "STG-048"); + assert!(err.as_string().unwrap().contains(detail)); + drop(patch); + assert_eq!(ns(&db).await, vec![0, 1, 2]); + } + db.close().unwrap(); + let patch = JsPatch::method( + &prototype("FileSystemSyncAccessHandle"), + "read", + "throw new DOMException('unreadable','NotReadableError')", + ); + assert_eq!( + code_of(&rejection(BrowserDb::open_opfs(name.clone()).await)), + "STG-048" + ); + drop(patch); + // A failed open released the real handle. + let db = open(&name).await; + assert_eq!(ns(&db).await, vec![0, 1, 2]); + db.close().unwrap(); + } + + #[wasm_bindgen_test] + async fn unsafe_offsets_fail_before_io_and_quota_names_survive() { + let name = unique("offset-quota"); + let db = open(&name).await; + let f = file(&db); + let io = js_sys::Object::new(); + js_sys::Reflect::set(&io, &"calls".into(), &0.into()).unwrap(); + let replacement = js_sys::Function::new_with_args( + "counter", + "return function(){counter.calls++; throw new Error('unexpected IO')}", + ) + .call1(&JsValue::NULL, &io) + .unwrap(); + let patch = JsPatch::new(f.handle_for_tests(), "write", &replacement); + let err = f.write_page(u64::MAX, &[0; 4096]).unwrap_err(); + assert_eq!(crate::MinigrafError::from(err).code(), "INT-048"); + assert_eq!( + js_sys::Reflect::get(&io, &"calls".into()).unwrap().as_f64(), + Some(0.0) + ); + drop(patch); + let patch = JsPatch::method( + f.handle_for_tests(), + "write", + "throw new DOMException('quota','QuotaExceededError')", + ); + let err = rejection(db.execute("(transact [[:q :n 9]])".into()).await); + assert_eq!(code_of(&err), "STG-048"); + let text = err.as_string().unwrap(); + assert!(text.contains("QuotaExceededError") && text.contains("write page")); + assert_eq!( + code_of(&rejection( + db.execute("(transact [[:q :n 10]])".into()).await + )), + "STG-046" + ); + drop(patch); + db.close().unwrap(); + let db = open(&name).await; + assert!(ns(&db).await.is_empty()); + db.close().unwrap(); + } + // ── AC1, AC4: entry point, persistence, files ────────────────────────── #[wasm_bindgen_test] @@ -436,6 +600,184 @@ mod worker { reopened.close().unwrap(); } + #[wasm_bindgen_test] + async fn deferred_checkpoint_and_import_reject_after_close_and_reopen() { + for mode in ["opfs", "idb", "memory"] { + let name = unique("deferred-close"); + let db = match mode { + "opfs" => open(&name).await, + "idb" => BrowserDb::open(&name).await.unwrap(), + _ => BrowserDb::open_in_memory().unwrap(), + }; + put(&db, 0..3).await; + // Rust async method futures are the bodies the JS promise wrapper + // defers. Leave them unpolled until after the same-name reopen. + let checkpoint = db.checkpoint(); + let import = db.import_graph(blob(&[])); + let execute = db.execute("(transact [[:late :n 9]])".into()); + db.close().unwrap(); + let reopened = match mode { + "opfs" => open(&name).await, + "idb" => BrowserDb::open(&name).await.unwrap(), + _ => BrowserDb::open_in_memory().unwrap(), + }; + for err in [ + rejection(checkpoint.await), + rejection(import.await), + rejection(execute.await).into(), + ] { + assert_eq!(code_of(&err), "API-018"); + } + let expected = if mode == "memory" { + vec![] + } else { + vec![0, 1, 2] + }; + assert_eq!(ns(&reopened).await, expected); + reopened.close().unwrap(); + } + } + + #[wasm_bindgen_test] + async fn memory_commit_and_failed_commit_keep_their_own_outcomes_after_close() { + let db = BrowserDb::open_in_memory().unwrap(); + let mut tx = db.begin_write().unwrap(); + tx.execute("(transact [[:c :n 1]])").unwrap(); + let commit = tx.commit(); + db.close().unwrap(); + let reopened = BrowserDb::open_in_memory().unwrap(); + JsFuture::from(commit) + .await + .expect("applied memory commit resolves"); + assert!(ns(&reopened).await.is_empty()); + reopened.close().unwrap(); + + let name = unique("failed-promise-close"); + let db = open(&name).await; + put(&db, 0..3).await; + *file(&db).faults.lock().unwrap() = Faults { + write_after: Some((0, WriteFault::Throw(0))), + ..Faults::default() + }; + let mut tx = db.begin_write().unwrap(); + tx.execute("(transact [[:bad :n 9]])").unwrap(); + let commit = tx.commit(); + // Attach a rejection handler before yielding to the browser. + let observed: js_sys::Promise = + js_sys::Function::new_with_args("p", "return p.catch(e => e)") + .call1(&JsValue::NULL, &commit) + .unwrap() + .unchecked_into(); + db.close().unwrap(); + let reopened = open(&name).await; + assert_eq!(code_of(&JsFuture::from(observed).await.unwrap()), "STG-048"); + assert_eq!(ns(&reopened).await, vec![0, 1, 2]); + reopened.close().unwrap(); + } + + #[wasm_bindgen_test] + async fn submitted_indexeddb_flush_settles_after_close_and_reopen() { + let name = unique("submitted-flush-close"); + let db = BrowserDb::open(&name).await.unwrap(); + let state = js_sys::Object::new(); + let handle = db.inner.borrow().idb.as_ref().unwrap().db.clone(); + // Queue real IndexedDB requests and let the browser commit them, but + // hold the completion callback so the old commit promise is pending. + let replacement = js_sys::Function::new_with_args("db,state", r#" + const original = db.transaction; + return function(...args) { + const tx = original.apply(this,args); + if (args[1] === 'readwrite') { + state.submitted = true; + const setter = Object.getOwnPropertyDescriptor(IDBTransaction.prototype,'oncomplete').set; + Object.defineProperty(tx,'oncomplete',{set(callback) { + setter.call(tx,event => { + state.completed = true; + state.resume = () => callback(event); + }); + }}); + } + return tx; + }; + "#).call2(&JsValue::NULL, &handle, &state).unwrap(); + let patch = JsPatch::new(&handle, "transaction", &replacement); + let mut tx = db.begin_write().unwrap(); + tx.execute("(transact [[:c :n 1]])").unwrap(); + let commit = tx.commit(); + JsFuture::from(js_sys::Promise::resolve(&JsValue::NULL)) + .await + .unwrap(); + JsFuture::from(js_sys::Promise::resolve(&JsValue::NULL)) + .await + .unwrap(); + assert!( + js_sys::Reflect::get(&state, &"submitted".into()) + .unwrap() + .is_truthy() + ); + db.close().unwrap(); + let reopened = BrowserDb::open(&name).await.unwrap(); + assert!( + js_sys::Reflect::get(&state, &"completed".into()) + .unwrap() + .is_truthy() + ); + assert_eq!(ns(&reopened).await, vec![1]); + // New writes must not be replaced by an old completion callback. + put(&reopened, 2..3).await; + let resume: js_sys::Function = js_sys::Reflect::get(&state, &"resume".into()) + .unwrap() + .unchecked_into(); + resume.call0(&JsValue::NULL).unwrap(); + JsFuture::from(commit) + .await + .expect("submitted flush keeps its outcome"); + assert_eq!(ns(&reopened).await, vec![1, 2]); + drop(patch); + reopened.close().unwrap(); + let reopened = BrowserDb::open(&name).await.unwrap(); + assert_eq!(ns(&reopened).await, vec![1, 2]); + reopened.close().unwrap(); + } + + #[wasm_bindgen_test] + async fn dropping_every_owner_releases_the_lock_and_never_saves_staged_facts() { + for failed_commit in [false, true] { + let name = unique("drop-no-save"); + let db = open(&name).await; + put(&db, 0..3).await; + let before = db.export_graph().unwrap().to_vec(); + let cursor = db.query("(query [:find ?n :where [?e :n ?n]])").unwrap(); + let pq = db.prepare("(query [:find ?n :where [?e :n ?n]])").unwrap(); + let mut tx = db.begin_write().unwrap(); + tx.execute("(transact [[:unsaved :n 99]])").unwrap(); + if failed_commit { + *file(&db).faults.lock().unwrap() = Faults { + write_after: Some((0, WriteFault::Throw(0))), + ..Faults::default() + }; + assert_eq!( + code_of(&rejection(JsFuture::from(tx.commit()).await)), + "STG-048" + ); + // The one-shot fault is gone: an unintended save on final drop + // could now succeed, so byte equality below catches a retry. + } + drop(db); + assert_eq!( + code_of(&rejection(BrowserDb::open_opfs(name.clone()).await)), + "STG-026" + ); + drop(cursor); + drop(pq); + drop(tx); + let db = open(&name).await; + assert_eq!(db.export_graph().unwrap().to_vec(), before); + assert_eq!(ns(&db).await, vec![0, 1, 2]); + db.close().unwrap(); + } + } + // ── AC10: failed commits ─────────────────────────────────────────────── #[wasm_bindgen_test] @@ -608,6 +950,63 @@ mod worker { db.close().unwrap(); } + #[wasm_bindgen_test] + async fn every_interrupted_initial_creation_exports_a_complete_image() { + let empty = crate::storage::meta::MetaPage::empty(1).encode(); + for prefix in [0, 16, 40, 512, 4096, 8192] { + let name = unique("initial-export"); + let mut bytes = vec![0; 8192]; + bytes[..4096].copy_from_slice(&empty); + write_raw(&format!("{name}.graph"), &bytes[..prefix]).await; + let db = open(&name).await; + assert!(ns(&db).await.is_empty()); + let exported = db + .export_graph() + .expect("export after initial interruption"); + assert_eq!(exported.length(), 8192); + image_of(exported.to_vec()).expect("complete export verifies"); + db.close().unwrap(); + } + } + + #[wasm_bindgen_test] + async fn nonempty_import_without_a_selectable_meta_preserves_destination() { + let empty = crate::storage::meta::MetaPage::empty(1).encode(); + for kind in ["zeros", "torn", "page-zero-only"] { + let name = unique("invalid-meta-import"); + let db = open(&name).await; + put(&db, 0..3).await; + let before = db.export_graph().unwrap().to_vec(); + let mut cursor = db.query("(query [:find ?n :where [?e :n ?n]])").unwrap(); + let pq = db.prepare("(query [:find ?n :where [?e :n ?n]])").unwrap(); + let mut bytes = vec![0; 8192]; + match kind { + "torn" => bytes[..24].copy_from_slice(&empty[..24]), + "page-zero-only" => bytes = empty.clone(), + _ => (), + } + let err = rejection(db.import_graph(blob(&bytes)).await); + if kind == "page-zero-only" { + assert_eq!(code_of(&err), "STG-044"); + } else { + assert_eq!(code_of(&err), "STG-032"); + } + assert_eq!(db.export_graph().unwrap().to_vec(), before); + assert!(cursor.next_batch(10).unwrap().is_some()); + assert!(pq.execute(JsValue::UNDEFINED).is_ok()); + assert_eq!( + code_of(&rejection(BrowserDb::open_opfs(name.clone()).await)), + "STG-026" + ); + assert_eq!(ns(&db).await, vec![0, 1, 2]); + db.close().unwrap(); + let reopened = open(&name).await; + assert_eq!(ns(&reopened).await, vec![0, 1, 2]); + assert_eq!(reopened.export_graph().unwrap().to_vec(), before); + reopened.close().unwrap(); + } + } + // ── AC11: portability ────────────────────────────────────────────────── #[wasm_bindgen_test] @@ -642,16 +1041,14 @@ mod worker { assert_portability_answers(&db, "native v8, reopened").await; db.close().unwrap(); - let v7: &[u8] = include_bytes!("../../tests/fixtures/compat.graph"); + let v7: &[u8] = include_bytes!("../../tests/fixtures/portability/native_v7.graph"); let name = unique("native-v7"); let db = open(&name).await; db.import_graph(blob(v7)).await.expect("import native v7"); - let names = exec(&db, "(query [:find ?n :where [?e :name ?n]])").await; - assert_eq!(rows(&names), serde_json::json!([["Alice"]])); + assert_portability_answers(&db, "native v7").await; db.close().unwrap(); let db = open(&name).await; - let names = exec(&db, "(query [:find ?n :where [?e :name ?n]])").await; - assert_eq!(rows(&names), serde_json::json!([["Alice"]])); + assert_portability_answers(&db, "native v7, reopened").await; db.close().unwrap(); } @@ -802,6 +1199,93 @@ mod worker { db.close().unwrap(); } + #[wasm_bindgen_test] + async fn admission_precedence_is_closed_then_busy_then_transaction_open() { + let name = unique("precedence"); + let db = open(&name).await; + put(&db, 0..3).await; + let mut cursor = db.query("(query [:find ?n :where [?e :n ?n]])").unwrap(); + let pq = db.prepare("(query [:find ?n :where [?e :n ?n]])").unwrap(); + let mut tx = db.begin_write().unwrap(); + let check_database = |code: &str| { + assert_eq!( + code_of(&rejection(db.query("(query [:find ?n :where [?e :n ?n]])"))), + code + ); + assert_eq!( + code_of(&rejection( + db.prepare("(query [:find ?n :where [?e :n ?n]])") + )), + code + ); + assert_eq!(code_of(&rejection(db.begin_write())), code); + }; + check_database("INT-001"); + assert_eq!( + code_of(&rejection(db.import_graph(blob(&[])).await)), + "INT-001" + ); + assert_eq!(code_of(&rejection(db.checkpoint().await)), "INT-001"); + // Exercise overlapping flags directly: normal admission prevents an + // import from starting under a transaction, but precedence must hold + // even when several refusal conditions apply. + db.inner.importing.set(true); + check_database("API-027"); + assert_eq!( + code_of(&rejection(pq.execute(JsValue::UNDEFINED))), + "API-027" + ); + assert_eq!(code_of(&rejection(cursor.next_batch(1))), "API-027"); + assert_eq!( + code_of(&rejection( + tx.execute("(query [:find ?n :where [?e :n ?n]])") + )), + "API-027" + ); + assert_eq!(code_of(&rejection(db.checkpoint().await)), "API-027"); + assert_eq!( + code_of(&rejection(db.import_graph(blob(&[])).await)), + "API-027" + ); + assert_eq!( + code_of(&rejection(db.register_predicate( + "p?", + js_sys::Function::new_no_args("return true").into() + ))), + "API-027" + ); + // A dependent invalidated by publication is closed even if the next + // import is busy and a transaction flag is also present. + db.inner.epoch.set(db.inner.epoch.get() + 1); + assert_eq!(code_of(&rejection(cursor.next_batch(1))), "API-018"); + assert_eq!( + code_of(&rejection(pq.execute(JsValue::UNDEFINED))), + "API-018" + ); + db.inner.importing.set(false); + db.close().unwrap(); + db.inner.importing.set(true); + check_database("API-018"); + assert_eq!(code_of(&rejection(db.checkpoint().await)), "API-018"); + assert_eq!( + code_of(&rejection(db.import_graph(blob(&[])).await)), + "API-018" + ); + assert_eq!( + code_of(&rejection(tx.execute("(transact [[:late :n 99]])"))), + "API-018" + ); + assert_eq!( + code_of(&rejection(JsFuture::from(tx.commit()).await)), + "API-018" + ); + cursor.close(); + tx.rollback(); + let reopened = open(&name).await; + assert_eq!(ns(&reopened).await, vec![0, 1, 2]); + reopened.close().unwrap(); + } + /// An image staged like an import, ready for [`recover_import`] tests. fn staged_bytes(image: &[u8], generation: u64) -> Vec { let pfs = image_of(image.to_vec()).unwrap(); @@ -835,6 +1319,8 @@ mod worker { let name = unique("interrupt"); let db = open(&name).await; put(&db, 0..3).await; + let mut cursor = db.query("(query [:find ?n :where [?e :n ?n]])").unwrap(); + let pq = db.prepare("(query [:find ?n :where [?e :n ?n]])").unwrap(); let faults = Faults { write_after: (!flush).then_some((k, WriteFault::Throw(16))), flush_after: flush.then_some(k), @@ -847,13 +1333,37 @@ mod worker { let _ = opfs::tests::take_armed(&format!("{name}.graph-import")); if result.is_ok() { assert_eq!(ns(&db).await, new); + assert_eq!(code_of(&rejection(cursor.next_batch(1))), "API-018"); + assert_eq!( + code_of(&rejection(pq.execute(JsValue::UNDEFINED))), + "API-018" + ); + assert_eq!( + code_of(&rejection(BrowserDb::open_opfs(name.clone()).await)), + "STG-026" + ); + assert!( + !entries().await.contains(&format!("{name}.graph-import")), + "successful import cleaned staging" + ); db.close().unwrap(); break; } if db.inner.closed.get() { published_failures += 1; + assert_eq!(code_of(&rejection(cursor.next_batch(1))), "API-018"); + assert_eq!( + code_of(&rejection(pq.execute(JsValue::UNDEFINED))), + "API-018" + ); } else { kept_old += 1; + assert!(cursor.next_batch(10).unwrap().is_some()); + assert!(pq.execute(JsValue::UNDEFINED).is_ok()); + assert_eq!( + code_of(&rejection(BrowserDb::open_opfs(name.clone()).await)), + "STG-026" + ); assert_eq!(ns(&db).await, old, "the old database, still open"); db.close().unwrap(); } @@ -861,6 +1371,10 @@ mod worker { let db = open(&name).await; let got = ns(&db).await; assert!(got == old || got == new, "old or new, whole"); + assert_eq!( + code_of(&rejection(BrowserDb::open_opfs(name.clone()).await)), + "STG-026" + ); db.close().unwrap(); let mine: Vec = entries() .await @@ -902,6 +1416,162 @@ mod worker { db.close().unwrap(); } + #[wasm_bindgen_test] + async fn staging_create_acquire_truncate_and_cleanup_failures() { + let other = BrowserDb::open_in_memory().unwrap(); + put(&other, 100..110).await; + let image = other.export_graph().unwrap(); + for boundary in ["create", "acquire", "truncate", "cleanup"] { + let name = unique("stage-boundary"); + let db = open(&name).await; + put(&db, 0..3).await; + let before = db.export_graph().unwrap().to_vec(); + let mut cursor = db.query("(query [:find ?n :where [?e :n ?n]])").unwrap(); + let pq = db.prepare("(query [:find ?n :where [?e :n ?n]])").unwrap(); + let patch = match boundary { + "create" => JsPatch::wrap( + &prototype("FileSystemDirectoryHandle"), + "getFileHandle", + "if(args[0].endsWith('.graph-import')) return Promise.reject(new DOMException('create','UnknownError')); return original.apply(this,args)", + ), + "acquire" => JsPatch::wrap( + &prototype("FileSystemFileHandle"), + "createSyncAccessHandle", + "if(this.name.endsWith('.graph-import')) return Promise.reject(new DOMException('acquire','UnknownError')); return original.apply(this,args)", + ), + "truncate" => JsPatch::method( + &prototype("FileSystemSyncAccessHandle"), + "truncate", + "throw new DOMException('truncate','UnknownError')", + ), + _ => JsPatch::wrap( + &prototype("FileSystemDirectoryHandle"), + "removeEntry", + "if(args[0].endsWith('.graph-import')) return Promise.reject(new DOMException('cleanup','UnknownError')); return original.apply(this,args)", + ), + }; + let result = db.import_graph(image.clone()).await; + drop(patch); + if boundary == "cleanup" { + result.expect("post-publication cleanup rejection must resolve import"); + assert_eq!(ns(&db).await, (100..110).collect::>()); + assert_eq!(code_of(&rejection(cursor.next_batch(1))), "API-018"); + assert_eq!( + code_of(&rejection(pq.execute(JsValue::UNDEFINED))), + "API-018" + ); + assert!( + entries().await.contains(&format!("{name}.graph-import")), + "cleanup really failed" + ); + } else { + assert_eq!(code_of(&rejection(result)), "STG-048"); + assert_eq!(db.export_graph().unwrap().to_vec(), before); + assert_eq!(ns(&db).await, vec![0, 1, 2]); + assert!(cursor.next_batch(10).unwrap().is_some()); + assert!(pq.execute(JsValue::UNDEFINED).is_ok()); + } + assert_eq!( + code_of(&rejection(BrowserDb::open_opfs(name.clone()).await)), + "STG-026" + ); + db.close().unwrap(); + let db = open(&name).await; + let expected = if boundary == "cleanup" { + (100..110).collect() + } else { + vec![0, 1, 2] + }; + assert_eq!(ns(&db).await, expected); + assert_eq!( + code_of(&rejection(BrowserDb::open_opfs(name.clone()).await)), + "STG-026" + ); + db.close().unwrap(); + let mine: Vec<_> = entries() + .await + .into_iter() + .filter(|e| e.starts_with(&name)) + .collect(); + assert_eq!(mine, vec![format!("{name}.graph")]); + } + } + + #[wasm_bindgen_test] + async fn recovery_copy_survives_every_second_interruption() { + let other = BrowserDb::open_in_memory().unwrap(); + put(&other, 100..400).await; + let image = other.export_graph().unwrap().to_vec(); + let new: Vec<_> = (100..400).collect(); + let mut interruptions = 0; + for boundary in ["write", "flush", "truncate"] { + for k in 0..80 { + let name = unique("recovery-sweep"); + let entry = format!("{name}.graph"); + let staging = format!("{name}.graph-import"); + let db = open(&name).await; + put(&db, 0..3).await; + let generation = db.inner.borrow().pfs.generation() + 10; + db.close().unwrap(); + write_raw(&staging, &staged_bytes(&image, generation | 1)).await; + let patch = if boundary == "truncate" { + Some(JsPatch::method( + &prototype("FileSystemSyncAccessHandle"), + "truncate", + "throw new DOMException('truncate','UnknownError')", + )) + } else { + arm( + &entry, + Faults { + write_after: (boundary == "write") + .then_some((k, WriteFault::Throw(16))), + flush_after: (boundary == "flush").then_some(k), + }, + ); + None + }; + let result = BrowserDb::open_opfs(name.clone()).await; + drop(patch); + if let Ok(db) = result { + assert_eq!(ns(&db).await, new); + db.close().unwrap(); + break; + } + interruptions += 1; + assert!( + entries().await.contains(&staging), + "recovery retained its source" + ); + // Refused open released both handles; the source remains intact. + let main = raw_file(&entry).await; + main.close(); + let stage = raw_file(&staging).await; + stage.close(); + let db = open(&name).await; + assert_eq!(ns(&db).await, new, "whole new database after retry"); + assert_eq!( + code_of(&rejection(BrowserDb::open_opfs(name.clone()).await)), + "STG-026" + ); + db.close().unwrap(); + let mine: Vec<_> = entries() + .await + .into_iter() + .filter(|e| e.starts_with(&name)) + .collect(); + assert_eq!(mine, vec![entry]); + if boundary == "truncate" { + break; + } + } + } + assert!( + interruptions > 10, + "swept data, meta, both flushes and truncate" + ); + } + /// A staging file left after its copy finished (a failed cleanup) is /// removed at the next open, never copied over later commits; one that /// never reached its publication point is removed too. From c43845eb1c10001d71ff71e2f711cb6fd9499c6d Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Sat, 14 Nov 2026 11:42:40 +0530 Subject: [PATCH 09/20] fix(browser): check import header access without slicing (#441) Use a checked first-page view during import admission so the browser strict lint gate also verifies the new validation path. Refs #441 Co-Authored-By: Codex (GPT-6.1-sol) --- src/browser/mod.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/browser/mod.rs b/src/browser/mod.rs index bdc78fb0f3..69834b2f2d 100644 --- a/src/browser/mod.rs +++ b/src/browser/mod.rs @@ -1323,8 +1323,12 @@ fn image_of(bytes: Vec) -> anyhow::Result have { crate::error::bail_coded!(ErrorCode::Stg044, meta.page_count, have); } - } else if crate::storage::LegacyHeaderV7::detect(&bytes[..crate::storage::PAGE_SIZE])? - .is_none() + } else if crate::storage::LegacyHeaderV7::detect( + bytes + .get(..crate::storage::PAGE_SIZE) + .unwrap_or(bytes.as_slice()), + )? + .is_none() { crate::error::bail_coded!(ErrorCode::Stg032); } From 12305fedfa1145c628937687ad426bac99b7b900 Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Sat, 14 Nov 2026 11:42:40 +0530 Subject: [PATCH 10/20] docs(browser): align OPFS guarantees and verification evidence (#441) Document the exact database and staging entry mapping, strict nonempty import admission and complete recovered exports. Record the expanded lifecycle and fault coverage, current native and browser counts, unchanged healthy-read memory measurement and the remaining WebKit limitation. Refs #441 Co-Authored-By: Codex (GPT-6.1-sol) --- CHANGELOG.md | 2 +- CLAUDE.md | 2 +- docs/DURABILITY.md | 21 ++++++++++++---- docs/TEST_COVERAGE.md | 40 +++++++++++++++++++----------- docs/benchmarks/441-opfs/README.md | 8 ++++-- 5 files changed, 49 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 837265bda2..35ce231036 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- **OPFS-backed `BrowserDb` persistence and explicit close** (#441). `await BrowserDb.openOpfs(name)` opens one `.graph` file through an exclusive sync access handle in a dedicated worker on a secure origin. It uses the default 256-page cache, reads pages on demand and commits by copy-on-write with data/meta flushes, without a WAL or a whole-file buffer. IndexedDB `open(name)` now also works in workers. The two modes have independent namespaces and no automatic fallback. OPFS starts at Tier 2 within the Tier 1 browser binding. All modes gain synchronous, idempotent `close()`, which saves nothing, discards an open transaction, invalidates dependents and immediately releases an OPFS lock. OPFS import validates the whole image, stages and publishes the complete old or new database across interruption, and invalidates earlier cursors/prepared queries after publication. OPFS recovery accepts the sole valid meta; unlike native recovery this can silently fall back after later meta rot. A failed commit is unconfirmed (it may be stored), leaves reads working, and refuses later writes with STG-046 until reopen. New registered errors API-026 (unavailable OPFS context), API-027 (busy database) and STG-048 (browser operation and exception name); lock contention is STG-026. No file-format change or new crate dependency. See [durability](docs/DURABILITY.md#browser-storage-modes) and [measurement status](docs/benchmarks/441-opfs/README.md). +- **OPFS-backed `BrowserDb` persistence and explicit close** (#441). `await BrowserDb.openOpfs(name)` opens one `.graph` file through an exclusive sync access handle in a dedicated worker on a secure origin. It uses the default 256-page cache, reads pages on demand and commits by copy-on-write with data/meta flushes, without a WAL or a whole-file buffer. IndexedDB `open(name)` now also works in workers. The two modes have independent namespaces and no automatic fallback. OPFS starts at Tier 2 within the Tier 1 browser binding. All modes gain synchronous, idempotent `close()`, which saves nothing, discards an open transaction, invalidates dependents and immediately releases an OPFS lock. OPFS import rejects nonempty images without a selectable meta before replacing anything (only an empty blob creates an empty database), validates the whole image, stages and publishes the complete old or new database across interruption, and invalidates earlier cursors/prepared queries after publication. Exports after interrupted initial creation fill an absent second meta slot with zeros and still propagate failed reads of existing pages. OPFS recovery accepts the sole valid meta; unlike native recovery this can silently fall back after later meta rot. A failed commit is unconfirmed (it may be stored), leaves reads working, and refuses later writes with STG-046 until reopen. New registered errors API-026 (unavailable OPFS context), API-027 (busy database) and STG-048 (browser operation and exception name); lock contention is STG-026. No file-format change or new crate dependency. See [durability](docs/DURABILITY.md#browser-storage-modes) and [measurement status](docs/benchmarks/441-opfs/README.md). - **`OpenOptions::allow_overrides`** (#869, part of #862): the embedder lists, as `Overridable` variants, the query keywords that may override the database's settings. The default, `Overridable::DEFAULTS` (`MaxDerivedFacts`, `MaxResults`), allows every keyword that could do so before, so no query changes. A query using a keyword the list leaves out is refused with the new **API-025** before it runs, from `execute`, `query`, `prepare` and `WriteTransaction::execute`; the list replaces the defaults rather than adding to them, and is not stored in the file. `BrowserDb` takes no options and uses the defaults. `Overridable` is `#[non_exhaustive]`. diff --git a/CLAUDE.md b/CLAUDE.md index d3447986b2..3d4d2c0ef4 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -250,7 +250,7 @@ Auto-migrates v7 → v8 on open (spec §9, with a backup meta page). v1–v6 are ## Test Coverage -**2169 tests** (2147 active, 22 ignored; unit + integration + doc). Engine answers once checked by the executor are frozen in `tests/answers/` (#551). +**2172 tests** (2149 active, 23 ignored; unit + integration + doc). Engine answers once checked by the executor are frozen in `tests/answers/` (#551). See `docs/TEST_COVERAGE.md` for the full per-file breakdown. The #760 deterministic storage suite adds three native unit tests in a separate `--cfg minigraf_shuttle` build, outside the default count. Run only `graph::storage::deterministic::` with that cfg; ordinary unit tests do not enter its scheduler. `deterministic-schedules.yml` runs relevant v3 PRs on Linux/macOS/Windows plus Linux MSRV, and larger nightly/manual searches with replay artifacts. The identical main workflow activates the v3 schedule after both PRs merge. See `docs/testing/deterministic-schedules.md` for scope, controls and findings. diff --git a/docs/DURABILITY.md b/docs/DURABILITY.md index f53900254f..93c69e5c82 100644 --- a/docs/DURABILITY.md +++ b/docs/DURABILITY.md @@ -72,7 +72,9 @@ and `importGraph()`; exported v8 bytes open natively without a WAL, and OPFS can import supported v7 files through migration. OPFS maps the name exactly to `.graph` in the origin's root directory. -Empty names, `.`, `..`, slash/backslash and reserved import names are API-017. +Empty names, `.`, `..`, slash/backslash and NUL are API-017. No suffix is +reserved: every database entry ends in `.graph`; an import staging entry is +`.graph-import`, which no database name maps to. The exclusive sync access handle is the lock: another open in the same worker, another worker or tab fails with STG-026. `close()` releases it immediately; worker termination also releases it. A refused open releases any handle it took. @@ -109,7 +111,12 @@ An already applied OPFS/in-memory transaction commit resolves after close; an IndexedDB commit whose flush has not been submitted rejects with API-018 and never submits it. An already submitted IndexedDB flush settles by its own result. -OPFS import first validates and verifies its complete input, then stages it in +OPFS import first validates and verifies its complete input. Only an empty blob +creates an empty database; a nonempty image without a selectable meta is rejected, +even if opening those bytes as a file could recover an interrupted creation. +A meta counting pages beyond the input is STG-044. Exports normalize a missing +second meta slot after interrupted creation, while failed reads of existing +pages remain errors. Import stages the verified image in `.graph-import`. Publication makes the new image recoverable before copying it over the main file; interruption and reopen finish with the complete old or new image. A pre-publication failure preserves the old database and dependents; @@ -123,13 +130,17 @@ OPFS reads healthy v8 pages on demand through a 256-page cache. The storage boun excludes query working state, results/export blobs, import input, v7 migration, transaction facts, pages and quarantine metadata held for long-open cursors, and recovery scans. [The measurement record](benchmarks/441-opfs/README.md) -separates inherited browser test results from outstanding engine and memory runs. +records the completed Chromium and Firefox memory checks: both used 1,245,184 +bytes of wasm memory for each OPFS size (zero growth between 4 MiB and 40 MiB). +WebKit remains unrun because its browser binary is unavailable. Both persistent modes use browser-managed origin quota and eviction. Applications may request `navigator.storage.persist()` themselves; Minigraf never requests it. Quota errors during OPFS commit are STG-048 and follow the failed-handle rule. -Private-browsing behavior is engine/policy dependent; checks for this change -remain unrun in the current sandbox, as recorded in the measurement record. +Private-browsing behavior is engine/policy dependent. Actual checks recorded +Chromium incognito opening OPFS and querying `[[42]]`, and Firefox private mode +rejecting with API-026 / SecurityError. WebKit private mode remains unrun; see +the measurement record for the exact engines and harness limitations. ## Opening after a crash diff --git a/docs/TEST_COVERAGE.md b/docs/TEST_COVERAGE.md index 043922cde7..669c89db0b 100644 --- a/docs/TEST_COVERAGE.md +++ b/docs/TEST_COVERAGE.md @@ -4,30 +4,30 @@ | | Default features | With `--features test-hooks` | |---|---|---| -| **Active** | 2,147 | 2,185 | -| **Ignored** | 22 | 31 | -| **Total** | 2,169 | 2,216 | +| **Active** | 2,149 | 2,187 | +| **Ignored** | 23 | 32 | +| **Total** | 2,172 | 2,219 | Counts above are the current Cargo test inventory (active versus ignored), including #441, #777, #789, #781, #794, #795, #837, #865, #869, #870, #861, #863, #871, #872 and #824 (Linux). Verification results below identify which suites ran. **#441 verification (2026-11-14)**: the full native default suite passes -(2,147 active, 22 ignored), including the grammar verifier and seven new +(2,149 active, 23 ignored), including the grammar verifier and eight new WAL-less/image unit tests. The WASI suite passes (1,222 active, seven ignored). The CI-equivalent -nightly branch-coverage run passes 2,125 tests with 18 ignored (doctests excluded), -95.71% line / 86.64% branch coverage, and every module floor. Its first attempt -omitted the lock helper override and failed two helper lookups; rebuilding the -helper and supplying `MINIGRAF_PID_NS_HELPER` resolved that harness setup error. +nightly branch-coverage run passes 2,127 tests with 19 ignored (doctests excluded), +95.68% line / 86.65% branch coverage, and every module floor. The lock helper is +supplied through `MINIGRAF_PID_NS_HELPER`. Native all-feature lib and browser/WASI lib strict clippy pass, as does formatting. The broader `--all-targets` lint fails on existing test-code lints on both the base -and this branch (including 21 library-test diagnostics); it is not reported as -passing. Rust 1.99.1's distribution binary is 1,269,568 bytes, below the unchanged +and this branch (including 21 library-test diagnostics); complete `--keep-going` comparison +against `9fd321e75` finds the same 4,322 diagnostics on each revision, with no +additions. It is not reported as passing. Rust 1.99.1's distribution binary is 1,269,568 bytes, below the unchanged 1,280,000-byte gate; [build sizes](benchmarks/441-opfs/build-sizes.md) record the older compiler's oversized base/control and the wasm increase. A nightly build with wasm atomics enabled fails with E0277 because the JS handle is not `Send + Sync`, preserving worker confinement. -Fresh Firefox 159.0.1 runs pass **101 worker** and **81 main-thread** tests. -The packaged consumer passes **52 checks per engine** on Chromium 140.0.7339.16 +Fresh Firefox 159.0.1 runs pass **113 worker** and **81 main-thread** tests. +The packaged consumer passes **54 checks per engine** on Chromium 140.0.7339.16 (Playwright) and Firefox 159.0.1 (installed Firefox via WebDriver, because the Playwright Firefox executable is absent). These include cross-worker locking, acknowledged-write recovery, six sequential-commit termination timings, four @@ -40,7 +40,8 @@ WebKit and WebKit private tests lack an installed engine and remain unrun. See [measurement evidence and reproduction](benchmarks/441-opfs/README.md). The native additions are `wal_less_crash_at_every_point_of_a_commit_opens_old_or_new`, -`wal_less_crash_while_creating_opens_empty`, `wal_less_fixed_cases`, +`wal_less_crash_while_creating_opens_empty`, +`initial_creation_exports_open_with_minigraf_and_verify`, `wal_less_fixed_cases`, `wal_less_open_outcomes_match_native`, `wal_less_short_foreign_file_is_stg_045_and_unmodified`, `wal_less_image_after_recovery_opens_with_minigraf_open` and @@ -48,10 +49,19 @@ The native additions are `wal_less_crash_at_every_point_of_a_commit_opens_old_or `storage::persistent_facts`. They cover write/sync/tear boundaries, uncertain meta publication, completed-unacknowledged predecessors, native STG-033 parity, refused-open immutability and portable export after recovery. -`tests/opfs_portability_test.rs` adds two active fixture checks and one ignored -native fixture generator; long values, retraction, transaction/valid-time queries +`tests/opfs_portability_test.rs` adds three active fixture checks and one ignored +native fixture generator; a second ignored generator in the storage unit tests +builds the real v7 portability fixture with the migration tests' legacy writer; long values, retraction, transaction/valid-time queries and `verify()` agree on checked-in native and browser images. +The OPFS worker tests additionally cover invalid nonempty meta imports without +replacement, complete export after every initial-creation prefix, missing/insecure/ +denied browser APIs, short/thrown reads and quota exception names, offsets refused +before I/O, deferred close/admission precedence, submitted IndexedDB completion after +reopen, drop without save, staging create/acquire/truncate/cleanup faults and every +write/flush/truncate boundary in a repeated recovery copy. The full v7 portability +query set runs before and after OPFS reopen. + **#863 nightly candidates**: four engine unit tests check the `test-hooks` path events and memory peaks (both builds). With `test-hooks` only, `path_events_test.rs` adds two integration tests and diff --git a/docs/benchmarks/441-opfs/README.md b/docs/benchmarks/441-opfs/README.md index 673fef98bb..092a5cd237 100644 --- a/docs/benchmarks/441-opfs/README.md +++ b/docs/benchmarks/441-opfs/README.md @@ -80,9 +80,13 @@ Build-size comparisons and exact toolchains are in [build-sizes.md](build-sizes. ## Browser execution and private browsing -- Firefox 159.0.1 core: **81 main-thread and 101 worker tests pass**. +The execution checks below were rerun after the import admission/export fixes. +The memory results above remain from the identified measured artifact: the +healthy OPFS open/read/cache paths are unchanged. + +- Firefox 159.0.1 core: **81 main-thread and 113 worker tests pass**. - Packaged consumer: **Chromium 140.0.7339.16 via Playwright** and - **Firefox 159.0.1 via WebDriver** each pass 52 checks, including six + **Firefox 159.0.1 via WebDriver** each pass 54 checks, including six sequential-commit termination timings, four import-interruption timings, cross-worker locking, IndexedDB in a worker and the README worker example. - Chrome core wasm-bindgen tests: not run. Cached ChromeDriver is version 156; From f3f29b35bcecc6bf3c54c8f5ea102af1e8d23bfb Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Sat, 14 Nov 2026 13:01:33 +0530 Subject: [PATCH 11/20] test(browser): retain dependents across repeated import recovery (#441) Start every recovery-copy interruption case from an actual published import whose main-file copy failed. Keep its earlier cursor and prepared query alive and assert their closed outcomes, lock exclusion and successful staging cleanup across the retry. Refs #441 Co-Authored-By: Codex (GPT-6.1-sol) --- src/browser/opfs_tests.rs | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/src/browser/opfs_tests.rs b/src/browser/opfs_tests.rs index 1ed89eeae9..8bca562b10 100644 --- a/src/browser/opfs_tests.rs +++ b/src/browser/opfs_tests.rs @@ -1511,9 +1511,20 @@ mod worker { let staging = format!("{name}.graph-import"); let db = open(&name).await; put(&db, 0..3).await; - let generation = db.inner.borrow().pfs.generation() + 10; - db.close().unwrap(); - write_raw(&staging, &staged_bytes(&image, generation | 1)).await; + let mut cursor = db.query("(query [:find ?n :where [?e :n ?n]])").unwrap(); + let pq = db.prepare("(query [:find ?n :where [?e :n ?n]])").unwrap(); + // First interruption: a real import publishes its staging + // meta, then fails on the first write into the main file. + *file(&db).faults.lock().unwrap() = Faults { + write_after: Some((0, WriteFault::Throw(0))), + ..Faults::default() + }; + assert_eq!( + code_of(&rejection(db.import_graph(blob(&image)).await)), + "STG-048" + ); + assert!(db.inner.closed.get()); + drop(db); let patch = if boundary == "truncate" { Some(JsPatch::method( &prototype("FileSystemSyncAccessHandle"), @@ -1533,8 +1544,18 @@ mod worker { }; let result = BrowserDb::open_opfs(name.clone()).await; drop(patch); + assert_eq!(code_of(&rejection(cursor.next_batch(1))), "API-018"); + assert_eq!( + code_of(&rejection(pq.execute(JsValue::UNDEFINED))), + "API-018" + ); if let Ok(db) = result { assert_eq!(ns(&db).await, new); + assert_eq!( + code_of(&rejection(BrowserDb::open_opfs(name.clone()).await)), + "STG-026" + ); + assert!(!entries().await.contains(&staging)); db.close().unwrap(); break; } From 3d2d5b700e258191360835f73fa8b873292c1de1 Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Sat, 14 Nov 2026 13:09:59 +0530 Subject: [PATCH 12/20] docs(browser): include compiler diagnostics in lint totals (#441) Distinguish the 4322 Clippy diagnostics from 23 unchanged compiler diagnostics in the complete base comparison. Neither category adds a diagnostic on the feature branch. Refs #441 Co-Authored-By: Codex (GPT-6.1-sol) --- docs/TEST_COVERAGE.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/TEST_COVERAGE.md b/docs/TEST_COVERAGE.md index 669c89db0b..28c61d7816 100644 --- a/docs/TEST_COVERAGE.md +++ b/docs/TEST_COVERAGE.md @@ -20,7 +20,8 @@ supplied through `MINIGRAF_PID_NS_HELPER`. Native all-feature lib and browser/WASI lib strict clippy pass, as does formatting. The broader `--all-targets` lint fails on existing test-code lints on both the base and this branch (including 21 library-test diagnostics); complete `--keep-going` comparison -against `9fd321e75` finds the same 4,322 diagnostics on each revision, with no +against `9fd321e75` finds the same 4,345 diagnostics on each revision +(4,322 Clippy lints and 23 compiler diagnostics), with no additions. It is not reported as passing. Rust 1.99.1's distribution binary is 1,269,568 bytes, below the unchanged 1,280,000-byte gate; [build sizes](benchmarks/441-opfs/build-sizes.md) record the older compiler's oversized base/control and the wasm increase. A nightly build From c3ed0475313a5d2f0dc78e018c69a2a184b6bb9c Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Sat, 14 Nov 2026 13:55:50 +0530 Subject: [PATCH 13/20] fix(browser): preserve shared import errors and clean worker helpers (#441) Keep shared storage error classification for foreign imports while refusing nonempty initial images that open would repair. Cover error parity and preserved destination state, clarify supported import images, and restrict OPFS test helpers to their worker build. Refs #441 Co-Authored-By: Codex (GPT-6) --- CLAUDE.md | 4 +- docs/DURABILITY.md | 8 +- docs/TEST_COVERAGE.md | 4 +- src/browser/mod.rs | 11 ++- src/browser/opfs.rs | 74 +++++++++---------- src/browser/opfs_tests.rs | 37 +++++++++- .../persistent_facts/browser_access.rs | 2 +- 7 files changed, 93 insertions(+), 47 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 3d4d2c0ef4..91dd562198 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -156,7 +156,7 @@ cargo run < demos/demo_negation.txt - `dict.rs`: `DictReader` (id ↔ UUID/ident, tx timestamps, long values; `uuids_of` walks dense id batches with `LeafCursor` and uses exact scalar reads for sparse/small batches, #530) and `Encoder` (checkpoint-time id assignment and key building) - `value_pages.rs`: append-only value pages for strings over 64 bytes and sets whose element encoding is over 64 bytes (#316; DICT dedup compares bytes, `long_bytes`/`find_long_bytes`) - `reader.rs`: `OnDiskReader` — covering reads of committed facts (`CommittedReader`, `KeySource`); query scans use the resumable `live_keys` to decide net-assert on index keys (#379) among the versions a `TxBound` keeps; `live_facts` is test-only, and production fact-level net-assert handles pending records (#532, superseded by #551); `KeySource::any_key` proves raw kind-prefix presence with one leaf seek, including historical/retracted keys (#641); `tx_order` (cached per reader), `last_tx_at`, `tx_runs` over the DICT's transactions (#551) - - `verify.rs`: integrity walk for `Minigraf::verify` (tree order, index digests, DICT, page accounting) and the source choice for `rebuild_indexes` (#373) + - `verify.rs`: integrity walk for `Minigraf::verify` (tree order, index digests, DICT, page accounting) and the source choice for `rebuild_indexes` (#373); also built with the `browser` feature for OPFS import verification - `meta.rs` / `page.rs` / `freelist.rs`: meta pages A/B, the common page header and allocator (skips quarantined ids), free-list chain - `pins.rs`: `Pins` / `GenerationPin` — generations held by open snapshots and the quarantine of pages checkpoints freed after them (#543) - `cache.rs`: LRU page cache (`PageCache`, default 256 pages) @@ -193,7 +193,7 @@ cargo run < demos/demo_negation.txt - `src/fact_log.rs`: `FactLog` / `FactFilter` / `FactRecord` / `FactOrder` — every fact version streamed by `Minigraf::fact_log` (#430): tx order (multi-pass, bounded window) or storage order, key-level filters; an open log pins the committed generation (`storage::pins`), so checkpoints run but do not reuse its pages (`API-013` deprecated, #543) - `src/log_writer.rs`: `LogWriter` — builds a new file from `FactRecord`s with their tx and valid-time kept (#431): tx-ordered appends with holes (`API-015`/`API-016`), batched copy-on-write checkpoints at tx boundaries, no WAL, built at `.partial` and renamed by `finish` (`STG-043` if the target exists) -7. **`src/wal.rs`** — Fact-level sidecar WAL, CRC32-protected entries, crash recovery; `append_entry` takes the `SyncMode` of each append (#871), and in tests the writer tracks its synced length (lost-sync fault, `Minigraf::power_loss`). Like `storage::verify` and `storage::dir_sync`, not compiled for wasm32 (no file backend there) +7. **`src/wal.rs`** — Fact-level sidecar WAL, CRC32-protected entries, crash recovery; `append_entry` takes the `SyncMode` of each append (#871), and in tests the writer tracks its synced length (lost-sync fault, `Minigraf::power_loss`). Like `storage::dir_sync`, not compiled for wasm32 (no file backend there) 8. **`src/error.rs`** — Structured error codes: `MinigrafError`, `ErrorCategory`, `ErrorCode` registry (PRS/QRY/STG/WAL/API/INT codes); must match `docs/ERROR_REFERENCE.md`. `MinigrafError::invalid_argument` (API-017) and `MinigrafError::closed` (API-018) exist for language bindings; browser OPFS adds API-026 (unavailable context), API-027 (busy) and STG-048 (failed browser storage operation) diff --git a/docs/DURABILITY.md b/docs/DURABILITY.md index 93c69e5c82..581daedd25 100644 --- a/docs/DURABILITY.md +++ b/docs/DURABILITY.md @@ -111,9 +111,11 @@ An already applied OPFS/in-memory transaction commit resolves after close; an IndexedDB commit whose flush has not been submitted rejects with API-018 and never submits it. An already submitted IndexedDB flush settles by its own result. -OPFS import first validates and verifies its complete input. Only an empty blob -creates an empty database; a nonempty image without a selectable meta is rejected, -even if opening those bytes as a file could recover an interrupted creation. +OPFS import admits an empty blob; a nonempty v8 image with a selectable meta +and complete, verified contents (including an empty v8 image); or a supported v7 +image that migrates completely. Nonempty torn or zero-filled initial Minigraf +images are rejected even if opening those bytes as a file could recover an +interrupted creation. A meta counting pages beyond the input is STG-044. Exports normalize a missing second meta slot after interrupted creation, while failed reads of existing pages remain errors. Import stages the verified image in diff --git a/docs/TEST_COVERAGE.md b/docs/TEST_COVERAGE.md index 28c61d7816..dd0fb3fb8a 100644 --- a/docs/TEST_COVERAGE.md +++ b/docs/TEST_COVERAGE.md @@ -728,13 +728,13 @@ These are not `#[test]`s; run them with `cargo kani` or `.github/workflows/kani. None of these are counted in the summary. The wasm32 targets run in CI (`.github/workflows/wasm.yml`). -- **OPFS worker mode** (#441; `src/browser/opfs_tests.rs`, 21 worker tests plus one main-thread test): +- **OPFS worker mode** (#441; `src/browser/opfs_tests.rs`, 34 worker tests plus one main-thread test): - AC1/AC4: `acknowledged_writes_survive_close_and_reopen`, `the_database_is_one_file_and_no_wal`. - AC2/AC8: `invalid_names_are_api_017_before_anything_is_created`, `a_second_open_is_stg_026_until_close`, `open_opfs_on_the_main_thread_is_api_026_and_creates_nothing`. - AC9: `after_close_every_call_is_api_018_but_disposal`, `close_from_a_user_function_during_a_query_is_busy`, `calls_issued_before_close_change_nothing_after_it`. - AC5/AC6/AC10: `a_failed_commit_is_stg_048_then_stg_046`, `a_torn_meta_reopens_at_the_previous_commit`, `open_outcomes_match_native_and_refusals_modify_nothing`. - AC7/AC11: `export_is_a_complete_file_that_verifies`, `native_v8_and_v7_files_import_and_survive_reopen`, `a_smaller_import_leaves_no_trailing_bytes`. - - AC12: `import_validation_rejects_before_replacing_anything`, `a_published_import_invalidates_earlier_cursors_and_prepared_queries`, `calls_admitted_while_importing_are_busy`, `an_interrupted_import_reopens_old_or_new`, `leftover_staging_files_are_removed_not_replayed`. + - AC12: `import_validation_rejects_before_replacing_anything`, `a_published_import_invalidates_earlier_cursors_and_prepared_queries`, `calls_admitted_while_importing_are_busy`, `an_interrupted_import_reopens_old_or_new`, `leftover_staging_files_are_removed_not_replayed`, `foreign_import_keeps_shared_error_and_preserves_destination` (in-memory/OPFS STG-002 parity; preserved bytes, data, dependents and lock). - AC13/AC14: `opfs_open_read_bound`, `opfs_cache_residency_bound`, `indexeddb_mode_works_in_a_worker`. - AC3 parity: the worker build runs the existing transaction, prepared-query and UDF suites against OPFS as well as IndexedDB. The main-thread build preserves the existing IndexedDB tests. Worker CI sets `RUSTFLAGS=--cfg minigraf_browser_worker` for Firefox and Chrome. diff --git a/src/browser/mod.rs b/src/browser/mod.rs index 69834b2f2d..c7b320dbc9 100644 --- a/src/browser/mod.rs +++ b/src/browser/mod.rs @@ -1307,6 +1307,7 @@ fn image_of(bytes: Vec) -> anyhow::Result) -> anyhow::Result> = bytes @@ -1343,6 +1348,9 @@ fn image_of(bytes: Vec) -> anyhow::Result String { thread_local! { static NEXT: std::cell::Cell = const { std::cell::Cell::new(0) }; diff --git a/src/browser/opfs.rs b/src/browser/opfs.rs index bc5c721a60..8662d2ad73 100644 --- a/src/browser/opfs.rs +++ b/src/browser/opfs.rs @@ -104,7 +104,7 @@ pub(crate) struct OpfsFile { /// The file's entry name, for error messages. name: String, open: AtomicBool, - #[cfg(test)] + #[cfg(all(test, minigraf_browser_worker))] pub(crate) faults: std::sync::Mutex, } @@ -114,7 +114,7 @@ impl OpfsFile { handle, name, open: AtomicBool::new(true), - #[cfg(test)] + #[cfg(all(test, minigraf_browser_worker))] faults: std::sync::Mutex::default(), } } @@ -153,7 +153,7 @@ impl OpfsFile { let op = || format!("read page {page_id} of {}", self.name); self.check_open("read")?; let off = offset(page_id)?; - #[cfg(test)] + #[cfg(all(test, minigraf_browser_worker))] tests::count_read(); let mut buf = vec![0u8; len]; let n = self @@ -174,7 +174,7 @@ impl OpfsFile { let op = || format!("write page {page_id} of {}", self.name); self.check_open("write")?; let off = offset(page_id)?; - #[cfg(test)] + #[cfg(all(test, minigraf_browser_worker))] if let Some(fault) = self.faults.lock().unwrap().on_write() { return self.inject_write(off, data, fault, op()); } @@ -192,7 +192,7 @@ impl OpfsFile { /// makes it, which is not a power-loss guarantee. pub(crate) fn flush(&self) -> Result<()> { self.check_open("flush")?; - #[cfg(test)] + #[cfg(all(test, minigraf_browser_worker))] if self.faults.lock().unwrap().on_flush() { return Err(storage_error( format!("flush {}", self.name), @@ -398,7 +398,7 @@ pub(crate) async fn open_file( match JsFuture::from(file.create_sync_access_handle()).await { Ok(h) => { let file = OpfsFile::new(h.unchecked_into(), entry.to_string()); - #[cfg(test)] + #[cfg(all(test, minigraf_browser_worker))] if let Some(faults) = tests::take_armed(entry) { *file.faults.lock().unwrap() = faults; } @@ -476,7 +476,37 @@ pub(crate) async fn recover_import( Ok(()) } -#[cfg(test)] +#[cfg(all(test, minigraf_browser_worker))] +impl OpfsFile { + /// The handle itself, for tests that write raw bytes. + pub(crate) fn handle_for_tests(&self) -> &FileSystemSyncAccessHandle { + &self.handle + } + + fn inject_write( + &self, + off: f64, + data: &[u8], + fault: tests::WriteFault, + op: String, + ) -> Result<()> { + let (n, throw) = match fault { + tests::WriteFault::Throw(n) => (n.min(data.len()), true), + tests::WriteFault::Short(n) => (n.min(data.len()), false), + }; + if n > 0 { + self.handle + .write_with_u8_array_and_options(&data[..n], &at(off)) + .map_err(|e| js_failure(op.clone(), &e))?; + } + if throw { + return Err(storage_error(op, "fault injection")); + } + Err(storage_error(op, "short read/write")) + } +} + +#[cfg(all(test, minigraf_browser_worker))] pub(crate) mod tests { //! Fault injection and read counting for the OPFS tests. use std::cell::Cell; @@ -556,33 +586,3 @@ pub(crate) mod tests { } } } - -#[cfg(test)] -impl OpfsFile { - /// The handle itself, for tests that write raw bytes. - pub(crate) fn handle_for_tests(&self) -> &FileSystemSyncAccessHandle { - &self.handle - } - - fn inject_write( - &self, - off: f64, - data: &[u8], - fault: tests::WriteFault, - op: String, - ) -> Result<()> { - let (n, throw) = match fault { - tests::WriteFault::Throw(n) => (n.min(data.len()), true), - tests::WriteFault::Short(n) => (n.min(data.len()), false), - }; - if n > 0 { - self.handle - .write_with_u8_array_and_options(&data[..n], &at(off)) - .map_err(|e| js_failure(op.clone(), &e))?; - } - if throw { - return Err(storage_error(op, "fault injection")); - } - Err(storage_error(op, "short read/write")) - } -} diff --git a/src/browser/opfs_tests.rs b/src/browser/opfs_tests.rs index 8bca562b10..d9ae85fe95 100644 --- a/src/browser/opfs_tests.rs +++ b/src/browser/opfs_tests.rs @@ -624,7 +624,7 @@ mod worker { for err in [ rejection(checkpoint.await), rejection(import.await), - rejection(execute.await).into(), + rejection(execute.await), ] { assert_eq!(code_of(&err), "API-018"); } @@ -1007,6 +1007,41 @@ mod worker { } } + #[wasm_bindgen_test] + async fn foreign_import_keeps_shared_error_and_preserves_destination() { + let bytes = vec![0xaa; 3 * crate::storage::PAGE_SIZE]; + let memory = BrowserDb::open_in_memory().unwrap(); + put(&memory, 0..3).await; + let memory_before = memory.export_graph().unwrap().to_vec(); + let memory_code = code_of(&rejection(memory.import_graph(blob(&bytes)).await)); + assert_eq!(memory_code, "STG-002"); + assert_eq!(memory.export_graph().unwrap().to_vec(), memory_before); + assert_eq!(ns(&memory).await, vec![0, 1, 2]); + memory.close().unwrap(); + + let name = unique("foreign-import"); + let db = open(&name).await; + put(&db, 0..3).await; + let before = db.export_graph().unwrap().to_vec(); + let mut cursor = db.query("(query [:find ?n :where [?e :n ?n]])").unwrap(); + let pq = db.prepare("(query [:find ?n :where [?e :n ?n]])").unwrap(); + let opfs_code = code_of(&rejection(db.import_graph(blob(&bytes)).await)); + assert_eq!(opfs_code, memory_code); + assert_eq!(db.export_graph().unwrap().to_vec(), before); + assert!(cursor.next_batch(10).unwrap().is_some()); + assert!(pq.execute(JsValue::UNDEFINED).is_ok()); + assert_eq!( + code_of(&rejection(BrowserDb::open_opfs(name.clone()).await)), + "STG-026" + ); + assert_eq!(ns(&db).await, vec![0, 1, 2]); + db.close().unwrap(); + let reopened = open(&name).await; + assert_eq!(reopened.export_graph().unwrap().to_vec(), before); + assert_eq!(ns(&reopened).await, vec![0, 1, 2]); + reopened.close().unwrap(); + } + // ── AC11: portability ────────────────────────────────────────────────── #[wasm_bindgen_test] diff --git a/src/storage/persistent_facts/browser_access.rs b/src/storage/persistent_facts/browser_access.rs index af21c7db19..e6c7f74fe1 100644 --- a/src/storage/persistent_facts/browser_access.rs +++ b/src/storage/persistent_facts/browser_access.rs @@ -61,7 +61,7 @@ impl PersistentFactStorage { } /// Pages in the page cache now (#441). - #[cfg(test)] + #[cfg(all(test, minigraf_browser_worker))] pub(crate) fn cached_page_count(&self) -> usize { self.page_cache.cached_page_count() } From dd80d40849ab17a70c407e2d03b2cf3d0bf71d18 Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Tue, 17 Nov 2026 12:55:23 +0530 Subject: [PATCH 14/20] docs(browser): list every input OPFS import admits in the changelog (#441) Refs #441 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01H2a7QjCHpikfsg5qwAV1p9 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 35ce231036..9889cae51f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- **OPFS-backed `BrowserDb` persistence and explicit close** (#441). `await BrowserDb.openOpfs(name)` opens one `.graph` file through an exclusive sync access handle in a dedicated worker on a secure origin. It uses the default 256-page cache, reads pages on demand and commits by copy-on-write with data/meta flushes, without a WAL or a whole-file buffer. IndexedDB `open(name)` now also works in workers. The two modes have independent namespaces and no automatic fallback. OPFS starts at Tier 2 within the Tier 1 browser binding. All modes gain synchronous, idempotent `close()`, which saves nothing, discards an open transaction, invalidates dependents and immediately releases an OPFS lock. OPFS import rejects nonempty images without a selectable meta before replacing anything (only an empty blob creates an empty database), validates the whole image, stages and publishes the complete old or new database across interruption, and invalidates earlier cursors/prepared queries after publication. Exports after interrupted initial creation fill an absent second meta slot with zeros and still propagate failed reads of existing pages. OPFS recovery accepts the sole valid meta; unlike native recovery this can silently fall back after later meta rot. A failed commit is unconfirmed (it may be stored), leaves reads working, and refuses later writes with STG-046 until reopen. New registered errors API-026 (unavailable OPFS context), API-027 (busy database) and STG-048 (browser operation and exception name); lock contention is STG-026. No file-format change or new crate dependency. See [durability](docs/DURABILITY.md#browser-storage-modes) and [measurement status](docs/benchmarks/441-opfs/README.md). +- **OPFS-backed `BrowserDb` persistence and explicit close** (#441). `await BrowserDb.openOpfs(name)` opens one `.graph` file through an exclusive sync access handle in a dedicated worker on a secure origin. It uses the default 256-page cache, reads pages on demand and commits by copy-on-write with data/meta flushes, without a WAL or a whole-file buffer. IndexedDB `open(name)` now also works in workers. The two modes have independent namespaces and no automatic fallback. OPFS starts at Tier 2 within the Tier 1 browser binding. All modes gain synchronous, idempotent `close()`, which saves nothing, discards an open transaction, invalidates dependents and immediately releases an OPFS lock. OPFS import admits an empty blob, a nonempty v8 image with a selectable meta and complete, verified contents (an empty v8 image included), or a supported v7 image that migrates completely, and rejects anything else (such as a torn or zero-filled initial image) before replacing anything; it validates the whole image, stages and publishes the complete old or new database across interruption, and invalidates earlier cursors/prepared queries after publication. Exports after interrupted initial creation fill an absent second meta slot with zeros and still propagate failed reads of existing pages. OPFS recovery accepts the sole valid meta; unlike native recovery this can silently fall back after later meta rot. A failed commit is unconfirmed (it may be stored), leaves reads working, and refuses later writes with STG-046 until reopen. New registered errors API-026 (unavailable OPFS context), API-027 (busy database) and STG-048 (browser operation and exception name); lock contention is STG-026. No file-format change or new crate dependency. See [durability](docs/DURABILITY.md#browser-storage-modes) and [measurement status](docs/benchmarks/441-opfs/README.md). - **`OpenOptions::allow_overrides`** (#869, part of #862): the embedder lists, as `Overridable` variants, the query keywords that may override the database's settings. The default, `Overridable::DEFAULTS` (`MaxDerivedFacts`, `MaxResults`), allows every keyword that could do so before, so no query changes. A query using a keyword the list leaves out is refused with the new **API-025** before it runs, from `execute`, `query`, `prepare` and `WriteTransaction::execute`; the list replaces the defaults rather than adding to them, and is not stored in the file. `BrowserDb` takes no options and uses the defaults. `Overridable` is `#[non_exhaustive]`. From e689e59ac454b3dc76b6d37c2ead6d9d70021f39 Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Tue, 17 Nov 2026 15:17:40 +0530 Subject: [PATCH 15/20] ci: run the browser suites in Safari on macOS (#441) Playwright's Linux WebKit has no OPFS in workers, so WebKit OPFS had no automated run. Add main-thread and worker lanes in real Safari via safaridriver, and record that Safari private browsing is not tested. Refs #441 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01H2a7QjCHpikfsg5qwAV1p9 --- .github/workflows/wasm.yml | 28 ++++++++++++++++++++++++++++ docs/benchmarks/441-opfs/README.md | 20 +++++++++++++++++++- 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/.github/workflows/wasm.yml b/.github/workflows/wasm.yml index f82e14aaf7..1e20a57754 100644 --- a/.github/workflows/wasm.yml +++ b/.github/workflows/wasm.yml @@ -61,6 +61,34 @@ jobs: - name: wasm-bindgen worker tests (OPFS and IndexedDB) run: wasm-pack test --headless --${{ matrix.browser }} --features browser --lib + safari: + name: Safari tests (${{ matrix.context }}) + # Real Safari through safaridriver: Playwright's Linux WebKit has no OPFS + # in workers, so this is the WebKit OPFS lane. Safari has no headless mode; + # the hosted runner's GUI session runs it. Private windows cannot be + # opened over WebDriver, so Safari private browsing is not covered here. + runs-on: macos-15 + strategy: + fail-fast: false + matrix: + include: + - { context: main thread, rustflags: "", args: "" } + - { context: dedicated worker, rustflags: "--cfg minigraf_browser_worker", args: "--lib" } + env: + RUSTFLAGS: ${{ matrix.rustflags }} + WASM_BINDGEN_TEST_TIMEOUT: 180 + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + targets: wasm32-unknown-unknown + - name: Install wasm-pack + run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh + - name: Enable safaridriver + run: sudo safaridriver --enable + - name: wasm-bindgen tests in Safari + run: wasm-pack test --headless --safari --features browser ${{ matrix.args }} + wasi: name: WASI tests (Wasmtime) runs-on: ubuntu-latest diff --git a/docs/benchmarks/441-opfs/README.md b/docs/benchmarks/441-opfs/README.md index 092a5cd237..cb51e39749 100644 --- a/docs/benchmarks/441-opfs/README.md +++ b/docs/benchmarks/441-opfs/README.md @@ -106,8 +106,26 @@ healthy OPFS open/read/cache paths are unchanged. - Firefox private: geckodriver launched with `browser.privatebrowsing.autostart = true`; `openOpfs` rejected with `[API-026] OPFS mode is not available here: the browser denies access to this origin's storage (SecurityError)`. -- WebKit private: unchecked because the engine is absent. +- WebKit private: **not tested** (see below). CI specifies Chrome/Firefox core main-thread and worker lanes and Chromium/Firefox/WebKit package-consumer lanes; this local record does not claim those remote CI jobs have run. + +### WebKit coverage in CI + +Playwright's Linux WebKit has no `navigator.storage.getDirectory()` in a +dedicated worker, so its consumer lane only checks that `openOpfs` rejects +with API-026 for that reason. WebKit OPFS is covered on `macos-15` instead: + +- core `wasm.yml` job `safari`: the main-thread and dedicated-worker + wasm-bindgen suites in real Safari through `safaridriver`; +- minigraf-wasm consumer lane `browser, macos-15, webkit`: the packaged + consumer test on Playwright's macOS WebKit, where OPFS is required (an + API-026 there fails the run). + +**Omitted: Safari private browsing.** Neither WebDriver nor Playwright can +open a Safari private window, so no automated run checks OPFS there, and no +manual check was made. What `openOpfs` does in a Safari private window +(open, or reject with API-026) is unverified. Firefox private and Chromium +incognito results are above. From 6813affd497d301a17866d5809b9a9ebfd388262 Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Tue, 17 Nov 2026 15:20:21 +0530 Subject: [PATCH 16/20] docs: cite WebKit's private-browsing OPFS note (#441) Refs #441 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01H2a7QjCHpikfsg5qwAV1p9 --- docs/benchmarks/441-opfs/README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/benchmarks/441-opfs/README.md b/docs/benchmarks/441-opfs/README.md index cb51e39749..8b2755181c 100644 --- a/docs/benchmarks/441-opfs/README.md +++ b/docs/benchmarks/441-opfs/README.md @@ -126,6 +126,8 @@ with API-026 for that reason. WebKit OPFS is covered on `macos-15` instead: **Omitted: Safari private browsing.** Neither WebDriver nor Playwright can open a Safari private window, so no automated run checks OPFS there, and no -manual check was made. What `openOpfs` does in a Safari private window -(open, or reject with API-026) is unverified. Firefox private and Chromium +manual check was made. WebKit documents OPFS as unavailable in Private +Browsing windows ([WebKit blog, 2022](https://webkit.org/blog/12257/the-file-system-access-api-with-origin-private-file-system/), +"Persistence"), so `openOpfs` is expected to reject with API-026 there, as +in Firefox private; that expectation is unverified. Firefox private and Chromium incognito results are above. From 85f2db533c0b8c3b0e0060f1a9e715c34eff89f8 Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Tue, 17 Nov 2026 15:36:00 +0530 Subject: [PATCH 17/20] ci: start safaridriver before the Safari tests (#441) The test runner's own launch timed out on the hosted runner. Refs #441 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01H2a7QjCHpikfsg5qwAV1p9 --- .github/workflows/wasm.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/wasm.yml b/.github/workflows/wasm.yml index 1e20a57754..3b1faec6e3 100644 --- a/.github/workflows/wasm.yml +++ b/.github/workflows/wasm.yml @@ -84,9 +84,21 @@ jobs: targets: wasm32-unknown-unknown - name: Install wasm-pack run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh - - name: Enable safaridriver - run: sudo safaridriver --enable + # The test runner's own safaridriver launch times out on hosted runners + # ("driver failed to bind port"), so start it here and point the runner + # at it with SAFARIDRIVER_REMOTE. + - name: Start safaridriver + run: | + sudo safaridriver --enable + safaridriver -p 4444 & + for _ in $(seq 1 60); do + curl -sf http://127.0.0.1:4444/status >/dev/null && exit 0 + sleep 1 + done + echo "::error::safaridriver did not start"; exit 1 - name: wasm-bindgen tests in Safari + env: + SAFARIDRIVER_REMOTE: http://127.0.0.1:4444 run: wasm-pack test --headless --safari --features browser ${{ matrix.args }} wasi: From bb84f39e770deba0bf2f32b3a60177af215d675a Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Tue, 17 Nov 2026 15:54:51 +0530 Subject: [PATCH 18/20] fix(browser): treat WebKit's InvalidStateError as the OPFS lock (#441) Safari throws InvalidStateError, not NoModificationAllowedError, when a sync access handle is already open on the file, so a second open was STG-048 instead of STG-026. Refs #441 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01H2a7QjCHpikfsg5qwAV1p9 --- src/browser/opfs.rs | 20 +++++++++++++++----- src/browser/opfs_tests.rs | 4 +++- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/browser/opfs.rs b/src/browser/opfs.rs index 8662d2ad73..61a6a46e5e 100644 --- a/src/browser/opfs.rs +++ b/src/browser/opfs.rs @@ -8,7 +8,8 @@ //! cache holds pages in memory. Commits are the storage layer's copy-on-write //! protocol on the file itself; there is no WAL. The handle is exclusive: while //! one is open on a file, acquiring another, from any worker or tab, fails with -//! `NoModificationAllowedError`, which is the database lock (STG-026). +//! `NoModificationAllowedError` (WebKit: `InvalidStateError`), which is the +//! database lock (STG-026). //! //! The database `name` is the file `.graph` in the origin's root //! directory. An import into OPFS mode stages the new database in @@ -404,10 +405,19 @@ pub(crate) async fn open_file( } Ok(Some(Arc::new(file))) } - Err(e) if exception_name(&e) == "NoModificationAllowedError" => bail_coded!( - ErrorCode::Stg026, - format!("OPFS file {entry} is open in another worker or tab") - ), + // The spec names NoModificationAllowedError for a held lock; WebKit + // (Safari) throws InvalidStateError ("access handle is active"). + Err(e) + if matches!( + exception_name(&e).as_str(), + "NoModificationAllowedError" | "InvalidStateError" + ) => + { + bail_coded!( + ErrorCode::Stg026, + format!("OPFS file {entry} is open in another worker or tab") + ) + } Err(e) => Err(js_failure(format!("acquire a handle on {entry}"), &e)), } } diff --git a/src/browser/opfs_tests.rs b/src/browser/opfs_tests.rs index d9ae85fe95..d99aa9556d 100644 --- a/src/browser/opfs_tests.rs +++ b/src/browser/opfs_tests.rs @@ -427,7 +427,9 @@ mod worker { let (a, b) = (unique("lock-a"), unique("lock-b")); let db = open(&a).await; let err = rejection(BrowserDb::open_opfs(a.clone()).await); - assert_eq!(code_of(&err), "STG-026"); + // The message names the browser's exception when it is not the lock. + let message = err.as_string().unwrap_or_default(); + assert_eq!(code_of(&err), "STG-026", "{message}"); let other = open(&b).await; put(&other, 0..1).await; // Dependents still referenced do not hold the lock after close(). From 4708f9b9e1a091e3ef54454502d71def2e0d1977 Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Tue, 17 Nov 2026 16:01:54 +0530 Subject: [PATCH 19/20] docs: name Safari's OPFS lock exception under STG-026 (#441) Refs #441 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01H2a7QjCHpikfsg5qwAV1p9 --- docs/ERROR_REFERENCE.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/ERROR_REFERENCE.md b/docs/ERROR_REFERENCE.md index 97106a7356..413d556664 100644 --- a/docs/ERROR_REFERENCE.md +++ b/docs/ERROR_REFERENCE.md @@ -2886,7 +2886,8 @@ Read-only handles (`OpenOptions::read_only`) share a file with each other, but n In browser OPFS mode, the same code means another tab or worker (or another handle in this worker) holds the file's exclusive sync access handle. The browser -reports `NoModificationAllowedError`; `close()` or worker termination releases it. +reports `NoModificationAllowedError` (Safari: `InvalidStateError`); `close()` or +worker termination releases it. There is no retry on acquisition and no fallback to IndexedDB. **Resolution**: From 4d874f5efcd807d4db6b1411014eb82f20c66f7d Mon Sep 17 00:00:00 2001 From: Aditya Mukhopadhyay Date: Tue, 17 Nov 2026 19:23:29 +0530 Subject: [PATCH 20/20] docs: record the CI browser results and the Safari lock amendment (#441) Refs #441 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01H2a7QjCHpikfsg5qwAV1p9 --- docs/benchmarks/441-opfs/README.md | 10 +++++++--- .../specs/2026-11-14-441-browser-opfs-design.md | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/benchmarks/441-opfs/README.md b/docs/benchmarks/441-opfs/README.md index 8b2755181c..f681caaa37 100644 --- a/docs/benchmarks/441-opfs/README.md +++ b/docs/benchmarks/441-opfs/README.md @@ -108,9 +108,13 @@ healthy OPFS open/read/cache paths are unchanged. `[API-026] OPFS mode is not available here: the browser denies access to this origin's storage (SecurityError)`. - WebKit private: **not tested** (see below). -CI specifies Chrome/Firefox core main-thread and worker lanes and -Chromium/Firefox/WebKit package-consumer lanes; this local record does not -claim those remote CI jobs have run. +CI results on the final PR revisions (core #899, minigraf-wasm #26): Chrome, +Firefox and Safari core suites pass on the main thread (81) and in a dedicated +worker (114); the packaged consumer test passes on Chromium, Firefox and +macOS WebKit (54 checks each), and Linux WebKit records its missing +`getDirectory()` as a limitation. Runs: +[core](https://github.com/project-minigraf/minigraf/actions/runs/44093820254), +[package](https://github.com/project-minigraf/minigraf-wasm/actions/runs/44065838738). ### WebKit coverage in CI diff --git a/docs/superpowers/specs/2026-11-14-441-browser-opfs-design.md b/docs/superpowers/specs/2026-11-14-441-browser-opfs-design.md index 421a36c0e9..ba2a5cb374 100644 --- a/docs/superpowers/specs/2026-11-14-441-browser-opfs-design.md +++ b/docs/superpowers/specs/2026-11-14-441-browser-opfs-design.md @@ -159,7 +159,7 @@ Test names are suggestions unless marked **(named)**. Every behavioural AC is co - **D6 No WAL, checkpoint per transaction,** as the issue says. - **D7 Promise shapes kept.** `execute`, `checkpoint`, `importGraph` and `tx.commit()` stay Promise-returning in OPFS mode: one class and one TypeScript surface; they resolve after the synchronous durable write. - **D8 Thread confinement without bespoke unsafe where the dependency already provides it** (AC16). -- **D9 Errors.** Lock contention (`NoModificationAllowedError` on acquiring the handle) is exactly STG-026. Invalid name and malformed import length are API-017. Three new registered codes, numbers assigned at implementation: (1) API, "OPFS mode is not available here": no dedicated worker, no OPFS or sync-access-handle API, insecure context, or the browser denies access to the origin's directory (for example `SecurityError` from `getDirectory()`); (2) API, "database is busy" (AC9, AC12); (3) storage, "browser storage operation failed": every other failed OPFS operation — creating or opening an entry, acquiring a handle for a reason other than the lock, read, write, flush, size, truncate, and staging steps of an import — with the operation and the exception name or a short-transfer note; quota exhaustion is here, never under (1) or STG-026. Post-publication cleanup failures are not reported (AC12). STG-003 is not reused. No uncoded or `INT-000` rejection from the new code paths; IndexedDB mode's existing uncoded messages are unchanged. Codes are never removed or recycled. +- **D9 Errors.** Lock contention (`NoModificationAllowedError` on acquiring the handle) is exactly STG-026. *Amended 2026-11-17:* Safari reports a held handle as `InvalidStateError`, so that name on acquisition is STG-026 too (found by the real-Safari CI lane). Invalid name and malformed import length are API-017. Three new registered codes, numbers assigned at implementation: (1) API, "OPFS mode is not available here": no dedicated worker, no OPFS or sync-access-handle API, insecure context, or the browser denies access to the origin's directory (for example `SecurityError` from `getDirectory()`); (2) API, "database is busy" (AC9, AC12); (3) storage, "browser storage operation failed": every other failed OPFS operation — creating or opening an entry, acquiring a handle for a reason other than the lock, read, write, flush, size, truncate, and staging steps of an import — with the operation and the exception name or a short-transfer note; quota exhaustion is here, never under (1) or STG-026. Post-publication cleanup failures are not reported (AC12). STG-003 is not reused. No uncoded or `INT-000` rejection from the new code paths; IndexedDB mode's existing uncoded messages are unchanged. Codes are never removed or recycled. - **D10 Page cache:** the default 256 pages in OPFS mode. - **D11 Tier:** OPFS mode is Tier 2 in v3.0.0. - **D12 IndexedDB failed-flush gap** is #894 (v3.0.0, data-integrity), not this mission. This mission must not make it worse.