AD-SDL · RyanTheRobothead · Dec 4, 2025 · Nov 24, 2025 · Nov 24, 2025 · Nov 24, 2025
diff --git a/.env.example b/.env.example
@@ -102,3 +102,22 @@
 # LOCATION_REDIS_HOST="localhost"
 # LOCATION_REDIS_PORT=6379
 # LOCATION_REDIS_PASSWORD=null
+
+### PostgreSQLBackupSettings
+
+# POSTGRES_BACKUP_DIR=".madsci/backups"
+# POSTGRES_MAX_BACKUPS=10
+# POSTGRES_VALIDATE_INTEGRITY=true
+# POSTGRES_COMPRESSION=true
+# DB_URL="postgresql://madsci:madsci@localhost:5432/resources"
+# POSTGRES_BACKUP_FORMAT="custom"
+
+### MongoDBBackupSettings
+
+# MONGODB_BACKUP_DIR=".madsci/backups"
+# MONGODB_MAX_BACKUPS=10
+# MONGODB_VALIDATE_INTEGRITY=true
+# MONGODB_COMPRESSION=true
+# MONGO_DB_URL="mongodb://localhost:27017"
+MONGODB_DATABASE=
+# MONGODB_COLLECTIONS=null
diff --git a/.gitignore b/.gitignore
@@ -303,6 +303,7 @@ test_experiment/
 # Output data
 .wei/
 .madsci/
+backups/
 
 #node modules
 **/node_modules
@@ -317,4 +318,8 @@ coverage.xml
 
 # Alembic migration versions (environment-specific)
 src/madsci_resource_manager/madsci/resource_manager/alembic/versions/*.py
+
+
+# Agentic coding
 .claude/
+agent_docs/
diff --git a/.justfile b/.justfile
@@ -36,6 +36,8 @@ checks:
   @pre-commit run --all-files || { echo "" && echo "Some checks failed! Running one more time to see if any automatic fixes worked:" && echo "" ; pre-commit run --all-files; }
 # Run the pre-commit checks
 check: checks
+ruff-unsafe:
+  @ruff check . --fix --unsafe-fixes
 
 # Build the project
 build: dcb
@@ -187,6 +189,8 @@ node_integration_tests:
 experiment_integration_tests:
   docker compose run --rm --no-deps workcell_manager python -m nbconvert --to notebook --inplace --stdout --execute ./notebooks/experiment_notebook.ipynb
 
+backup_integration_tests:
+  docker compose run --rm --no-deps workcell_manager python -m nbconvert --to notebook --inplace --stdout --execute ./notebooks/backup_and_migration.ipynb
 
 # Run the integration tests
-integration_tests: node_integration_tests experiment_integration_tests
+integration_tests: node_integration_tests experiment_integration_tests backup_integration_tests
diff --git a/AGENTS.md b/AGENTS.md
@@ -126,6 +126,109 @@ The `AbstractManagerBase` class provides:
 - When generating new IDs, use `new_ulid_str()` from `madsci.common.utils`
 - Example usage: `resource_id = new_ulid_str()`
 
+### Database Patterns
+
+MADSci uses two database systems optimized for different use cases:
+
+#### Database Types
+- **PostgreSQL**: Used by Resource Manager for relational data with strict schemas
+- **MongoDB**: Used by Event, Data, Experiment, and Workcell Managers for flexible document storage
+
+#### Backup and Restore
+
+All backup tools are centralized in `madsci_common` for maximum reusability:
+
+```python
+# PostgreSQL backups
+from madsci.common.backup_tools import PostgreSQLBackupTool
+from madsci.common.types.backup_types import PostgreSQLBackupSettings
+
+settings = PostgreSQLBackupSettings(
+    db_url="postgresql://localhost/resources",
+    backup_dir=Path("./backups"),
+    max_backups=10,
+    validate_integrity=True
+)
+backup_tool = PostgreSQLBackupTool(settings)
+backup_path = backup_tool.create_backup("pre_migration")
+
+# MongoDB backups
+from madsci.common.backup_tools import MongoDBBackupTool
+from madsci.common.types.backup_types import MongoDBBackupSettings
+
+settings = MongoDBBackupSettings(
+    mongo_db_url=AnyUrl("mongodb://localhost:27017"),
+    database="events",
+    backup_dir=Path("./backups"),
+    max_backups=10
+)
+backup_tool = MongoDBBackupTool(settings)
+backup_path = backup_tool.create_backup("hourly")
+```
+
+**CLI Usage:**
+```bash
+# Unified CLI (auto-detects database type)
+madsci-backup create --db-url postgresql://localhost/resources
+madsci-backup create --db-url mongodb://localhost:27017/events
+
+# Database-specific CLIs
+madsci-postgres-backup create --db-url postgresql://localhost/resources
+madsci-mongodb-backup create --mongo-url mongodb://localhost:27017 --database events
+```
+
+#### Database Connections
+
+**PostgreSQL** (using SQLModel):
+```python
+from sqlmodel import Session, create_engine
+
+engine = create_engine(
+    db_url,
+    pool_size=20,        # Connection pool size
+    pool_pre_ping=True   # Verify connections before use
+)
+
+with Session(engine) as session:
+    # Perform operations
+    session.commit()
+```
+
+**MongoDB** (using pymongo):
+```python
+from pymongo import MongoClient
+
+with MongoClient(mongo_url) as client:
+    db = client[database_name]
+    collection = db[collection_name]
+    # Perform operations
+```
+
+#### Database Migrations
+
+**PostgreSQL migrations** (Resource Manager):
+- Uses Alembic for schema version management
+- Automatic backups before migrations
+- Auto-restore on migration failure
+```bash
+python -m madsci.resource_manager.migration_tool --db-url postgresql://localhost/resources
+```
+
+**MongoDB migrations** (per manager):
+- Handle index creation and schema validation
+- Manager-specific migration tools
+- Automatic pre-migration backups
+
+#### Best Practices
+
+1. **Always use ULID for IDs**: `resource_id = new_ulid_str()`
+2. **Backup before migrations**: Automatic with migration tools
+3. **Use connection pooling**: Configure appropriate pool sizes
+4. **Environment variables for config**: Never hardcode connection strings
+5. **Validate backups**: Use `validate_backup_integrity()` for critical backups
+6. **Test migrations first**: Always test in development before production
+
+
 ### Node Development
 Laboratory instruments implement the Node interface:
 1. Inherit from `AbstractNodeModule`

diff --git a/Configuration.md b/Configuration.md
@@ -170,3 +170,34 @@ Settings for the LocationManager.
 | `LOCATION_REDIS_HOST`         | `string`               | `"localhost"`              | The host of the Redis server for state storage.      | `"localhost"`              |
 | `LOCATION_REDIS_PORT`         | `integer`              | `6379`                     | The port of the Redis server for state storage.      | `6379`                     |
 | `LOCATION_REDIS_PASSWORD`     | `string` \| `NoneType` | `null`                     | The password for the Redis server (if required).     | `null`                     |
+
+## PostgreSQLBackupSettings
+
+PostgreSQL-specific backup settings.
+
+**Environment Prefix**: `POSTGRES_`
+
+| Name                          | Type      | Default                                                 | Description                                   | Example                                                 |
+|-------------------------------|-----------|---------------------------------------------------------|-----------------------------------------------|---------------------------------------------------------|
+| `POSTGRES_BACKUP_DIR`         | `Path`    | `".madsci/backups"`                                     | Directory for storing backups                 | `".madsci/backups"`                                     |
+| `POSTGRES_MAX_BACKUPS`        | `integer` | `10`                                                    | Maximum number of backups to retain           | `10`                                                    |
+| `POSTGRES_VALIDATE_INTEGRITY` | `boolean` | `true`                                                  | Perform integrity validation after backup     | `true`                                                  |
+| `POSTGRES_COMPRESSION`        | `boolean` | `true`                                                  | Enable backup compression                     | `true`                                                  |
+| `DB_URL` \| `DB_URL`          | `string`  | `"postgresql://madsci:madsci@localhost:5432/resources"` | PostgreSQL connection URL                     | `"postgresql://madsci:madsci@localhost:5432/resources"` |
+| `POSTGRES_BACKUP_FORMAT`      | `string`  | `"custom"`                                              | pg_dump format: custom, plain, directory, tar | `"custom"`                                              |
+
+## MongoDBBackupSettings
+
+MongoDB-specific backup settings.
+
+**Environment Prefix**: `MONGODB_`
+
+| Name                             | Type                  | Default                       | Description                                  | Example                       |
+|----------------------------------|-----------------------|-------------------------------|----------------------------------------------|-------------------------------|
+| `MONGODB_BACKUP_DIR`             | `Path`                | `".madsci/backups"`           | Directory for storing backups                | `".madsci/backups"`           |
+| `MONGODB_MAX_BACKUPS`            | `integer`             | `10`                          | Maximum number of backups to retain          | `10`                          |
+| `MONGODB_VALIDATE_INTEGRITY`     | `boolean`             | `true`                        | Perform integrity validation after backup    | `true`                        |
+| `MONGODB_COMPRESSION`            | `boolean`             | `true`                        | Enable backup compression                    | `true`                        |
+| `MONGO_DB_URL` \| `MONGO_DB_URL` | `AnyUrl`              | `"mongodb://localhost:27017"` | MongoDB connection URL                       | `"mongodb://localhost:27017"` |
+| `MONGODB_DATABASE`               | `string`              | *required*                    | Database name to backup                      |                               |
+| `MONGODB_COLLECTIONS`            | `array` \| `NoneType` | `null`                        | Specific collections to backup (all if None) | `null`                        |
diff --git a/compose.yaml b/compose.yaml
@@ -62,7 +62,7 @@ services:
     ports:
       - 5432:5432
     volumes:
-      - ${REPO_PATH:-.}/.madsci/postgres:/var/lib/postgresql/data
+      - ${REPO_PATH:-.}/.madsci/postgresql/data:/var/lib/postgresql/data
   minio:
     container_name: minio
     image: minio/minio