AD-SDL · RyanTheRobothead · Dec 4, 2025 · Aug 8, 2025 · Aug 8, 2025 · Aug 8, 2025
diff --git a/.env.example b/.env.example
@@ -41,15 +41,18 @@
 
 # DATA_SERVER_URL="http://localhost:8004/"
 # DATA_MANAGER_DEFINITION="data.manager.yaml"
-# DATA_DB_URL="mongodb://localhost:27017"
+# DATA_DATABASE_NAME="madsci_data"
+# DATA_COLLECTION_NAME="datapoints"
+# MONGO_DB_URL="mongodb://localhost:27017"
 # DATA_FILE_STORAGE_PATH="~/.madsci/datapoints"
 
 ### EventManagerSettings
 
 # EVENT_SERVER_URL="http://localhost:8001"
 # EVENT_MANAGER_DEFINITION="event.manager.yaml"
-# EVENT_DB_URL="mongodb://localhost:27017"
-# EVENT_COLLECTION_NAME="madsci_events"
+# MONGO_DB_URL="mongodb://localhost:27017"
+# EVENT_DATABASE_NAME="madsci_events"
+# EVENT_COLLECTION_NAME="events"
 # EVENT_ALERT_LEVEL=40
 # EVENT_EMAIL_ALERTS=null
 
@@ -65,14 +68,18 @@
 # WORKCELL_NODE_UPDATE_INTERVAL=1.0
 # WORKCELL_COLD_START_DELAY=0
 # WORKCELL_SCHEDULER="madsci.workcell_manager.schedulers.default_scheduler"
-# WORKCELL_MONGO_URL=null
+# MONGO_DB_URL="mongodb://localhost:27017"
+# WORKCELL_DATABASE_NAME="madsci_workcells"
+# WORKCELL_COLLECTION_NAME="archived_workflows"
 # WORKCELL_GET_ACTION_RESULT_RETRIES=3
 
 ### ExperimentManagerSettings
 
 # EXPERIMENT_SERVER_URL="http://localhost:8002/"
 # EXPERIMENT_MANAGER_DEFINITION="experiment.manager.yaml"
-# EXPERIMENT_DB_URL="mongodb://localhost:27017"
+# MONGO_DB_URL="mongodb://localhost:27017"
+# EXPERIMENT_DATABASE_NAME="madsci_experiments"
+# EXPERIMENT_COLLECTION_NAME="experiments"
 
 ### ResourceManagerSettings
 
@@ -93,3 +100,22 @@
 # LOCATION_REDIS_HOST="localhost"
 # LOCATION_REDIS_PORT=6379
 # LOCATION_REDIS_PASSWORD=null
+
+### PostgreSQLBackupSettings
+
+# POSTGRES_BACKUP_DIR=".madsci/backups"
+# POSTGRES_MAX_BACKUPS=10
+# POSTGRES_VALIDATE_INTEGRITY=true
+# POSTGRES_COMPRESSION=true
+# DB_URL="postgresql://madsci:madsci@localhost:5432/resources"
+# POSTGRES_BACKUP_FORMAT="custom"
+
+### MongoDBBackupSettings
+
+# MONGODB_BACKUP_DIR=".madsci/backups"
+# MONGODB_MAX_BACKUPS=10
+# MONGODB_VALIDATE_INTEGRITY=true
+# MONGODB_COMPRESSION=true
+# MONGO_DB_URL="mongodb://localhost:27017"
+MONGODB_DATABASE=
+# MONGODB_COLLECTIONS=null
diff --git a/.gitignore b/.gitignore
@@ -303,6 +303,7 @@ test_experiment/
 # Output data
 .wei/
 .madsci/
+backups/
 
 #node modules
 **/node_modules
@@ -314,4 +315,11 @@ coverage.xml
 .coverage
 
 .mcp.json
+
+# Alembic migration versions (environment-specific)
+src/madsci_resource_manager/madsci/resource_manager/alembic/versions/*.py
+
+
+# Agentic coding
 .claude/
+agent_docs/
diff --git a/.justfile b/.justfile
@@ -36,6 +36,8 @@ checks:
   @pre-commit run --all-files || { echo "" && echo "Some checks failed! Running one more time to see if any automatic fixes worked:" && echo "" ; pre-commit run --all-files; }
 # Run the pre-commit checks
 check: checks
+ruff-unsafe:
+  @ruff check . --fix --unsafe-fixes
 
 # Build the project
 build: dcb
@@ -187,6 +189,8 @@ node_integration_tests:
 experiment_integration_tests:
   docker compose run --rm --no-deps workcell_manager python -m nbconvert --to notebook --inplace --stdout --execute ./notebooks/experiment_notebook.ipynb
 
+backup_integration_tests:
+  docker compose run --rm --no-deps workcell_manager python -m nbconvert --to notebook --inplace --stdout --execute ./notebooks/backup_and_migration.ipynb
 
 # Run the integration tests
-integration_tests: node_integration_tests experiment_integration_tests
+integration_tests: node_integration_tests experiment_integration_tests backup_integration_tests
diff --git a/AGENTS.md b/AGENTS.md
@@ -126,6 +126,109 @@ The `AbstractManagerBase` class provides:
 - When generating new IDs, use `new_ulid_str()` from `madsci.common.utils`
 - Example usage: `resource_id = new_ulid_str()`
 
+### Database Patterns
+
+MADSci uses two database systems optimized for different use cases:
+
+#### Database Types
+- **PostgreSQL**: Used by Resource Manager for relational data with strict schemas
+- **MongoDB**: Used by Event, Data, Experiment, and Workcell Managers for flexible document storage
+
+#### Backup and Restore
+
+All backup tools are centralized in `madsci_common` for maximum reusability:
+
+```python
+# PostgreSQL backups
+from madsci.common.backup_tools import PostgreSQLBackupTool
+from madsci.common.types.backup_types import PostgreSQLBackupSettings
+
+settings = PostgreSQLBackupSettings(
+    db_url="postgresql://localhost/resources",
+    backup_dir=Path("./backups"),
+    max_backups=10,
+    validate_integrity=True
+)
+backup_tool = PostgreSQLBackupTool(settings)
+backup_path = backup_tool.create_backup("pre_migration")
+
+# MongoDB backups
+from madsci.common.backup_tools import MongoDBBackupTool
+from madsci.common.types.backup_types import MongoDBBackupSettings
+
+settings = MongoDBBackupSettings(
+    mongo_db_url=AnyUrl("mongodb://localhost:27017"),
+    database="events",
+    backup_dir=Path("./backups"),
+    max_backups=10
+)
+backup_tool = MongoDBBackupTool(settings)
+backup_path = backup_tool.create_backup("hourly")
+```
+
+**CLI Usage:**
+```bash
+# Unified CLI (auto-detects database type)
+madsci-backup create --db-url postgresql://localhost/resources
+madsci-backup create --db-url mongodb://localhost:27017/events
+
+# Database-specific CLIs
+madsci-postgres-backup create --db-url postgresql://localhost/resources
+madsci-mongodb-backup create --mongo-url mongodb://localhost:27017 --database events
+```
+
+#### Database Connections
+
+**PostgreSQL** (using SQLModel):
+```python
+from sqlmodel import Session, create_engine
+
+engine = create_engine(
+    db_url,
+    pool_size=20,        # Connection pool size
+    pool_pre_ping=True   # Verify connections before use
+)
+
+with Session(engine) as session:
+    # Perform operations
+    session.commit()
+```
+
+**MongoDB** (using pymongo):
+```python
+from pymongo import MongoClient
+
+with MongoClient(mongo_url) as client:
+    db = client[database_name]
+    collection = db[collection_name]
+    # Perform operations
+```
+
+#### Database Migrations
+
+**PostgreSQL migrations** (Resource Manager):
+- Uses Alembic for schema version management
+- Automatic backups before migrations
+- Auto-restore on migration failure
+```bash
+python -m madsci.resource_manager.migration_tool --db-url postgresql://localhost/resources
+```
+
+**MongoDB migrations** (per manager):
+- Handle index creation and schema validation
+- Manager-specific migration tools
+- Automatic pre-migration backups
+
+#### Best Practices
+
+1. **Always use ULID for IDs**: `resource_id = new_ulid_str()`
+2. **Backup before migrations**: Automatic with migration tools
+3. **Use connection pooling**: Configure appropriate pool sizes
+4. **Environment variables for config**: Never hardcode connection strings
+5. **Validate backups**: Use `validate_backup_integrity()` for critical backups
+6. **Test migrations first**: Always test in development before production
+
+
 ### Node Development
 Laboratory instruments implement the Node interface:
 1. Inherit from `AbstractNodeModule`