AD-SDL · RyanTheRobothead · Dec 15, 2025 · Dec 11, 2025 · Dec 12, 2025 · Dec 12, 2025
diff --git a/.env.example b/.env.example
@@ -82,6 +82,7 @@
 # WORKCELL_REDIS_PASSWORD=null
 # WORKCELL_SCHEDULER_UPDATE_INTERVAL=5.0
 # WORKCELL_NODE_UPDATE_INTERVAL=2.0
+# WORKCELL_RECONNECT_ATTEMPT_INTERVAL=1200.0
 # WORKCELL_NODE_INFO_UPDATE_INTERVAL=60.0
 # WORKCELL_COLD_START_DELAY=0
 # WORKCELL_SCHEDULER="madsci.workcell_manager.schedulers.default_scheduler"

diff --git a/Configuration.md b/Configuration.md
@@ -118,6 +118,7 @@ Settings for the MADSci Workcell Manager.
 | `WORKCELL_REDIS_PASSWORD`                                                      | `string` \| `NoneType`           | `null`                                                   | The password for the redis server.                                                                                                                                                       | `null`                                                   |
 | `WORKCELL_SCHEDULER_UPDATE_INTERVAL`                                           | `number`                         | `5.0`                                                    | The interval at which the scheduler runs, in seconds. Must be >= node_update_interval                                                                                                    | `5.0`                                                    |
 | `WORKCELL_NODE_UPDATE_INTERVAL`                                                | `number`                         | `2.0`                                                    | The interval at which the workcell queries its node's states and status, in seconds. Must be <= scheduler_update_interval                                                                | `2.0`                                                    |
+| `WORKCELL_RECONNECT_ATTEMPT_INTERVAL`                                          | `number`                         | `1200.0`                                                 | The interval at which the workcell resets disconnected nodes, in seconds.                                                                                                                | `1200.0`                                                 |
 | `WORKCELL_NODE_INFO_UPDATE_INTERVAL`                                           | `number`                         | `60.0`                                                   | The interval at which the workcell queries its node's info, in seconds. Node info changes infrequently, so this can be much larger than node_update_interval to reduce network overhead. | `60.0`                                                   |
 | `WORKCELL_COLD_START_DELAY`                                                    | `integer`                        | `0`                                                      | How long the Workcell engine should sleep on startup                                                                                                                                     | `0`                                                      |
 | `WORKCELL_SCHEDULER`                                                           | `string`                         | `"madsci.workcell_manager.schedulers.default_scheduler"` | Scheduler module that contains a Scheduler class that inherits from AbstractScheduler to use                                                                                             | `"madsci.workcell_manager.schedulers.default_scheduler"` |

diff --git a/pdm.lock b/pdm.lock
diff --git a/src/madsci_common/madsci/common/types/node_types.py b/src/madsci_common/madsci/common/types/node_types.py
@@ -368,6 +368,11 @@ class NodeStatus(MadsciBaseModel):
         title="Node Errored",
         description="Whether the node is in an errored state.",
     )
+    disconnected: bool = Field(
+        default=False,
+        title="Node Disconnected",
+        description="Whether the node is disconnected from the workcell manager",
+    )
     errors: list[Error] = Field(
         default_factory=list,
         title="Node Errors",
@@ -406,6 +411,8 @@ def ready(self) -> bool:
             ready = False
         if self.paused:
             ready = False
+        if self.disconnected:
+            ready = False
         if len(self.waiting_for_config) > 0:
             ready = False
         return ready
@@ -421,6 +428,8 @@ def description(self) -> str:
             reasons.append("Node is locked")
         if self.errored:
             reasons.append("Node is in an error state")
+        if self.disconnected:
+            reasons.append("Node is disconnected")
         if self.initializing:
             reasons.append("Node is initializing")
         if self.paused:

diff --git a/src/madsci_common/madsci/common/types/workcell_types.py b/src/madsci_common/madsci/common/types/workcell_types.py
@@ -200,6 +200,11 @@ class WorkcellManagerSettings(
         title="Node Update Interval",
         description="The interval at which the workcell queries its node's states and status, in seconds. Must be <= scheduler_update_interval",
     )
+    reconnect_attempt_interval: float = Field(
+        default=1200.0,
+        title="Reconnect Attempt Interval",
+        description="The interval at which the workcell resets disconnected nodes, in seconds.",
+    )
     node_info_update_interval: float = Field(
         default=60.0,
         title="Node Info Update Interval",

diff --git a/src/madsci_workcell_manager/madsci/workcell_manager/workcell_engine.py b/src/madsci_workcell_manager/madsci/workcell_manager/workcell_engine.py
@@ -86,6 +86,7 @@ def spin(self) -> None:
         self.update_active_nodes(self.state_handler, update_info=True)
         node_tick = time.time()
         info_tick = time.time()
+        reconnect_tick = time.time()
         scheduler_tick = time.time()
         while True and not self.state_handler.shutdown:
             try:
@@ -107,6 +108,13 @@ def spin(self) -> None:
                     node_tick = time.time()
                     if should_update_info:
                         info_tick = time.time()
+                if (
+                    time.time() - reconnect_tick
+                    > self.workcell_settings.reconnect_attempt_interval
+                ):
+                    self.reset_disconnects()
+                    reconnect_tick = time.time()
+
                 if (
                     time.time() - scheduler_tick
                     > self.workcell_settings.scheduler_update_interval
@@ -583,10 +591,11 @@ def update_active_nodes(
         with concurrent.futures.ThreadPoolExecutor() as executor:
             node_futures = []
             for node_name, node in state_manager.get_nodes().items():
-                node_future = executor.submit(
-                    self.update_node, node_name, node, state_manager, update_info
-                )
-                node_futures.append(node_future)
+                if node.status is None or not node.status.disconnected:
+                    node_future = executor.submit(
+                        self.update_node, node_name, node, state_manager, update_info
+                    )
+                    node_futures.append(node_future)
 
             # Wait for all node updates to complete
             concurrent.futures.wait(node_futures)
@@ -617,7 +626,7 @@ def update_node(
                 state_manager.set_node(node_name, node)
         except Exception as e:
             error = Error.from_exception(e)
-            node.status = NodeStatus(errored=True, errors=[error])
+            node.status = NodeStatus(errored=True, errors=[error], disconnected=True)
             with state_manager.wc_state_lock():
                 state_manager.set_node(node_name, node)
             with ownership_context(
@@ -630,3 +639,11 @@ def update_node(
                         event_data=node.status,
                     )
                 )
+
+    def reset_disconnects(self) -> None:
+        """Reset all disconnected nodes to initializing state."""
+        with self.state_handler.wc_state_lock():
+            for name, node in self.state_handler.get_nodes().items():
+                node.status = NodeStatus()
+                node.status.initializing = True
+                self.state_handler.set_node(name, node)
diff --git a/src/madsci_workcell_manager/madsci/workcell_manager/workcell_server.py b/src/madsci_workcell_manager/madsci/workcell_manager/workcell_server.py
@@ -310,9 +310,16 @@ def send_admin_command_to_node(
         self, command: str, node: str
     ) -> AdminCommandResponse:
         """Send admin command to a node."""
-        node = self.state_handler.get_node(node)
-        if command in node.info.capabilities.admin_commands:
-            client = find_node_client(node.node_url)
+        node_object = self.state_handler.get_node(node)
+        if command == "reset":
+            with self.state_handler.wc_state_lock():
+                # Clear errors on reset command
+                node_object.status.errored = False
+                node_object.status.disconnected = False
+                node_object.status.errors = []
+                self.state_handler.set_node(node_name=node, node=node_object)
+        if command in node_object.info.capabilities.admin_commands:
+            client = find_node_client(node_object.node_url)
             return client.send_admin_command(command)
         raise HTTPException(
             status_code=400, detail="Node cannot perform that admin command"

diff --git a/src/madsci_workcell_manager/tests/test_workcell_engine.py b/src/madsci_workcell_manager/tests/test_workcell_engine.py
@@ -110,7 +110,9 @@ def engine(state_handler: WorkcellStateHandler) -> Engine:
         mock_location_client.return_value = mock_location_client_instance
 
         warnings.simplefilter("ignore", UserWarning)
-        return Engine(state_handler=state_handler, data_client=DataClient())
+        engine = Engine(state_handler=state_handler, data_client=DataClient())
+        engine.state_handler.set_node(node_name="node1", node=test_node)
+        return engine
 
 
 def test_engine_initialization(engine: Engine) -> None:
@@ -125,6 +127,21 @@ def test_run_next_step_no_ready_workflows(engine: Engine) -> None:
     assert workflow is None
 
 
+def test_disconnect_node_on_connection_failure(engine: Engine) -> None:
+    """Test run_next_step when no workflows are ready."""
+    with patch(
+        "madsci.client.node.rest_node_client.RestNodeClient.get_status",
+        side_effect=Exception("Connection failed"),
+    ):
+        engine.update_active_nodes(engine.state_handler)
+        for node in engine.state_handler.get_nodes().values():
+            assert node.status.disconnected is True
+        engine.reset_disconnects()
+        for node in engine.state_handler.get_nodes().values():
+            assert node.status.initializing is True
+            assert node.status.disconnected is False
+
+
 def test_run_next_step_with_ready_workflow(
     engine: Engine, state_handler: WorkcellStateHandler
 ) -> None:
@@ -155,10 +172,6 @@ def test_run_single_step(engine: Engine, state_handler: WorkcellStateHandler) ->
         status=WorkflowStatus(running=True),
     )
     state_handler.set_active_workflow(workflow)
-    state_handler.set_node(
-        node_name="node1",
-        node=test_node,
-    )
     with patch(
         "madsci.workcell_manager.workcell_engine.find_node_client"
     ) as mock_client:
@@ -209,10 +222,6 @@ def test_run_single_step_with_update_parameters(
         status=WorkflowStatus(running=True),
     )
     state_handler.set_active_workflow(workflow)
-    state_handler.set_node(
-        node_name="node1",
-        node=test_node,
-    )
     with patch(
         "madsci.workcell_manager.workcell_engine.find_node_client"
     ) as mock_client:
@@ -243,10 +252,6 @@ def test_run_single_step_of_workflow_with_multiple_steps(
         status=WorkflowStatus(running=True),
     )
     state_handler.set_active_workflow(workflow)
-    state_handler.set_node(
-        node_name="node1",
-        node=test_node,
-    )
     with patch(
         "madsci.workcell_manager.workcell_engine.find_node_client"
     ) as mock_client:
@@ -310,9 +315,7 @@ def test_finalize_step_failure(
     assert finalized_workflow.steps[0].status == ActionStatus.FAILED
 
 
-def test_handle_data_and_files_with_data(
-    engine: Engine, state_handler: WorkcellStateHandler
-) -> None:
+def test_handle_data_and_files_with_data(engine: Engine) -> None:
     """Test handle_data_and_files with data points."""
     step = Step(
         name="Test Step",
@@ -325,10 +328,6 @@ def test_handle_data_and_files_with_data(
         steps=[step],
         status=WorkflowStatus(running=True),
     )
-    state_handler.set_node(
-        node_name="node1",
-        node=test_node,
-    )
     action_result = ActionSucceeded(json_result=42)
 
     # Create a mock datapoint that will be returned by submit_datapoint
@@ -352,9 +351,7 @@ def test_handle_data_and_files_with_data(
         )
 
 
-def test_handle_data_and_files_with_files(
-    engine: Engine, state_handler: WorkcellStateHandler
-) -> None:
+def test_handle_data_and_files_with_files(engine: Engine) -> None:
     """Test handle_data_and_files with file points."""
     step = Step(
         name="Test Step",
@@ -368,10 +365,6 @@ def test_handle_data_and_files_with_files(
         steps=[step],
         status=WorkflowStatus(running=True),
     )
-    state_handler.set_node(
-        node_name="node1",
-        node=test_node,
-    )
     action_result = ActionSucceeded(files=Path("/path/to/file"))
 
     with (
@@ -398,10 +391,6 @@ def test_run_step_send_action_exception_then_get_action_result_success(
         status=WorkflowStatus(running=True),
     )
     state_handler.set_active_workflow(workflow)
-    state_handler.set_node(
-        node_name="node1",
-        node=test_node,
-    )
 
     with patch(
         "madsci.workcell_manager.workcell_engine.find_node_client"
@@ -436,10 +425,6 @@ def test_run_step_send_action_and_get_action_result_fail(
         status=WorkflowStatus(running=True),
     )
     state_handler.set_active_workflow(workflow)
-    state_handler.set_node(
-        node_name="node1",
-        node=test_node,
-    )
 
     with patch(
         "madsci.workcell_manager.workcell_engine.find_node_client"

diff --git a/src/madsci_workcell_manager/tests/test_workcell_server.py b/src/madsci_workcell_manager/tests/test_workcell_server.py
@@ -122,6 +122,9 @@ def test_send_admin_command(test_client: TestClient) -> None:
         response = client.post("/admin/reset")
         assert response.status_code == 200
         assert isinstance(response.json(), list)
+        for node in client.get("/nodes").json().values():
+            valid_node = Node.model_validate(node)
+            assert valid_node.status.initializing
 
 
 def test_get_active_workflows(test_client: TestClient) -> None: