aclk/pluginsd: safely tear down hosts and preserve requested retries - #23867

sequenceDiagram
    participant CD as Collector Daemon
    participant DS as Data Store
    participant WS as Worker Spawner
    participant PSR as Parser

    loop Worker Lifecycle
        CD->>WS: Spawn plugin process
        WS-->>CD: PID (unsafe.pi)
        CD->>PSR: Start parsing data
        PSR->>CD: retry request (transient condition)
        alt Retry requested
            CD->>WS: Request graceful exit (QUIT/SIGTERM)
            WS-->>CD: worker_ret_code
            alt worker_ret_code == -1 (forced kill)
                CD->>CD: Disable retry?<br/>No - keep retries enabled
                CD->>CD: Log warning<br/>Restart with longer backoff
                CD->>CD: Sleep update_every * 10
            else worker_ret_code != -1 (clean exit)
                CD->>CD: Sleep update_every
            end
            CD->>WS: Respawn plugin
        else No retry requested
            alt worker_ret_code == 0 (success)
                CD->>CD: Handle success
            else worker_ret_code != 0 (unexpected exit)
                CD->>CD: Handle failure/crash
            end
        end
    end

sequenceDiagram
    participant WD as Worker Thread
    participant POP as spawn_popen
    participant SS as Spawn Server
    participant BP as Backend (POSIX/UV/Nofork/Windows)
    participant CFG as Config Parser

    Note over WD,CFG: Collector lifecycle & kill outcome tracking

    WD->>POP: spawn_popen_kill_ex(pi, 3s, &outcome)
    POP->>POP: Close file descriptors
    POP->>SS: spawn_server_exec_kill_ex(si, 3s, &outcome)
    
    alt Backend = POSIX/UV
        SS->>BP: Send SIGTERM
        BP-->>SS: Signal sent
        SS->>BP: spawn_server_exec_timedwait(grace_ms)
        alt Child exited within grace
            BP-->>SS: EXITED
            SS-->>POP: outcome = SPAWN_KILL_EXITED
        else Child still running
            SS->>BP: Send SIGKILL
            BP-->>SS: Signal sent
            SS->>BP: spawn_server_exec_timedwait(grace_ms)
            alt Child confirmed gone
                BP-->>SS: EXITED
                SS-->>POP: outcome = SPAWN_KILL_FORCED_EXITED
            else Still not confirmed
                BP-->>SS: TIMEDWAIT_TIMEOUT
                alt Deferred cleanup available
                    SS->>BP: Defer waitpid to reaper
                    SS-->>POP: outcome = SPAWN_KILL_UNKNOWN
                else No reaper slot
                    SS->>BP: spawn_server_exec_wait (unbounded)
                    BP-->>SS: Child reaped
                    SS-->>POP: outcome = SPAWN_KILL_FORCED_EXITED
                end
            end
        end
    else Backend = Windows
        SS->>BP: WaitForSingleObject(3s)
        alt Child exited within grace
            BP-->>SS: WAIT_OBJECT_0
            SS->>BP: TerminateChildProcesses
            SS->>BP: WaitForSingleObject(INFINITE)
            alt Handle valid
                BP-->>SS: WAIT_OBJECT_0
                SS-->>POP: outcome = SPAWN_KILL_EXITED
            else Handle failed
                BP-->>SS: WAIT_FAILED
                SS-->>POP: outcome = SPAWN_KILL_UNKNOWN
            end
        else Still running
            SS->>BP: kill(SIGTERM) + TerminateProcess
            SS->>BP: TerminateChildProcesses
            SS->>BP: WaitForSingleObject(INFINITE)
            alt Handle valid
                BP-->>SS: WAIT_OBJECT_0
                SS-->>POP: outcome = SPAWN_KILL_FORCED_EXITED
            else Handle failed
                BP-->>SS: WAIT_FAILED
                SS-->>POP: outcome = SPAWN_KILL_UNKNOWN
            end
        end
    else Backend = Nofork
        SS->>BP: Close fds, signal child
        alt Child exited
            BP-->>SS: EXITED
            SS-->>POP: outcome = SPAWN_KILL_EXITED
        else Need SIGKILL
            SS->>BP: Send SIGKILL
            BA-->>SS: Signal sent
            BP-->>SS: Timed wait
            alt Confirmed gone
                SS-->>POP: outcome = SPAWN_KILL_FORCED_EXITED
            else Not confirmed
                SS-->>POP: outcome = SPAWN_KILL_UNKNOWN
            end
        end
    end

    POP-->>WD: return code + outcome

    WD->>WD: Analyze kill_outcome

    alt SPAWN_KILL_UNKNOWN
        Note over WD: Process may still be running<br/>Refusing restart
        WD->>WD: Log error, disable plugin, set pid=0
    else SPAWN_KILL_EXITED or FORCED_EXITED
        alt retry == true (parser requested)
            WD->>WD: Log why (clean exit vs forced)
            WD->>WD: pluginsd_sleep_backoff(cd)
            WD->>WD: Restart collector
        else retry == false
            alt worker_ret_code == 0
                WD->>WD: Handle success, backoff before restart
            else error
                WD->>WD: Handle error, disable if needed, backoff
            end
        end
    end

    Note over CFG,WD: Config load with clamping
    CFG->>CFG: GET update_every from config
    CFG->>CFG: Clamp to [UPDATE_EVERY_MIN, UPDATE_EVERY_MAX]
    alt Out of range
        CFG->>CFG: Log warning, use clamped value
    end
    CFG-->>WD: update_every (now safe for backoff math)

    Note over WD: Backoff sleep uses usec_t<br/>avoids int overflow on large values