abahmed · abahmed · Jun 10, 2026 · Jun 10, 2026 · Jun 10, 2026 · Jun 10, 2026
@@ -4,3 +4,11 @@ updates:
     directory: "/"
     schedule:
       interval: "daily"
+  - package-ecosystem: "docker"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"
@@ -0,0 +1,36 @@
+name: Publish LLM image
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: "model image version, e.g. v2"
+        required: true
+
+jobs:
+  push_llm_image:
+    name: Push kwatch-llm image to GHCR
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+      - name: Free disk space
+        run: docker buildx prune -af && df -h
+      - uses: docker/setup-buildx-action@v3
+      - uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - uses: docker/build-push-action@v6
+        with:
+          context: deploy/llm
+          platforms: linux/amd64,linux/arm64
+          push: true
+          build-args: |
+            MODEL_VERSION=${{ github.event.inputs.version }}
+          tags: |
+            ghcr.io/abahmed/kwatch-llm:${{ github.event.inputs.version }}
+            ghcr.io/abahmed/kwatch-llm:latest
@@ -8,6 +8,9 @@ jobs:
   push_to_registry:
     name: Push Docker image to GitHub Container Registry
     runs-on: ubuntu-latest
+    permissions:
+      packages: write
+      contents: read
     steps:
       - name: Check out the repo
         uses: actions/checkout@v4

@@ -29,4 +29,5 @@ config.yaml
 *.DS_Store
 
 # debug
-__debug_bin*
+__debug_bin*
+/kwatch
@@ -6,10 +6,10 @@
             "type": "go",
             "request": "launch",
             "mode": "auto",
-            "program": "${workspaceRoot}",
+            "program": "${workspaceRoot}/cmd/kwatch",
             "showLog": true,
             "env": {
-                "CONFIG_FILE": "config.yaml"
+                "CONFIG_FILE": "${workspaceRoot}/config.yaml"
             }
         }
     ]

@@ -0,0 +1,121 @@
+# Changelog
+
+All notable changes to this project are documented in this file.
+
+## [Unreleased]
+
+### Added
+
+- **AI incident enrichment (opt-in, default-off)**: self-hosted LLM sidecar
+  (`kwatch-llm` — Qwen2.5-Coder-1.5B-Instruct via Ollama) appends root-cause
+  analysis to incident alerts. Everything runs in-cluster; no external API
+  calls. Configuration: `llm.enabled` (default `false`). Circuit breaker
+  (3-failure / 60s cooldown), single-flight enrich channel, 15s timeout.
+  - `internal/llm/` — client, prompt builder with CD-1 grounding,
+    `selectRelevant` log extraction, `redact` (password/token/JWT scrubbing)
+  - Baked model image at `deploy/llm/` with Makefile target & CI
+    (`publish-llm.yml`)
+  - Helm: `config.llm.enabled`, `llm.nativeSidecar`, replica guard
+  - Metrics: `kwatch_llm_enrich_total`, `kwatch_llm_enrich_failed_total`,
+    `kwatch_llm_enrich_skipped_total`
+
+- **Cheap-depth enhancements (always-on, no LLM needed)**:
+  | ID | What |
+  |----|------|
+  | CD-1 | Grounding fields in enrichment (occurrences, affected pods, duration) |
+  | CD-2 | Correlation surfaces affected-pod count + per-node breakdown |
+  | CD-3 | Built-in signature hints (Postgres OOM, TLS handshake, disk-full, timeout, RBAC, OOM-without-limit) |
+  | CD-4 | `runbooks:` URL appended to incident hint |
+  | CD-5 | Investigate commands (`kubectl logs`/`kubectl describe`) + dashboard deep-link |
+
+### Fixed
+
+#### Phase 0 bugs
+
+- **CR-1**: LifecycleHook data race — `MarkResolved`, `RemovePod`,
+  `ResolveByResource`, and `checkLifecycle` renotify now clone the incident
+  under lock before passing it to the hook callback.
+- **CR-2**: Node inhibition cleared during hold-down — `ResolveByResource` no
+  longer unconditionally deletes `activeNodeIncidents`; the hold-down pathway
+  re-inserts the inhibition entry when a resource-level incident remains open.
+- **PVC-3**: Swallowed node-level error auto-resolves PVCs — added
+  `incomplete` flag; `getNodeUsage` errors set `incomplete=true` and the
+  resolve-by-absence path skips incomplete cycles.
+- **BUG-1**: CronJob false positives — split nil `LastScheduleTime` and
+  staleness branches; new CronJob only alerts if `CreationTimestamp` > 24h ago;
+  stale check only fires when `LastScheduleTime` is non-nil and > 24h old.
+- **BUG-2**: PodStatusFilter "Added" casing — switched to
+  `strings.EqualFold(ctx.EvType, "Added")`.
+- **BUG-3**: OOMKilled hint for containers with no memory limit — added
+  dedicated hint string when `resources.limits.memory` is nil or zero.
+- **BUG-4**: Per-signal `IncludeEvents`/`IncludeLogs` overwritten by
+  `eventWithConfig` — removed dead `IncludeEvents`/`IncludeLogs` fields from
+  the `Signal` struct.
+- **BUG-5**: `tls_sweep` clock — changed `time.Until(expiry)` to
+  `expiry.Sub(now)`.
+- **BUG-6**: Init container failures now use `InitContainerError` hint — added
+  `IsInit bool` to `ContainerContext`, set when iterating
+  `InitContainerStatuses`, and `buildContainerHint` prefers the
+  `InitContainerError` hint when `IsInit && exitCode != 0`.
+- **BUG-7**: Crashloop incidents dropped on transient clean exit — added
+  `!ctx.Container.HasRestarts &&` guard to the skip condition for
+  `Terminated{Completed|ExitCode 0|143}`.
+- **F2**: Drop-oldest channel send can block — the drain-receive path is now
+  non-blocking (`select` with `default`).
+- **SCAN-1**: `ActiveCount` overcount — now iterates `e.state` counting only
+  `State != StateResolved` instead of returning `len(e.state)`.
+- **SCAN-3**: AlertManager `Init` races with concurrent `NotifyIncident` calls
+  — added `a.mu` mutex; `Start` snapshots entries; `Notify`, `NotifyEvent`,
+  `NotifyIncident` snapshot entries; `shutdown` is idempotent.
+- **LOG-1**: Container detection log level — changed `klog.InfoS` to
+  `klog.V(2).InfoS` in `execute_containers_filters.go`.
+- **SQ-1**: Remove `startupQuiet` — deleted field from config and engine;
+  removed wiring in `main.go` and CRD types.
+- **PVC-1**: Kubelet proxy calls use request context — threaded
+  `context.Context` through `PvcMonitor.checkUsage` → `getNodeUsage` →
+  `GetNodeSummary`/`GetNodes`/`GetPVNameFromPVC`/`GetPodContainerLogs`/`GetPodEvents`;
+  each wraps the call with `context.WithTimeout(ctx, 10s)`.
+- **PVC-2**: N+1 PVC API fan-out — `checkUsage` builds a `pvByPVC` map
+  (ns/name → spec.VolumeName) once per cycle via one `List` call.
+- **PVC-4**: Nil PodRef panic — added `if pod.PodRef == nil { continue }`.
+- **PVC-5**: Division by zero — added `if vol.CapacityBytes <= 0 { continue }`.
+- **HTTP-1**: Discord/Telegram no timeout — replaced `&http.Client{}` with
+  `k8s.GetDefaultClient()` (shared client with proper transport and timeout).
+- **HTTP-2**: Unbounded log fetch when `maxRecentLogLines == 0` — default tail
+  of 500 lines and 1 MB `LimitBytes` cap.
+- **HB-1**: Heartbeat ping not tied to request context — `ping()` now accepts
+  `context.Context` and creates the HTTP request with `NewRequestWithContext`.
+- **MAIN-1/MAIN-2**: Shutdown/exit — replaced `os.Exit(0)` with a `stop`
+  channel; lost-leader exits with code 1; graceful shutdown uses a fresh
+  timeout context.
+- **HEALTH-1**: `/readyz` static OK — added `ready atomic.Bool`;
+  `SetReady(true)` at end of `runLeaderTasks`; returns 503 when not ready.
+- **Severity default table**: Added `defaultSeverityByReason` map with
+  `Evicted → medium`, `ImagePullBackOff → medium`. Looked up in
+  `resolveSeverity` between user `SeverityByReason` and owner-kind check.
+- **Escalation + default severities**: Added `severityForTier(tierIdx, current)`
+  that computes tier-based severity, preferring the higher of the tier and the
+  current severity. Prevents double-escalation when a default reason severity
+  is already set.
+
+### Added
+
+- **NEW-4**: `internal/integration/controller_fault_test.go` — integration test
+  (build tag `integration`) with fake clientset that verifies the controller
+  tolerates processing pod events with no providers configured.
+
+### Removed
+
+- `correlation.startupQuiet` config field and CRD field.
+- `IncludeEvents`/`IncludeLogs` fields from `event.Signal` struct (dead code).
+- `StartupQuiet` field from `config/config.go` and `api/v1alpha1/types.go`.
+
+### Changed
+
+- `kubernetes.Interface` methods (`GetNodes`, `GetPVNameFromPVC`,
+  `GetNodeSummary`, `GetPodContainerLogs`, `GetPodEvents`) now accept
+  `context.Context` as the first parameter.
+- `discord.Verify()`, `telegram.Verify()`, `telegram.sendByTelegramApi()` now
+  use `k8s.GetDefaultClient()` instead of `&http.Client{}`.
+- `HeartbeatMonitor.ping()` now requires a `context.Context` parameter.
+- `PvcMonitor.checkUsage()` now requires a `context.Context` parameter.
@@ -99,3 +99,7 @@ docker-build:
 # Docker build with latest tag
 docker-build-latest:
 	docker build -t kwatch:latest -t kwatch:$(VERSION) .
+
+# Build LLM sidecar image
+llm-image:
+	PUSH=$(PUSH) TAG=$(TAG) deploy/llm/build.sh
@@ -0,0 +1,3 @@
+// +k8s:deepcopy-gen=package
+
+package v1alpha1
@@ -0,0 +1,111 @@
+package v1alpha1
+
+import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+// KwatchConfig is the schema for the kwatch deployment configuration.
+type KwatchConfig struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec KwatchConfigSpec `json:"spec"`
+}
+
+// KwatchConfigSpec defines the desired kwatch configuration.
+type KwatchConfigSpec struct {
+	MaxRecentLogLines            int64                  `json:"maxRecentLogLines,omitempty"`
+	IgnoreFailedGracefulShutdown bool                   `json:"ignoreFailedGracefulShutdown,omitempty"`
+	Namespaces                   []string               `json:"namespaces,omitempty"`
+	Reasons                      []string               `json:"reasons,omitempty"`
+	IgnoreContainerNames         []string               `json:"ignoreContainerNames,omitempty"`
+	IgnorePodNames               []string               `json:"ignorePodNames,omitempty"`
+	IgnoreLogPatterns            []string               `json:"ignoreLogPatterns,omitempty"`
+	SeverityByOwnerKind          map[string]string      `json:"severityByOwnerKind,omitempty"`
+	PendingPodThreshold          int                    `json:"pendingPodThreshold,omitempty"`
+	ResyncSeconds                int                    `json:"resyncSeconds,omitempty"`
+	Silences                     []SilenceRule          `json:"silences,omitempty"`
+	Correlation                  CorrelationConfig      `json:"correlation,omitempty"`
+	PvcMonitor                   PvcMonitorConfig       `json:"pvcMonitor,omitempty"`
+	NodeMonitor                  NodeMonitorConfig      `json:"nodeMonitor,omitempty"`
+	RolloutMonitor               RolloutMonitorConfig   `json:"rolloutMonitor,omitempty"`
+	DaemonSetMonitor             DaemonSetMonitorConfig `json:"daemonSetMonitor,omitempty"`
+	JobMonitor                   JobMonitorConfig       `json:"jobMonitor,omitempty"`
+	CronJobMonitor               CronJobMonitorConfig   `json:"cronJobMonitor,omitempty"`
+	HeartbeatMonitor             HeartbeatMonitorConfig `json:"heartbeatMonitor,omitempty"`
+	HealthCheck                  HealthCheckConfig      `json:"healthCheck,omitempty"`
+	App                          AppConfig              `json:"app,omitempty"`
+	Workers                      int                    `json:"workers,omitempty"`
+}
+
+type CorrelationConfig struct {
+	Window            int `json:"window,omitempty"`
+	Cooldown          int `json:"cooldown,omitempty"`
+	StaleThreshold    int `json:"staleThreshold,omitempty"`
+	LifecycleInterval int `json:"lifecycleInterval,omitempty"`
+}
+
+type PvcMonitorConfig struct {
+	Enabled           bool    `json:"enabled,omitempty"`
+	Interval          int     `json:"interval,omitempty"`
+	Threshold         float64 `json:"threshold,omitempty"`
+	CriticalThreshold float64 `json:"criticalThreshold,omitempty"`
+}
+
+type NodeMonitorConfig struct {
+	Enabled bool `json:"enabled,omitempty"`
+}
+
+type RolloutMonitorConfig struct {
+	Enabled bool `json:"enabled,omitempty"`
+}
+
+type DaemonSetMonitorConfig struct {
+	Enabled bool `json:"enabled,omitempty"`
+}
+
+type JobMonitorConfig struct {
+	Enabled bool `json:"enabled,omitempty"`
+}
+
+type CronJobMonitorConfig struct {
+	Enabled bool `json:"enabled,omitempty"`
+}
+
+type HeartbeatMonitorConfig struct {
+	Enabled  bool   `json:"enabled,omitempty"`
+	Interval int    `json:"interval,omitempty"`
+	URL      string `json:"url,omitempty"`
+}
+
+type HealthCheckConfig struct {
+	Enabled bool `json:"enabled,omitempty"`
+	Port    int  `json:"port,omitempty"`
+}
+
+type AppConfig struct {
+	ClusterName           string `json:"clusterName,omitempty"`
+	ProxyURL              string `json:"proxyURL,omitempty"`
+	DisableStartupMessage bool   `json:"disableStartupMessage,omitempty"`
+	LogFormatter          string `json:"logFormatter,omitempty"`
+}
+
+type SilenceRule struct {
+	Namespaces        []string `json:"namespaces,omitempty"`
+	Reasons           []string `json:"reasons,omitempty"`
+	PodNamePatterns   []string `json:"podNamePatterns,omitempty"`
+	ContainerNames    []string `json:"containerNames,omitempty"`
+	LogPatterns       []string `json:"logPatterns,omitempty"`
+	ContainerMessages []string `json:"containerMessages,omitempty"`
+	NodeReasons       []string `json:"nodeReasons,omitempty"`
+	NodeMessages      []string `json:"nodeMessages,omitempty"`
+}
+
+// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
+
+// KwatchConfigList contains a list of KwatchConfig.
+type KwatchConfigList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []KwatchConfig `json:"items"`
+}