diff --git a/scripts/verify-nightly-provenance.sh b/scripts/verify-nightly-provenance.sh index c77105ef5..5d27d702c 100755 --- a/scripts/verify-nightly-provenance.sh +++ b/scripts/verify-nightly-provenance.sh @@ -5,12 +5,13 @@ # Promotion (promote.yml) only lets a channel point at a version the nightly # release workflow actually produced. A version passes when the GitHub Actions # API shows a successful run of the nightly workflow whose head commit matches -# the staged version's short SHA; otherwise this script exits non-zero and the -# promotion fails before the channel pointer is touched. +# the staged version's short SHA AND whose smoke-success aggregator job actually +# ran (conclusion "success", not "skipped"); otherwise this script exits +# non-zero and the promotion fails before the channel pointer is touched. # -# A nightly run that skipped its build because the SHA was already staged still -# counts: the run pointed the `nightly` channel at that SHA, so the version went -# through the nightly path either way. +# A no-op nightly run that skipped its build (the SHA was already staged) also +# skips the smoke jobs, so it does NOT pass verification — the version must +# have been smoked by a prior run that actually built and tested it. # # Usage: # scripts/verify-nightly-provenance.sh --sha SHORTSHA [options] @@ -72,18 +73,43 @@ command -v gh >/dev/null 2>&1 || die "gh not found" sha_lc=$(printf '%s' "$SHA" | tr '[:upper:]' '[:lower:]') -# One "head_sha html_url" line per successful run. Capture before grepping so a -# grep -m1 early exit can't SIGPIPE gh under pipefail. +# One "head_sha run_id html_url" line per successful run. Capture before +# grepping so a grep -m1 early exit can't SIGPIPE gh under pipefail. runs=$(gh api --paginate \ "repos/${REPO}/actions/workflows/${WORKFLOW_FILE}/runs?status=success&per_page=100" \ - --jq '.workflow_runs[] | "\(.head_sha) \(.html_url)"') + --jq '.workflow_runs[] | "\(.head_sha) \(.id) \(.html_url)"') # Anchor the prefix match to the head_sha field (hex run, then the space -# before the URL) so the input can only ever match a prefix of the commit SHA. +# before the run_id) so the input can only ever match a prefix of the commit SHA. match=$(printf '%s\n' "$runs" | grep -m1 -E "^${sha_lc}[0-9a-f]* " || true) [ -n "$match" ] || die "version '$SHA' was not built by a successful ${WORKFLOW_FILE} run in ${REPO}; \ promote a nightly-built version, or re-dispatch with the emergency provenance override" -printf 'verify-nightly-provenance: %s was built by %s run %s\n' \ - "$SHA" "$WORKFLOW_FILE" "${match#* }" >&2 +# Extract run_id (second field) and html_url (third field onwards). +run_id="${match#* }" # drop head_sha +run_id="${run_id%% *}" # keep only run_id +run_url="${match#* }" # drop head_sha +run_url="${run_url#* }" # drop run_id, leaving html_url + +# Verify that smoke tests actually ran (not skipped). A no-op nightly run where +# the check job found an already-staged version skips all build/smoke jobs; the +# overall run still succeeds, but we must not promote a version that bypassed +# the smoke test suite. The smoke-success aggregator job exists with conclusion +# "success" only when the smoke jobs actually executed and passed. +smoke_conclusion=$(gh api \ + "repos/${REPO}/actions/runs/${run_id}/jobs?per_page=100" \ + --jq '.jobs[] | select(.name == "smoke-success") | .conclusion' 2>/dev/null || true) + +if [ "$smoke_conclusion" != "success" ]; then + if [ -z "$smoke_conclusion" ]; then + die "version '$SHA' in ${WORKFLOW_FILE} run ${run_url} has no smoke-success job; \ +smoke tests must complete before promotion" + else + die "version '$SHA' in ${WORKFLOW_FILE} run ${run_url} has smoke-success with conclusion '$smoke_conclusion' (not 'success'); \ +smoke tests must pass before promotion" + fi +fi + +printf 'verify-nightly-provenance: %s was built by %s run %s (smoke-success: %s)\n' \ + "$SHA" "$WORKFLOW_FILE" "$run_url" "$smoke_conclusion" >&2 diff --git a/scripts/verify-nightly-provenance_test.sh b/scripts/verify-nightly-provenance_test.sh index 7e0157a73..98f97b163 100755 --- a/scripts/verify-nightly-provenance_test.sh +++ b/scripts/verify-nightly-provenance_test.sh @@ -23,12 +23,25 @@ cat >"$root/bin/gh" <<'EOF' printf '%s\n' "$@" >>"${GH_STUB_ARGS:?}" code="${GH_STUB_EXIT:-0}" [ "$code" -eq 0 ] || exit "$code" -cat "${GH_STUB_RUNS:?}" + +# Route requests to the appropriate canned response based on the API path. +# $2 is the API path (api [opts...]). +case "$2" in + *"/actions/runs/"*"/jobs"*) + # Jobs endpoint: return canned jobs response. + cat "${GH_STUB_JOBS:?}" + ;; + *) + # Workflow runs endpoint: return canned runs list. + cat "${GH_STUB_RUNS:?}" + ;; +esac EOF chmod +x "$root/bin/gh" export PATH="$root/bin:$PATH" export GH_STUB_ARGS="$root/gh-args" export GH_STUB_RUNS="$root/runs" +export GH_STUB_JOBS="$root/jobs" unset GH_STUB_EXIT pass=0 fail=0 @@ -52,12 +65,16 @@ expect() { fi } -# One successful nightly run per line: " ". +# One successful nightly run per line: " ". sha_a="dd20ee67e6349ca8573df43e16abd407a617a0df" sha_b="0badf00d5eed5eed5eed5eed5eed5eed5eed5eed" cat >"$GH_STUB_RUNS" <"$GH_STUB_JOBS" <"$GH_STUB_ARGS" @@ -95,6 +112,8 @@ expect 0 "https://example.test/runs/2" "match on a later line is found" \ -- "$script" --sha 0badf00d --repo acme/widgets expect 0 "https://example.test/runs/1" "uppercase input is normalized" \ -- "$script" --sha DD20EE67 --repo acme/widgets +expect 0 "smoke-success: success" "success message includes smoke-success status" \ + -- "$script" --sha dd20ee67 --repo acme/widgets expect 1 "was not built by a successful" "unknown sha fails and mentions the override" \ -- "$script" --sha deadbeef --repo acme/widgets expect 1 "provenance override" "failure message points at the emergency override" \ @@ -110,6 +129,46 @@ if grep -q "repos/acme/widgets/actions/workflows/custom.yml/runs?status=success" else bad "queries the requested repo and workflow (args: $(cat "$GH_STUB_ARGS"))" fi +# Verify jobs endpoint is queried for the matched run. +if grep -q "repos/acme/widgets/actions/runs/1001/jobs" "$GH_STUB_ARGS"; then + ok "queries jobs endpoint for matched run" +else + bad "queries jobs endpoint for matched run (args: $(cat "$GH_STUB_ARGS"))" +fi + +# ── Smoke-success job verification ─────────────────────────────────────────── +# Reset to valid runs data for smoke tests. +cat >"$GH_STUB_RUNS" <"$GH_STUB_JOBS" +expect 1 "conclusion 'skipped'" "skipped smoke-success fails with clear message" \ + -- "$script" --sha dd20ee67 --repo acme/widgets +expect 1 "smoke tests must pass" "skipped smoke-success mentions smoke tests must pass" \ + -- "$script" --sha dd20ee67 --repo acme/widgets + +# Missing smoke-success: malformed run without the aggregator job. +: >"$GH_STUB_JOBS" +expect 1 "no smoke-success job" "missing smoke-success job fails with clear message" \ + -- "$script" --sha dd20ee67 --repo acme/widgets +expect 1 "smoke tests must complete" "missing smoke-success mentions tests must complete" \ + -- "$script" --sha dd20ee67 --repo acme/widgets + +# Failed smoke-success: smoke tests ran but failed. +echo "failure" >"$GH_STUB_JOBS" +expect 1 "conclusion 'failure'" "failed smoke-success fails with clear message" \ + -- "$script" --sha dd20ee67 --repo acme/widgets + +# Cancelled smoke-success: run was cancelled mid-flight. +echo "cancelled" >"$GH_STUB_JOBS" +expect 1 "conclusion 'cancelled'" "cancelled smoke-success fails with clear message" \ + -- "$script" --sha dd20ee67 --repo acme/widgets + +# Restore success for remaining tests. +echo "success" >"$GH_STUB_JOBS" # ── gh failure propagation ─────────────────────────────────────────────────── expect 22 "" "gh API failure propagates the failing exit code" \