Repository navigation
ci: discover the installed MSVC instance for examples #1573
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Dynamic Provider Testing Workflow | |
| # Automatically discovers and tests all providers without manual configuration | |
| # Uses `vx test --ci` for comprehensive end-to-end testing | |
| # | |
| # Key features: | |
| # - Dynamic provider discovery via .github/scripts/discover-providers.sh | |
| # - Automatic platform filtering (skips Windows-only on Linux, etc.) | |
| # - Smart chunking for parallel execution | |
| # - Weekly scheduled runs to catch upstream changes | |
| name: Test Providers | |
| on: | |
| push: | |
| branches: [main] | |
| paths: | |
| - "crates/vx-providers/**" | |
| - "crates/vx-runtime/**" | |
| - "crates/vx-installer/**" | |
| - "crates/vx-starlark/**" | |
| - "crates/vx-star-metadata/**" | |
| - "crates/vx-manifest/**" | |
| - ".github/workflows/test-providers.yml" | |
| - ".github/scripts/discover-providers.sh" | |
| - ".github/scripts/summarize-test-results.sh" | |
| pull_request: | |
| branches: [main] | |
| workflow_dispatch: | |
| inputs: | |
| runtimes: | |
| description: "Specific runtimes to test (comma-separated, empty for all)" | |
| required: false | |
| default: "" | |
| skip: | |
| description: "Runtimes to skip (comma-separated)" | |
| required: false | |
| default: "" | |
| chunk_size: | |
| description: "Number of runtimes per test job (default: 8)" | |
| required: false | |
| default: "8" | |
| schedule: | |
| - cron: "0 6 * * 1" # Every Monday at 6:00 UTC | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| # ============================================================================ | |
| # Stage 0: Check if relevant files changed (ensures required check always reports) | |
| # ============================================================================ | |
| check-changes: | |
| name: Check Changes | |
| runs-on: ubuntu-latest | |
| outputs: | |
| should_run: ${{ steps.filter.outputs.should_run }} | |
| changed_file_count: ${{ steps.filter.outputs.changed_file_count }} | |
| base: ${{ steps.filter.outputs.base }} | |
| provider_subset: ${{ steps.scope.outputs.provider_subset }} | |
| changed_providers: ${{ steps.scope.outputs.changed_providers }} | |
| steps: | |
| # fetch-depth: 0 is required: change detection diffs against the PR base | |
| # SHA, which a default shallow clone does not contain. | |
| - uses: actions/checkout@v7 | |
| with: | |
| fetch-depth: 0 | |
| - name: Detect file changes | |
| id: filter | |
| shell: bash | |
| # Event values are passed through env (not interpolated into the script) so a | |
| # crafted branch or ref name cannot inject shell commands. | |
| env: | |
| EVENT_NAME: ${{ github.event_name }} | |
| PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} | |
| PR_CHANGED_FILES: ${{ github.event.pull_request.changed_files }} | |
| PUSH_BEFORE: ${{ github.event.before }} | |
| # Used by the compare-API cross-check below. Only read access to the | |
| # repository is needed, which the workflow's `contents` permission grants. | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| # --------------------------------------------------------------------------- | |
| # Resolve the base commit for the diff. | |
| # | |
| # pull_request: `base.sha` is the base branch TIP at event time; once the base | |
| # branch moves ahead of the branch point, diffing against it reports the | |
| # branch as reverting base-only commits and over-includes unrelated files. | |
| # push: `before` is the pre-push tip, which is the correct base. It is the | |
| # zero SHA for a newly created branch and is empty for non-push events. | |
| # workflow_dispatch / schedule: no base is available in the payload, so fall | |
| # back to HEAD~1. This is the same resolution order as ci.yml. | |
| # --------------------------------------------------------------------------- | |
| case "$EVENT_NAME" in | |
| pull_request) BASE="$PR_BASE_SHA" ;; | |
| push) BASE="$PUSH_BEFORE" ;; | |
| *) BASE="" ;; | |
| esac | |
| if [ -z "$BASE" ] || [ "$BASE" = "0000000000000000000000000000000000000000" ]; then | |
| BASE="HEAD~1" | |
| fi | |
| # Fail loudly instead of silently reporting "nothing changed". Swallowing the | |
| # diff error is what made every downstream job skip: an unresolvable BASE | |
| # produced an empty file list, so the pattern below matched false and the | |
| # Provider Success gate reported green with no work done. | |
| if ! git cat-file -e "${BASE}^{commit}" 2>/dev/null; then | |
| echo "::error::Cannot resolve base commit '${BASE}'. The checkout is too shallow to diff against it." >&2 | |
| exit 1 | |
| fi | |
| if [ "$EVENT_NAME" = "pull_request" ]; then | |
| # `base.sha` is on the base branch, so the merge-base with HEAD is the real | |
| # branch point. Only needed when the two are not the same commit. | |
| MERGE_BASE=$(git merge-base "$BASE" HEAD) || { | |
| echo "::error::Cannot compute merge-base between '${BASE}' and HEAD." >&2 | |
| exit 1 | |
| } | |
| if [ -n "$MERGE_BASE" ] && [ "$MERGE_BASE" != "$BASE" ]; then | |
| echo "Base '${BASE}' is ahead of the branch point; using merge-base '${MERGE_BASE}'." | |
| BASE="$MERGE_BASE" | |
| fi | |
| fi | |
| # No `2>/dev/null || echo ""` here: the shell runs with -e, so a failing diff | |
| # aborts this job instead of yielding an empty changeset. | |
| CHANGED=$(git diff --name-only "$BASE" HEAD) | |
| CHANGED_COUNT=$(printf '%s' "$CHANGED" | grep -c . || true) | |
| # --------------------------------------------------------------------------- | |
| # Cross-check #1: the event payload. | |
| # | |
| # An empty changeset makes the pattern below match false, which skips every | |
| # job and lets provider-success report green with no work done. That is only | |
| # correct when there really is nothing to diff. The payload tells us how many | |
| # files the event itself reports as changed, so "payload says N > 0, we | |
| # detected 0" means detection is broken — fail instead of going green. | |
| # | |
| # This check only bites on `pull_request`: push payloads can report 0 files for | |
| # a push that really changed files, and `workflow_dispatch` carries no file list | |
| # at all. Those events are covered by cross-check #2 below. | |
| # --------------------------------------------------------------------------- | |
| EXPECTED="$PR_CHANGED_FILES" | |
| if [ "$EVENT_NAME" = "push" ]; then | |
| # Sum the file lists the push payload carries for its commits. The payload | |
| # may omit older commits on very large pushes, so this is a lower bound: | |
| # any file it does list really is part of the push. | |
| EXPECTED=$(jq -r '[.commits[]? | ((.added // []) + (.modified // []) + (.removed // []))] | add // [] | length' "$GITHUB_EVENT_PATH" 2>/dev/null || echo "") | |
| fi | |
| # --------------------------------------------------------------------------- | |
| # Cross-check #2: sources that do not come from the event payload. | |
| # | |
| # Without this, every event other than `pull_request` had no working | |
| # empty-changeset guard at all: the payload check above no-ops on push (its | |
| # file lists can report 0 files for a push that really changed files) and on | |
| # workflow_dispatch (no file list at all), so a base that resolves to | |
| # "nothing to diff" could still skip every job and let the run go green with | |
| # nothing tested. | |
| # | |
| # Two independent sources, both reported in the log so the verdict is never | |
| # left at "unknown": | |
| # 1. `GET /repos/{owner}/{repo}/compare/{base}...{head}` — computed | |
| # server-side, shares nothing with the local diff or the payload. Only the | |
| # zero / non-zero signal is used, so neither the API's 300-file cap nor its | |
| # three-dot (merge-base) form can affect the verdict. | |
| # 2. The commit range `BASE..HEAD`. If it holds commits, an empty file diff | |
| # means the diff did not run, not that nothing changed. | |
| # | |
| # The payload check is deliberately left as the only guard for `pull_request`, | |
| # where it is already effective. | |
| # --------------------------------------------------------------------------- | |
| CROSS_EVENT="skipped" | |
| CROSS_FILES="" | |
| CROSS_COMMITS="" | |
| if [ "$EVENT_NAME" != "pull_request" ]; then | |
| CROSS_EVENT="run" | |
| HEAD_SHA=$(git rev-parse HEAD) | |
| # Resolve BASE to a concrete SHA before handing it to the API. `HEAD~1`, the | |
| # fallback used when the payload carries no base, is meaningful to git but | |
| # the compare endpoint would resolve it against the default branch and | |
| # silently report a different range than the one diffed locally. | |
| BASE_SHA=$(git rev-parse "${BASE}^{commit}") | |
| # Only trust the API answer when it is a usable number; an empty or | |
| # non-numeric value means the call did not produce a file count. | |
| API_FILES=$(gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_SHA}...${HEAD_SHA}" \ | |
| --jq '(.files // []) | length' 2>/dev/null || echo "") | |
| if [ -n "$API_FILES" ] && [ "$API_FILES" -ge 0 ] 2>/dev/null; then | |
| CROSS_FILES="$API_FILES" | |
| fi | |
| CROSS_COMMITS=$(git rev-list --count "${BASE_SHA}..${HEAD_SHA}" 2>/dev/null || echo "") | |
| if [ "$CHANGED_COUNT" -eq 0 ] && [ -n "$CROSS_FILES" ] && [ "$CROSS_FILES" -gt 0 ] 2>/dev/null; then | |
| echo "::error::Change detection found 0 files against '${BASE}', but GitHub compare '${BASE_SHA}...${HEAD_SHA}' reports ${CROSS_FILES} changed file(s). Refusing to report an empty changeset and skip every job." >&2 | |
| exit 1 | |
| fi | |
| if [ "$CHANGED_COUNT" -eq 0 ] && [ -n "$CROSS_COMMITS" ] && [ "$CROSS_COMMITS" -gt 0 ] 2>/dev/null; then | |
| echo "::error::Change detection found 0 files against '${BASE}', but '${BASE_SHA}..${HEAD_SHA}' holds ${CROSS_COMMITS} commit(s). Refusing to report an empty changeset and skip every job." >&2 | |
| exit 1 | |
| fi | |
| fi | |
| # Written to the log and to the job summary so the resolved base and the | |
| # changed-file list stay verifiable from `gh run view --log`. | |
| { | |
| echo "### Change detection" | |
| echo "" | |
| echo "- event: \`${EVENT_NAME}\`" | |
| echo "- base: \`${BASE}\`" | |
| echo "- changed files: **${CHANGED_COUNT}**" | |
| echo "- payload reported: \`${EXPECTED:-unknown}\`" | |
| if [ "$CROSS_EVENT" = "run" ]; then | |
| echo "- independent cross-check: compare API \`${CROSS_FILES:-unavailable}\` file(s), commit range \`${CROSS_COMMITS:-unavailable}\` commit(s)" | |
| else | |
| echo "- independent cross-check: n/a (\`pull_request\` is covered by the payload cross-check)" | |
| fi | |
| echo "" | |
| echo '```' | |
| echo "$CHANGED" | |
| echo '```' | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| echo "event=$EVENT_NAME" | |
| echo "base=$BASE" | |
| echo "changed_file_count=$CHANGED_COUNT" | |
| echo "payload_changed_files=${EXPECTED:-unknown}" | |
| echo "cross_check_event=${CROSS_EVENT}" | |
| echo "cross_check_files=${CROSS_FILES:-unavailable}" | |
| echo "cross_check_commits=${CROSS_COMMITS:-unavailable}" | |
| echo "changed_files:" | |
| echo "$CHANGED" | |
| echo "base=$BASE" >> $GITHUB_OUTPUT | |
| echo "changed_file_count=$CHANGED_COUNT" >> $GITHUB_OUTPUT | |
| if [ -n "$EXPECTED" ] && [ "$EXPECTED" != "null" ] && [ "$EXPECTED" -gt 0 ] 2>/dev/null; then | |
| if [ "$CHANGED_COUNT" -eq 0 ]; then | |
| echo "::error::Change detection found 0 files against '${BASE}', but the ${EVENT_NAME} payload reports ${EXPECTED} changed file(s). Refusing to report an empty changeset and skip every job." >&2 | |
| exit 1 | |
| fi | |
| fi | |
| if [ "$CHANGED_COUNT" -eq 0 ]; then | |
| echo "::warning::No files detected as changed against '${BASE}'. Every downstream job will be skipped; this is only correct if the event really carries no file changes." | |
| fi | |
| if echo "$CHANGED" | grep -qE '^(crates/vx-(providers|runtime|installer|starlark|star-metadata|manifest)/|\.github/(workflows/test-providers\.yml|scripts/(discover-providers|summarize-test-results)\.sh))'; then | |
| echo "should_run=true" >> $GITHUB_OUTPUT | |
| else | |
| echo "should_run=false" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Determine provider scope | |
| id: scope | |
| shell: bash | |
| # The base is resolved by the detection step above and passed through env so a | |
| # crafted base branch name cannot inject shell commands. | |
| env: | |
| EVENT_NAME: ${{ github.event_name }} | |
| BASE: ${{ steps.filter.outputs.base }} | |
| run: | | |
| echo "provider_subset=false" >> "$GITHUB_OUTPUT" | |
| echo "changed_providers=" >> "$GITHUB_OUTPUT" | |
| if [[ "$EVENT_NAME" != "pull_request" ]]; then | |
| exit 0 | |
| fi | |
| # The full history fetched above keeps this base resolvable, so the diff no | |
| # longer needs its own shallow `git fetch`. Fail loudly if it still cannot | |
| # run: an empty file list degrades to a full matrix run, which hides a | |
| # broken checkout behind extra work instead of reporting it. | |
| if ! git cat-file -e "${BASE}^{commit}" 2>/dev/null; then | |
| echo "::error::Cannot resolve base commit '${BASE}' for provider scoping." >&2 | |
| exit 1 | |
| fi | |
| mapfile -t relevant_files < <(git diff --name-only "$BASE"...HEAD -- \ | |
| 'crates/vx-providers/**' \ | |
| 'crates/vx-runtime/**' \ | |
| 'crates/vx-installer/**' \ | |
| 'crates/vx-starlark/**' \ | |
| 'crates/vx-star-metadata/**' \ | |
| 'crates/vx-manifest/**' \ | |
| '.github/workflows/test-providers.yml' \ | |
| '.github/scripts/discover-providers.sh' \ | |
| '.github/scripts/summarize-test-results.sh') | |
| if [[ ${#relevant_files[@]} -eq 0 ]]; then | |
| exit 0 | |
| fi | |
| mapfile -t changed_providers < <(printf '%s\n' "${relevant_files[@]}" | awk -F/ '/^crates\/vx-providers\/[^/]+\// {print $3}' | sort -u) | |
| non_provider_changes=0 | |
| for file in "${relevant_files[@]}"; do | |
| if [[ "$file" != crates/vx-providers/* ]]; then | |
| non_provider_changes=1 | |
| break | |
| fi | |
| done | |
| echo "### Provider Scope" >> "$GITHUB_STEP_SUMMARY" | |
| if [[ ${#changed_providers[@]} -gt 0 && $non_provider_changes -eq 0 ]]; then | |
| changed_csv=$(printf '%s\n' "${changed_providers[@]}" | paste -sd, -) | |
| echo "provider_subset=true" >> "$GITHUB_OUTPUT" | |
| echo "changed_providers=$changed_csv" >> "$GITHUB_OUTPUT" | |
| echo "- Mode: changed providers only" >> "$GITHUB_STEP_SUMMARY" | |
| echo "- Providers: $changed_csv" >> "$GITHUB_STEP_SUMMARY" | |
| else | |
| echo "- Mode: full provider matrix" >> "$GITHUB_STEP_SUMMARY" | |
| fi | |
| # Build VX and discover all testable providers | |
| build-and-discover: | |
| name: Build & Discover | |
| needs: check-changes | |
| if: needs.check-changes.outputs.should_run == 'true' || github.event_name == 'workflow_dispatch' || github.event_name == 'schedule' | |
| runs-on: ubuntu-latest | |
| env: | |
| RUSTC_WRAPPER: sccache | |
| SCCACHE_DIR: ${{ github.workspace }}/.cache/sccache | |
| outputs: | |
| matrix-linux: ${{ steps.discover.outputs.matrix-linux }} | |
| matrix-macos: ${{ steps.discover.outputs.matrix-macos }} | |
| matrix-windows: ${{ steps.discover.outputs.matrix-windows }} | |
| has-linux: ${{ steps.discover.outputs.has-linux }} | |
| has-macos: ${{ steps.discover.outputs.has-macos }} | |
| has-windows: ${{ steps.discover.outputs.has-windows }} | |
| total-runtimes: ${{ steps.discover.outputs.total-runtimes }} | |
| testable-runtimes: ${{ steps.discover.outputs.testable-runtimes }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v7 | |
| - name: Setup Rust + sccache | |
| uses: ./.github/actions/setup-rust-sccache | |
| with: | |
| shared-key: provider-build | |
| - name: Build VX | |
| run: cargo build --release | |
| - name: Discover providers and generate matrix | |
| id: discover | |
| run: | | |
| chmod +x .github/scripts/discover-providers.sh | |
| # Build arguments | |
| ARGS="--chunk-size ${{ inputs.chunk_size || '8' }}" | |
| ARGS="$ARGS --output $GITHUB_OUTPUT" | |
| ARGS="$ARGS --summary $GITHUB_STEP_SUMMARY" | |
| if [ -n "${{ inputs.skip }}" ]; then | |
| ARGS="$ARGS --skip '${{ inputs.skip }}'" | |
| fi | |
| if [ -n "${{ inputs.runtimes }}" ]; then | |
| ARGS="$ARGS --runtimes '${{ inputs.runtimes }}'" | |
| elif [ "${{ needs.check-changes.outputs.provider_subset }}" = "true" ] && [ -n "${{ needs.check-changes.outputs.changed_providers }}" ]; then | |
| ARGS="$ARGS --providers '${{ needs.check-changes.outputs.changed_providers }}'" | |
| fi | |
| # Run discovery script | |
| eval ".github/scripts/discover-providers.sh $ARGS" | |
| - name: Validate provider discovery | |
| shell: bash | |
| run: | | |
| echo "Discovered runtimes: ${{ steps.discover.outputs.total-runtimes }}" | |
| echo "Testable runtimes: ${{ steps.discover.outputs.testable-runtimes }}" | |
| if [[ "${{ steps.discover.outputs.testable-runtimes }}" == "0" ]]; then | |
| echo "No testable runtimes were discovered. Provider E2E coverage would be silently skipped." >&2 | |
| exit 1 | |
| fi | |
| - name: Upload VX binary (Linux) | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: vx-linux | |
| path: target/release/vx | |
| retention-days: 1 | |
| # Build for other platforms | |
| build-macos: | |
| name: Build (macOS) | |
| runs-on: macos-latest | |
| needs: build-and-discover | |
| if: needs.build-and-discover.outputs.has-macos == 'true' | |
| env: | |
| RUSTC_WRAPPER: sccache | |
| SCCACHE_DIR: ${{ github.workspace }}/.cache/sccache | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - name: Setup Rust + sccache | |
| uses: ./.github/actions/setup-rust-sccache | |
| with: | |
| shared-key: provider-build | |
| - run: cargo build --release | |
| - uses: actions/upload-artifact@v7 | |
| with: | |
| name: vx-macos | |
| path: target/release/vx | |
| retention-days: 1 | |
| build-windows: | |
| name: Build (Windows) | |
| runs-on: windows-latest | |
| needs: build-and-discover | |
| if: needs.build-and-discover.outputs.has-windows == 'true' | |
| env: | |
| RUSTC_WRAPPER: sccache | |
| SCCACHE_DIR: ${{ github.workspace }}/.cache/sccache | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - name: Setup Rust + sccache | |
| uses: ./.github/actions/setup-rust-sccache | |
| with: | |
| shared-key: provider-build | |
| - run: cargo build --release | |
| - uses: actions/upload-artifact@v7 | |
| with: | |
| name: vx-windows | |
| path: target/release/vx.exe | |
| retention-days: 1 | |
| # Test on Linux | |
| test-linux: | |
| name: Test Linux (${{ matrix.chunk }}) | |
| runs-on: ubuntu-latest | |
| needs: build-and-discover | |
| if: needs.build-and-discover.outputs.has-linux == 'true' | |
| timeout-minutes: 45 | |
| env: | |
| VX_TEST_ROOT: ${{ github.workspace }}/.cache/vx-provider-linux | |
| VX_HOME: ${{ github.workspace }}/.cache/vx-provider-linux | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| chunk: ${{ fromJSON(needs.build-and-discover.outputs.matrix-linux) }} | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - name: Download VX binary | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: vx-linux | |
| path: ./bin | |
| - name: Make binary executable | |
| run: chmod +x ./bin/vx | |
| - name: Cache VX runtime store | |
| uses: actions/cache@v6 | |
| with: | |
| path: ${{ env.VX_TEST_ROOT }} | |
| key: ${{ runner.os }}-vx-provider-e2e-${{ hashFiles('Cargo.lock', '.github/workflows/test-providers.yml', '.github/scripts/discover-providers.sh', 'crates/vx-providers/**/provider.star', 'crates/vx-runtime/**', 'crates/vx-installer/**', 'crates/vx-starlark/**', 'crates/vx-star-metadata/**', 'crates/vx-manifest/**') }}-${{ strategy.job-index }} | |
| restore-keys: | | |
| ${{ runner.os }}-vx-provider-e2e- | |
| - name: Test providers | |
| run: | | |
| echo "Testing providers: ${{ matrix.chunk }}" | |
| # Pre-install uv so pip packages (pre-commit, rez, meson) can use it. | |
| ./bin/vx install uv --force | |
| set +e | |
| ./bin/vx test --ci --ci-runtimes "${{ matrix.chunk }}" --vx-root "$VX_TEST_ROOT" --keep-going --verbose --json > test-report.json | |
| status=$? | |
| cat test-report.json | |
| if [[ $status -ne 0 ]]; then | |
| ./bin/vx test --ci --ci-runtimes "${{ matrix.chunk }}" --vx-root "$VX_TEST_ROOT" --keep-going --detailed || true | |
| fi | |
| exit $status | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| RUST_LOG: warn,vx_installer=info | |
| - name: Upload test report | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: report-linux-${{ strategy.job-index }} | |
| path: test-report.json | |
| retention-days: 7 | |
| # Test on macOS | |
| test-macos: | |
| name: Test macOS (${{ matrix.chunk }}) | |
| runs-on: macos-latest | |
| needs: [build-and-discover, build-macos] | |
| if: needs.build-and-discover.outputs.has-macos == 'true' | |
| timeout-minutes: 45 | |
| env: | |
| VX_TEST_ROOT: ${{ github.workspace }}/.cache/vx-provider-macos | |
| VX_HOME: ${{ github.workspace }}/.cache/vx-provider-macos | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| chunk: ${{ fromJSON(needs.build-and-discover.outputs.matrix-macos) }} | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - name: Download VX binary | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: vx-macos | |
| path: ./bin | |
| - name: Make binary executable | |
| run: chmod +x ./bin/vx | |
| - name: Cache VX runtime store | |
| uses: actions/cache@v6 | |
| with: | |
| path: ${{ env.VX_TEST_ROOT }} | |
| key: ${{ runner.os }}-vx-provider-e2e-${{ hashFiles('Cargo.lock', '.github/workflows/test-providers.yml', '.github/scripts/discover-providers.sh', 'crates/vx-providers/**/provider.star', 'crates/vx-runtime/**', 'crates/vx-installer/**', 'crates/vx-starlark/**', 'crates/vx-star-metadata/**', 'crates/vx-manifest/**') }}-${{ strategy.job-index }} | |
| restore-keys: | | |
| ${{ runner.os }}-vx-provider-e2e- | |
| - name: Test providers | |
| run: | | |
| echo "Testing providers: ${{ matrix.chunk }}" | |
| # Pre-install uv so pip packages (pre-commit, rez, meson) can use it. | |
| ./bin/vx install uv --force | |
| set +e | |
| ./bin/vx test --ci --ci-runtimes "${{ matrix.chunk }}" --vx-root "$VX_TEST_ROOT" --keep-going --verbose --json > test-report.json | |
| status=$? | |
| cat test-report.json | |
| if [[ $status -ne 0 ]]; then | |
| ./bin/vx test --ci --ci-runtimes "${{ matrix.chunk }}" --vx-root "$VX_TEST_ROOT" --keep-going --detailed || true | |
| fi | |
| exit $status | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| RUST_LOG: warn,vx_installer=info | |
| - name: Upload test report | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: report-macos-${{ strategy.job-index }} | |
| path: test-report.json | |
| retention-days: 7 | |
| # Test on Windows | |
| test-windows: | |
| name: Test Windows (${{ matrix.chunk }}) | |
| runs-on: windows-latest | |
| needs: [build-and-discover, build-windows] | |
| if: needs.build-and-discover.outputs.has-windows == 'true' | |
| timeout-minutes: 45 | |
| env: | |
| VX_TEST_ROOT: ${{ github.workspace }}\.cache\vx-provider-windows | |
| VX_HOME: ${{ github.workspace }}\.cache\vx-provider-windows | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| chunk: ${{ fromJSON(needs.build-and-discover.outputs.matrix-windows) }} | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - name: Download VX binary | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: vx-windows | |
| path: ./bin | |
| - name: Cache VX runtime store | |
| uses: actions/cache@v6 | |
| with: | |
| path: ${{ env.VX_TEST_ROOT }} | |
| key: ${{ runner.os }}-vx-provider-e2e-${{ hashFiles('Cargo.lock', '.github/workflows/test-providers.yml', '.github/scripts/discover-providers.sh', 'crates/vx-providers/**/provider.star', 'crates/vx-runtime/**', 'crates/vx-installer/**', 'crates/vx-starlark/**', 'crates/vx-star-metadata/**', 'crates/vx-manifest/**') }}-${{ strategy.job-index }} | |
| restore-keys: | | |
| ${{ runner.os }}-vx-provider-e2e- | |
| - name: Test providers | |
| run: | | |
| echo "Testing providers: ${{ matrix.chunk }}" | |
| # Pre-install uv so pip packages (pre-commit, rez, meson) can use it. | |
| .\bin\vx.exe install uv --force | |
| .\bin\vx.exe test --ci --ci-runtimes "${{ matrix.chunk }}" --vx-root "$env:VX_TEST_ROOT" --keep-going --verbose --json > test-report.json | |
| $status = $LASTEXITCODE | |
| if (Test-Path test-report.json) { Get-Content test-report.json } | |
| if ($status -ne 0) { | |
| .\bin\vx.exe test --ci --ci-runtimes "${{ matrix.chunk }}" --vx-root "$env:VX_TEST_ROOT" --keep-going --detailed | |
| } | |
| exit $status | |
| shell: pwsh | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| RUST_LOG: warn,vx_installer=info | |
| - name: Upload test report | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: report-windows-${{ strategy.job-index }} | |
| path: test-report.json | |
| retention-days: 7 | |
| # Quick smoke test for PRs (run subset of essential providers) | |
| quick-test: | |
| name: Quick Smoke Test | |
| runs-on: ubuntu-latest | |
| needs: [check-changes, build-and-discover] | |
| if: github.event_name == 'pull_request' && needs.check-changes.outputs.should_run == 'true' && needs.check-changes.outputs.provider_subset != 'true' | |
| timeout-minutes: 15 | |
| env: | |
| VX_TEST_ROOT: ${{ github.workspace }}/.cache/vx-quick | |
| VX_HOME: ${{ github.workspace }}/.cache/vx-quick | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - name: Download VX binary | |
| uses: actions/download-artifact@v8 | |
| with: | |
| name: vx-linux | |
| path: ./bin | |
| - name: Make binary executable | |
| run: chmod +x ./bin/vx | |
| - name: Cache VX runtime store | |
| uses: actions/cache@v6 | |
| with: | |
| path: ${{ env.VX_TEST_ROOT }} | |
| key: ${{ runner.os }}-vx-provider-quick-${{ hashFiles('Cargo.lock', '.github/workflows/test-providers.yml', '.github/scripts/discover-providers.sh', 'crates/vx-providers/**/provider.star', 'crates/vx-runtime/**', 'crates/vx-installer/**', 'crates/vx-starlark/**', 'crates/vx-star-metadata/**', 'crates/vx-manifest/**') }} | |
| restore-keys: | | |
| ${{ runner.os }}-vx-provider-quick- | |
| - name: Quick provider test | |
| run: | | |
| # Test essential providers for fast PR feedback when shared runtime logic changes | |
| ./bin/vx test --ci --ci-runtimes node,go,uv,just,python --vx-root "$VX_TEST_ROOT" --verbose | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| # Summary job | |
| summary: | |
| name: Test Summary | |
| runs-on: ubuntu-latest | |
| needs: [test-linux, test-macos, test-windows] | |
| if: always() && (needs.test-linux.result != 'skipped' || needs.test-macos.result != 'skipped' || needs.test-windows.result != 'skipped') | |
| steps: | |
| - uses: actions/checkout@v7 | |
| - name: Download all reports | |
| uses: actions/download-artifact@v8 | |
| with: | |
| pattern: report-* | |
| merge-multiple: false | |
| - name: Generate summary | |
| run: | | |
| chmod +x .github/scripts/summarize-test-results.sh | |
| .github/scripts/summarize-test-results.sh \ | |
| --reports-dir . \ | |
| --summary "$GITHUB_STEP_SUMMARY" \ | |
| --fail-on-error | |
| provider-success: | |
| name: Provider Success | |
| runs-on: ubuntu-latest | |
| needs: | |
| - check-changes | |
| - build-and-discover | |
| - build-macos | |
| - build-windows | |
| - test-linux | |
| - test-macos | |
| - test-windows | |
| - quick-test | |
| - summary | |
| if: always() | |
| steps: | |
| - name: Check provider workflow results | |
| shell: bash | |
| run: | | |
| echo "## Provider Workflow Results" >> "$GITHUB_STEP_SUMMARY" | |
| echo "" >> "$GITHUB_STEP_SUMMARY" | |
| failed=0 | |
| any_matrix=0 | |
| check_job() { | |
| local job_name="$1" | |
| local result="$2" | |
| if [[ "$result" == "success" ]]; then | |
| echo "- ✅$job_name" >> "$GITHUB_STEP_SUMMARY" | |
| elif [[ "$result" == "skipped" ]]; then | |
| echo "- ⏭️ $job_name (skipped)" >> "$GITHUB_STEP_SUMMARY" | |
| else | |
| echo "- ❌$job_name ($result)" >> "$GITHUB_STEP_SUMMARY" | |
| failed=1 | |
| fi | |
| } | |
| # Check change detection first: when it fails it emits no `should_run` | |
| # output, and reading that as "nothing changed" is exactly what made this | |
| # gate report green with every test job skipped. | |
| check_job "check-changes" "${{ needs.check-changes.result }}" | |
| if [[ $failed -eq 1 ]]; then | |
| echo "" >> "$GITHUB_STEP_SUMMARY" | |
| echo "❌**Provider workflow failed**" >> "$GITHUB_STEP_SUMMARY" | |
| exit 1 | |
| fi | |
| # If no relevant files changed, all jobs are skipped — that's a pass | |
| if [[ "${{ needs.check-changes.outputs.should_run }}" != "true" && "${{ github.event_name }}" != "workflow_dispatch" && "${{ github.event_name }}" != "schedule" ]]; then | |
| echo "No relevant files changed — skipping provider tests" >> "$GITHUB_STEP_SUMMARY" | |
| exit 0 | |
| fi | |
| check_job "build-and-discover" "${{ needs.build-and-discover.result }}" | |
| if [[ "${{ needs.build-and-discover.outputs.testable-runtimes }}" == "0" ]]; then | |
| echo "- ❌no testable runtimes discovered" >> "$GITHUB_STEP_SUMMARY" | |
| failed=1 | |
| fi | |
| echo "" >> "$GITHUB_STEP_SUMMARY" | |
| echo "### Matrix Jobs" >> "$GITHUB_STEP_SUMMARY" | |
| check_job "build-macos" "${{ needs.build-macos.result }}" | |
| check_job "build-windows" "${{ needs.build-windows.result }}" | |
| check_job "test-linux" "${{ needs.test-linux.result }}" | |
| check_job "test-macos" "${{ needs.test-macos.result }}" | |
| check_job "test-windows" "${{ needs.test-windows.result }}" | |
| for result in "${{ needs.test-linux.result }}" "${{ needs.test-macos.result }}" "${{ needs.test-windows.result }}"; do | |
| if [[ "$result" != "skipped" ]]; then | |
| any_matrix=1 | |
| break | |
| fi | |
| done | |
| if [[ $any_matrix -eq 0 ]]; then | |
| echo "- ❌all provider matrix jobs were skipped" >> "$GITHUB_STEP_SUMMARY" | |
| failed=1 | |
| fi | |
| echo "" >> "$GITHUB_STEP_SUMMARY" | |
| echo "### Fast Feedback" >> "$GITHUB_STEP_SUMMARY" | |
| if [[ "${{ github.event_name }}" == "pull_request" ]]; then | |
| check_job "quick-test" "${{ needs.quick-test.result }}" | |
| else | |
| echo "- ⏭️ quick-test (pull_request only)" >> "$GITHUB_STEP_SUMMARY" | |
| fi | |
| if [[ $any_matrix -eq 1 ]]; then | |
| check_job "summary" "${{ needs.summary.result }}" | |
| else | |
| echo "- ⏭️ summary (no matrix jobs ran)" >> "$GITHUB_STEP_SUMMARY" | |
| fi | |
| if [[ $failed -eq 1 ]]; then | |
| echo "" >> "$GITHUB_STEP_SUMMARY" | |
| echo "❌**Provider workflow failed**" >> "$GITHUB_STEP_SUMMARY" | |
| exit 1 | |
| fi | |
| echo "" >> "$GITHUB_STEP_SUMMARY" | |
| echo "✅**Provider workflow passed**" >> "$GITHUB_STEP_SUMMARY" |