Run MIT Integration Tests v2 #4204
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Run MIT Integration Tests v2 | |
| concurrency: | |
| group: Run-MIT-Integration-Tests-${{ github.workflow }}-${{ github.head_ref || github.event.workflow_run.head_branch || github.run_id }} | |
| cancel-in-progress: true | |
| on: | |
| merge_group: | |
| types: [checks_requested] | |
| push: | |
| tags: | |
| - "v*.*.*" | |
| permissions: | |
| contents: read | |
| env: | |
| # Test Environment Variables | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} | |
| EXA_API_KEY: ${{ secrets.EXA_API_KEY }} | |
| CONFLUENCE_TEST_SPACE_URL: ${{ vars.CONFLUENCE_TEST_SPACE_URL }} | |
| CONFLUENCE_USER_NAME: ${{ vars.CONFLUENCE_USER_NAME }} | |
| CONFLUENCE_ACCESS_TOKEN: ${{ secrets.CONFLUENCE_ACCESS_TOKEN }} | |
| CONFLUENCE_ACCESS_TOKEN_SCOPED: ${{ secrets.CONFLUENCE_ACCESS_TOKEN_SCOPED }} | |
| JIRA_BASE_URL: ${{ secrets.JIRA_BASE_URL }} | |
| JIRA_USER_EMAIL: ${{ secrets.JIRA_USER_EMAIL }} | |
| JIRA_API_TOKEN: ${{ secrets.JIRA_API_TOKEN }} | |
| JIRA_API_TOKEN_SCOPED: ${{ secrets.JIRA_API_TOKEN_SCOPED }} | |
| PERM_SYNC_SHAREPOINT_CLIENT_ID: ${{ secrets.PERM_SYNC_SHAREPOINT_CLIENT_ID }} | |
| PERM_SYNC_SHAREPOINT_PRIVATE_KEY: ${{ secrets.PERM_SYNC_SHAREPOINT_PRIVATE_KEY }} | |
| PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD: ${{ secrets.PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD }} | |
| PERM_SYNC_SHAREPOINT_DIRECTORY_ID: ${{ secrets.PERM_SYNC_SHAREPOINT_DIRECTORY_ID }} | |
| jobs: | |
| discover-test-dirs: | |
| # NOTE: Github-hosted runners have about 20s faster queue times and are preferred here. | |
| runs-on: ubuntu-slim | |
| timeout-minutes: 45 | |
| outputs: | |
| test-dirs: ${{ steps.set-matrix.outputs.test-dirs }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # ratchet:actions/checkout@v6 | |
| with: | |
| persist-credentials: false | |
| - name: Discover test directories | |
| id: set-matrix | |
| run: | | |
| # Find all leaf-level directories in both test directories | |
| tests_dirs=$(find backend/tests/integration/tests -mindepth 1 -maxdepth 1 -type d ! -name "__pycache__" -exec basename {} \; | sort) | |
| connector_dirs=$(find backend/tests/integration/connector_job_tests -mindepth 1 -maxdepth 1 -type d ! -name "__pycache__" -exec basename {} \; | sort) | |
| # Create JSON array with directory info | |
| all_dirs="" | |
| for dir in $tests_dirs; do | |
| all_dirs="$all_dirs{\"path\":\"tests/$dir\",\"name\":\"tests-$dir\"}," | |
| done | |
| for dir in $connector_dirs; do | |
| all_dirs="$all_dirs{\"path\":\"connector_job_tests/$dir\",\"name\":\"connector-$dir\"}," | |
| done | |
| # Remove trailing comma and wrap in array | |
| all_dirs="[${all_dirs%,}]" | |
| echo "test-dirs=$all_dirs" >> $GITHUB_OUTPUT | |
| build-backend-image: | |
| runs-on: [runs-on, runner=1cpu-linux-arm64, "run-id=${{ github.run_id }}-build-backend-image", "extras=ecr-cache"] | |
| timeout-minutes: 45 | |
| steps: | |
| - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2 | |
| - name: Checkout code | |
| uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # ratchet:actions/checkout@v6 | |
| with: | |
| persist-credentials: false | |
| - name: Format branch name for cache | |
| id: format-branch | |
| env: | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| REF_NAME: ${{ github.ref_name }} | |
| run: | | |
| if [ -n "${PR_NUMBER}" ]; then | |
| CACHE_SUFFIX="${PR_NUMBER}" | |
| else | |
| # shellcheck disable=SC2001 | |
| CACHE_SUFFIX=$(echo "${REF_NAME}" | sed 's/[^A-Za-z0-9._-]/-/g') | |
| fi | |
| echo "cache-suffix=${CACHE_SUFFIX}" >> $GITHUB_OUTPUT | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # ratchet:docker/setup-buildx-action@v3 | |
| # needed for pulling Vespa, Redis, Postgres, and Minio images | |
| # otherwise, we hit the "Unauthenticated users" limit | |
| # https://docs.docker.com/docker-hub/usage/ | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # ratchet:docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_TOKEN }} | |
| - name: Build and push Backend Docker image | |
| uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # ratchet:docker/build-push-action@v6 | |
| with: | |
| context: ./backend | |
| file: ./backend/Dockerfile | |
| push: true | |
| tags: ${{ env.RUNS_ON_ECR_CACHE }}:integration-test-backend-test-${{ github.run_id }} | |
| cache-from: | | |
| type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache-${{ github.event.pull_request.head.sha || github.sha }} | |
| type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache-${{ steps.format-branch.outputs.cache-suffix }} | |
| type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache | |
| type=registry,ref=onyxdotapp/onyx-backend:latest | |
| cache-to: | | |
| type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache-${{ github.event.pull_request.head.sha || github.sha }},mode=max | |
| type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache-${{ steps.format-branch.outputs.cache-suffix }},mode=max | |
| type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:backend-cache,mode=max | |
| no-cache: ${{ vars.DOCKER_NO_CACHE == 'true' }} | |
| build-model-server-image: | |
| runs-on: [runs-on, runner=1cpu-linux-arm64, "run-id=${{ github.run_id }}-build-model-server-image", "extras=ecr-cache"] | |
| timeout-minutes: 45 | |
| steps: | |
| - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2 | |
| - name: Checkout code | |
| uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # ratchet:actions/checkout@v6 | |
| with: | |
| persist-credentials: false | |
| - name: Format branch name for cache | |
| id: format-branch | |
| env: | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| REF_NAME: ${{ github.ref_name }} | |
| run: | | |
| if [ -n "${PR_NUMBER}" ]; then | |
| CACHE_SUFFIX="${PR_NUMBER}" | |
| else | |
| # shellcheck disable=SC2001 | |
| CACHE_SUFFIX=$(echo "${REF_NAME}" | sed 's/[^A-Za-z0-9._-]/-/g') | |
| fi | |
| echo "cache-suffix=${CACHE_SUFFIX}" >> $GITHUB_OUTPUT | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # ratchet:docker/setup-buildx-action@v3 | |
| # needed for pulling Vespa, Redis, Postgres, and Minio images | |
| # otherwise, we hit the "Unauthenticated users" limit | |
| # https://docs.docker.com/docker-hub/usage/ | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # ratchet:docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_TOKEN }} | |
| - name: Build and push Model Server Docker image | |
| uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # ratchet:docker/build-push-action@v6 | |
| with: | |
| context: ./backend | |
| file: ./backend/Dockerfile.model_server | |
| push: true | |
| tags: ${{ env.RUNS_ON_ECR_CACHE }}:integration-test-model-server-test-${{ github.run_id }} | |
| cache-from: | | |
| type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-${{ github.event.pull_request.head.sha || github.sha }} | |
| type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-${{ steps.format-branch.outputs.cache-suffix }} | |
| type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache | |
| type=registry,ref=onyxdotapp/onyx-model-server:latest | |
| cache-to: | | |
| type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-${{ github.event.pull_request.head.sha || github.sha }},mode=max | |
| type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache-${{ steps.format-branch.outputs.cache-suffix }},mode=max | |
| type=registry,ref=${{ env.RUNS_ON_ECR_CACHE }}:model-server-cache,mode=max | |
| build-integration-image: | |
| runs-on: [runs-on, runner=2cpu-linux-arm64, "run-id=${{ github.run_id }}-build-integration-image", "extras=ecr-cache"] | |
| timeout-minutes: 45 | |
| steps: | |
| - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2 | |
| - name: Checkout code | |
| uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # ratchet:actions/checkout@v6 | |
| with: | |
| persist-credentials: false | |
| - name: Format branch name for cache | |
| id: format-branch | |
| env: | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| REF_NAME: ${{ github.ref_name }} | |
| run: | | |
| if [ -n "${PR_NUMBER}" ]; then | |
| CACHE_SUFFIX="${PR_NUMBER}" | |
| else | |
| # shellcheck disable=SC2001 | |
| CACHE_SUFFIX=$(echo "${REF_NAME}" | sed 's/[^A-Za-z0-9._-]/-/g') | |
| fi | |
| echo "cache-suffix=${CACHE_SUFFIX}" >> $GITHUB_OUTPUT | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # ratchet:docker/setup-buildx-action@v3 | |
| # needed for pulling openapitools/openapi-generator-cli | |
| # otherwise, we hit the "Unauthenticated users" limit | |
| # https://docs.docker.com/docker-hub/usage/ | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # ratchet:docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_TOKEN }} | |
| - name: Build and push integration test image with Docker Bake | |
| env: | |
| INTEGRATION_REPOSITORY: ${{ env.RUNS_ON_ECR_CACHE }} | |
| TAG: integration-test-${{ github.run_id }} | |
| CACHE_SUFFIX: ${{ steps.format-branch.outputs.cache-suffix }} | |
| HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }} | |
| run: | | |
| cd backend && docker buildx bake --push \ | |
| --set backend.cache-from=type=registry,ref=${RUNS_ON_ECR_CACHE}:backend-cache-${HEAD_SHA} \ | |
| --set backend.cache-from=type=registry,ref=${RUNS_ON_ECR_CACHE}:backend-cache-${CACHE_SUFFIX} \ | |
| --set backend.cache-from=type=registry,ref=${RUNS_ON_ECR_CACHE}:backend-cache \ | |
| --set backend.cache-from=type=registry,ref=onyxdotapp/onyx-backend:latest \ | |
| --set backend.cache-to=type=registry,ref=${RUNS_ON_ECR_CACHE}:backend-cache-${HEAD_SHA},mode=max \ | |
| --set backend.cache-to=type=registry,ref=${RUNS_ON_ECR_CACHE}:backend-cache-${CACHE_SUFFIX},mode=max \ | |
| --set backend.cache-to=type=registry,ref=${RUNS_ON_ECR_CACHE}:backend-cache,mode=max \ | |
| --set integration.cache-from=type=registry,ref=${RUNS_ON_ECR_CACHE}:integration-cache-${HEAD_SHA} \ | |
| --set integration.cache-from=type=registry,ref=${RUNS_ON_ECR_CACHE}:integration-cache-${CACHE_SUFFIX} \ | |
| --set integration.cache-from=type=registry,ref=${RUNS_ON_ECR_CACHE}:integration-cache \ | |
| --set integration.cache-to=type=registry,ref=${RUNS_ON_ECR_CACHE}:integration-cache-${HEAD_SHA},mode=max \ | |
| --set integration.cache-to=type=registry,ref=${RUNS_ON_ECR_CACHE}:integration-cache-${CACHE_SUFFIX},mode=max \ | |
| --set integration.cache-to=type=registry,ref=${RUNS_ON_ECR_CACHE}:integration-cache,mode=max \ | |
| integration | |
| integration-tests-mit: | |
| needs: | |
| [ | |
| discover-test-dirs, | |
| build-backend-image, | |
| build-model-server-image, | |
| build-integration-image, | |
| ] | |
| runs-on: | |
| - runs-on | |
| - runner=4cpu-linux-arm64 | |
| - ${{ format('run-id={0}-integration-tests-mit-job-{1}', github.run_id, strategy['job-index']) }} | |
| - extras=ecr-cache | |
| timeout-minutes: 45 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| test-dir: ${{ fromJson(needs.discover-test-dirs.outputs.test-dirs) }} | |
| steps: | |
| - uses: runs-on/action@cd2b598b0515d39d78c38a02d529db87d2196d1e # ratchet:runs-on/action@v2 | |
| - name: Checkout code | |
| uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # ratchet:actions/checkout@v6 | |
| with: | |
| persist-credentials: false | |
| # needed for pulling Vespa, Redis, Postgres, and Minio images | |
| # otherwise, we hit the "Unauthenticated users" limit | |
| # https://docs.docker.com/docker-hub/usage/ | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@5e57cd118135c172c3672efd75eb46360885c0ef # ratchet:docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_TOKEN }} | |
| # NOTE: Use pre-ping/null pool to reduce flakiness due to dropped connections | |
| # NOTE: don't need web server for integration tests | |
| - name: Create .env file for Docker Compose | |
| env: | |
| ECR_CACHE: ${{ env.RUNS_ON_ECR_CACHE }} | |
| RUN_ID: ${{ github.run_id }} | |
| run: | | |
| cat <<EOF > deployment/docker_compose/.env | |
| AUTH_TYPE=basic | |
| POSTGRES_POOL_PRE_PING=true | |
| POSTGRES_USE_NULL_POOL=true | |
| REQUIRE_EMAIL_VERIFICATION=false | |
| DISABLE_TELEMETRY=true | |
| ONYX_BACKEND_IMAGE=${ECR_CACHE}:integration-test-backend-test-${RUN_ID} | |
| ONYX_MODEL_SERVER_IMAGE=${ECR_CACHE}:integration-test-model-server-test-${RUN_ID} | |
| INTEGRATION_TESTS_MODE=true | |
| MCP_SERVER_ENABLED=true | |
| EOF | |
| - name: Start Docker containers | |
| run: | | |
| cd deployment/docker_compose | |
| docker compose -f docker-compose.yml -f docker-compose.dev.yml up \ | |
| relational_db \ | |
| index \ | |
| cache \ | |
| minio \ | |
| api_server \ | |
| inference_model_server \ | |
| indexing_model_server \ | |
| mcp_server \ | |
| background \ | |
| -d | |
| id: start_docker | |
| - name: Wait for services to be ready | |
| run: | | |
| echo "Starting wait-for-service script..." | |
| wait_for_service() { | |
| local url=$1 | |
| local label=$2 | |
| local timeout=${3:-300} # default 5 minutes | |
| local start_time | |
| start_time=$(date +%s) | |
| while true; do | |
| local current_time | |
| current_time=$(date +%s) | |
| local elapsed_time=$((current_time - start_time)) | |
| if [ $elapsed_time -ge $timeout ]; then | |
| echo "Timeout reached. ${label} did not become ready in $timeout seconds." | |
| exit 1 | |
| fi | |
| local response | |
| response=$(curl -s -o /dev/null -w "%{http_code}" "$url" || echo "curl_error") | |
| if [ "$response" = "200" ]; then | |
| echo "${label} is ready!" | |
| break | |
| elif [ "$response" = "curl_error" ]; then | |
| echo "Curl encountered an error while checking ${label}. Retrying in 5 seconds..." | |
| else | |
| echo "${label} not ready yet (HTTP status $response). Retrying in 5 seconds..." | |
| fi | |
| sleep 5 | |
| done | |
| } | |
| wait_for_service "http://localhost:8080/health" "API server" | |
| test_dir="${{ matrix.test-dir.path }}" | |
| if [ "$test_dir" = "tests/mcp" ]; then | |
| wait_for_service "http://localhost:8090/health" "MCP server" | |
| else | |
| echo "Skipping MCP server wait for non-MCP suite: $test_dir" | |
| fi | |
| echo "Finished waiting for services." | |
| - name: Start Mock Services | |
| run: | | |
| cd backend/tests/integration/mock_services | |
| docker compose -f docker-compose.mock-it-services.yml \ | |
| -p mock-it-services-stack up -d | |
| # NOTE: Use pre-ping/null to reduce flakiness due to dropped connections | |
| - name: Run Integration Tests for ${{ matrix.test-dir.name }} | |
| uses: nick-fields/retry@ce71cc2ab81d554ebbe88c79ab5975992d79ba08 # ratchet:nick-fields/retry@v3 | |
| with: | |
| timeout_minutes: 20 | |
| max_attempts: 3 | |
| retry_wait_seconds: 10 | |
| command: | | |
| echo "Running integration tests for ${{ matrix.test-dir.path }}..." | |
| docker run --rm --network onyx_default \ | |
| --name test-runner \ | |
| -e POSTGRES_HOST=relational_db \ | |
| -e POSTGRES_USER=postgres \ | |
| -e POSTGRES_PASSWORD=password \ | |
| -e POSTGRES_DB=postgres \ | |
| -e DB_READONLY_USER=db_readonly_user \ | |
| -e DB_READONLY_PASSWORD=password \ | |
| -e POSTGRES_POOL_PRE_PING=true \ | |
| -e POSTGRES_USE_NULL_POOL=true \ | |
| -e VESPA_HOST=index \ | |
| -e REDIS_HOST=cache \ | |
| -e API_SERVER_HOST=api_server \ | |
| -e MCP_SERVER_HOST=mcp_server \ | |
| -e MCP_SERVER_PORT=8090 \ | |
| -e OPENAI_API_KEY=${OPENAI_API_KEY} \ | |
| -e EXA_API_KEY=${EXA_API_KEY} \ | |
| -e SLACK_BOT_TOKEN=${SLACK_BOT_TOKEN} \ | |
| -e CONFLUENCE_TEST_SPACE_URL=${CONFLUENCE_TEST_SPACE_URL} \ | |
| -e CONFLUENCE_USER_NAME=${CONFLUENCE_USER_NAME} \ | |
| -e CONFLUENCE_ACCESS_TOKEN=${CONFLUENCE_ACCESS_TOKEN} \ | |
| -e CONFLUENCE_ACCESS_TOKEN_SCOPED=${CONFLUENCE_ACCESS_TOKEN_SCOPED} \ | |
| -e JIRA_BASE_URL=${JIRA_BASE_URL} \ | |
| -e JIRA_USER_EMAIL=${JIRA_USER_EMAIL} \ | |
| -e JIRA_API_TOKEN=${JIRA_API_TOKEN} \ | |
| -e JIRA_API_TOKEN_SCOPED=${JIRA_API_TOKEN_SCOPED} \ | |
| -e PERM_SYNC_SHAREPOINT_CLIENT_ID=${PERM_SYNC_SHAREPOINT_CLIENT_ID} \ | |
| -e PERM_SYNC_SHAREPOINT_PRIVATE_KEY="${PERM_SYNC_SHAREPOINT_PRIVATE_KEY}" \ | |
| -e PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD=${PERM_SYNC_SHAREPOINT_CERTIFICATE_PASSWORD} \ | |
| -e PERM_SYNC_SHAREPOINT_DIRECTORY_ID=${PERM_SYNC_SHAREPOINT_DIRECTORY_ID} \ | |
| -e TEST_WEB_HOSTNAME=test-runner \ | |
| -e MOCK_CONNECTOR_SERVER_HOST=mock_connector_server \ | |
| -e MOCK_CONNECTOR_SERVER_PORT=8001 \ | |
| ${{ env.RUNS_ON_ECR_CACHE }}:integration-test-${{ github.run_id }} \ | |
| /app/tests/integration/${{ matrix.test-dir.path }} | |
| # ------------------------------------------------------------ | |
| # Always gather logs BEFORE "down": | |
| - name: Dump API server logs | |
| if: always() | |
| run: | | |
| cd deployment/docker_compose | |
| docker compose logs --no-color api_server > $GITHUB_WORKSPACE/api_server.log || true | |
| - name: Dump all-container logs (optional) | |
| if: always() | |
| run: | | |
| cd deployment/docker_compose | |
| docker compose logs --no-color > $GITHUB_WORKSPACE/docker-compose.log || true | |
| - name: Upload logs | |
| if: always() | |
| uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 # ratchet:actions/upload-artifact@v4 | |
| with: | |
| name: docker-all-logs-${{ matrix.test-dir.name }} | |
| path: ${{ github.workspace }}/docker-compose.log | |
| # ------------------------------------------------------------ | |
| required: | |
| # NOTE: Github-hosted runners have about 20s faster queue times and are preferred here. | |
| runs-on: ubuntu-slim | |
| timeout-minutes: 45 | |
| needs: [integration-tests-mit] | |
| if: ${{ always() }} | |
| steps: | |
| - name: Check job status | |
| if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'skipped') }} | |
| run: exit 1 |