E2E schedule #19995
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: E2E CI | |
| run-name: E2E ${{ github.event_name }} ${{ inputs.providers }} | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| test_cases: | |
| description: 'Comma-separated list of test cases (vm)' | |
| required: true | |
| default: 'vm,github_runner_ubuntu_2404,github_runner_ubuntu_2204,postgres_standard,postgres_ha,postgres_upgrade,postgres_firewall,kubernetes' | |
| type: string | |
| providers: | |
| description: 'Comma-separated list of providers (metal,aws)' | |
| required: true | |
| default: 'metal,aws,gcp' | |
| type: string | |
| schedule: | |
| - cron: '0 */2 * * *' | |
| jobs: | |
| setup: | |
| if: vars.RUN_E2E == 'true' | |
| runs-on: ubuntu-slim | |
| outputs: | |
| providers: ${{ steps.set-providers.outputs.providers }} | |
| has_github_runner: ${{ steps.set-providers.outputs.has_github_runner }} | |
| steps: | |
| - name: Set provider matrix | |
| id: set-providers | |
| run: | | |
| providers="metal,aws,gcp" | |
| if [[ "${{ github.event_name }}" == "workflow_dispatch" && -n "${{ inputs.providers }}" ]]; then | |
| providers="${{ inputs.providers }}" | |
| fi | |
| # Convert comma-separated list to JSON array | |
| has_github_runner=false | |
| if [[ "${{ github.event_name }}" != "workflow_dispatch" || "${{ inputs.test_cases }}" == *"github_runner"* ]]; then | |
| has_github_runner=true | |
| fi | |
| echo "providers=$(echo "$providers" | jq -Rc 'split(",")')" >> $GITHUB_OUTPUT | |
| echo "has_github_runner=$has_github_runner" >> $GITHUB_OUTPUT | |
| e2e: | |
| needs: setup | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| provider: ${{ fromJson(needs.setup.outputs.providers) }} | |
| name: E2E - ${{ matrix.provider }} | |
| runs-on: ubicloud-standard-4 | |
| environment: E2E-${{ matrix.provider }} | |
| timeout-minutes: 93 | |
| concurrency: ${{ (matrix.provider == 'metal' && 'E2E-metal') || (matrix.provider == 'gcp' && format('E2E-gcp-{0}', github.run_id)) || (needs.setup.outputs.has_github_runner == 'true' && 'E2E-aws') || format('E2E-aws-{0}', github.run_id) }} | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v6 | |
| - name: Setup SSH access | |
| uses: ubicloud/ssh-runner@v2 | |
| with: | |
| public-ssh-key: ${{ secrets.OPERATOR_SSH_PUBLIC_KEYS }} | |
| wait-minutes: 0 | |
| - name: Set up Clover | |
| uses: ./.github/actions/setup-clover | |
| with: | |
| setup-node: true | |
| - name: Build assets | |
| run: npm run prod | |
| - name: Install cloudflared | |
| if: needs.setup.outputs.has_github_runner == 'true' | |
| run: curl -L --output cloudflared.deb https://github.com/cloudflare/cloudflared/releases/latest/download/cloudflared-linux-amd64.deb && sudo dpkg -i cloudflared.deb | |
| - name: Set cloudflared token | |
| if: needs.setup.outputs.has_github_runner == 'true' | |
| run: sudo cloudflared service install ${{ secrets.CLOUDFLARED_TOKEN }} | |
| - name: Install mailpit | |
| run: | | |
| curl -sfL -m 90 -o mailpit.tar.gz https://github.com/axllent/mailpit/releases/download/v1.27.0/mailpit-linux-amd64.tar.gz | |
| tar zxf mailpit.tar.gz | |
| mv mailpit /usr/local/bin/ | |
| sudo systemd-run --setenv="MP_UI_AUTH=${{ secrets.MAIL_USER }}:${{ secrets.MAIL_PASSWORD }}" \ | |
| --setenv="MP_SMTP_AUTH=${{ secrets.MAIL_USER }}:${{ secrets.MAIL_PASSWORD }}" --setenv="MP_SMTP_AUTH_ALLOW_INSECURE=true" \ | |
| --unit mailpit -- mailpit | |
| - name: Install foreman | |
| run: gem install foreman | |
| - name: Modify Procfile to add retry logic | |
| run: | | |
| sed -i "s/^\([^:]*\): \(.*\)/\1: bash -c 'for i in {1..4}; do \2 \&\& break || echo \"Attempt \$i failed, retrying...\"; sleep 1; done'/" Procfile | |
| - name: Add e2e script to Procfile | |
| run: | | |
| test_cases="$(yq -r '[.[].name] | join(",")' config/e2e_test_cases.yml)" | |
| if [[ "${{ github.event_name }}" == "workflow_dispatch" && -n "${{ inputs.test_cases }}" ]]; then | |
| test_cases="${{ inputs.test_cases }}" | |
| fi | |
| echo "Running e2e ${{ matrix.provider }} test cases:$test_cases" | |
| echo "e2e: bin/e2e --provider=${{ matrix.provider }} --test-cases=$test_cases" >> Procfile | |
| - name: Persist env vars to .env.rb | |
| run: | | |
| cat <<'EOF' >> .env.rb | |
| ENV["CLOVER_DATABASE_URL"] = "postgres:///clover_test?user=clover" | |
| ENV["RACK_ENV"] = "production" | |
| ENV["IS_E2E"] = "true" | |
| ENV["GITHUB_RUNNER_SERVICE_PROJECT_ID"] = "89841457-35a2-8ed2-a88a-424ea5ddd83b" | |
| ENV["GITHUB_RUNNER_AWS_LOCATION_ID"] = "88adf710-4d8b-8c20-a480-618cbbef802d" | |
| ENV["POSTGRES_SERVICE_PROJECT_ID"] = "546a1ed8-53e5-86d2-966c-fb782d2ae5ab" | |
| ENV["KUBERNETES_SERVICE_PROJECT_ID"] = "4e762e09-2a22-82d2-93c2-cfbefbda8c69" | |
| ENV["MINIO_SERVICE_PROJECT_ID"] = "37ad2a8d-0220-4e8b-a67f-38db95d6c657" | |
| ENV["VM_POOL_PROJECT_ID"] = "e4b93fb7-5125-8ad2-9d29-730ff4506bc3" | |
| EOF | |
| - name: Run tests | |
| env: | |
| BASE_URL: https://${{ secrets.CLOUDFLARED_ADDRESS }} | |
| ALLOW_UNSPREAD_SERVERS: "true" | |
| DISPATCHER_MAX_THREADS: 16 | |
| PROVIDER_RESOURCE_TAG_VALUE: ${{ github.run_id }} | |
| E2E_HETZNER_SERVER_ID: ${{ secrets.E2E_HETZNER_SERVER_ID }} | |
| E2E_GITHUB_INSTALLATION_ID: ${{ secrets.E2E_GH_INSTALLATION_ID }} | |
| E2E_AWS_ACCESS_KEY: ${{ secrets.E2E_AWS_ACCESS_KEY }} | |
| E2E_AWS_SECRET_KEY: ${{ secrets.E2E_AWS_SECRET_KEY }} | |
| E2E_GCP_CREDENTIALS_BASE64_JSON: ${{ secrets.E2E_GCP_CREDENTIALS_BASE64_JSON }} | |
| E2E_CACHE_PROXY_DOWNLOAD_URL: ${{ secrets.E2E_CACHE_PROXY_DOWNLOAD_URL }} | |
| HETZNER_SSH_PUBLIC_KEY: ${{ secrets.HETZNER_SSH_PUBLIC_KEY }} | |
| HETZNER_SSH_PRIVATE_KEY: ${{ secrets.HETZNER_SSH_PRIVATE_KEY }} | |
| HETZNER_USER: ${{ secrets.HETZNER_USER }} | |
| HETZNER_PASSWORD: ${{ secrets.HETZNER_PASSWORD }} | |
| UBICLOUD_IMAGES_BLOB_STORAGE_ENDPOINT: ${{ secrets.UBICLOUD_IMAGES_BLOB_STORAGE_ENDPOINT }} | |
| UBICLOUD_IMAGES_BLOB_STORAGE_ACCESS_KEY: ${{ secrets.UBICLOUD_IMAGES_BLOB_STORAGE_ACCESS_KEY }} | |
| UBICLOUD_IMAGES_BLOB_STORAGE_SECRET_KEY: ${{ secrets.UBICLOUD_IMAGES_BLOB_STORAGE_SECRET_KEY }} | |
| UBICLOUD_IMAGES_BLOB_STORAGE_CERTS: ${{ secrets.UBICLOUD_IMAGES_BLOB_STORAGE_CERTS }} | |
| UBICLOUD_IMAGES_R2_BUCKET_NAME: ${{ secrets.UBICLOUD_IMAGES_R2_BUCKET_NAME }} | |
| UBICLOUD_IMAGES_R2_ENDPOINT: ${{ secrets.UBICLOUD_IMAGES_R2_ENDPOINT }} | |
| UBICLOUD_IMAGES_R2_ACCESS_KEY: ${{ secrets.UBICLOUD_IMAGES_R2_ACCESS_KEY }} | |
| UBICLOUD_IMAGES_R2_SECRET_KEY: ${{ secrets.UBICLOUD_IMAGES_R2_SECRET_KEY }} | |
| GITHUB_APP_NAME: ${{ secrets.GH_APP_NAME }} | |
| GITHUB_APP_ID: ${{ secrets.GH_APP_ID }} | |
| GITHUB_APP_CLIENT_ID: ${{ secrets.GH_APP_CLIENT_ID }} | |
| GITHUB_APP_CLIENT_SECRET: ${{ secrets.GH_APP_CLIENT_SECRET }} | |
| GITHUB_APP_PRIVATE_KEY: ${{ secrets.GH_APP_PRIVATE_KEY }} | |
| GITHUB_APP_WEBHOOK_SECRET: ${{ secrets.GH_APP_WEBHOOK_SECRET }} | |
| GITHUB_CACHE_BLOB_STORAGE_ENDPOINT: ${{ secrets.GH_CACHE_BLOB_STORAGE_ENDPOINT }} | |
| GITHUB_CACHE_BLOB_STORAGE_REGION: ${{ secrets.GH_CACHE_BLOB_STORAGE_REGION }} | |
| GITHUB_CACHE_BLOB_STORAGE_ACCESS_KEY: ${{ secrets.GH_CACHE_BLOB_STORAGE_ACCESS_KEY }} | |
| GITHUB_CACHE_BLOB_STORAGE_SECRET_KEY: ${{ secrets.GH_CACHE_BLOB_STORAGE_SECRET_KEY }} | |
| GITHUB_CACHE_BLOB_STORAGE_ACCOUNT_ID: ${{ secrets.GH_CACHE_BLOB_STORAGE_ACCOUNT_ID }} | |
| GITHUB_CACHE_BLOB_STORAGE_API_KEY: ${{ secrets.GH_CACHE_BLOB_STORAGE_API_KEY }} | |
| MAIL_FROM: test@example.com | |
| SMTP_USER: ${{ secrets.MAIL_USER }} | |
| SMTP_PASSWORD: ${{ secrets.MAIL_PASSWORD }} | |
| SMTP_HOSTNAME: localhost | |
| SMTP_PORT: 1025 | |
| SMTP_TLS: false | |
| STRIPE_PUBLIC_KEY: ${{ secrets.STRIPE_PUBLIC_KEY }} | |
| STRIPE_SECRET_KEY: ${{ secrets.STRIPE_SECRET_KEY }} | |
| VHOST_BLOCK_BACKEND_VERSION: v0.4.2 | |
| OPERATOR_SSH_PUBLIC_KEYS: ${{ secrets.OPERATOR_SSH_PUBLIC_KEYS }} | |
| run: | | |
| set -o pipefail | |
| timeout 83m foreman start -m all=1,respirate=2 | tee foreman.log | grep --line-buffered -F "e2e.1" | |
| - name: Print logs | |
| if: always() | |
| run: grep -h -v -E "sleep_duration_sec|monitor.1" foreman.log | |
| - name: Extract GitHub workflow run | |
| if: always() | |
| continue-on-error: true | |
| run: | | |
| html_url="$(grep 'github_workflow_run' foreman.log | cut -d'|' -f2 | jq -r '.github_workflow_run.html_url' | head -1)" | |
| if [ -n "$html_url" ] && [ "$html_url" != "null" ]; then | |
| echo "### GitHub workflow run: ${html_url}" >> $GITHUB_STEP_SUMMARY | |
| fi | |
| - name: Extract image download time | |
| if: always() && matrix.provider == 'metal' | |
| id: extract_download | |
| run: | | |
| download_time="$(grep "VmHost.wait_download_boot_images" foreman.log | cut -d'|' -f2 | tail -1)" | |
| echo "### Image downloaded in $download_time" >> $GITHUB_STEP_SUMMARY | |
| echo "DOWNLOAD_TIME=$download_time" >> $GITHUB_OUTPUT | |
| - name: Extract last line | |
| if: always() | |
| id: extract_last | |
| run: | | |
| last_line="$(grep "e2e.1" foreman.log | tail -2 | head -1 | cut -d'|' -f2,4,5)" | |
| echo "### Last line: $last_line" >> $GITHUB_STEP_SUMMARY | |
| echo "LAST_LINE=$last_line" >> $GITHUB_OUTPUT | |
| - name: Extract vm provision times | |
| if: always() | |
| continue-on-error: true | |
| run: | | |
| echo "## VM Provisioning Times" >> $GITHUB_STEP_SUMMARY | |
| echo '| UBID | Name | Boot Image | Duration |' >> $GITHUB_STEP_SUMMARY | |
| echo '|---|---|---|---|' >> $GITHUB_STEP_SUMMARY | |
| grep "vm provisioned" foreman.log | cut -d'|' -f2 | jq -r '[.thread, .vm.name, .vm.boot_image, .provision.duration] | "| " + join(" | ") + " |"' >> $GITHUB_STEP_SUMMARY | |
| - name: Extract runner queue times | |
| if: always() | |
| continue-on-error: true | |
| run: | | |
| echo "## Runner Queue Times" >> $GITHUB_STEP_SUMMARY | |
| echo '| Runner | Label | VM | Duration |' >> $GITHUB_STEP_SUMMARY | |
| echo '|---|---|---|---|' >> $GITHUB_STEP_SUMMARY | |
| grep "runner_started" foreman.log | cut -d'|' -f2 | jq -r '[.runner_started.ubid, .runner_started.label, .runner_started.vm_ubid, .runner_started.duration] | "| " + join(" | ") + " |"' >> $GITHUB_STEP_SUMMARY | |
| - name: Extract exception | |
| if: always() | |
| continue-on-error: true | |
| run: | | |
| echo "## Exceptions" >> $GITHUB_STEP_SUMMARY | |
| echo '| Time | Thread | Prog | Class | Message | First Backtrace Line |' >> $GITHUB_STEP_SUMMARY | |
| echo '|---|---|---|---|---|---|' >> $GITHUB_STEP_SUMMARY | |
| grep 'respirate.*"exception"' foreman.log | cut -d'|' -f2 | jq -r '[.time, .thread, .prog_label, .exception.class, (.exception.message | gsub("\n"; " ")), (.exception.backtrace[0] // "" | gsub("\n"; " "))] | "| " + join(" | ") + " |"' >> $GITHUB_STEP_SUMMARY | |
| - name: Upload logs | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: e2e-${{ matrix.provider }}-${{ github.run_id }}-logs | |
| path: foreman.log | |
| - name: Dump PostgreSQL database | |
| if: always() | |
| run: | | |
| pg_dump "postgres:///clover_test?user=clover" \ | |
| --table='archived_record*' \ | |
| --table=vm \ | |
| --table=vm_host \ | |
| --table=load_balancer \ | |
| --table=load_balancer_port \ | |
| --table=load_balancer_vm_port \ | |
| --table=load_balancers_vms \ | |
| --table=kubernetes_cluster \ | |
| --table=kubernetes_node \ | |
| --table=kubernetes_nodepool \ | |
| --table=postgres_server \ | |
| --table=postgres_resource \ | |
| --table=postgres_timeline \ | |
| --table=access_control_entry \ | |
| --table=access_tag \ | |
| --table=action_tag \ | |
| --table=action_type \ | |
| --table=object_tag \ | |
| --table=subject_tag \ | |
| --table='applied_*_tag' \ | |
| --table=project \ | |
| --table=billing_record \ | |
| --table=billing_info \ | |
| --table=billing_rate \ | |
| --table=page \ | |
| --table=semaphore \ | |
| --table=strand \ | |
| --table=sshable \ | |
| --table=github_installation \ | |
| --table=github_runner \ | |
| --table=github_cache_entry \ | |
| --table=github_custom_label \ | |
| > clover_test_dump.sql | |
| - name: Upload database dump | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: e2e-${{ matrix.provider }}-${{ github.run_id }}-db-dump | |
| path: clover_test_dump.sql | |
| - name: Send notification if failed | |
| if: ${{ failure() && github.ref_name == 'main' }} | |
| uses: slackapi/slack-github-action@v3.0.1 | |
| with: | |
| webhook: ${{ secrets.SLACK_WEBHOOK_PAGER_URL }} | |
| webhook-type: incoming-webhook | |
| payload: | | |
| text: "*E2E Tests Failed* :this-is-fine-fire:" | |
| attachments: | |
| - color: "E33122" | |
| fields: | |
| - title: "Event" | |
| short: true | |
| value: "${{ github.event_name }}" | |
| - title: "Provider" | |
| short: true | |
| value: "${{ matrix.provider }}" | |
| - title: "Action" | |
| short: true | |
| value: "<${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}|${{ github.workflow }}>" | |
| - title: "Download Time" | |
| short: true | |
| value: "${{ steps.extract_download.outputs.DOWNLOAD_TIME }}" | |
| - title: "Last Line" | |
| short: false | |
| value: "${{ steps.extract_last.outputs.LAST_LINE }}" | |
| - name: Sleep 10 minutes after failed notification | |
| if: ${{ failure() && github.ref_name == 'main' }} | |
| run: sleep 600 | |
| - name: Stop cloudflared | |
| if: always() | |
| run: sudo systemctl stop cloudflared || true | |
| cleanup-aws: | |
| needs: [setup, e2e] | |
| if: always() && contains(needs.setup.outputs.providers, 'aws') | |
| runs-on: ubicloud-standard-4 | |
| environment: E2E-aws | |
| timeout-minutes: 20 | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v6 | |
| - name: Download e2e logs | |
| uses: actions/download-artifact@v7 | |
| with: | |
| name: e2e-aws-${{ github.run_id }}-logs | |
| - name: Cleanup AWS resources | |
| env: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.E2E_AWS_ACCESS_KEY }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.E2E_AWS_SECRET_KEY }} | |
| run: | | |
| log_instance_ids=$(grep -oh '"instance_id":"[^"]*"' foreman.log 2>/dev/null | cut -d'"' -f4 | sort -u | tr ',' '\n' | paste -sd, -) | |
| if [ -z "$log_instance_ids" ]; then | |
| echo "No instance IDs found in logs" | |
| exit 0 | |
| fi | |
| echo "Found instances in logs: $log_instance_ids" | |
| for region in eu-central-1 us-east-1 us-west-2; do | |
| echo "Cleaning up resources in $region" | |
| # Find instances that exist in this region (filter doesn't error for non-existent IDs) | |
| instance_ids=$(aws ec2 describe-instances --region $region \ | |
| --filters "Name=instance-id,Values=$log_instance_ids" \ | |
| --query "Reservations[].Instances[].InstanceId" --output text | tr '\t' ' ' || true) | |
| if [ -z "$(echo "$instance_ids" | tr -d ' ')" ]; then | |
| echo "No instances found in $region" | |
| continue | |
| fi | |
| echo "Found instances in $region: $instance_ids" | |
| # Fetch associated resources before terminating instances | |
| eni_ids=$(aws ec2 describe-instances --instance-ids $instance_ids --region $region \ | |
| --query "Reservations[].Instances[].NetworkInterfaces[].NetworkInterfaceId" --output text | sort -u || true) | |
| eip_ids=$(aws ec2 describe-addresses --region $region \ | |
| --filters "Name=instance-id,Values=$(echo $instance_ids | tr ' ' ',')" \ | |
| --query "Addresses[].AllocationId" --output text | sort -u || true) | |
| vpc_ids=$(aws ec2 describe-instances --instance-ids $instance_ids --region $region \ | |
| --query "Reservations[].Instances[].VpcId" --output text | sort -u || true) | |
| echo "Found ENIs: $eni_ids" | |
| echo "Found EIPs: $eip_ids" | |
| echo "Found VPCs: $vpc_ids" | |
| echo "Terminating instances: $instance_ids" | |
| aws ec2 terminate-instances --instance-ids $instance_ids --region $region || true | |
| aws ec2 wait instance-terminated --instance-ids $instance_ids --region $region || true | |
| for eni_id in $eni_ids; do | |
| echo "Deleting Network Interface: $eni_id" | |
| aws ec2 delete-network-interface --network-interface-id $eni_id --region $region || true | |
| done | |
| for eip_id in $eip_ids; do | |
| echo "Releasing Elastic IP: $eip_id" | |
| aws ec2 release-address --allocation-id $eip_id --region $region || true | |
| done | |
| for vpc_id in $vpc_ids; do | |
| echo "Deleting VPC: $vpc_id" | |
| for igw_id in $(aws ec2 describe-internet-gateways --filters "Name=attachment.vpc-id,Values=$vpc_id" --query 'InternetGateways[].InternetGatewayId' --output text --region $region); do | |
| aws ec2 detach-internet-gateway --internet-gateway-id $igw_id --vpc-id $vpc_id --region $region || true | |
| aws ec2 delete-internet-gateway --internet-gateway-id $igw_id --region $region || true | |
| done & | |
| for subnet_id in $(aws ec2 describe-subnets --filters "Name=vpc-id,Values=$vpc_id" --query 'Subnets[].SubnetId' --output text --region $region); do | |
| aws ec2 delete-subnet --subnet-id $subnet_id --region $region || true | |
| done & | |
| for sg_id in $(aws ec2 describe-security-groups --filters "Name=vpc-id,Values=$vpc_id" --query 'SecurityGroups[?GroupName!=`default`].GroupId' --output text --region $region); do | |
| aws ec2 delete-security-group --group-id $sg_id --region $region || true | |
| done & | |
| for rt_id in $(aws ec2 describe-route-tables --filters "Name=vpc-id,Values=$vpc_id" --query 'RouteTables[?Associations[0].Main!=`true`].RouteTableId' --output text --region $region); do | |
| aws ec2 delete-route-table --route-table-id $rt_id --region $region || true | |
| done & | |
| wait | |
| aws ec2 delete-vpc --vpc-id $vpc_id --region $region || true | |
| done | |
| done | |
| cleanup-gcp: | |
| needs: [setup, e2e] | |
| if: always() && contains(needs.setup.outputs.providers, 'gcp') | |
| runs-on: ubicloud-standard-4 | |
| environment: E2E-gcp | |
| timeout-minutes: 20 | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v6 | |
| - name: Download e2e logs | |
| uses: actions/download-artifact@v7 | |
| with: | |
| name: e2e-gcp-${{ github.run_id }}-logs | |
| - name: Cleanup GCP resources | |
| uses: ./.github/actions/gcp-cleanup-e2e | |
| with: | |
| credentials-base64: ${{ secrets.E2E_GCP_CREDENTIALS_BASE64_JSON }} |