Ray refactoring #544
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: backend checks | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| paths: | |
| - "backend/**" | |
| - ".github/workflows/backend_checks.yml" | |
| - "docker/**" | |
| - "tools/**" | |
| jobs: | |
| backend-checks: | |
| runs-on: self-hosted | |
| env: | |
| RAY_HAS_CHANGED: false | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Checkout PR | |
| if: github.event_name == 'pull_request' | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: gh pr checkout ${{ github.event.pull_request.number }} | |
| - name: Check for changes in ray (pull request) | |
| id: check_ray_changes_pr | |
| if: github.event_name == 'pull_request' | |
| run: | | |
| git fetch origin ${{ github.event.pull_request.base.ref }} | |
| base_branch="origin/${{ github.event.pull_request.base.ref }}" | |
| if git diff --name-only $base_branch HEAD | grep -q -e '^ray/'; then | |
| echo "RAY_HAS_CHANGED=true" >> $GITHUB_ENV | |
| fi | |
| - name: Check for changes in ray (push) | |
| id: check_ray_changes_push | |
| if: github.event_name == 'push' | |
| run: | | |
| if git diff --name-only ${{ github.event.before }} ${{ github.sha }} | grep -q -e '^ray/'; then | |
| echo "RAY_HAS_CHANGED=true" >> $GITHUB_ENV | |
| fi | |
| - name: Prepare environment | |
| run: | | |
| ./bin/setup-folders.sh | |
| ./bin/setup-envs.sh --project_name action-runner --port_prefix 131 | |
| sed -i 's/\(DATS_BACKEND_DOCKER_VERSION=[0-9.]*\)/\1-${{ github.run_id }}/' docker/.env | |
| sed -i 's/^RAY_DEVICE_IDS=.*/RAY_DEVICE_IDS=1/' docker/.env | |
| if [ "${{ env.RAY_HAS_CHANGED }}" = "true" ]; then | |
| sed -i 's/\(DATS_RAY_DOCKER_VERSION=[0-9.]*\)/\1-${{ github.run_id }}/' docker/.env | |
| sed -i 's/ray_network/ray_network_${{ github.run_id }}/g' docker/compose.ray.yml | |
| sed -i 's/ray_network/ray_network_${{ github.run_id }}/g' docker/compose.yml | |
| echo "RAY_CACHE_DIR=$HOME/ray_cache" >> docker/.env | |
| mkdir -p $HOME/ray_cache | |
| chmod 777 $HOME/ray_cache | |
| fi | |
| - name: Build & Start Docker Containers | |
| working-directory: docker | |
| run: | | |
| source .env | |
| if [ "${{ env.RAY_HAS_CHANGED }}" = "true" ]; then | |
| docker compose -f compose.ray.yml build | |
| docker compose -f compose.ray.yml up --wait | |
| fi | |
| COMPOSE_PROFILES="rq,backend" docker compose build | |
| COMPOSE_PROFILES="rq" docker compose up --wait --quiet-pull | |
| - name: Check 1 - pytest runs without errors | |
| working-directory: docker | |
| run: | | |
| source .env | |
| docker compose run --user $(id -u):$(id -g) dats-backend-api /dats_code/.venv/bin/python -m pytest /dats_code/test | |
| - name: Check 2 - Database migrates without errors database | |
| working-directory: docker | |
| run: | | |
| source .env | |
| docker compose run --user $(id -u):$(id -g) dats-backend-api /dats_code/.venv/bin/python src/migrations/run_migrations.py | |
| - name: Check 3 - Database schema is up-to-date after migration | |
| working-directory: docker | |
| run: | | |
| source .env | |
| docker compose run --user $(id -u):$(id -g) dats-backend-api /dats_code/.venv/bin/alembic -c src/migrations/alembic.ini check | |
| - name: Check 4 - install uv environment | |
| working-directory: backend | |
| run: | | |
| uv sync | |
| - name: Check 5 - pyright runs without errors | |
| working-directory: backend | |
| run: | | |
| uv run pyright | |
| - name: Start Remaining Docker Containers | |
| working-directory: docker | |
| run: | | |
| source .env | |
| COMPOSE_PROFILES="rq,backend" docker compose up --wait --quiet-pull | |
| - name: Check 6 - Test End-2-End importer script | |
| working-directory: backend | |
| env: | |
| TESTDATA_PASSWORD: ${{ secrets.TESTDATA_PASSWORD }} | |
| run: | | |
| source .venv/bin/activate | |
| cd ../tools/importer | |
| uv pip install -r requirements.txt | |
| wget -q http://ltdata1.informatik.uni-hamburg.de/dwts/totalitarismo.zip | |
| unzip -q -P "$TESTDATA_PASSWORD" totalitarismo.zip | |
| python dats_importer.py --input_dir json --backend_url http://localhost:13120/ --is_json --doctype text | |
| python dats_importer.py --input_dir images --backend_url http://localhost:13120/ --doctype image | |
| - name: Cleanup | |
| working-directory: docker | |
| if: always() | |
| run: | | |
| source .env | |
| if [ "${{ env.RAY_HAS_CHANGED }}" = "true" ]; then | |
| docker compose -f compose.ray.yml down -v | |
| RAY_IMAGE=uhhlt/dats_ray:$(grep -oP 'DATS_RAY_DOCKER_VERSION=\K.*' .env) | |
| docker rmi $RAY_IMAGE | |
| fi | |
| docker compose down -v --remove-orphans | |
| BACKEND_IMAGE=uhhlt/dats_backend:$(grep -oP 'DATS_BACKEND_DOCKER_VERSION=\K.*' .env) | |
| docker rmi $BACKEND_IMAGE |