Add json type #829
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: slow_tests | |
| on: | |
| # Schedule the workflow to run at 00:30 UTC+8 every day | |
| # https://docs.github.com/zh/actions/writing-workflows/choosing-when-your-workflow-runs/events-that-trigger-workflows#schedule | |
| schedule: | |
| - cron: '30 16 * * *' # utc-8: 16:30 | |
| # The only difference between pull_request and pull_request_target is the context in which the workflow runs: | |
| # — pull_request_target workflows use the workflow files from the default branch, and secrets are available. | |
| # — pull_request workflows use the workflow files from the pull request branch, and secrets are unavailable. | |
| pull_request: | |
| types: [ synchronize, ready_for_review ] | |
| paths-ignore: | |
| - 'docs/**' | |
| - '*.md' | |
| permissions: | |
| contents: read | |
| actions: read | |
| checks: write | |
| statuses: write | |
| # https://docs.github.com/en/actions/using-jobs/using-concurrency | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| download_code: | |
| name: download_code | |
| if: ${{ github.event_name != 'pull_request' || (github.event.pull_request.draft == false && contains(github.event.pull_request.labels.*.name, 'slow-test')) }} | |
| runs-on: [ "self-hosted", "slow-test" ] | |
| steps: | |
| - name: Ensure workspace ownership | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| echo "Workflow triggered by ${GITHUB_EVENT_NAME}" | |
| if [[ ${GITHUB_EVENT_NAME} == 'pull_request' ]]; then | |
| echo "PR labels: ${{ join(github.event.pull_request.labels.*.name, ', ') }}" | |
| fi | |
| echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE} | |
| - name: Check out code | |
| uses: actions/checkout@v6 | |
| with: | |
| ref: ${{ (github.event_name == 'pull_request' || github.event_name == 'pull_request_target') && format('refs/pull/{0}/merge', github.event.pull_request.number) || github.sha }} | |
| fetch-depth: 0 | |
| fetch-tags: false | |
| submodules: recursive | |
| - name: Backup code | |
| run: | | |
| tar --create --gzip --file=infinity.tgz .git $(git ls-files --recurse-submodules) | |
| ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID} | |
| echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> ${GITHUB_ENV} | |
| rm -rf ${ARTIFACTS_DIR} && mkdir -p ${ARTIFACTS_DIR} && cp infinity.tgz ${ARTIFACTS_DIR}/ | |
| debug_build: | |
| name: debug_build | |
| runs-on: [ "self-hosted", "infinity-build" ] | |
| needs: download_code | |
| steps: | |
| - name: Prepare code | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE} | |
| ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID} | |
| echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> ${GITHUB_ENV} | |
| tar --extract --gzip --file=${ARTIFACTS_DIR}/infinity.tgz && git reset --hard && git clean -ffdx | |
| - name: Start builder container | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| BUILDER_CONTAINER=infinity_build_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ') | |
| echo "BUILDER_CONTAINER=${BUILDER_CONTAINER}" >> ${GITHUB_ENV} | |
| TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')} | |
| sudo docker run --privileged -d --name ${BUILDER_CONTAINER} -e TZ=${TZ} -e UV_INDEX=https://pypi.tuna.tsinghua.edu.cn/simple -v ${PWD}:/infinity -v ${PWD}/resource:/usr/share/infinity/resource infiniflow/infinity_builder:ubuntu20_clang20 | |
| - name: Build debug version | |
| if: ${{ !cancelled() && !failure() }} | |
| run: sudo docker exec ${BUILDER_CONTAINER} bash -c "git config --global safe.directory \"*\" && cd /infinity && rm -rf cmake-build-debug && mkdir -p cmake-build-debug && cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug -S /infinity -B /infinity/cmake-build-debug && cmake --build /infinity/cmake-build-debug -j 6 --target infinity test_main" | |
| - name: Build debug-tsan version | |
| if: ${{ !cancelled() && !failure() }} | |
| run: sudo docker exec ${BUILDER_CONTAINER} bash -c "git config --global safe.directory \"*\" && cd /infinity && rm -rf cmake-build-debug-tsan && mkdir -p cmake-build-debug-tsan && cmake -G Ninja -DCMAKE_BUILD_TYPE=Debug -DENABLE_SANITIZER_THREAD=ON -S /infinity -B /infinity/cmake-build-debug-tsan && cmake --build /infinity/cmake-build-debug-tsan -j 6 --target infinity" | |
| - name: Upload artifacts | |
| if: ${{ !cancelled() && !failure() }} | |
| run: tar --create --file=- cmake-build-debug/src/{infinity,test_main} cmake-build-debug-tsan/src/infinity | tar --extract --directory=${ARTIFACTS_DIR} --file=- | |
| - name: Remove build container | |
| if: always() | |
| run: | | |
| if [[ -n "${BUILDER_CONTAINER}" ]]; then | |
| sudo docker rm -f -v "${BUILDER_CONTAINER}" | |
| fi | |
| debug_unit_test: | |
| name: debug_unit_test | |
| runs-on: [ "self-hosted", "slow-test" ] | |
| needs: debug_build | |
| steps: | |
| - name: Prepare code and artifacts | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE} | |
| ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID} | |
| echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> ${GITHUB_ENV} | |
| echo "INFINITY_ROOT_PATH=${PWD}" >> ${GITHUB_ENV} | |
| tar xzf ${ARTIFACTS_DIR}/infinity.tgz && git reset --hard && git clean -ffdx | |
| tar --directory=${ARTIFACTS_DIR} --create --file=- cmake-build-debug/src/test_main | tar --extract --file=- | |
| sudo mkdir -p /usr/share/infinity/ && sudo cp -rf resource /usr/share/infinity/ && sudo chown -R ${USER} /usr/share/infinity/resource | |
| sudo rm -rf /var/infinity && sudo mkdir -p /var/infinity && sudo chown -R ${USER} /var/infinity | |
| - name: Start minio container | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| MINIO_CONTAINER=minio_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ') | |
| MINIO_VOLUME_PATH=${PWD}/${MINIO_CONTAINER} | |
| echo "MINIO_CONTAINER=${MINIO_CONTAINER}" >> ${GITHUB_ENV} | |
| echo "MINIO_VOLUME_PATH=${MINIO_VOLUME_PATH}" >> ${GITHUB_ENV} | |
| rm -rf ${MINIO_VOLUME_PATH} && mkdir -p ${MINIO_VOLUME_PATH} | |
| sudo docker run -d --name ${MINIO_CONTAINER} --network container:$(hostname) -e "MINIO_ROOT_PASSWORD=minioadmin" -e "MINIO_ROOT_USER=minioadmin" -v ${MINIO_VOLUME_PATH}:/data quay.io/minio/minio server /data --console-address ":9006" --address ":9005" | |
| sleep 5s | |
| if ! sudo docker ps --filter "name=${MINIO_CONTAINER}" --filter "status=running" | grep -q ${MINIO_CONTAINER}; then | |
| echo "Minio container is not running" | |
| exit 1 | |
| fi | |
| - name: Unit test debug version | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| ASAN_OPTIONS=detect_leaks=0 cmake-build-debug/src/test_main --gtest_also_run_disabled_tests --gtest_filter=*SLOW_* > debug_unit_test_stdout.log 2> debug_unit_test_stderr.log | |
| - name: Collect infinity log | |
| if: ${{ !cancelled() }} | |
| run: | | |
| mkdir -p ${ARTIFACTS_DIR}/debug_unit_test && cp debug_unit_test*.log ${ARTIFACTS_DIR}/debug_unit_test/ | |
| echo "debug unit test log" && tail -n 1000 debug_unit_test*.log | |
| - name: Remove minio container | |
| if: always() | |
| run: | | |
| if [[ -n "${MINIO_CONTAINER}" ]]; then sudo docker rm -f -v "${MINIO_CONTAINER}"; fi | |
| if [[ -n "${MINIO_VOLUME_PATH}" ]]; then sudo rm -rf "${MINIO_VOLUME_PATH}"; fi | |
| debug_tsan_restart_test: | |
| name: debug_tsan_restart_test | |
| runs-on: [ "self-hosted", "slow-test" ] | |
| needs: debug_build | |
| steps: | |
| - name: Prepare code and artifacts | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE} | |
| ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID} | |
| echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> ${GITHUB_ENV} | |
| echo "INFINITY_ROOT_PATH=${PWD}" >> ${GITHUB_ENV} | |
| tar --extract --gzip --file=${ARTIFACTS_DIR}/infinity.tgz && git reset --hard && git clean -ffdx | |
| tar --directory=${ARTIFACTS_DIR} --create --file=- cmake-build-debug-tsan/src/infinity | tar --extract --file=- | |
| sudo mkdir -p /usr/share/infinity/ && sudo cp -rf resource /usr/share/infinity/ && sudo chown -R ${USER} /usr/share/infinity/resource | |
| sudo rm -rf /var/infinity && sudo mkdir -p /var/infinity && sudo chown -R ${USER} /var/infinity | |
| - name: Prepare restart test data | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| tar --directory=${RUNNER_WORKSPACE_PREFIX} --create --file=- benchmark/sift_1m/{sift_base.fvecs,sift_query.fvecs} benchmark/enwiki/enwiki-10w.csv | tar --directory=test/data --extract --file=- | |
| - name: Install pysdk | |
| if: ${{ !cancelled() && !failure() }} | |
| run: uv sync --python 3.11 --frozen --extra test | |
| - name: Run restart test | |
| if: ${{ !cancelled() && !failure() }} | |
| run: ASAN_OPTIONS=detect_leaks=0 TSAN_OPTIONS=log_path=restart_test_tsan.log uv run python3 tools/run_restart_test.py --infinity_path=cmake-build-debug-tsan/src/infinity | |
| - name: Collect infinity log | |
| if: ${{ !cancelled() }} | |
| run: | | |
| mkdir -p ${ARTIFACTS_DIR}/debug_tsan_restart_test && sudo chown -R ${USER} . && cp restart_test*.log* ${ARTIFACTS_DIR}/debug_tsan_restart_test/ | |
| echo "restart_test log(last 1000 lines):" && tail -n 1000 restart_test*.log* | |
| debug_tsan_cluster_test: | |
| name: debug_tsan_cluster_test | |
| runs-on: [ "self-hosted", "slow-test" ] | |
| needs: debug_build | |
| steps: | |
| - name: Prepare code and artifacts | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE} | |
| ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID} | |
| echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> ${GITHUB_ENV} | |
| echo "INFINITY_ROOT_PATH=${PWD}" >> ${GITHUB_ENV} | |
| tar --extract --gzip --file=${ARTIFACTS_DIR}/infinity.tgz && git reset --hard && git clean -ffdx | |
| tar --directory=${ARTIFACTS_DIR} --create --file=- cmake-build-debug-tsan/src/infinity | tar --extract --file=- | |
| sudo mkdir -p /usr/share/infinity/ && sudo cp -rf resource /usr/share/infinity/ && sudo chown -R ${USER} /usr/share/infinity/resource | |
| sudo rm -rf /var/infinity && sudo mkdir -p /var/infinity && sudo chown -R ${USER} /var/infinity | |
| - name: Install pysdk | |
| if: ${{ !cancelled() && !failure() }} | |
| run: uv sync --python 3.11 --frozen --extra test | |
| - name: Run cluster test | |
| if: ${{ !cancelled() && !failure() }} | |
| run: uv run python3 tools/run_cluster_test.py --infinity_path=cmake-build-debug-tsan/src/infinity | |
| - name: Collect infinity log | |
| if: ${{ !cancelled() }} | |
| run: | | |
| mkdir -p ${ARTIFACTS_DIR}/debug_tsan_cluster_test | |
| sudo find /var/infinity/ -name '*.log' -type f | sudo tar --create --gzip --file=cluster_test.tgz --files-from=- | |
| tar --directory=${ARTIFACTS_DIR}/debug_tsan_cluster_test --strip-components=2 --extract --gzip --file=cluster_test.tgz | |
| debug_tsan_parallel_test: | |
| name: debug_tsan_parallel_test | |
| runs-on: [ "self-hosted", "slow-test" ] | |
| needs: | |
| - debug_build | |
| steps: | |
| - name: Prepare code and artifacts | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE} | |
| ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID} | |
| echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> ${GITHUB_ENV} | |
| echo "INFINITY_ROOT_PATH=${PWD}" >> ${GITHUB_ENV} | |
| tar xzf ${ARTIFACTS_DIR}/infinity.tgz && git reset --hard && git clean -ffdx | |
| tar -C ${ARTIFACTS_DIR} -cf - cmake-build-debug-tsan/src/infinity | tar -xf - | |
| sudo mkdir -p /usr/share/infinity/ && sudo cp -rf resource /usr/share/infinity/ && sudo chown -R ${USER} /usr/share/infinity/resource | |
| sudo rm -rf /var/infinity && sudo mkdir -p /var/infinity && sudo chown -R ${USER} /var/infinity | |
| - name: Install pysdk | |
| if: ${{ !cancelled() && !failure() }} | |
| run: uv sync --python 3.11 --frozen --extra test | |
| - name: Start infinity | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| nohup bash -c "ASAN_OPTIONS=detect_leaks=0 TSAN_OPTIONS=log_path=debug_tsan_parallel_tsan.log cmake-build-debug-tsan/src/infinity --config=conf/pytest_parallel_infinity_conf.toml 2> debug_tsan_parallel_stderr.log" & | |
| ./scripts/wait_for_infinity.sh | |
| - name: Run parallel test | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| uv run python3 tools/run_parallel_test.py | |
| - name: Collect infinity log | |
| if: ${{ !cancelled() }} | |
| run: | | |
| mkdir -p ${ARTIFACTS_DIR}/debug_tsan_parallel_test && sudo chown -R ${USER} . && cp debug_tsan_parallel*.log ${ARTIFACTS_DIR}/debug_tsan_parallel_test/ | |
| cp /var/infinity/log/infinity.log* ${ARTIFACTS_DIR}/debug_tsan_parallel_test/ | |
| echo "debug tsan parallel log" && tail -n 1000 debug_tsan_parallel*.log | |
| - name: Stop infinity | |
| if: always() | |
| run: python3 scripts/timeout_kill.py 10 $(pgrep infinity) | |
| reldeb_build: | |
| name: reldeb_build | |
| runs-on: [ "self-hosted", "infinity-build" ] | |
| needs: download_code | |
| steps: | |
| - name: Prepare code | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE} | |
| ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID} | |
| echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> ${GITHUB_ENV} | |
| tar xzf ${ARTIFACTS_DIR}/infinity.tgz && git reset --hard && git clean -ffdx | |
| - name: Start builder container | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| BUILDER_CONTAINER=infinity_build_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ') | |
| echo "BUILDER_CONTAINER=${BUILDER_CONTAINER}" >> ${GITHUB_ENV} | |
| TZ=${TZ:-$(readlink -f /etc/localtime | awk -F '/zoneinfo/' '{print $2}')} | |
| sudo docker run --privileged -d --name ${BUILDER_CONTAINER} -e TZ=${TZ} -e UV_INDEX=https://pypi.tuna.tsinghua.edu.cn/simple -v ${PWD}:/infinity -v ${PWD}/resource:/usr/share/infinity/resource infiniflow/infinity_builder:ubuntu20_clang20 | |
| - name: Build reldeb version | |
| if: ${{ !cancelled() && !failure() }} | |
| run: sudo docker exec ${BUILDER_CONTAINER} bash -c "git config --global safe.directory \"*\" && cd /infinity && rm -rf cmake-build-reldeb && mkdir -p cmake-build-reldeb && cmake -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -S /infinity -B /infinity/cmake-build-reldeb && cmake --build /infinity/cmake-build-reldeb -j 6 --target infinity test_main" | |
| - name: Upload artifacts | |
| if: ${{ !cancelled() && !failure() }} | |
| run: tar --create --file=- cmake-build-reldeb/src/{infinity,test_main} | tar --extract --directory=${ARTIFACTS_DIR} --file=- | |
| - name: Remove build container | |
| if: always() | |
| run: | | |
| if [[ -n "${BUILDER_CONTAINER}" ]]; then | |
| sudo docker rm -f -v "${BUILDER_CONTAINER}" | |
| fi | |
| reldeb_unit_test: | |
| name: reldeb_unit_test | |
| runs-on: [ "self-hosted", "slow-test" ] | |
| needs: reldeb_build | |
| steps: | |
| - name: Prepare code and artifacts | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE} | |
| ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID} | |
| echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> ${GITHUB_ENV} | |
| echo "INFINITY_ROOT_PATH=${PWD}" >> ${GITHUB_ENV} | |
| tar --extract --gzip --file=${ARTIFACTS_DIR}/infinity.tgz && git reset --hard && git clean -ffdx | |
| tar --directory=${ARTIFACTS_DIR} --create --file=- cmake-build-reldeb/src/test_main | tar --extract --file=- | |
| sudo mkdir -p /usr/share/infinity/ && sudo cp -rf resource /usr/share/infinity/ && sudo chown -R ${USER} /usr/share/infinity/resource | |
| sudo rm -rf /var/infinity && sudo mkdir -p /var/infinity && sudo chown -R ${USER} /var/infinity | |
| - name: Start minio container | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| MINIO_CONTAINER=minio_$(od -An -N4 -tx4 /dev/urandom | tr -d ' ') | |
| MINIO_VOLUME_PATH=${PWD}/${MINIO_CONTAINER} | |
| echo "MINIO_CONTAINER=${MINIO_CONTAINER}" >> ${GITHUB_ENV} | |
| echo "MINIO_VOLUME_PATH=${MINIO_VOLUME_PATH}" >> ${GITHUB_ENV} | |
| rm -rf ${MINIO_VOLUME_PATH} && mkdir -p ${MINIO_VOLUME_PATH} | |
| sudo docker run -d --name ${MINIO_CONTAINER} --network container:$(hostname) -e "MINIO_ROOT_PASSWORD=minioadmin" -e "MINIO_ROOT_USER=minioadmin" -v ${MINIO_VOLUME_PATH}:/data quay.io/minio/minio server /data --console-address ":9006" --address ":9005" | |
| sleep 5s | |
| if ! sudo docker ps --filter "name=${MINIO_CONTAINER}" --filter "status=running" | grep -q ${MINIO_CONTAINER}; then | |
| echo "Minio container is not running" | |
| exit 1 | |
| fi | |
| - name: Unit test | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| cmake-build-reldeb/src/test_main --gtest_also_run_disabled_tests --gtest_filter=*SLOW_* > reldeb_unit_test_stdout.log 2> reldeb_unit_test_stderr.log | |
| - name: Collect infinity log | |
| if: ${{ !cancelled() }} | |
| run: | | |
| mkdir -p ${ARTIFACTS_DIR}/reldeb_unit_test && cp reldeb_unit_test*.log ${ARTIFACTS_DIR}/reldeb_unit_test/ | |
| echo "reldeb unit test log" && tail -n 1000 reldeb_unit_test*.log | |
| - name: Remove minio container | |
| if: always() | |
| run: | | |
| if [[ -n "${MINIO_CONTAINER}" ]]; then sudo docker rm -f -v "${MINIO_CONTAINER}"; fi | |
| if [[ -n "${MINIO_VOLUME_PATH}" ]]; then sudo rm -rf "${MINIO_VOLUME_PATH}"; fi | |
| reldeb_parallel_test: | |
| name: reldeb_parallel_test | |
| runs-on: [ "self-hosted", "slow-test" ] | |
| needs: | |
| - reldeb_build | |
| steps: | |
| - name: Prepare code and artifacts | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE} | |
| ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID} | |
| echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> ${GITHUB_ENV} | |
| echo "INFINITY_ROOT_PATH=${PWD}" >> ${GITHUB_ENV} | |
| tar --extract --gzip --file=${ARTIFACTS_DIR}/infinity.tgz && git reset --hard && git clean -ffdx | |
| tar --directory=${ARTIFACTS_DIR} --create --file=- cmake-build-reldeb/src/infinity | tar --extract --file=- | |
| sudo mkdir -p /usr/share/infinity/ && sudo cp -rf resource /usr/share/infinity/ && sudo chown -R ${USER} /usr/share/infinity/resource | |
| sudo rm -rf /var/infinity && sudo mkdir -p /var/infinity && sudo chown -R ${USER} /var/infinity | |
| - name: Install pysdk | |
| if: ${{ !cancelled() && !failure() }} | |
| run: uv sync --python 3.11 --frozen --extra test | |
| - name: Start infinity | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| nohup bash -c "cmake-build-reldeb/src/infinity --config=conf/pytest_parallel_infinity_conf.toml 2> reldeb_parallel_stderr.log" & | |
| ./scripts/wait_for_infinity.sh | |
| - name: Run parallel test | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| uv run python3 tools/run_pytest_parallel_continuous.py | |
| - name: Collect infinity log | |
| if: ${{ !cancelled() }} | |
| run: | | |
| mkdir -p ${ARTIFACTS_DIR}/reldeb_parallel_test && cp reldeb_parallel_stderr.log ${ARTIFACTS_DIR}/reldeb_parallel_test/ | |
| cp /var/infinity/log/infinity.log* ${ARTIFACTS_DIR}/reldeb_parallel_test/ | |
| echo "reldeb parallel log" && tail -n 1000 reldeb_parallel*.log | |
| - name: Stop infinity | |
| if: always() | |
| run: python3 scripts/timeout_kill.py 10 $(pgrep infinity) | |
| reldeb_restart_test: | |
| name: reldeb_restart_test | |
| runs-on: [ "self-hosted", "slow-test" ] | |
| needs: reldeb_build | |
| steps: | |
| - name: Prepare code and artifacts | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| echo "chown -R ${USER} ${GITHUB_WORKSPACE}" && sudo chown -R ${USER} ${GITHUB_WORKSPACE} | |
| ARTIFACTS_DIR=${RUNNER_WORKSPACE_PREFIX}/artifacts/${GITHUB_REPOSITORY}/${GITHUB_RUN_ID} | |
| echo "ARTIFACTS_DIR=${ARTIFACTS_DIR}" >> ${GITHUB_ENV} | |
| echo "INFINITY_ROOT_PATH=${PWD}" >> ${GITHUB_ENV} | |
| tar xzf ${ARTIFACTS_DIR}/infinity.tgz && git reset --hard && git clean -ffdx | |
| tar -C ${ARTIFACTS_DIR} -cf - cmake-build-reldeb/src/infinity | tar -xf - | |
| sudo mkdir -p /usr/share/infinity/ && sudo cp -rf resource /usr/share/infinity/ && sudo chown -R ${USER} /usr/share/infinity/resource | |
| sudo rm -rf /var/infinity && sudo mkdir -p /var/infinity && sudo chown -R ${USER} /var/infinity | |
| - name: Prepare restart test data | |
| if: ${{ !cancelled() && !failure() }} | |
| run: | | |
| tar --directory=${RUNNER_WORKSPACE_PREFIX} --create --file=- benchmark/sift_1m/{sift_base.fvecs,sift_query.fvecs} benchmark/enwiki/enwiki-10w.csv | tar --directory=test/data --extract --file=- | |
| - name: Install pysdk | |
| if: ${{ !cancelled() && !failure() }} | |
| run: uv sync --python 3.11 --frozen --extra test | |
| - name: Run restart test | |
| if: ${{ !cancelled() && !failure() }} | |
| run: uv run python3 tools/run_restart_test_continuously.py --infinity_path=cmake-build-reldeb/src/infinity | |
| - name: Collect infinity log | |
| if: ${{ !cancelled() }} | |
| run: | | |
| mkdir -p ${ARTIFACTS_DIR}/reldeb_restart_test | |
| cp restart_test*.log* ${ARTIFACTS_DIR}/reldeb_restart_test/ | |
| echo "reldeb_restart_test log(last 1000 lines):" && tail -n 1000 restart_test*.log* |