release_build #114
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # FIXME: update this workflow | |
| name: release_build | |
| on: # yamllint disable-line rule:truthy | |
| workflow_dispatch: | |
| inputs: | |
| prerelease: | |
| description: 'pre-release' | |
| required: false | |
| type: boolean | |
| default: false | |
| # schedule: | |
| # # build at 15:30UTC,8:30PST, 23:30CST, on every Tuesday and Friday | |
| # - cron: '30 15 * * 4' #was 30 15 * * 2,5 | |
| jobs: | |
| create_release: | |
| needs: update_version | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Create GitHub release with notes | |
| uses: softprops/action-gh-release@v2.5.0 | |
| with: | |
| token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| tag_name: ${{ needs.update_version.outputs.tag_name }} | |
| prerelease: ${{ inputs.prerelease }} | |
| generate_release_notes: true | |
| body: | | |
| ## Docker Images Available | |
| ```bash | |
| docker pull d.timeplus.com/timeplus-io/proton:latest | |
| ``` | |
| For detailed usage and more information, check out the Timeplus documentation: https://docs.timeplus.com/ | |
| update_version: | |
| uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
| with: | |
| ec2-instance-type: c5.large | |
| ec2-image-id: ami-042a37e33a285c22b | |
| submodules: 'false' | |
| run_mode: 'start' # start ec2 on demand instance | |
| upjob: update_version | |
| command: | | |
| cd $GITHUB_WORKSPACE | |
| # git config | |
| git config user.name "proton-robot" | |
| git config user.email "proton_robot@timeplus.io" | |
| # update version | |
| if [ "${{ inputs.prerelease }}" == "true" ]; then | |
| ./release --rc --version patch | |
| else | |
| ./release --version patch | |
| fi | |
| # get proton tag first | |
| PROTON_TAG=`grep "SET(VERSION_DESCRIBE" $GITHUB_WORKSPACE/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_DESCRIBE \(.*\)$/\1/' | sed 's/[) ].*//'` | |
| echo "Proton tag: $PROTON_TAG" | |
| echo "tag_name=$PROTON_TAG" >> $GITHUB_OUTPUT | |
| secrets: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| AWS_S3_BUCKET: ${{ vars.AWS_S3_BUCKET_WEST_2 }} | |
| GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| Build_Linux_X86_64: | |
| needs: [update_version, create_release] | |
| uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
| with: | |
| ec2-instance-type: c7i.8xlarge | |
| ec2-image-id: ${{ vars.X64_AMI }} | |
| submodules: 'true' | |
| run_mode: 'start' # start ec2 on demand instance | |
| upload_files: | | |
| proton-*-Linux-x86_64.tar.gz | |
| proton-*-Linux-x86_64-debug-symbols.tar.gz | |
| python-Linux-x86_64.tar.gz | |
| prerelease: ${{ inputs.prerelease }} | |
| ref: ${{ needs.update_version.outputs.tag_name }} | |
| upjob: Build_Linux_X86_64 | |
| generate_release_notes: false | |
| release_body: '' | |
| upload_only: true | |
| command: | | |
| cd $GITHUB_WORKSPACE | |
| # git config | |
| git config user.name "proton-robot" | |
| git config user.email "proton_robot@timeplus.io" | |
| # prepare build cache | |
| aws s3 cp --no-progress s3://tp-internal2/proton-oss/cache.tar.gz . | |
| mkdir $GITHUB_WORKSPACE/ccache | |
| tar -zxf ./cache.tar.gz -C $GITHUB_WORKSPACE/ccache | |
| rm cache.tar.gz | |
| # compiling | |
| ./docker/packager/packager --package-type binary --docker-image-version clang-21 --build-type relwithdebinfo --proton-build --enable-proton-local --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output | |
| if [ ! -f "$GITHUB_WORKSPACE/output/proton" ]; then | |
| echo "Compiling proton Failed" | |
| exit 127 | |
| fi | |
| # get proton tag first | |
| PROTON_TAG=`grep "SET(VERSION_DESCRIBE" $GITHUB_WORKSPACE/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_DESCRIBE \(.*\)$/\1/' | sed 's/[) ].*//'` | |
| echo "Proton tag: $PROTON_TAG" | |
| echo "tag_name=$PROTON_TAG" >> $GITHUB_OUTPUT | |
| PROTON_BINARY=proton-$PROTON_TAG-Linux-x86_64 | |
| PROTON_DEBUG=proton-$PROTON_TAG-Linux-x86_64-debug-symbols.tar.gz | |
| echo "Building: $PROTON_BINARY" | |
| echo "Debug package: $PROTON_DEBUG" | |
| # Show original size | |
| echo "Original binary size:" | |
| ls -lh $GITHUB_WORKSPACE/output/proton | |
| # Work directly with the final filename - just rename once | |
| cd $GITHUB_WORKSPACE | |
| mv output/proton $PROTON_BINARY | |
| # Extract debug symbols from the final binary | |
| echo "Extracting debug symbols..." | |
| docker run --rm -v $GITHUB_WORKSPACE:/work -w /work timeplus/proton-binary-builder:clang-21 \ | |
| /usr/bin/objcopy --only-keep-debug $PROTON_BINARY ${PROTON_BINARY}.debug | |
| # Create debug symbols package | |
| echo "Creating debug symbols package..." | |
| chmod 0644 ${PROTON_BINARY}.debug | |
| tar -czf $PROTON_DEBUG ${PROTON_BINARY}.debug | |
| rm ${PROTON_BINARY}.debug | |
| # Strip the binary in-place (conservative approach) | |
| echo "Stripping binary in conservative approach..." | |
| docker run --rm -v $GITHUB_WORKSPACE:/work -w /work timeplus/proton-binary-builder:clang-21 \ | |
| /usr/bin/llvm-strip-21 --strip-debug --remove-section=.comment --remove-section=.note $PROTON_BINARY | |
| # Show size comparison | |
| echo "Final sizes:" | |
| ls -lh $PROTON_BINARY $PROTON_DEBUG | |
| # Package the binary into a tar.gz for faster downloads | |
| echo "Packaging binary tarball..." | |
| tar -czf ${PROTON_BINARY}.tar.gz $PROTON_BINARY | |
| ls -lh ${PROTON_BINARY}.tar.gz | |
| # Start static server for docker build | |
| docker run --name static-server -p 8080:80 -v $GITHUB_WORKSPACE:/usr/share/nginx/html:ro -d nginx | |
| cd $GITHUB_WORKSPACE/docker/server | |
| # prepare files to be copied to the image | |
| mkdir -p resources/protos/google/protobuf | |
| cp -r $GITHUB_WORKSPACE/contrib/google-protobuf/src/google/protobuf/*.proto ./resources/protos/google/protobuf/ | |
| rm -rf resources/protos/google/protobuf/unittest_* | |
| # build docker image for both Docker Hub and GHCR (using the stripped binary) | |
| docker build . --network host --build-arg single_binary_location_url=http://localhost:8080/$PROTON_BINARY \ | |
| -t timeplus/proton:${GITHUB_SHA}_amd64 \ | |
| -t ghcr.io/timeplus-io/proton:${GITHUB_SHA}_amd64 | |
| # push docker image to both registries | |
| docker push timeplus/proton:${GITHUB_SHA}_amd64 | |
| docker push ghcr.io/timeplus-io/proton:${GITHUB_SHA}_amd64 | |
| # upload build cache | |
| tar -zcf ./cache.tar.gz -C $GITHUB_WORKSPACE/ccache . | |
| aws s3 cp --no-progress ./cache.tar.gz s3://tp-internal2/proton-oss/ | |
| # prepare python package | |
| mkdir python-Linux-x86_64 | |
| cd python-Linux-x86_64 | |
| mkdir lib | |
| mkdir bin | |
| docker pull python:3.10 | |
| # warning: do not use double quote here. It will be remove when the run_command workflow rending this command script | |
| docker run --name python_packages python:3.10 sh -c 'pip install --upgrade truststore; pip install --force-reinstall --no-cache-dir -t /usr/local/lib/python3.10/site-packages/ timeplus-neutrino==0.1.12;' | |
| docker cp python_packages:/usr/local/lib/libpython3.so ./lib/libpython3.so | |
| docker cp python_packages:/usr/local/lib/libpython3.10.so.1.0 ./lib/libpython3.10.so.1.0 | |
| docker cp python_packages:/usr/local/lib/libpython3.10.so ./lib/libpython3.10.so | |
| docker cp python_packages:/usr/local/lib/python3.10 ./lib/python3.10 | |
| docker cp python_packages:/usr/local/bin/python3.10 ./bin/python3.10 | |
| tar -zcf $GITHUB_WORKSPACE/python-Linux-x86_64.tar.gz . | |
| secrets: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| AWS_S3_BUCKET: ${{ vars.AWS_S3_BUCKET_WEST_2 }} | |
| GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| Build_Linux_Arm64: | |
| needs: [update_version, create_release] | |
| uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
| with: | |
| ec2-instance-type: c6g.8xlarge | |
| ec2-image-id: ${{ vars.ARM_AMI }} | |
| submodules: 'true' | |
| run_mode: 'start' # start ec2 on demand instance | |
| upload_files: | | |
| proton-*-Linux-aarch64.tar.gz | |
| prerelease: ${{ inputs.prerelease }} | |
| ref: ${{ needs.update_version.outputs.tag_name }} | |
| upjob: Build_Linux_Arm64 | |
| generate_release_notes: false | |
| release_body: '' | |
| upload_only: true | |
| command: | | |
| cd $GITHUB_WORKSPACE | |
| # git config | |
| git config user.name "proton-robot" | |
| git config user.email "proton_robot@timeplus.io" | |
| # prepare build cache | |
| aws s3 cp --no-progress s3://tp-internal2/proton-oss/cache-arm.tar.gz . | |
| mkdir $GITHUB_WORKSPACE/ccache | |
| tar -zxf ./cache-arm.tar.gz -C $GITHUB_WORKSPACE/ccache | |
| rm cache-arm.tar.gz | |
| # compiling | |
| ./docker/packager/packager --package-type binary --docker-image-version clang-21 --build-type release --proton-build --disable-python-udf --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output | |
| if [ ! -f "$GITHUB_WORKSPACE/output/proton" ]; then | |
| echo "Compiling proton Failed" | |
| exit 127 | |
| fi | |
| # strip | |
| ls -lh $GITHUB_WORKSPACE/output/proton | |
| docker run --rm -v $GITHUB_WORKSPACE/output:/work -w /work timeplus/proton-binary-builder:clang-21 /usr/bin/llvm-strip-21 proton | |
| ls -lh $GITHUB_WORKSPACE/output/proton | |
| # get proton tag | |
| PROTON_TAG=`grep "SET(VERSION_DESCRIBE" $GITHUB_WORKSPACE/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_DESCRIBE \(.*\)$/\1/' | sed 's/[) ].*//'` | |
| echo "Proton tag: $PROTON_TAG" | |
| echo "tag_name=$PROTON_TAG" >> $GITHUB_OUTPUT | |
| PROTON_BINARY=proton-$PROTON_TAG-Linux-aarch64 | |
| echo "Proton Binary Name: $PROTON_BINARY" | |
| docker run --name static-server -p 8080:80 -v $GITHUB_WORKSPACE/output:/usr/share/nginx/html:ro -d nginx | |
| cd $GITHUB_WORKSPACE/docker/server | |
| # prepare files to be copied to the image | |
| mkdir -p resources/protos/google/protobuf | |
| cp -r $GITHUB_WORKSPACE/contrib/google-protobuf/src/google/protobuf/*.proto ./resources/protos/google/protobuf/ | |
| rm -rf resources/protos/google/protobuf/unittest_* | |
| # build docker image (skip Python UDF pieces on ARM64) | |
| docker build . --network host \ | |
| --build-arg ENABLE_PYTHON_UDF=0 \ | |
| --build-arg single_binary_location_url=http://localhost:8080/proton \ | |
| -t timeplus/proton:${GITHUB_SHA}_arm64v8 \ | |
| -t ghcr.io/timeplus-io/proton:${GITHUB_SHA}_arm64v8 | |
| # push docker image to both registries | |
| docker push timeplus/proton:${GITHUB_SHA}_arm64v8 | |
| docker push ghcr.io/timeplus-io/proton:${GITHUB_SHA}_arm64v8 | |
| # upload build cache | |
| tar -zcf ./cache-arm.tar.gz -C $GITHUB_WORKSPACE/ccache . | |
| aws s3 cp --no-progress ./cache-arm.tar.gz s3://tp-internal2/proton-oss/ | |
| mv $GITHUB_WORKSPACE/output/proton $GITHUB_WORKSPACE/$PROTON_BINARY | |
| # Package the binary into a tar.gz for faster downloads | |
| echo "Packaging binary tarball..." | |
| cd $GITHUB_WORKSPACE | |
| tar -czf ${PROTON_BINARY}.tar.gz $PROTON_BINARY | |
| ls -lh ${PROTON_BINARY}.tar.gz | |
| secrets: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| AWS_S3_BUCKET: ${{ vars.AWS_S3_BUCKET_WEST_2 }} | |
| GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| CommitTag: | |
| needs: [Build_Linux_X86_64, Build_Linux_Arm64] | |
| uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
| with: | |
| ec2-instance-type: c5.large | |
| ec2-image-id: ami-042a37e33a285c22b | |
| submodules: false | |
| upjob: CommitTag | |
| command: | | |
| cd $GITHUB_WORKSPACE | |
| # git config | |
| git config user.name "proton-robot" | |
| git config user.email "proton_robot@timeplus.io" | |
| # Pull images from both registries | |
| docker pull timeplus/proton:${GITHUB_SHA}_amd64 | |
| docker pull timeplus/proton:${GITHUB_SHA}_arm64v8 | |
| docker pull ghcr.io/timeplus-io/proton:${GITHUB_SHA}_amd64 | |
| docker pull ghcr.io/timeplus-io/proton:${GITHUB_SHA}_arm64v8 | |
| # Create and push Docker Hub manifests | |
| docker manifest create timeplus/proton:develop \ | |
| timeplus/proton:${GITHUB_SHA}_amd64 \ | |
| timeplus/proton:${GITHUB_SHA}_arm64v8 | |
| docker manifest create timeplus/proton:latest \ | |
| timeplus/proton:${GITHUB_SHA}_amd64 \ | |
| timeplus/proton:${GITHUB_SHA}_arm64v8 | |
| docker manifest push timeplus/proton:develop | |
| docker manifest push timeplus/proton:latest | |
| # Create and push GHCR manifests | |
| docker manifest create ghcr.io/timeplus-io/proton:develop \ | |
| ghcr.io/timeplus-io/proton:${GITHUB_SHA}_amd64 \ | |
| ghcr.io/timeplus-io/proton:${GITHUB_SHA}_arm64v8 | |
| docker manifest create ghcr.io/timeplus-io/proton:latest \ | |
| ghcr.io/timeplus-io/proton:${GITHUB_SHA}_amd64 \ | |
| ghcr.io/timeplus-io/proton:${GITHUB_SHA}_arm64v8 | |
| docker manifest push ghcr.io/timeplus-io/proton:develop | |
| docker manifest push ghcr.io/timeplus-io/proton:latest | |
| # Get version tag and create version-specific manifests for both registries | |
| PROTON_TAG="$(docker history --no-trunc timeplus/proton:${GITHUB_SHA}_amd64 | grep -Eo 'ARG version=[0-9]+.[0-9]+.[0-9]+(-[a-zA-Z0-9.]+)?' -o | grep -Eo '[0-9]+.[0-9]+.[0-9]+(-[a-zA-Z0-9.]+)?')" | |
| # Docker Hub version tag | |
| docker manifest create timeplus/proton:$PROTON_TAG \ | |
| timeplus/proton:${GITHUB_SHA}_amd64 \ | |
| timeplus/proton:${GITHUB_SHA}_arm64v8 | |
| docker manifest push timeplus/proton:$PROTON_TAG | |
| # GHCR version tag | |
| docker manifest create ghcr.io/timeplus-io/proton:$PROTON_TAG \ | |
| ghcr.io/timeplus-io/proton:${GITHUB_SHA}_amd64 \ | |
| ghcr.io/timeplus-io/proton:${GITHUB_SHA}_arm64v8 | |
| docker manifest push ghcr.io/timeplus-io/proton:$PROTON_TAG | |
| export REF="${GITHUB_REF#refs/heads/}" | |
| echo "branch_name: $REF" | |
| curl -L \ | |
| -X POST \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "Authorization: token $GH_PERSONAL_ACCESS_TOKEN"\ | |
| -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| https://api.github.com/repos/timeplus-io/proton/actions/workflows/manual_trigger_test.yml/dispatches \ | |
| -d "{\"ref\":\"$REF\",\"inputs\":{\"arch\": \"x64\", \"tag\":\"$PROTON_TAG\", \"test_category\":\"scope.release\", \"nodes\":\"1\"}}\"" | |
| curl -L \ | |
| -X POST \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "Authorization: token $GH_PERSONAL_ACCESS_TOKEN"\ | |
| -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| https://api.github.com/repos/timeplus-io/proton/actions/workflows/manual_trigger_test.yml/dispatches \ | |
| -d "{\"ref\":\"$REF\",\"inputs\":{\"arch\": \"arm\", \"tag\":\"$PROTON_TAG\", \"test_category\":\"scope.release\", \"nodes\":\"1\"}}\"" | |
| curl -L \ | |
| -X POST \ | |
| -H "Accept: application/vnd.github+json" \ | |
| -H "Authorization: token $GH_PERSONAL_ACCESS_TOKEN"\ | |
| -H "X-GitHub-Api-Version: 2022-11-28" \ | |
| https://api.github.com/repos/timeplus-io/proton/actions/workflows/manual_trigger_test.yml/dispatches \ | |
| -d "{\"ref\":\"$REF\",\"inputs\":{\"arch\": \"x64\", \"tag\":\"$PROTON_TAG\", \"test_category\":\"smoke\", \"has_feature_enabled\":\"hybrid\", \"nodes\":\"1\"}}\"" | |
| secrets: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| AWS_S3_BUCKET: ${{ vars.AWS_S3_BUCKET_WEST_2 }} | |
| GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| Build_Darwin_X86_64: | |
| needs: [update_version, create_release] | |
| uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
| with: | |
| ec2-instance-type: c6i.8xlarge | |
| ec2-image-id: ami-042a37e33a285c22b | |
| submodules: 'true' | |
| run_mode: 'start' # start ec2 on demand instance | |
| upload_files: | | |
| proton-*-Darwin-x86_64.tar.gz | |
| prerelease: ${{ inputs.prerelease }} | |
| ref: ${{ needs.update_version.outputs.tag_name }} | |
| upjob: Build_Darwin_X86_64 | |
| generate_release_notes: false | |
| release_body: '' | |
| upload_only: true | |
| command: | | |
| cd $GITHUB_WORKSPACE | |
| # git config | |
| git config user.name "proton-robot" | |
| git config user.email "proton_robot@timeplus.io" | |
| # download the pre-built binary of v8 (this is only for cross-compile) | |
| aws s3 cp --no-progress s3://tp-internal2/proton-oss/cross-compile-prebuilt-binary/138/v8-cmake-x64.tar.gz $GITHUB_WORKSPACE/contrib/v8-cmake/ | |
| tar -zxf $GITHUB_WORKSPACE/contrib/v8-cmake/v8-cmake-x64.tar.gz -C $GITHUB_WORKSPACE/contrib/v8-cmake/ | |
| rm $GITHUB_WORKSPACE/contrib/v8-cmake/v8-cmake-x64.tar.gz | |
| chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/bytecode_builtins_list_generator | |
| chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/mksnapshot | |
| chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/torque | |
| # prepare build cache | |
| mkdir $GITHUB_WORKSPACE/ccache | |
| echo "max_size = 100.0G" > $GITHUB_WORKSPACE/ccache/ccache.conf | |
| # compiling | |
| ./docker/packager/packager --package-type binary --docker-image-version clang-18 --build-type release --proton-build --disable-python-udf --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output --compiler clang-18-darwin | |
| if [ ! -f "$GITHUB_WORKSPACE/output/proton" ]; then | |
| echo "Compiling proton Failed" | |
| exit 127 | |
| fi | |
| # strip | |
| ls -lh $GITHUB_WORKSPACE/output/proton | |
| docker run --rm -v $GITHUB_WORKSPACE/output:/work -w /work timeplus/proton-binary-builder:clang-18 /usr/bin/llvm-strip-18 proton | |
| ls -lh $GITHUB_WORKSPACE/output/proton | |
| # clean ccache and build footprint | |
| rm -rf $GITHUB_WORKSPACE/ccache | |
| rm -rf $GITHUB_WORKSPACE/build_docker | |
| # get proton tag | |
| PROTON_TAG=`grep "SET(VERSION_DESCRIBE" $GITHUB_WORKSPACE/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_DESCRIBE \(.*\)$/\1/' | sed 's/[) ].*//'` | |
| echo "Proton tag: $PROTON_TAG" | |
| echo "tag_name=$PROTON_TAG" >> $GITHUB_OUTPUT | |
| PROTON_BINARY=proton-$PROTON_TAG-Darwin-x86_64 | |
| echo "Proton Binary Name: $PROTON_BINARY" | |
| sha256sum $GITHUB_WORKSPACE/output/proton | |
| mv $GITHUB_WORKSPACE/output/proton $GITHUB_WORKSPACE/$PROTON_BINARY | |
| # Package the binary into a tar.gz for faster downloads | |
| echo "Packaging binary tarball..." | |
| cd $GITHUB_WORKSPACE | |
| tar -czf ${PROTON_BINARY}.tar.gz $PROTON_BINARY | |
| ls -lh ${PROTON_BINARY}.tar.gz | |
| secrets: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| AWS_S3_BUCKET: ${{ vars.AWS_S3_BUCKET_WEST_2 }} | |
| GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| Build_Darwin_Arm64: | |
| if: false | |
| needs: [update_version, create_release] | |
| uses: timeplus-io/proton/.github/workflows/run_command.yml@develop | |
| with: | |
| ec2-instance-type: c6g.4xlarge | |
| ec2-image-id: ami-0f3dbc4cc9994fdee | |
| submodules: 'true' | |
| run_mode: 'start' # start ec2 on demand instance | |
| upload_files: proton-*-Darwin-arm64 | |
| prerelease: ${{ inputs.prerelease }} | |
| ref: ${{ needs.update_version.outputs.tag_name }} | |
| upjob: Build_Darwin_Arm64 | |
| generate_release_notes: false | |
| release_body: '' | |
| upload_only: true | |
| command: | | |
| cd $GITHUB_WORKSPACE | |
| # git config | |
| git config user.name "proton-robot" | |
| git config user.email "proton_robot@timeplus.io" | |
| # download the pre-built binary of v8 (this is only for cross-compile) | |
| aws s3 cp --no-progress s3://tp-internal2/proton-oss/cross-compile-prebuilt-binary/138/v8-cmake-arm64.tar.gz $GITHUB_WORKSPACE/contrib/v8-cmake/ | |
| tar -zxf $GITHUB_WORKSPACE/contrib/v8-cmake/v8-cmake-arm64.tar.gz -C $GITHUB_WORKSPACE/contrib/v8-cmake/ | |
| rm $GITHUB_WORKSPACE/contrib/v8-cmake/v8-cmake-arm64.tar.gz | |
| chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/bytecode_builtins_list_generator | |
| chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/mksnapshot | |
| chmod a+x $GITHUB_WORKSPACE/contrib/v8-cmake/torque | |
| # prepare build cache | |
| mkdir $GITHUB_WORKSPACE/ccache | |
| echo "max_size = 100.0G" > $GITHUB_WORKSPACE/ccache/ccache.conf | |
| # compiling | |
| ./docker/packager/packager --package-type binary --docker-image-version clang-21 --build-type release --proton-build --disable-python-udf --cache ccache --ccache_dir $GITHUB_WORKSPACE/ccache --output-dir $GITHUB_WORKSPACE/output --compiler clang-21-darwin-aarch64 | |
| if [ ! -f "$GITHUB_WORKSPACE/output/proton" ]; then | |
| echo "Compiling proton Failed" | |
| exit 127 | |
| fi | |
| # strip | |
| ls -lh $GITHUB_WORKSPACE/output/proton | |
| docker run --rm -v $GITHUB_WORKSPACE/output:/work -w /work timeplus/proton-binary-builder:clang-21 /usr/bin/llvm-strip-21 proton | |
| ls -lh $GITHUB_WORKSPACE/output/proton | |
| # clean ccache and build footprint | |
| rm -rf $GITHUB_WORKSPACE/ccache | |
| rm -rf $GITHUB_WORKSPACE/build_docker | |
| # get proton tag | |
| PROTON_TAG=`grep "SET(VERSION_DESCRIBE" $GITHUB_WORKSPACE/cmake/autogenerated_versions.txt | sed 's/^.*VERSION_DESCRIBE \(.*\)$/\1/' | sed 's/[) ].*//'` | |
| echo "Proton tag: $PROTON_TAG" | |
| echo "tag_name=$PROTON_TAG" >> $GITHUB_OUTPUT | |
| PROTON_BINARY=proton-$PROTON_TAG-Darwin-arm64 | |
| echo "Proton Binary Name: $PROTON_BINARY" | |
| sha256sum $GITHUB_WORKSPACE/output/proton | |
| mv $GITHUB_WORKSPACE/output/proton $GITHUB_WORKSPACE/$PROTON_BINARY | |
| secrets: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: ${{ secrets.AWS_REGION }} | |
| AWS_S3_BUCKET: ${{ vars.AWS_S3_BUCKET_WEST_2 }} | |
| GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} | |
| DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} | |
| Build_Native_Darwin_arm64: | |
| needs: Build_Linux_X86_64 # Gate the job behind the Linux x86_64 job so the release exists before uploads. | |
| runs-on: [self-hosted, macOS, ARM64] | |
| env: | |
| build_directory: ${{ github.workspace }}/build | |
| build_type: RelWithDebInfo | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v5.0.1 | |
| with: | |
| token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| fetch-depth: 0 # Fetch all history including all tags | |
| submodules: true | |
| # Checkout the latest tag | |
| - name: Checkout latest tag | |
| run: | | |
| git fetch --all --tags | |
| REF="${GITHUB_REF#refs/heads/}" | |
| LATEST_TAG=$(git describe --tags --abbrev=0 origin/$REF) | |
| echo "Checking out tag $LATEST_TAG" | |
| git checkout $LATEST_TAG | |
| echo "LATEST_TAG=$LATEST_TAG" >> $GITHUB_ENV | |
| # Setup build environment and configure CMake | |
| - name: Create and Configure Build | |
| run: | | |
| mkdir -p ${{ env.build_directory }} | |
| export CC=$(brew --prefix llvm@21)/bin/clang | |
| export CXX=$(brew --prefix llvm@21)/bin/clang++ | |
| export PATH=$(brew --prefix llvm@21)/bin:$PATH | |
| cmake -B ${{ env.build_directory }} -G "Ninja" -DCMAKE_BUILD_TYPE=${{ env.build_type }} -DENABLE_TESTS=OFF -DENABLE_UTILS=OFF -DENABLE_EXAMPLES=OFF -DENABLE_PROTON_LOCAL=OFF -DENABLE_PULSAR=ON -DENABLE_PYTHON_UDF=ON -DENABLE_PROTON_PYTHON=ON | |
| # Compile the project using Ninja | |
| - name: Build with Ninja | |
| run: cmake --build ${{ env.build_directory }} | |
| # Strip the binary to reduce its size and rename it | |
| - name: Strip and Rename binary | |
| run: | | |
| ORIGINAL_BINARY=${{ env.build_directory }}/programs/proton | |
| STRIPPED_BINARY=${{ env.build_directory }}/programs/proton-${{ env.LATEST_TAG }}-Darwin-arm64 | |
| ls -lh $ORIGINAL_BINARY | |
| /opt/homebrew/opt/llvm@21/bin/llvm-strip $ORIGINAL_BINARY -o $STRIPPED_BINARY | |
| ls -lh $STRIPPED_BINARY | |
| # Set up AWS credentials for S3 upload | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v6.0.0 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ secrets.AWS_REGION }} | |
| # Upload the stripped binary to an AWS S3 bucket with retries | |
| - name: Upload Artifact To S3 | |
| run: | | |
| STRIPPED_BINARY=${{ env.build_directory }}/programs/proton-${{ env.LATEST_TAG }}-Darwin-arm64 | |
| S3_PATH="s3://tp-internal2/proton-oss/native_build_macOS/" | |
| # Function to upload with retry | |
| upload_with_retry() { | |
| local max_attempts=5 | |
| local attempt=1 | |
| local wait_time=15 | |
| while [ $attempt -le $max_attempts ]; do | |
| echo "S3 upload attempt $attempt of $max_attempts for $(basename $1)" | |
| if aws s3 cp \ | |
| --no-progress \ | |
| --cli-connect-timeout 30 \ | |
| --cli-read-timeout 600 \ | |
| --only-show-errors \ | |
| "$1" "$2"; then | |
| echo "Upload completed successfully" | |
| return 0 | |
| else | |
| local status=$? | |
| echo "Upload failed on attempt $attempt with status $status" | |
| if [ $attempt -eq $max_attempts ]; then | |
| echo "All attempts failed" | |
| return 1 | |
| fi | |
| echo "Waiting $wait_time seconds before retry..." | |
| sleep $wait_time | |
| wait_time=$((wait_time * 2)) # Exponential backoff | |
| attempt=$((attempt + 1)) | |
| fi | |
| done | |
| } | |
| # Upload binary with retry | |
| if ! upload_with_retry "$STRIPPED_BINARY" "${S3_PATH}$(basename $STRIPPED_BINARY)"; then | |
| echo "Binary upload failed after all attempts" | |
| exit 1 | |
| fi | |
| # Prepare python3.10 | |
| aws s3 cp --no-progress ${S3_PATH}python-Darwin-arm64.tar.gz ${{ env.build_directory }}/programs/python-Darwin-arm64.tar.gz | |
| # Print SHA256 Checksum | |
| - name: Print SHA256 Checksum | |
| run: | | |
| STRIPPED_BINARY=${{ env.build_directory }}/programs/proton-${{ env.LATEST_TAG }}-Darwin-arm64 | |
| shasum -a 256 $STRIPPED_BINARY | |
| # Release the binary using GitHub CLI with retry logic | |
| - name: Release binary using GitHub CLI | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} | |
| run: | | |
| STRIPPED_BINARY=${{ env.build_directory }}/programs/proton-${{ env.LATEST_TAG }}-Darwin-arm64 | |
| PYTHON_PACKAGE=${{ env.build_directory }}/programs/python-Darwin-arm64.tar.gz | |
| TARBALL=${STRIPPED_BINARY}.tar.gz | |
| # Create tar.gz package for faster downloads | |
| echo "Packaging binary tarball..." | |
| tar -czf "$TARBALL" -C "$(dirname "$STRIPPED_BINARY")" "$(basename "$STRIPPED_BINARY")" | |
| ls -lh "$TARBALL" | |
| # Function to upload asset with retry | |
| upload_asset_with_retry() { | |
| local max_attempts=5 | |
| local attempt=1 | |
| local wait_time=15 | |
| local file_path="$1" | |
| local tag="$2" | |
| while [ $attempt -le $max_attempts ]; do | |
| echo "Upload attempt $attempt of $max_attempts for $(basename $file_path)" | |
| if gh release upload "$tag" "$file_path" --clobber --repo ${{ github.repository }}; then | |
| echo "Upload completed successfully" | |
| return 0 | |
| else | |
| local status=$? | |
| echo "Upload failed on attempt $attempt with status $status" | |
| if [ $attempt -eq $max_attempts ]; then | |
| echo "All upload attempts failed" | |
| return 1 | |
| fi | |
| echo "Waiting $wait_time seconds before retry..." | |
| sleep $wait_time | |
| wait_time=$((wait_time * 2)) # Exponential backoff | |
| attempt=$((attempt + 1)) | |
| fi | |
| done | |
| } | |
| # Upload tarball with retry | |
| if ! upload_asset_with_retry "$TARBALL" "${{ env.LATEST_TAG }}"; then | |
| echo "Tarball upload failed after all attempts" | |
| exit 1 | |
| fi | |
| # Upload Python package with retry | |
| if ! upload_asset_with_retry "$PYTHON_PACKAGE" "${{ env.LATEST_TAG }}"; then | |
| echo "Python package upload failed after all attempts" | |
| exit 1 | |
| fi | |
| Update_Homebrew_Tap: | |
| if: ${{ !inputs.prerelease }} | |
| needs: | |
| - update_version | |
| - Build_Darwin_X86_64 | |
| - Build_Native_Darwin_arm64 | |
| uses: ./.github/workflows/update_homebrew_tap.yml | |
| with: | |
| tag_name: ${{ needs.update_version.outputs.tag_name }} | |
| secrets: | |
| GH_PERSONAL_ACCESS_TOKEN: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} |