Merge pull request #3765 from fosrl/dev #129

Workflow file for this run

	name: Public CICD Pipeline

	# CI/CD workflow for building, publishing, mirroring, signing container images and building release binaries.
	# Actions are pinned to specific SHAs to reduce supply-chain risk. This workflow triggers on tag push events.

	permissions:
	contents: read
	packages: write # for GHCR push
	id-token: write # for Cosign Keyless (OIDC) Signing

	# Required secrets:
	# - DOCKER_HUB_USERNAME / DOCKER_HUB_ACCESS_TOKEN: push to Docker Hub
	# - GITHUB_TOKEN: used for GHCR login and OIDC keyless signing
	# - COSIGN_PRIVATE_KEY / COSIGN_PASSWORD / COSIGN_PUBLIC_KEY: for key-based signing

	on:
	push:
	tags:
	- "[0-9]+.[0-9]+.[0-9]+"
	- "[0-9]+.[0-9]+.[0-9]+-rc.[0-9]+"

	concurrency:
	group: ${{ github.ref }}
	cancel-in-progress: true

	jobs:
	pre-run:
	runs-on: ubuntu-latest
	permissions: write-all
	steps:
	- name: Configure AWS credentials
	uses: aws-actions/configure-aws-credentials@v6
	with:
	role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_ROLE_NAME }}
	role-duration-seconds: 3600
	aws-region: ${{ secrets.AWS_REGION }}

	- name: Verify AWS identity
	run: aws sts get-caller-identity

	- name: Start EC2 instances
	run: \|
	aws ec2 start-instances --instance-ids ${{ secrets.EC2_INSTANCE_ID_ARM_RUNNER }}
	aws ec2 start-instances --instance-ids ${{ secrets.EC2_INSTANCE_ID_AMD_RUNNER }}
	echo "EC2 instances started"


	release-arm:
	name: Build and Release (ARM64)
	runs-on: [self-hosted, linux, arm64, us-east-1]
	needs: [pre-run]
	if: >-
	${{
	needs.pre-run.result == 'success'
	}}
	# Job-level timeout to avoid runaway or stuck runs
	timeout-minutes: 120
	env:
	# Target images
	DOCKERHUB_IMAGE: docker.io/fosrl/${{ github.event.repository.name }}
	GHCR_IMAGE: ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}

	steps:
	- name: Checkout code
	uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

	- name: Monitor storage space
	run: \|
	THRESHOLD=75
	USED_SPACE=$(df / \| grep / \| awk '{ print $5 }' \| sed 's/%//g')
	echo "Used space: $USED_SPACE%"
	if [ "$USED_SPACE" -ge "$THRESHOLD" ]; then
	echo "Used space is below the threshold of 75% free. Running Docker system prune."
	echo y \| docker system prune -a
	else
	echo "Storage space is above the threshold. No action needed."
	fi

	- name: Log in to Docker Hub
	uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
	with:
	registry: docker.io
	username: ${{ secrets.DOCKER_HUB_USERNAME }}
	password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}

	- name: Extract tag name
	id: get-tag
	run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
	shell: bash

	- name: Update version in package.json
	run: \|
	TAG=${{ env.TAG }}
	sed -i "s/export const APP_VERSION = \".*\";/export const APP_VERSION = \"$TAG\";/" server/lib/consts.ts
	cat server/lib/consts.ts
	shell: bash

	- name: Check if release candidate
	id: check-rc
	run: \|
	TAG=${{ env.TAG }}
	if [[ "$TAG" == "-rc." ]]; then
	echo "IS_RC=true" >> $GITHUB_ENV
	else
	echo "IS_RC=false" >> $GITHUB_ENV
	fi
	shell: bash

	- name: Build and push Docker images (Docker Hub - ARM64)
	run: \|
	TAG=${{ env.TAG }}
	if [ "$IS_RC" = "true" ]; then
	make build-rc-arm tag=$TAG
	else
	make build-release-arm tag=$TAG
	fi
	echo "Built & pushed ARM64 images to: ${{ env.DOCKERHUB_IMAGE }}:${TAG}"
	shell: bash

	release-amd:
	name: Build and Release (AMD64)
	runs-on: [self-hosted, linux, x64, us-east-1]
	needs: [pre-run]
	if: >-
	${{
	needs.pre-run.result == 'success'
	}}
	# Job-level timeout to avoid runaway or stuck runs
	timeout-minutes: 120
	env:
	# Target images
	DOCKERHUB_IMAGE: docker.io/fosrl/${{ github.event.repository.name }}
	GHCR_IMAGE: ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}

	steps:
	- name: Checkout code
	uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

	- name: Monitor storage space
	run: \|
	THRESHOLD=75
	USED_SPACE=$(df / \| grep / \| awk '{ print $5 }' \| sed 's/%//g')
	echo "Used space: $USED_SPACE%"
	if [ "$USED_SPACE" -ge "$THRESHOLD" ]; then
	echo "Used space is below the threshold of 75% free. Running Docker system prune."
	echo y \| docker system prune -a
	else
	echo "Storage space is above the threshold. No action needed."
	fi

	- name: Log in to Docker Hub
	uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
	with:
	registry: docker.io
	username: ${{ secrets.DOCKER_HUB_USERNAME }}
	password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}

	- name: Extract tag name
	id: get-tag
	run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
	shell: bash

	- name: Update version in package.json
	run: \|
	TAG=${{ env.TAG }}
	sed -i "s/export const APP_VERSION = \".*\";/export const APP_VERSION = \"$TAG\";/" server/lib/consts.ts
	cat server/lib/consts.ts
	shell: bash

	- name: Check if release candidate
	id: check-rc
	run: \|
	TAG=${{ env.TAG }}
	if [[ "$TAG" == "-rc." ]]; then
	echo "IS_RC=true" >> $GITHUB_ENV
	else
	echo "IS_RC=false" >> $GITHUB_ENV
	fi
	shell: bash

	- name: Build and push Docker images (Docker Hub - AMD64)
	run: \|
	TAG=${{ env.TAG }}
	if [ "$IS_RC" = "true" ]; then
	make build-rc-amd tag=$TAG
	else
	make build-release-amd tag=$TAG
	fi
	echo "Built & pushed AMD64 images to: ${{ env.DOCKERHUB_IMAGE }}:${TAG}"
	shell: bash

	create-manifest:
	name: Create Multi-Arch Manifests
	runs-on: [self-hosted, linux, x64, us-east-1]
	needs: [release-arm, release-amd]
	if: >-
	${{
	needs.release-arm.result == 'success' &&
	needs.release-amd.result == 'success'
	}}
	timeout-minutes: 30
	steps:
	- name: Checkout code
	uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

	- name: Log in to Docker Hub
	uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
	with:
	registry: docker.io
	username: ${{ secrets.DOCKER_HUB_USERNAME }}
	password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}

	- name: Extract tag name
	id: get-tag
	run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
	shell: bash

	- name: Check if release candidate
	id: check-rc
	run: \|
	TAG=${{ env.TAG }}
	if [[ "$TAG" == "-rc." ]]; then
	echo "IS_RC=true" >> $GITHUB_ENV
	else
	echo "IS_RC=false" >> $GITHUB_ENV
	fi
	shell: bash

	- name: Create multi-arch manifests
	run: \|
	TAG=${{ env.TAG }}
	if [ "$IS_RC" = "true" ]; then
	make create-manifests-rc tag=$TAG
	else
	make create-manifests tag=$TAG
	fi
	echo "Created multi-arch manifests for tag: ${TAG}"
	shell: bash

	sign-and-package:
	name: Sign and Package
	runs-on: [self-hosted, linux, x64, us-east-1]
	needs: [release-arm, release-amd, create-manifest]
	if: >-
	${{
	needs.release-arm.result == 'success' &&
	needs.release-amd.result == 'success' &&
	needs.create-manifest.result == 'success'
	}}
	# Job-level timeout to avoid runaway or stuck runs
	timeout-minutes: 120
	env:
	# Target images
	DOCKERHUB_IMAGE: docker.io/fosrl/${{ github.event.repository.name }}
	GHCR_IMAGE: ghcr.io/${{ github.repository_owner }}/${{ github.event.repository.name }}

	steps:
	- name: Checkout code
	uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

	- name: Extract tag name
	id: get-tag
	run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV
	shell: bash

	- name: Install Go
	uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
	with:
	go-version: 1.26

	- name: Update version in package.json
	run: \|
	TAG=${{ env.TAG }}
	sed -i "s/export const APP_VERSION = \".*\";/export const APP_VERSION = \"$TAG\";/" server/lib/consts.ts
	cat server/lib/consts.ts
	shell: bash

	- name: Pull latest Gerbil version
	id: get-gerbil-tag
	run: \|
	LATEST_TAG=$(curl -s https://api.github.com/repos/fosrl/gerbil/tags \| jq -r '.[0].name')
	echo "LATEST_GERBIL_TAG=$LATEST_TAG" >> $GITHUB_ENV
	shell: bash

	- name: Pull latest Badger version
	id: get-badger-tag
	run: \|
	LATEST_TAG=$(curl -s https://api.github.com/repos/fosrl/badger/tags \| jq -r '.[0].name')
	echo "LATEST_BADGER_TAG=$LATEST_TAG" >> $GITHUB_ENV
	shell: bash

	- name: Build installer
	working-directory: install
	run: \|
	make go-build-release \
	PANGOLIN_VERSION=${{ env.TAG }} \
	GERBIL_VERSION=${{ env.LATEST_GERBIL_TAG }} \
	BADGER_VERSION=${{ env.LATEST_BADGER_TAG }}
	shell: bash

	- name: Upload artifacts from /install/bin
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
	with:
	name: install-bin
	path: install/bin/

	- name: Install skopeo + jq
	# skopeo: copy/inspect images between registries
	# jq: JSON parsing tool used to extract digest values
	run: \|
	sudo apt-get update -y
	sudo apt-get install -y skopeo jq
	skopeo --version
	shell: bash

	- name: Login to GHCR
	env:
	REGISTRY_AUTH_FILE: ${{ runner.temp }}/containers/auth.json
	run: \|
	mkdir -p "$(dirname "$REGISTRY_AUTH_FILE")"
	skopeo login ghcr.io -u "${{ github.actor }}" -p "${{ secrets.GITHUB_TOKEN }}"
	shell: bash

	- name: Copy tags from Docker Hub to GHCR
	# Mirror the already-built images (all architectures) to GHCR so we can sign them
	# Wait a bit for both architectures to be available in Docker Hub manifest
	env:
	REGISTRY_AUTH_FILE: ${{ runner.temp }}/containers/auth.json
	run: \|
	set -euo pipefail
	TAG=${{ env.TAG }}
	MAJOR_TAG=$(echo $TAG \| cut -d. -f1)
	MINOR_TAG=$(echo $TAG \| cut -d. -f1,2)

	echo "Waiting for multi-arch manifests to be ready..."
	sleep 30

	# Determine if this is an RC release
	IS_RC="false"
	if [[ "$TAG" == "-rc." ]]; then
	IS_RC="true"
	fi

	if [ "$IS_RC" = "true" ]; then
	echo "RC release detected - copying version-specific tags only"

	# SQLite OSS
	echo "Copying ${{ env.DOCKERHUB_IMAGE }}:${TAG} -> ${{ env.GHCR_IMAGE }}:${TAG}"
	skopeo copy --all --retry-times 3 \
	docker://$DOCKERHUB_IMAGE:$TAG \
	docker://$GHCR_IMAGE:$TAG

	# PostgreSQL OSS
	echo "Copying ${{ env.DOCKERHUB_IMAGE }}:postgresql-${TAG} -> ${{ env.GHCR_IMAGE }}:postgresql-${TAG}"
	skopeo copy --all --retry-times 3 \
	docker://$DOCKERHUB_IMAGE:postgresql-$TAG \
	docker://$GHCR_IMAGE:postgresql-$TAG

	# SQLite Enterprise
	echo "Copying ${{ env.DOCKERHUB_IMAGE }}:ee-${TAG} -> ${{ env.GHCR_IMAGE }}:ee-${TAG}"
	skopeo copy --all --retry-times 3 \
	docker://$DOCKERHUB_IMAGE:ee-$TAG \
	docker://$GHCR_IMAGE:ee-$TAG

	# PostgreSQL Enterprise
	echo "Copying ${{ env.DOCKERHUB_IMAGE }}:ee-postgresql-${TAG} -> ${{ env.GHCR_IMAGE }}:ee-postgresql-${TAG}"
	skopeo copy --all --retry-times 3 \
	docker://$DOCKERHUB_IMAGE:ee-postgresql-$TAG \
	docker://$GHCR_IMAGE:ee-postgresql-$TAG
	else
	echo "Regular release detected - copying all tags (latest, major, minor, full version)"

	# SQLite OSS - all tags
	for TAG_SUFFIX in "latest" "$MAJOR_TAG" "$MINOR_TAG" "$TAG"; do
	echo "Copying ${{ env.DOCKERHUB_IMAGE }}:${TAG_SUFFIX} -> ${{ env.GHCR_IMAGE }}:${TAG_SUFFIX}"
	skopeo copy --all --retry-times 3 \
	docker://$DOCKERHUB_IMAGE:$TAG_SUFFIX \
	docker://$GHCR_IMAGE:$TAG_SUFFIX
	done

	# PostgreSQL OSS - all tags
	for TAG_SUFFIX in "latest" "$MAJOR_TAG" "$MINOR_TAG" "$TAG"; do
	echo "Copying ${{ env.DOCKERHUB_IMAGE }}:postgresql-${TAG_SUFFIX} -> ${{ env.GHCR_IMAGE }}:postgresql-${TAG_SUFFIX}"
	skopeo copy --all --retry-times 3 \
	docker://$DOCKERHUB_IMAGE:postgresql-$TAG_SUFFIX \
	docker://$GHCR_IMAGE:postgresql-$TAG_SUFFIX
	done

	# SQLite Enterprise - all tags
	for TAG_SUFFIX in "latest" "$MAJOR_TAG" "$MINOR_TAG" "$TAG"; do
	echo "Copying ${{ env.DOCKERHUB_IMAGE }}:ee-${TAG_SUFFIX} -> ${{ env.GHCR_IMAGE }}:ee-${TAG_SUFFIX}"
	skopeo copy --all --retry-times 3 \
	docker://$DOCKERHUB_IMAGE:ee-$TAG_SUFFIX \
	docker://$GHCR_IMAGE:ee-$TAG_SUFFIX
	done

	# PostgreSQL Enterprise - all tags
	for TAG_SUFFIX in "latest" "$MAJOR_TAG" "$MINOR_TAG" "$TAG"; do
	echo "Copying ${{ env.DOCKERHUB_IMAGE }}:ee-postgresql-${TAG_SUFFIX} -> ${{ env.GHCR_IMAGE }}:ee-postgresql-${TAG_SUFFIX}"
	skopeo copy --all --retry-times 3 \
	docker://$DOCKERHUB_IMAGE:ee-postgresql-$TAG_SUFFIX \
	docker://$GHCR_IMAGE:ee-postgresql-$TAG_SUFFIX
	done
	fi

	echo "All images copied successfully to GHCR!"
	shell: bash

	- name: Login to GitHub Container Registry (for cosign)
	uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
	with:
	registry: ghcr.io
	username: ${{ github.actor }}
	password: ${{ secrets.GITHUB_TOKEN }}

	- name: Install cosign
	# cosign is used to sign container images using keyless (OIDC) signing
	uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6 # v4.1.2
	with:
	cosign-release: v3.0.6

	- name: Sign (GHCR, keyless)
	# Sign each GHCR image by digest using keyless (OIDC) signing via Sigstore/Rekor.
	# Signatures are stored in the registry alongside the image.
	env:
	TAG: ${{ env.TAG }}
	COSIGN_YES: "true"
	run: \|
	set -euo pipefail

	# Determine if this is an RC release
	IS_RC="false"
	if [[ "$TAG" == "-rc." ]]; then
	IS_RC="true"
	fi

	# Define image variants to sign
	if [ "$IS_RC" = "true" ]; then
	echo "RC release - signing version-specific tags only"
	IMAGE_TAGS=(
	"${TAG}"
	"postgresql-${TAG}"
	"ee-${TAG}"
	"ee-postgresql-${TAG}"
	)
	else
	echo "Regular release - signing all tags"
	MAJOR_TAG=$(echo $TAG \| cut -d. -f1)
	MINOR_TAG=$(echo $TAG \| cut -d. -f1,2)
	IMAGE_TAGS=(
	"latest" "$MAJOR_TAG" "$MINOR_TAG" "$TAG"
	"postgresql-latest" "postgresql-$MAJOR_TAG" "postgresql-$MINOR_TAG" "postgresql-$TAG"
	"ee-latest" "ee-$MAJOR_TAG" "ee-$MINOR_TAG" "ee-$TAG"
	"ee-postgresql-latest" "ee-postgresql-$MAJOR_TAG" "ee-postgresql-$MINOR_TAG" "ee-postgresql-$TAG"
	)
	fi

	FAILED_TAGS=()
	SUCCESSFUL_TAGS=()

	for IMAGE_TAG in "${IMAGE_TAGS[@]}"; do
	echo "Processing ${GHCR_IMAGE}:${IMAGE_TAG}"
	TAG_FAILED=false

	(
	set -e
	DIGEST="$(skopeo inspect --retry-times 3 docker://${GHCR_IMAGE}:${IMAGE_TAG} \| jq -r '.Digest')"
	REF="${GHCR_IMAGE}@${DIGEST}"
	echo "Resolved digest: ${REF}"

	echo "==> cosign sign (keyless) --recursive ${REF}"
	cosign sign --recursive "${REF}"
	) \|\| TAG_FAILED=true

	if [ "$TAG_FAILED" = "true" ]; then
	echo "⚠️ WARNING: Failed to sign ${GHCR_IMAGE}:${IMAGE_TAG}"
	FAILED_TAGS+=("${GHCR_IMAGE}:${IMAGE_TAG}")
	else
	echo "✓ Successfully signed ${GHCR_IMAGE}:${IMAGE_TAG}"
	SUCCESSFUL_TAGS+=("${GHCR_IMAGE}:${IMAGE_TAG}")
	fi
	done

	echo ""
	echo "=========================================="
	echo "Sign Summary"
	echo "=========================================="
	echo "Successful: ${#SUCCESSFUL_TAGS[@]}"
	echo "Failed: ${#FAILED_TAGS[@]}"

	if [ ${#FAILED_TAGS[@]} -gt 0 ]; then
	echo "Failed tags:"
	for tag in "${FAILED_TAGS[@]}"; do
	echo " - $tag"
	done
	echo "⚠️ WARNING: Some tags failed to sign, but continuing anyway"
	else
	echo "✓ All images signed successfully!"
	fi
	shell: bash

	post-run:
	needs: [pre-run, release-arm, release-amd, create-manifest, sign-and-package]
	if: >-
	${{
	always() &&
	needs.pre-run.result == 'success' &&
	(needs.release-arm.result == 'success' \|\| needs.release-arm.result == 'skipped' \|\| needs.release-arm.result == 'failure') &&
	(needs.release-amd.result == 'success' \|\| needs.release-amd.result == 'skipped' \|\| needs.release-amd.result == 'failure') &&
	(needs.create-manifest.result == 'success' \|\| needs.create-manifest.result == 'skipped' \|\| needs.create-manifest.result == 'failure') &&
	(needs.sign-and-package.result == 'success' \|\| needs.sign-and-package.result == 'skipped' \|\| needs.sign-and-package.result == 'failure')
	}}
	runs-on: ubuntu-latest
	permissions: write-all
	steps:
	- name: Configure AWS credentials
	uses: aws-actions/configure-aws-credentials@v6
	with:
	role-to-assume: arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID }}:role/${{ secrets.AWS_ROLE_NAME }}
	role-duration-seconds: 3600
	aws-region: ${{ secrets.AWS_REGION }}

	- name: Verify AWS identity
	run: aws sts get-caller-identity

	- name: Stop EC2 instances
	run: \|
	aws ec2 stop-instances --instance-ids ${{ secrets.EC2_INSTANCE_ID_ARM_RUNNER }}
	aws ec2 stop-instances --instance-ids ${{ secrets.EC2_INSTANCE_ID_AMD_RUNNER }}
	echo "EC2 instances stopped"

Navigation Menu

Merge pull request #3765 from fosrl/dev #129

Workflow file

Merge pull request #3765 from fosrl/dev #129

Uh oh!

Workflow file for this run