Skip to content
Rotki Development Builds

Rotki Development Builds #1889

Sign in to view logs

Rotki Development Builds

Rotki Development Builds #1889

Workflow file for this run

.github/workflows/rotki_dev_builds.yml at 20c089d
name: Rotki Development Builds
on:
schedule:
- cron: "0 0 * * *"
push:
branches:
- build
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
cancel-in-progress: true
env:
PYTHON_MACOS: 11
permissions: { }
jobs:
build-linux:
name: Build linux binary
if: ${{ github.repository == 'rotki/rotki' || github.event_name != 'schedule' }}
env:
CI: true
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
runs-on: ubuntu-22.04
timeout-minutes: 30 # observed max 13m
permissions:
contents: read
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 0
persist-credentials: false
- name: Load env
uses: rotki/action-env@49f59e650cfdc8796c4ac6533ec044b190d83f9a # v4.0.0
with:
env_file: .github/.env.ci
- name: Setup python
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: ${{ env.PYTHON_VERSION }}
freethreaded: true
- name: Install uv
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
with:
enable-cache: false
version: ${{ env.UV_VERSION }}
cache-dependency-glob: "uv.lock"
- name: Setup pnpm
uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
with:
package_json_file: frontend/package.json
- name: Setup node
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version-file: 'frontend/.nvmrc'
- name: Package
id: packaging
run: |
echo "::group::packaging dependencies"
uv sync --locked --no-dev --group packaging
echo "::endgroup::"
uv run --no-sync package.py --build full
- name: Upload files
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: linux-app
path: |
dist/*.AppImage
dist/*.AppImage.sha512
build-macos:
name: 'Build macOS binary ${{ matrix.os.arch }}'
if: ${{ github.repository == 'rotki/rotki' || github.event_name != 'schedule' }}
strategy:
fail-fast: false
matrix:
os:
- runner: macos-15
arch: 'arm64'
target: 11
platform: macosx-11.0-arm64
uv-platform: aarch64-apple-darwin
- runner: macos-15-intel
arch: 'x86_64'
target: 11
platform: macosx-11.0-x86_64
uv-platform: x86_64-apple-darwin
env:
CI: true
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
ARCHFLAGS: -arch ${{ matrix.os.arch }}
_PYTHON_HOST_PLATFORM: ${{ matrix.os.platform }}
MACOSX_DEPLOYMENT_TARGET: ${{ matrix.os.target }}
# uv otherwise picks wheels for the runner's macOS, e.g. a numpy that needs macOS 14
UV_TARGET_PLATFORM: ${{ matrix.os.uv-platform }}
# pyyaml builds from source and would link Homebrew's libyaml, built for the runner's macOS
PYYAML_FORCE_LIBYAML: 0
CMAKE_OSX_ARCHITECTURES: ${{ matrix.os.arch }}
NODE_OPTIONS: --max-old-space-size=4096
environment: macos_sign
runs-on: ${{ matrix.os.runner }}
# observed max: arm64 18m, x86_64 29m. x86_64 used to hang forever in
# PyInstaller's codesign when the build keychain auto-locked after 300 s;
# package.py now raises that limit.
timeout-minutes: 45
permissions:
contents: read
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 0
persist-credentials: false
- name: Load env
uses: rotki/action-env@49f59e650cfdc8796c4ac6533ec044b190d83f9a # v4.0.0
with:
env_file: .github/.env.ci
- name: Add rust target for building deps
if: ${{ matrix.os.arch == 'arm64' }}
run: rustup target add aarch64-apple-darwin
- name: Cache python pkg
uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
with:
path: ~/python*.pkg
key: ${{ runner.os }}-python-${{ env.PYTHON_VERSION }}-${{ env.PYTHON_MACOS }}-${{ matrix.os.arch }}
- name: Set up python
run: packaging/setup-macos-python.sh "${PYTHON_VERSION}" "${PYTHON_MACOS}"
- name: Install uv
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
with:
enable-cache: false
version: ${{ env.UV_VERSION }}
cache-dependency-glob: "uv.lock"
- name: Setup pnpm
uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
with:
package_json_file: frontend/package.json
- name: Setup node
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version-file: 'frontend/.nvmrc'
- name: Package
id: packaging
run: |
echo "::group::packaging dependencies"
uv sync --locked --no-dev --group packaging --python-platform "${UV_TARGET_PLATFORM}"
echo "::endgroup::"
uv run --no-sync ./package.py --build full
env:
CERTIFICATE_OSX_APPLICATION: ${{ secrets.CERTIFICATE_OSX_APPLICATION }}
CSC_KEY_PASSWORD: ${{ secrets.CERTIFICATE_PASSWORD }}
IDENTITY: ${{ secrets.IDENTITY }}
APPLEID: ${{ secrets.APPLEID }}
APPLEIDPASS: ${{ secrets.APPLEIDPASS }}
- name: Upload files (${{ matrix.os.arch }})
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: macos-app-${{ matrix.os.arch }}
path: |
dist/rotki-darwin_*.dmg
dist/rotki-darwin_*.dmg.sha512
- name: Upload colibri files
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: macos-colibri-${{ matrix.os.arch }}
path: |
dist/colibri
build-windows:
name: Build windows binary
if: ${{ github.repository == 'rotki/rotki' || github.event_name != 'schedule' }}
env:
CI: true
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
runs-on: windows-latest
timeout-minutes: 45 # observed max 24m
environment: windows_sign
permissions:
contents: read
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 0
persist-credentials: false
- name: Load env
uses: rotki/action-env@49f59e650cfdc8796c4ac6533ec044b190d83f9a # v4.0.0
with:
env_file: .github/.env.ci
- name: Set up python
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: ${{ env.PYTHON_VERSION }}
freethreaded: true
- name: Install uv
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
with:
enable-cache: false
version: ${{ env.UV_VERSION }}
cache-dependency-glob: "uv.lock"
- name: Setup pnpm
uses: pnpm/action-setup@ea17c68df8912ef543352723c149a84f56e3d413 # v6.1.0
with:
package_json_file: frontend/package.json
- name: Setup node
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version-file: 'frontend/.nvmrc'
- name: Build rotki
id: packaging
run: |
echo ::group::"packaging dependencies"
uv sync --locked --no-dev --group packaging
echo ::endgroup::
uv run --no-sync package.py --build full
shell: powershell
- name: Upload files
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: windows-app
path: |
dist\rotki-win32*.exe
dist\rotki-win32*.exe.sha512
build-docker:
name: Build docker images
if: ${{ github.repository == 'rotki/rotki' || github.event_name != 'schedule' }}
runs-on: ${{ matrix.platform.runner }}
timeout-minutes: 30 # observed max 9m warm; extra air for a cold build cache
environment: docker
permissions:
contents: read
strategy:
fail-fast: false
matrix:
platform:
- name: linux/amd64
runner: ubuntu-24.04
- name: linux/arm64
runner: ubuntu-24.04-arm
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 0
persist-credentials: false
- name: Prepare
id: prepare
run: |
platform=${{ matrix.platform.name }}
echo "PLATFORM_PAIR=${platform//\//-}" >> "$GITHUB_ENV"
- name: Docker meta
id: meta
uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0
with:
images: |
${{ github.repository }}
- name: Login to DockerHub
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
with:
username: ${{ secrets.DOCKERHUB_USER }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
- name: Rotki Version
id: rotki_version
run: |
ROTKI_VERSION=$(cat .bumpversion.cfg | grep 'current_version = ' | sed -n -e 's/current_version = //p')
POSTFIX=$(if git describe --tags --exact-match "$REVISION" &>/dev/null; then echo ''; else echo '.dev'; fi)
ROTKI_VERSION=${ROTKI_VERSION}${POSTFIX}$(date +'%Y%m%d')
echo "version=${ROTKI_VERSION}" >> "$GITHUB_OUTPUT"
- name: Build and push by digest
id: build
uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0
with:
context: .
file: ./Dockerfile
platforms: ${{ matrix.platform.name }}
provenance: mode=max
sbom: true
build-args: |
REVISION=${{ github.sha }}
ROTKI_VERSION=${{ steps.rotki_version.outputs.version }}
labels: ${{ steps.meta.outputs.labels }}
tags: ${{ github.repository }}
outputs: type=image,"name=${{ github.repository }}",push-by-digest=true,name-canonical=true,push=true
- name: Export digest
env:
TEMP: ${{ runner.temp }}
DIGEST: ${{ steps.build.outputs.digest }}
run: |
mkdir -p "${TEMP}/digests"
digest="${DIGEST}"
touch "${TEMP}/digests/${digest#sha256:}"
- name: Upload digest
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: digests-${{ env.PLATFORM_PAIR }}
path: ${{ runner.temp }}/digests/*
if-no-files-found: error
retention-days: 1
docker-merge:
runs-on: ubuntu-latest
timeout-minutes: 15 # observed max 36s; the prune step lives here
needs:
- build-docker
environment: docker
steps:
- name: Download digests
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
path: ${{ runner.temp }}/digests
pattern: digests-*
merge-multiple: true
- name: Login to Docker Hub
uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0
with:
username: ${{ secrets.DOCKERHUB_USER }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
- name: Docker meta
id: meta
uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0
with:
images: |
${{ github.repository }}
tags: |
type=schedule,pattern=nightly
type=schedule,pattern=nightly-{{date 'YYYYMMDD'}}
type=edge,branch=build
type=sha,prefix=edge-,format=short,enable=${{ github.ref == 'refs/heads/build' }}
- name: Record the digests these tags are about to replace
if: ${{ github.repository == 'rotki/rotki' }}
env:
REPOSITORY: ${{ github.repository }}
TAGS: ${{ steps.meta.outputs.tags }}
run: |
set -euo pipefail
# Pushing a tag that already exists leaves whatever it pointed at
# before untagged, and an untagged manifest can no longer be listed,
# so the prune step would never see it again. Record what each tag
# resolves to now, while it is still reachable; the prune step then
# deletes the ones nothing references any more. This is the re-run
# case: a second scheduled run on the same day reuses
# nightly-YYYYMMDD, and a rebuild of the same commit reuses
# edge-<sha>. The repo is public, so resolving a tag needs no auth.
# "<tag> <index-digest> <child,child,...>" per line. The children are
# recorded too: a rebuild can produce a new index over byte-identical
# arch manifests, and deleting the old index must not take a child the
# newly pushed one still references.
: > "${RUNNER_TEMP}/superseded.txt"
while read -r ref; do
[ -n "$ref" ] || continue
tag="${ref##*:}"
# Only a 404 is benign here (the tag does not exist yet). Anything
# else -- a 429 from this unauthenticated endpoint, a 5xx, a DNS
# failure -- would silently lose the record of a manifest the push
# is about to orphan, which is the exact leak this step prevents.
code="$(curl -sS -o "${RUNNER_TEMP}/tag.json" -w '%{http_code}' \
"https://hub.docker.com/v2/repositories/${REPOSITORY}/tags/${tag}/" || true)"
case "$code" in
200) ;;
404) echo "no current ${tag}, nothing it can supersede"; continue ;;
*) echo "cannot resolve ${tag}: HTTP ${code}" >&2; exit 1 ;;
esac
jq -r --arg tag "$tag" '
(.digest // empty) as $d
| select($d != "")
| "\($tag) \($d) \([.images[]?.digest // empty] | join(","))"
' "${RUNNER_TEMP}/tag.json" >> "${RUNNER_TEMP}/superseded.txt"
done <<< "$TAGS"
cat "${RUNNER_TEMP}/superseded.txt"
- name: Create manifest list and push
working-directory: ${{ runner.temp }}/digests
env:
REPOSITORY: ${{ github.repository }}
run: |
docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
$(printf "${REPOSITORY}@sha256:%s " *)
- name: Inspect image
env:
REPOSITORY: ${{ github.repository }}
VERSION: ${{ steps.meta.outputs.version }}
run: |
docker buildx imagetools inspect "${REPOSITORY}:${VERSION}"
- name: Prune old dated dev-build images
if: ${{ github.repository == 'rotki/rotki' }}
# Backstop for anything the script does not anticipate. Every failure it
# does anticipate is already a warning with an explanation, so reaching
# this is itself a sign the script needs a case adding.
continue-on-error: true
env:
DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
REPOSITORY: ${{ github.repository }}
KEEP: 2
run: |
set -euo pipefail
# Keep only the newest $KEEP dated dev-build images per family
# (nightly-YYYYMMDD, edge-<short-sha>) and delete the rest, image and
# all. A tag delete via the hub.docker.com API only drops the tag and
# leaves the multi-arch index + its arch/attestation children as
# untagged leftovers (Docker Hub does not garbage-collect them, and its
# image-management API is unreachable with a token on this account).
# The OCI registry manifest DELETE, however, works: deleting an index
# digest removes the index, every child manifest it references, and the
# tag pointing at it. So we list tags via the Hub API but reclaim via
# the registry.
#
# Token hygiene throughout: secrets are read from the environment (never
# on an argv), derived tokens are masked in the log, and every auth
# header/credential is handed to curl through a config on a file
# descriptor (printf is a shell builtin) so it never lands on argv or
# disk. The rolling `nightly`/`edge` tags and all release tags do not
# match the dated patterns and are never selected.
# Reclaiming storage is housekeeping, not part of publishing the image:
# the push has already succeeded by the time we get here. So every
# failure below is reported as a warning and the step still exits 0 --
# a stuck prune must not turn the nightly red. The cost is that it can
# go unnoticed, so each warning says plainly that storage keeps growing
# until it is fixed.
# Every deliberate stop below is an `exit 0`, so a non-zero exit means
# the script hit something it does not handle. `continue-on-error`
# would leave that green and silent, which is how a prune stops working
# without anyone noticing.
report_crash() {
local rc=$?
[ "$rc" -eq 0 ] && return 0
echo "::warning title=Docker prune crashed::The prune script exited unexpectedly with status ${rc}, so it did not finish and nothing after that point ran. This is a bug in the step itself, not a Docker Hub problem."
}
trap report_crash EXIT
# Decode a JWT payload (base64url, padding stripped) to stdout.
jwt_payload() {
local p="${1#*.}"
p="${p%%.*}"
case $(( ${#p} % 4 )) in
2) p="${p}==" ;;
3) p="${p}=" ;;
esac
printf '%s' "$p" | tr '_-' '/+' | base64 -d 2>/dev/null
}
# --- Hub JWT: lists the tags and their index digests ---
hub_jwt="$(jq -nc '{username: env.DOCKERHUB_USER, password: env.DOCKERHUB_TOKEN}' \
| curl -fsS -H 'Content-Type: application/json' -d @- \
https://hub.docker.com/v2/users/login \
| jq -r '.token')" || true
if [ -z "$hub_jwt" ] || [ "$hub_jwt" = 'null' ]; then
echo '::warning title=Docker prune skipped::Docker Hub login failed, so nothing was pruned this run and untagged images keep accumulating. Check the DOCKERHUB_USER/DOCKERHUB_TOKEN secrets.'
exit 0
fi
echo "::add-mask::${hub_jwt}"
# EVERY tag in the repo, as "<last_updated> <name> <index-digest>" per
# line. All of them, not just the first page: the collateral guard
# below can only protect a tag it has seen, so a release tag left off
# by pagination would be one we could delete by accident. NOTE: Docker
# Hub inverts the usual convention -- ordering=last_updated returns
# NEWEST first (a leading '-' gives oldest); do not "fix" it to
# -last_updated. The ISO 8601 timestamp sorts lexically the same as
# chronologically.
# Two record kinds per page:
# T <last_updated> <name> <index-digest> -- candidates come from these
# R <digest> <name> -- <digest> is referenced by <name>
# An R row is emitted for a tag's index AND for every child manifest it
# lists, because deleting an index takes its children with it. Roughly
# a third of this repo's tags carry a null top-level digest (older
# single-arch releases); those are skipped as candidates but still
# contribute their children as references.
declare -a lines=() refs=() kids=()
url="https://hub.docker.com/v2/repositories/${REPOSITORY}/tags/?page_size=100&ordering=last_updated"
for _page in $(seq 20); do
[ -n "$url" ] || break
body="$(curl -fsS \
--config <(printf 'header = "Authorization: JWT %s"\n' "$hub_jwt") "$url")"
# jq's status is checked explicitly: inside <(...) it would be
# discarded, and a page that silently parsed to nothing is a tag the
# guard cannot see.
rows="$(jq -r '
.results[] as $t
| ( "T \($t.last_updated) \($t.name) \($t.digest // "-")" ),
( select($t.digest != null) | "R \($t.digest) \($t.name)" ),
( $t.images[]? | select(.digest != null) | "R \(.digest) \($t.name)" ),
( $t.images[]? | select(.digest != null) | "C \($t.name) \(.digest)" )
' <<< "$body")"
mapfile -t -O "${#lines[@]}" lines < <(grep '^T ' <<< "$rows" | cut -d' ' -f2-)
mapfile -t -O "${#refs[@]}" refs < <(grep '^R ' <<< "$rows" | cut -d' ' -f2-)
mapfile -t -O "${#kids[@]}" kids < <(grep '^C ' <<< "$rows" | cut -d' ' -f2-)
url="$(jq -r '.next // empty' <<< "$body")"
done
# Both of these mean the collateral guard below is working from an
# incomplete picture, and a tag it cannot see is a tag it could delete
# by accident. Stop, but without failing the job.
if [ -n "$url" ]; then
echo '::warning title=Docker prune skipped::The tag listing did not terminate within 20 pages, so the guard that protects release tags is incomplete and nothing was pruned this run.'
exit 0
fi
if [ "${#lines[@]}" -eq 0 ]; then
echo '::warning title=Docker prune skipped::The tag listing came back empty, so nothing was pruned this run.'
exit 0
fi
echo "listed ${#lines[@]} tags, ${#refs[@]} digest references"
# digest -> " name name ", covering indexes and children alike.
declare -A referenced_by=()
for r in "${refs[@]}"; do
referenced_by["${r%% *}"]="${referenced_by[${r%% *}]:-} ${r#* }"
done
# tag name -> " child child ", so a candidate's children can be checked
# against the references above.
declare -A children_of=()
for c in "${kids[@]}"; do
children_of["${c%% *}"]="${children_of[${c%% *}]:-} ${c#* }"
done
# Select stale dated tags (all but the newest $KEEP per family), keeping
# each tag paired with its index digest. Patterns are anchored on the
# name field (single spaces around it) so the timestamp/digest cannot
# match.
declare -a stale_names=() stale_digests=() stale_kinds=() stale_kids=()
# Names whose references do not count as protection. A superseded
# candidate is deliberately NOT in here: its name now belongs to the
# freshly pushed tag, which is very much being kept.
own_names=''
for pattern in ' nightly-[0-9]{8} ' ' edge-[0-9a-f]{7,} '; do
while read -r _ts name digest; do
[ -n "$name" ] || continue
[ "$digest" != '-' ] || { echo "skip ${name}: no index digest listed"; continue; }
stale_names+=("$name"); stale_digests+=("$digest")
stale_kinds+=('tag'); stale_kids+=("${children_of[$name]:-}")
# Only a live tag being pruned may discount its own references.
own_names="${own_names} ${name}"
done < <(printf '%s\n' "${lines[@]}" | grep -E "$pattern" | sort -r | tail -n "+$((KEEP + 1))")
done
# Plus the digests this run's own push superseded, recorded before the
# push because they are unlistable afterwards. Only the dated families
# qualify: a rolling tag's predecessor is always still held by its own
# dated tag, so taking rolling tags here would add a candidate that no
# pattern ever vouched for.
if [ -f "${RUNNER_TEMP}/superseded.txt" ]; then
while read -r tag digest kids; do
[ -n "$digest" ] || continue
case " ${tag} " in
' nightly-'[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]' ') ;;
' edge-'*) [[ ${tag#edge-} =~ ^[0-9a-f]{7,}$ ]] || continue ;;
*) continue ;;
esac
stale_names+=("$tag"); stale_digests+=("$digest")
stale_kinds+=('superseded'); stale_kids+=("${kids//,/ }")
done < "${RUNNER_TEMP}/superseded.txt"
fi
if [ "${#stale_digests[@]}" -eq 0 ]; then
echo 'No stale dated dev-build images to prune.'
exit 0
fi
# Registry token with delete scope, minted only now that there is work
# to do: these expire in about five minutes, and the tag listing above
# can take several requests.
reg_token="$(curl -fsSL \
--config <(printf 'user = "%s:%s"\n' "$DOCKERHUB_USER" "$DOCKERHUB_TOKEN") \
"https://auth.docker.io/token?service=registry.docker.io&scope=repository:${REPOSITORY}:pull,push,delete" \
| jq -r '.token')"
if [ -z "$reg_token" ] || [ "$reg_token" = 'null' ]; then
echo '::warning title=Docker prune skipped::Could not mint a registry token, so nothing was pruned this run.'
exit 0
fi
echo "::add-mask::${reg_token}"
# What the registry ACTUALLY granted, which is not what we asked for.
# auth.docker.io answers 200 with a valid token even when it cannot
# authenticate the caller at all: it silently narrows the scope rather
# than failing, so neither curl's -f nor the emptiness check above can
# see a credential or permission problem. Left unreported, the only
# symptom is a bare 401 on the first DELETE, which looks like a
# transient registry error and is not.
reg_access="$(jwt_payload "$reg_token" \
| jq -c '[.access[]? | select(.type == "repository" and .name == env.REPOSITORY)] | first // {}' \
2>/dev/null || true)"
[ -n "$reg_access" ] || reg_access='{}'
reg_actions="$(jq -r '[.actions[]?] | sort | join(",")' <<< "$reg_access" 2>/dev/null || true)"
echo "registry token grants [${reg_actions:-unreadable}] on ${REPOSITORY}"
# A token issued to nobody carries the anonymous pull-rate parameters.
# That is the fingerprint that separates "our credentials never
# arrived" from "our credentials arrived and delete was refused" --
# the two produce an identical 401 at the DELETE.
if [ -z "$reg_actions" ]; then
echo '::warning title=Docker prune: token scope unreadable::Could not read the grant out of the registry token, so this run cannot say in advance whether deletes will work. That is a defect in this check rather than evidence of a credential problem; the deletes below may well succeed.'
elif [ -n "$(jq -r '.parameters.pull_limit // empty' <<< "$reg_access" 2>/dev/null || true)" ]; then
echo "::warning title=Docker prune: credentials did not authenticate::The registry token came back anonymous (it carries the anonymous pull-rate parameters and grants [${reg_actions:-none}]). DOCKERHUB_USER/DOCKERHUB_TOKEN did not reach the auth service in usable form, so every delete below will fail with 401. Note the Hub login earlier in this step passes its credentials as a JSON body while this call passes them through a curl config, so a secret containing a quote or backslash breaks here and only here."
elif [[ ",${reg_actions}," != *,delete,* ]]; then
echo "::warning title=Docker prune: delete not granted::Authentication succeeded but the registry granted only [${reg_actions:-none}], not delete, so every delete below will fail with 401. The PAT's own scopes are not the whole story: on an organisation repository the account's role caps the grant, and deleting manifests needs admin on the repository -- a read/write member gets [pull,push] no matter what the PAT says."
fi
# Three independent layers stand between a candidate and an
# irreversible delete, so that no single one has to be perfect:
# 1. the name matched a dated pattern (already enforced above)
# 2. the tag still resolves the way we think it does (per candidate)
# 3. neither the index nor any child it carries is referenced by a
# tag we are keeping (the listing)
# Layer 3 is the only one that catches a release cut from the same
# commit as a dev build, where v<x.y.z> and edge-<sha> land on one
# index. That is a permanent skip, and correctly so.
declare -a done_digests=() failed_names=()
first_reason=''
for i in "${!stale_digests[@]}"; do
name="${stale_names[$i]}"
digest="${stale_digests[$i]}"
kind="${stale_kinds[$i]}"
# Two candidates can share one index. The first delete already took
# it, and a second attempt would 404 and fail the job.
case " ${done_digests[*]} " in
*" ${digest} "*)
echo "skip ${name} (${digest}): already deleted"
continue
;;
esac
# Layer 2. A live stale tag must still point where the listing said;
# a superseded one must NOT, since the push moved it. Either way a
# mismatch means something changed under us, so we bail rather than
# guess.
code="$(curl -sS -o "${RUNNER_TEMP}/now.json" -w '%{http_code}' \
--config <(printf 'header = "Authorization: JWT %s"\n' "$hub_jwt") \
"https://hub.docker.com/v2/repositories/${REPOSITORY}/tags/${name}/" || true)"
case "$code" in
200) now="$(jq -r '.digest // "-"' "${RUNNER_TEMP}/now.json")" ;;
404) now='-' ;;
*)
echo "cannot re-resolve ${name}: HTTP ${code}, left alone (without a confirmed digest the safety check cannot run)"
failed_names+=("$name")
[ -n "$first_reason" ] || first_reason="HTTP ${code} re-resolving the tag on Docker Hub, so its digest could not be confirmed."
continue
;;
esac
if [ "$kind" = 'tag' ] && [ "$now" != "$digest" ]; then
echo "skip ${name} (${digest}): tag now resolves to ${now}"
continue
fi
if [ "$kind" = 'superseded' ] && [ "$now" = "$digest" ]; then
echo "skip ${name} (${digest}): still the current digest, not superseded"
continue
fi
# Layer 3, over the index and every child it would take with it.
protected=''
for d in "$digest" ${stale_kids[$i]}; do
for other in ${referenced_by[$d]:-}; do
case " ${own_names} " in *" ${other} "*) continue ;; esac
case " ${protected} " in *" ${other} "*) continue ;; esac
protected="${protected} ${other}"
done
done
if [ -n "$protected" ]; then
echo "skip ${name} (${digest}): shared with kept tag(s)${protected}"
continue
fi
# Tolerate curl's own exit status so a transport failure reports as
# code 000 through the message below, rather than killing the step
# with no indication of which digest it died on. The body and the
# headers are kept: the registry names the reason in an OCI error
# object and in the WWW-Authenticate challenge, and without them a
# 401 is indistinguishable from any other kind of refusal.
# Truncated first: curl may write neither file when it never gets a
# response, and stale content from the previous candidate would then
# annotate this one with someone else's error.
: > "${RUNNER_TEMP}/delete-body.json"
: > "${RUNNER_TEMP}/delete-head.txt"
code="$(curl -sS -o "${RUNNER_TEMP}/delete-body.json" -D "${RUNNER_TEMP}/delete-head.txt" \
-w '%{http_code}' -X DELETE \
--config <(printf 'header = "Authorization: Bearer %s"\n' "$reg_token") \
"https://registry-1.docker.io/v2/${REPOSITORY}/manifests/${digest}" || true)"
if [ "$code" = '202' ]; then
done_digests+=("$digest")
echo "deleted ${REPOSITORY}:${name} (${digest})"
continue
fi
if [ "$code" = '404' ]; then
# Already reclaimed, by an earlier run or by a candidate in this one
# that shared the index. Nothing is left behind, so this is not a
# failure and must not be counted as one.
done_digests+=("$digest")
echo "skip ${name} (${digest}): already gone from the registry"
continue
fi
detail="$(jq -r '[.errors[]? | "\(.code // "?"): \(.message // "?")"] | join("; ")' \
"${RUNNER_TEMP}/delete-body.json" 2>/dev/null || true)"
challenge="$(sed -n 's/^[Ww][Ww][Ww]-[Aa]uthenticate: *//p' \
"${RUNNER_TEMP}/delete-head.txt" 2>/dev/null | tr -d '\r' || true)"
case "$code" in
401)
reason="the registry rejected the token. It was granted [${reg_actions:-unreadable}] on ${REPOSITORY}, and a delete needs 'delete' in that list. Either the credentials did not authenticate (see the token warning earlier in this step) or the account lacks delete rights on the repository -- on an org repo that means admin, which a read/write member does not have regardless of the PAT's own scopes."
;;
403)
reason='authenticated, but this account is not allowed to delete in this repository. Check its role on the organisation and on the repository.'
;;
405|501)
reason='the registry says manifest deletion is not allowed here, which would mean Docker Hub withdrew the registry DELETE this whole step depends on. That needs a different approach, not a retry.'
;;
429)
reason='rate limited by Docker Hub. Harmless on its own; the next run retries.'
;;
000)
reason='no HTTP response at all (DNS, TLS or connection failure). Transient, the next run retries.'
;;
5*)
reason='Docker Hub server error. Transient, the next run retries.'
;;
*)
reason='unexpected status.'
;;
esac
# Plain log line, not an annotation: GitHub keeps only the first ten
# warnings per step, and when every candidate fails -- the case this
# reporting exists for -- the per-image ones would crowd out the
# summary below, which is the only place the cause is named.
failed_names+=("$name")
[ -n "$first_reason" ] || first_reason="HTTP ${code}: ${reason}"
echo "FAILED to delete ${REPOSITORY}:${name} (${digest}): HTTP ${code}: ${reason}${detail:+ Registry said: ${detail}.}${challenge:+ Challenge: ${challenge}.}"
done
if [ "${#failed_names[@]}" -gt 0 ]; then
echo "::warning title=Docker prune incomplete::${#failed_names[@]} of ${#stale_digests[@]} stale dev-build image(s) could not be reclaimed (${failed_names[*]}). First cause: ${first_reason} The images themselves published fine, and the nightly is deliberately not failed for this, so nothing else will flag it: registry storage keeps growing every night until this is resolved."
fi
notify:
name: 'Success check'
if: ${{ always() && (github.repository == 'rotki/rotki' || github.event_name != 'schedule') }}
needs: [ 'build-linux', 'build-windows', 'build-macos', 'docker-merge' ]
runs-on: ubuntu-latest
timeout-minutes: 10 # observed max 42s
environment: nightly_notify
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 1
persist-credentials: false
- name: Check if any task failed
run: |
data=($(echo "${RESULT}" | sed 's/[][,"]//g'))
echo "Processing ${#data[@]} entries"
for idx in "${!data[@]}"
do
echo "Processing entry $idx: ${data[$idx]}"
if [[ ${data[$idx]} == "failure" ]]; then
./.github/scripts/notifier.py --webhook "$DISCORD_WEBHOOK" --run-id "$GITHUB_RUN_ID"
exit 1;
fi
done
env:
RESULT: ${{ toJSON(needs.*.result) }}
DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }}