Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions .github/ci/compose.smoke.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
# Layered over docker-compose.yml by the smoke script, on a hosted runner only.

services:
traefik:
volumes:
# traefik.yml with the ACME directory pointed at nothing, so a CI run never asks
# Let's Encrypt for a certificate. Same mount target as the real file, so it replaces
# that entry rather than adding one.
- ${CI_TRAEFIK_STATIC:?Set by the smoke script}:/etc/traefik/traefik.yml:ro

# A hosted runner has no GPU and no nvidia runtime. Both servers start without one and
# fall back to software transcoding, which is enough to show the image boots.
plex:
runtime: runc
jellyfin:
runtime: runc
40 changes: 40 additions & 0 deletions .github/ci/dummy-env
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
#!/usr/bin/env bash
#
# Writes the .env files the compose projects need, from their .env.example, with placeholder
# values and scratch directories in place of the host's paths. No real secret is involved:
# a compose file that grows a required variable without .env.example learning about it fails
# here, which is the point.
#
# Usage: dummy-env SCRATCH_DIR

set -euo pipefail

# shellcheck source=.github/ci/lib
source "$(dirname "${BASH_SOURCE[0]}")/lib"

scratch=${1:?usage: dummy-env SCRATCH_DIR}

# A checkout on a real machine has a real .env, and this would replace it.
if [[ ${CI:-} != true ]]; then
echo "dummy-env overwrites .env files; it only runs with CI=true" >&2
exit 1
fi

mkdir -p "$scratch"/{config,storage,immich-library,immich-pgdata}

# deploy-dagger exports CONFIG_DIR and STORAGE_DIR rather than keeping them in .env, so
# .env.example has neither. AIOStreams and Garage both reject a key that is not 64 hex
# characters, and Garage exits without saying why.
hex64=$(printf '0123456789abcdef%.0s' 1 2 3 4)
{
sed -e "s/^AIOSTREAMS_SECRET_KEY=.*/AIOSTREAMS_SECRET_KEY=$hex64/" \
-e "s/^GARAGE_RPC_SECRET=.*/GARAGE_RPC_SECRET=$hex64/" \
"$repo_root/.env.example"
printf 'CONFIG_DIR=%s\n' "$scratch/config"
printf 'STORAGE_DIR=%s\n' "$scratch/storage"
} > "$repo_root/.env"

sed -e "s|^UPLOAD_LOCATION=.*|UPLOAD_LOCATION=$scratch/immich-library|" \
-e "s|^DB_DATA_LOCATION=.*|DB_DATA_LOCATION=$scratch/immich-pgdata|" \
-e "s|^DB_PASSWORD=.*|DB_PASSWORD=ciPlaceholderPassword|" \
"$repo_root/immich/.env.example" > "$repo_root/immich/.env"
85 changes: 85 additions & 0 deletions .github/ci/images
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
#!/usr/bin/env bash
#
# Checks that each image the compose files name can actually be pulled: the pinned digest
# exists in the registry and has a linux/amd64 build, and the tag beside it exists too. A
# digest that was mistyped, garbage-collected upstream, or published without an amd64 build
# fails here rather than at `docker compose pull` on dagger.
#
# A tag that has moved on from its pinned digest is reported but does not fail: that is the
# normal state of a floating tag such as postgres:17-alpine between Renovate's digest PRs.
#
# Needs the .env files from dummy-env.

set -euo pipefail

# shellcheck source=.github/ci/lib
source "$(dirname "${BASH_SOURCE[0]}")/lib"

# Images this job cannot see. td-server is a private package pushed by dominics/td's own
# deploy workflow, and this repository's token has no read access to it.
skip=(ghcr.io/dominics/td-server)

# manifest REF: the registry's descriptor for REF as JSON (digest, and for a multi-platform
# image the list of per-platform manifests).
manifest() {
docker buildx imagetools inspect "$1" --format '{{json .Manifest}}'
}

failed=0

refs=$(for dir in "${compose_dirs[@]}"; do (cd "$dir" && docker compose config --images); done | sort -u)

while read -r ref; do
repo=${ref%%@*}
digest=
[[ $ref == *@* ]] && digest=${ref#*@}
tag=
# A colon after the last slash is a tag; one before it is a registry port.
if [[ ${repo##*/} == *:* ]]; then
tag=${repo##*:}
repo=${repo%:*}
fi

if [[ " ${skip[*]} " == *" $repo "* ]]; then
notice "$ref: skipped, not readable from this repository"
continue
fi

if [[ -n $digest ]]; then
if ! pinned=$(manifest "$repo@$digest"); then
error "$ref: the pinned digest does not resolve"
failed=1
continue
fi
# A single-platform manifest carries no platform list; the registry gives no cheaper
# way to ask, and the publishers here all ship an index.
if jq -e 'has("manifests")' <<<"$pinned" >/dev/null &&
! jq -e '[.manifests[] | select(.platform.os == "linux" and .platform.architecture == "amd64")] | length > 0' <<<"$pinned" >/dev/null; then
error "$ref: the pinned digest has no linux/amd64 build"
failed=1
continue
fi
fi

if [[ -n $tag ]]; then
if ! tagged=$(manifest "$repo:$tag"); then
error "$ref: the tag does not resolve"
failed=1
continue
fi
current=$(jq -r .digest <<<"$tagged")
if [[ -n $digest && $current != "$digest" ]]; then
warning "$repo:$tag now points at $current, not the pinned $digest"
fi
elif [[ -z $digest ]]; then
if ! manifest "$repo:latest" >/dev/null; then
error "$ref: does not resolve"
failed=1
continue
fi
fi

printf 'ok %s\n' "$ref"
done <<<"$refs"

exit "$failed"
69 changes: 69 additions & 0 deletions .github/ci/lib
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
# Shared by the scripts beside it. Sourced, not run.

repo_root=$(git rev-parse --show-toplevel)

# The compose projects in this repository that have images Renovate updates: the main stack
# at the root, and Immich with a project of its own.
compose_dirs=("$repo_root" "$repo_root/immich")

# GitHub Actions log annotations; plain lines anywhere else.
notice() { printf '::notice::%s\n' "$*"; }
warning() { printf '::warning::%s\n' "$*"; }
error() { printf '::error::%s\n' "$*"; }

group() { printf '::group::%s\n' "$*"; }
endgroup() { printf '::endgroup::\n'; }

# What a failing step leaves behind for whoever reads the log: container states, then the
# tail of each container's output. Run from the compose project's directory.
dump_compose() {
group "docker compose ps"
docker compose ps -a || true
endgroup
group "docker compose logs (last 60 lines of each)"
docker compose logs --no-color --tail 60 || true
endgroup
}

# probe NAME URL CODES [curl args...]
#
# Polls URL until curl reports a status matching the extended regex CODES, for up to two
# minutes. A status of 000 is curl failing to get an HTTP response at all.
probe() {
local name=$1 url=$2 codes=$3 code=000
shift 3
for _ in $(seq 60); do
code=$(curl -sk -o /dev/null -w '%{http_code}' --max-time 10 "$@" "$url" || true)
if [[ $code =~ ^($codes)$ ]]; then
printf 'ok %-16s %s -> %s\n' "$name" "$url" "$code"
return 0
fi
sleep 2
done
error "$name: $url answered $code, wanted $codes"
return 1
}

# assert_settled
#
# All containers of the compose project in the current directory are running, have never
# been restarted, and are not unhealthy. `restart: unless-stopped` turns a crash at boot into
# a loop that still reads as "running" at the moment it is looked at; the restart count is
# what gives it away.
assert_settled() {
local ids bad
ids=$(docker compose ps -aq)
if [[ -z $ids ]]; then
error "no containers in this compose project"
return 1
fi
# shellcheck disable=SC2086
bad=$(docker inspect $ids --format \
'{{.Name}} state={{.State.Status}} restarts={{.RestartCount}} health={{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' \
| grep -Ev 'state=running restarts=0 health=(healthy|none|starting)$' || true)
if [[ -n $bad ]]; then
error "containers that did not settle: $bad"
return 1
fi
echo "ok all containers running, none restarted"
}
111 changes: 111 additions & 0 deletions .github/ci/smoke
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
#!/usr/bin/env bash
#
# Starts the main stack on empty volumes with placeholder secrets, and checks that each
# service came up and stayed up, that Traefik built the routers and middlewares its config
# declares, and that requests are routed the way dynamic.yml and the container labels say.
#
# Needs the .env files from dummy-env. Usage: smoke SCRATCH_DIR

set -euo pipefail

# shellcheck source=.github/ci/lib
source "$(dirname "${BASH_SOURCE[0]}")/lib"

scratch=${1:?usage: smoke SCRATCH_DIR}
cd "$repo_root"

# Everything except td-server, whose image is private to dominics/td.
services=(
traefik redlib td-replays garage
ryot-db ryot
sonarr radarr lidarr prowlarr sabnzbd nzbdav
aiostreams-lan aiostreams-ts
plex jellyfin
)

# Static config cannot be split across sources, so the CI variant is the real file with
# one key added to each resolver.
export CI_TRAEFIK_STATIC=$scratch/traefik.yml
yq '.certificatesResolvers[].acme.caServer = "https://127.0.0.1:9/directory"' traefik.yml > "$CI_TRAEFIK_STATIC"

export COMPOSE_FILE=docker-compose.yml:.github/ci/compose.smoke.yml

trap 'status=$?; [[ $status -eq 0 ]] || dump_compose; exit $status' EXIT

docker compose up -d --quiet-pull --wait --wait-timeout 600 "${services[@]}"

failed=0
check() { "$@" || failed=1; }

group "Services answer"
# The *arrs and Jellyfin have an unauthenticated health endpoint; the rest are held to
# "an HTTP response that is not a server error".
check probe sonarr http://127.0.0.1:8989/ping 200
check probe radarr http://127.0.0.1:7878/ping 200
check probe lidarr http://127.0.0.1:8686/ping 200
check probe prowlarr http://127.0.0.1:9696/ping 200
check probe jellyfin http://127.0.0.1:8096/health 200
check probe plex http://127.0.0.1:32400/identity 200
check probe nzbdav http://127.0.0.1:3000/ '[234]..'
check probe aiostreams-ts http://127.0.0.1:8084/ '[234]..'
check probe redlib http://127.0.0.1:8090/ '[2345]..'
check probe garage http://127.0.0.1:3900/ '[234]..'
# Ryot only listens once its migrations have run against ryot-db.
check probe ryot http://127.0.0.1:8000/ '[234]..'
endgroup

group "Traefik built its configuration"
check probe traefik-api http://127.0.0.1:8080/api/overview 200
for kind in routers middlewares services; do
objects=$(curl -fsS "http://127.0.0.1:8080/api/http/$kind?per_page=1000")
jq -r '.[] | "\(.status)\t\(.name)"' <<<"$objects"
if jq -e '[.[] | select(.status != "enabled")] | length > 0' <<<"$objects" >/dev/null; then
error "traefik: $kind not enabled: $(jq -c '[.[] | select(.status != "enabled") | {name, error}]' <<<"$objects")"
failed=1
fi
done
# Names from dynamic.yml and the td-replays labels. A router that fails to parse is listed
# with an error above; one that is silently dropped is caught here.
for want in routers/cdn-station@file routers/cdn-station-www@file routers/aiostreams@file \
routers/traged-replays@docker middlewares/geoblock-nz@file \
middlewares/www-to-bare@file middlewares/tailscale-only@file; do
check probe "$want" "http://127.0.0.1:8080/api/http/$want" 200
done
endgroup

group "Requests are routed"
# The web entry point redirects to https.
check probe web-redirect http://127.0.0.1/ '30[1278]' -H 'Host: statio.nz'

# www.statio.nz passes the geoblock plugin (the runner is a private address, which
# allowLocalRequests lets through) and is redirected to the bare name.
location=$(curl -sk -o /dev/null -w '%{redirect_url}' --max-time 20 \
--resolve www.statio.nz:443:127.0.0.1 https://www.statio.nz/ci-probe || true)
if [[ $location == https://statio.nz/ci-probe ]]; then
echo "ok www-to-bare -> $location"
else
error "www.statio.nz/ci-probe redirected to '$location', wanted https://statio.nz/ci-probe"
failed=1
fi

# The AIOStreams route is gated to the tailnet and the LAN, and the runner is on neither.
check probe tailscale-only https://aiostreams.media.varspool.com/ 403 \
--resolve aiostreams.media.varspool.com:443:127.0.0.1

# /replays/ reaches the td-replays nginx through the docker provider. The world volume is
# empty, so nginx answers 404, with the CORS header only it adds.
headers=$(curl -sk -o /dev/null -D - --max-time 20 \
--resolve ws.traged.commo.nz:443:127.0.0.1 https://ws.traged.commo.nz/replays/index.tdindex || true)
if grep -qi '^access-control-allow-origin: \*' <<<"$headers"; then
echo "ok traged-replays -> td-replays answered"
else
error "ws.traged.commo.nz/replays/ did not reach td-replays: $(head -1 <<<"$headers")"
failed=1
fi
endgroup

group "Containers stayed up"
check assert_settled
endgroup

exit "$failed"
26 changes: 26 additions & 0 deletions .github/ci/smoke-immich
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
#!/usr/bin/env bash
#
# Starts Immich's server with its Postgres and Valkey on empty volumes, and checks the API
# answers, which it only does once the server has reached both and run its migrations.
# The machine-learning container is left out: it is several gigabytes, and the server
# starts without it.
#
# Needs the .env files from dummy-env.

set -euo pipefail

# shellcheck source=.github/ci/lib
source "$(dirname "${BASH_SOURCE[0]}")/lib"

cd "$repo_root/immich"

trap 'status=$?; [[ $status -eq 0 ]] || dump_compose; exit $status' EXIT

# The server exits if Postgres is not accepting connections yet and is restarted until it
# is, which is harmless on dagger but reads as a failure to `--wait`. The compose file
# follows upstream's and has no health condition on depends_on, so the order is made here.
docker compose up -d --quiet-pull --wait --wait-timeout 300 redis database
docker compose up -d --quiet-pull --wait --wait-timeout 600 immich-server

probe immich-server http://127.0.0.1:2283/api/server/ping 200
assert_settled
Loading
Loading