From 128ca9d8b34749fc5c90ad29fefe5d06d66db174 Mon Sep 17 00:00:00 2001 From: Miles Ward <1+mward4@noreply.localhost> Date: Mon, 28 Sep 2026 14:33:27 -0400 Subject: [PATCH] ci: only a never-published version can fail the sync run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Runs have been red since 2026-09-19 even though every release shipped: the plan step re-plans the oldest 11 versions as "stale" on every run (pnpm and the node base image moved on), and those versions can no longer be rebuilt at all — their floating dependency ranges now resolve to @deepseek-ai packages that have moved past them, so they fail the boot smoke test ~67s in. Step 8 then failed the run for them, and every 6-hour run went red for ~13 minutes of builds that could never succeed. A green run meant nothing, and a red one no longer pointed at anything real. Split the plan into two kinds of build and treat them differently: NEW - the registry has no image for this version. This is an upstream release that has not shipped here yet, which is the whole job. REFRESH - the registry already has the image; it is only being moved onto a newer pnpm/base/recipe. Best-effort. * Plan NEW first so a new release lands in the first minutes of the run instead of queueing behind refreshes that may all be unbuildable. * A failed NEW version is recorded in a `missing` output and later reported by the final gate; a failed REFRESH stays a warning and keeps its image. * The final gate now fails only on `missing` or on a current version with no image in the registry — a missed upstream release is still visible and red. Also removes two early aborts that silently skipped the prune and the release publish whenever the current version failed to build: step 4 no longer exits after a failed build, and the :latest step records its miss instead of exiting. Publishing now always happens, so the releases for the versions that did ship are in place even when the run legitimately ends red. --- .gitea/workflows/sync-upstream.yml | 66 ++++++++++++++++++++++-------- 1 file changed, 50 insertions(+), 16 deletions(-) diff --git a/.gitea/workflows/sync-upstream.yml b/.gitea/workflows/sync-upstream.yml index 7ef1c7cb80..081e7f4dd0 100644 --- a/.gitea/workflows/sync-upstream.yml +++ b/.gitea/workflows/sync-upstream.yml @@ -13,8 +13,16 @@ name: Sync upstream and build images # 4. Publish one Gitea release per built version on the mirrored upstream # tag, titled like upstream, carrying upstream's own notes plus a short # image footer. Updated in place when the image is rebuilt. -# 5. A version that fails to build or push fails the run: a missed upstream -# release must be visible, not a warning in a green run. +# 5. A version that fails to build or push fails the run, but only when the +# registry has never had it: a missed upstream release must be visible, +# not a warning in a green run. A *refresh* rebuild — the registry +# already has an image for that version and is only being moved onto a +# newer pnpm/base/recipe — is best-effort: it warns, leaves the existing +# image alone, and cannot turn the run red. Old versions cannot always +# be rebuilt any more, because their floating dependency ranges now +# resolve to packages that have moved past them, and a doomed 0.1.x +# refresh must not hide the 0.2.x release that did ship. Every failure +# stays listed in the run either way. # # Images are pushed with regctl, in 50 MB chunks. The registry sits behind # Cloudflare, which rejects any request body over 100 MB; `docker push` sends a @@ -143,23 +151,35 @@ jobs: done < <(sort -V /tmp/upstream-versions) echo "candidates:$CANDIDATES" - BUILDS="" + # Two kinds of build, and they do not matter equally. NEW is a + # version the registry has never had — an upstream release that has + # not shipped here yet, which is the whole point of this job. + # REFRESH is an already-published version being moved onto newer + # toolchain. Plan NEW first, so a new release lands in the first + # minutes of the run instead of queueing behind refreshes that may + # all be unbuildable. + NEW="" + REFRESH="" for V in $CANDIDATES; do if ! tag_exists "$V"; then - echo "$V: missing -> build"; BUILDS="$BUILDS $V" + echo "$V: missing -> build"; NEW="$NEW $V" elif forced "$V"; then - echo "$V: rebuild requested"; BUILDS="$BUILDS $V" + echo "$V: rebuild requested"; REFRESH="$REFRESH $V" else HAVE=$(tag_toolchain "$V") - if [ "$HAVE" != "$WANT" ]; then echo "$V: stale ($HAVE) -> rebuild"; BUILDS="$BUILDS $V"; fi + if [ "$HAVE" != "$WANT" ]; then echo "$V: stale ($HAVE) -> rebuild"; REFRESH="$REFRESH $V"; fi fi done + BUILDS="$NEW $REFRESH" # Newest release candidate = what :latest points at. CURRENT=$(printf '%s\n' $CANDIDATES | grep -v alpha | sort -V | tail -1) + echo "new (must ship):$NEW" + echo "refresh (best-effort):$REFRESH" echo "planned builds:$BUILDS" echo "current (:latest): $CURRENT" { echo "builds=${BUILDS}" + echo "new=${NEW}" echo "current=${CURRENT}" echo "pnpm=${PNPM_LATEST}" echo "base_digest=${BASE_DIGEST}" @@ -174,7 +194,6 @@ jobs: PNPM_VERSION: ${{ steps.plan.outputs.pnpm }} BASE_DIGEST: ${{ steps.plan.outputs.base_digest }} RECIPE_HASH: ${{ steps.plan.outputs.recipe_hash }} - CURRENT: ${{ steps.plan.outputs.current }} CI_RUN: ${{ github.run_number }} run: | set -u @@ -262,6 +281,7 @@ jobs: ) FAILED="" + MISSING="" for V in ${{ steps.plan.outputs.builds }}; do if build_one "$V"; then echo "::notice::built ${IMAGE}:${V}" @@ -269,17 +289,24 @@ jobs: docker rm -f "$SMOKE" 2>/dev/null || true; docker volume rm -f "$SMOKE" 2>/dev/null || true echo "::warning::${V} did not build or push; any existing image for it is left untouched" FAILED="$FAILED $V" + # Missing from the registry AND failed to build = an upstream + # release we failed to ship. Record it for the final gate. + case " ${{ steps.plan.outputs.new }} " in *" $V "*) MISSING="$MISSING $V";; esac fi done echo "failed=${FAILED}" >> "$GITHUB_OUTPUT" - if [ -n "$FAILED" ]; then - echo "failed versions:$FAILED" - case " $FAILED " in *" $CURRENT "*) echo "::error::the current version (${CURRENT}) failed to build"; exit 1;; esac - fi + echo "missing=${MISSING}" >> "$GITHUB_OUTPUT" + # Deliberately no exit here. Steps 5-7 still have to run so the + # releases for every version that did ship are in place even when + # this run ends red; the single red/green decision belongs to the + # last step. Aborting here is what used to leave a new release + # unpublished. + [ -z "$FAILED" ] || echo "failed versions:$FAILED" # `:latest` = the newest release-candidate version. A manifest copy # through the registry API: identical digest, nothing rebuilt. - name: Point :latest at the current version + id: latest env: TOKEN: ${{ secrets.REGISTRY_TOKEN }} OWNER: ${{ github.repository_owner }} @@ -290,7 +317,9 @@ jobs: ACCEPT='application/vnd.oci.image.index.v1+json,application/vnd.docker.distribution.manifest.list.v2+json,application/vnd.oci.image.manifest.v1+json,application/vnd.docker.distribution.manifest.v2+json' digest_of() { curl -sI -u "${OWNER}:${TOKEN}" -H "Accept: $ACCEPT" "https://${REGISTRY}/v2/${IMAGE}/manifests/$1" | grep -i '^docker-content-digest' | awk '{print $2}' | tr -d '\r'; } WANT=$(digest_of "$CURRENT") - [ -n "$WANT" ] || { echo "::error::${CURRENT} has no image in the registry"; exit 1; } + # Do not exit non-zero here either: that would skip both the prune + # and the release publish. Record the miss, let the gate decide. + [ -n "$WANT" ] || { echo "::error::${CURRENT} has no image in the registry, so :latest cannot be moved"; echo "latest_failed=1" >> "$GITHUB_OUTPUT"; exit 0; } HAVE=$(digest_of latest || true) echo "latest -> ${CURRENT} (${WANT}); currently ${HAVE:-unset}" [ "$WANT" != "$HAVE" ] || { echo "latest already current"; exit 0; } @@ -349,9 +378,14 @@ jobs: "$BASE_IMAGE" node - < .gitea/scripts/publish-releases.js # Last, so :latest and the releases for everything that did ship are - # already in place when the run turns red. - - name: Fail the run if a version did not ship - if: steps.build.outputs.failed != '' + # already in place before the run turns red — and the only thing that + # turns it red is a version this registry has never published. Refreshes + # of versions we already ship stay warnings. + - name: Fail the run if a new version did not ship + if: steps.build.outputs.missing != '' || steps.latest.outputs.latest_failed != '' run: | - echo "::error::these versions did not build or push:${{ steps.build.outputs.failed }}" + [ -z "${{ steps.build.outputs.missing }}" ] || \ + echo "::error::never-published versions did not build or push:${{ steps.build.outputs.missing }}" + [ "${{ steps.latest.outputs.latest_failed }}" != "1" ] || \ + echo "::error::${{ steps.plan.outputs.current }} has no image in the registry, so :latest was not moved" exit 1