From 88d91c1cc5faca6992b475c5ef9e8b2d6fdb7eec Mon Sep 17 00:00:00 2001 From: Burer Date: Wed, 16 Sep 2026 08:35:34 +0300 Subject: [PATCH 1/5] ci: compact older builds on the storage server Every push to main publishes four images per language plus the unpacked build tree, and nothing ever shrinks: kolibri.raw alone is a fixed 128 MiB apiece, and kolibri.img lands three times over. Keep the newest build as it is - bare images and a browsable tree - and reduce every older one to zipped images. Distribution kits, checksums and build logs stay untouched. Capped per run, since the first pass has the whole backlog to get through, and skips anything touched in the last half hour so a concurrent deploy is never compacted mid-upload. Assisted-by: Claude Opus 5 --- .gitea/ansible/playbooks/deploy.yml | 24 ++++++++++++ .gitea/utils/compact-builds.sh | 57 +++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+) create mode 100755 .gitea/utils/compact-builds.sh diff --git a/.gitea/ansible/playbooks/deploy.yml b/.gitea/ansible/playbooks/deploy.yml index b48bedf05..4e7b4d02f 100644 --- a/.gitea/ansible/playbooks/deploy.yml +++ b/.gitea/ansible/playbooks/deploy.yml @@ -54,3 +54,27 @@ src: "{{ stable_artifact_path }}/" dest: "{{ storage_path }}/../" mode: preserve + + # older builds shrink to zipped images and lose their unpacked tree; the newest keeps both + - name: Looking for zip on the storage server + ansible.builtin.shell: command -v zip + register: zip_present + changed_when: false + failed_when: false + + - name: Requiring zip for compaction + ansible.builtin.assert: + that: zip_present.rc == 0 + fail_msg: >- + zip is not installed on the storage server, so older builds cannot + be compacted. Install it and re-run this build. + + - name: Compacting older builds + ansible.builtin.script: + cmd: >- + ../../utils/compact-builds.sh + {{ storage_path | quote }} + {{ version | quote }} + {{ compact_max | default(20) | int }} + register: compaction + changed_when: "'compacted 0 of' not in compaction.stdout" diff --git a/.gitea/utils/compact-builds.sh b/.gitea/utils/compact-builds.sh new file mode 100755 index 000000000..6c2cd2699 --- /dev/null +++ b/.gitea/utils/compact-builds.sh @@ -0,0 +1,57 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0-only +# SPDX-FileCopyrightText: 2026 KolibriOS team + +# Shrink older builds on the storage server: zip their images and drop their +# unpacked tree, which is rebuildable from the commit. The current build is +# left alone. Distribution kits, checksums and build logs are never touched - +# note that sha256sums.txt keeps naming the bare images, so a compacted build +# has to be unzipped before verifying. +# +# Layout: ///kolibrios--.{img,iso,raw} -> .zip +# ///data/ -> removed +# +# Usage: compact-builds.sh [max-per-run] + +set -euo pipefail + +root=${1:?storage root required} +current=${2:?current version required} +# a raw image is 128 MiB apiece, so cap the first pass over the whole backlog +max=${3:-20} +# a build untouched for this long is not being uploaded by a concurrent deploy +quiet_minutes=30 + +# images and the unpacked tree both sit exactly two levels below +leftovers() { + find "$1" -mindepth 2 -maxdepth 2 \ + \( -type d -name data \ + -o -type f \( -name '*.img' -o -name '*.iso' -o -name '*.raw' \) \) \ + -print -quit +} + +compacted=0 +pending=0 + +while IFS= read -r version; do + dir="$root/$version" + [ -n "$(leftovers "$dir")" ] || continue + + pending=$((pending + 1)) + [ "$compacted" -lt "$max" ] || continue + + find "$dir" -mindepth 2 -maxdepth 2 -type d -name data -exec rm -rf {} + + + # zip -m drops the source only once the archive is written + while IFS= read -r -d '' image; do + ( cd "$(dirname "$image")" \ + && zip -9 -q -m "$(basename "$image").zip" "$(basename "$image")" ) + done < <(find "$dir" -mindepth 2 -maxdepth 2 -type f \ + \( -name '*.img' -o -name '*.iso' -o -name '*.raw' \) -print0) + + compacted=$((compacted + 1)) + echo "compacted $version" +done < <(find "$root" -mindepth 1 -maxdepth 1 -type d \ + ! -name "$current" -mmin "+$quiet_minutes" -printf '%f\n' | sort) + +echo "compacted $compacted of $pending older build(s), $((pending - compacted)) left for the next run" -- 2.54.0 From 39e29bfbe3afb48613119773b7e470ef7064443d Mon Sep 17 00:00:00 2001 From: Burer Date: Fri, 25 Sep 2026 13:52:38 +0300 Subject: [PATCH 2/5] ci: pick builds to compact by commit count, not mtime - keep the newest 5 builds uncompacted, ordered by the commit count in the directory name; mtime says nothing about build order after a re-upload - name the leftovers() parameter so it does not read as the script's own $1 - lower the per-run cap to 10 builds --- .gitea/ansible/playbooks/deploy.yml | 6 +++--- .gitea/utils/compact-builds.sh | 26 ++++++++++++-------------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/.gitea/ansible/playbooks/deploy.yml b/.gitea/ansible/playbooks/deploy.yml index 4e7b4d02f..d98d3f14d 100644 --- a/.gitea/ansible/playbooks/deploy.yml +++ b/.gitea/ansible/playbooks/deploy.yml @@ -55,7 +55,7 @@ dest: "{{ storage_path }}/../" mode: preserve - # older builds shrink to zipped images and lose their unpacked tree; the newest keeps both + # older builds shrink to zipped images and lose their unpacked tree; the newest few keep both - name: Looking for zip on the storage server ansible.builtin.shell: command -v zip register: zip_present @@ -74,7 +74,7 @@ cmd: >- ../../utils/compact-builds.sh {{ storage_path | quote }} - {{ version | quote }} - {{ compact_max | default(20) | int }} + {{ compact_keep | default(5) | int }} + {{ compact_max | default(10) | int }} register: compaction changed_when: "'compacted 0 of' not in compaction.stdout" diff --git a/.gitea/utils/compact-builds.sh b/.gitea/utils/compact-builds.sh index 6c2cd2699..0e973dbec 100755 --- a/.gitea/utils/compact-builds.sh +++ b/.gitea/utils/compact-builds.sh @@ -3,28 +3,25 @@ # SPDX-FileCopyrightText: 2026 KolibriOS team # Shrink older builds on the storage server: zip their images and drop their -# unpacked tree, which is rebuildable from the commit. The current build is -# left alone. Distribution kits, checksums and build logs are never touched - -# note that sha256sums.txt keeps naming the bare images, so a compacted build -# has to be unzipped before verifying. +# unpacked tree, which is rebuildable from the commit. sha256sums.txt keeps +# naming the bare images, so a compacted build has to be unzipped to verify. # # Layout: ///kolibrios--.{img,iso,raw} -> .zip # ///data/ -> removed # -# Usage: compact-builds.sh [max-per-run] +# Usage: compact-builds.sh [keep] [max-per-run] set -euo pipefail root=${1:?storage root required} -current=${2:?current version required} -# a raw image is 128 MiB apiece, so cap the first pass over the whole backlog -max=${3:-20} -# a build untouched for this long is not being uploaded by a concurrent deploy -quiet_minutes=30 +# the build just published is always among these, so an upload in flight is safe +keep=${2:-5} +# a build is hundreds of MiB to re-compress: cap the first pass over the backlog +max=${3:-10} -# images and the unpacked tree both sit exactly two levels below leftovers() { - find "$1" -mindepth 2 -maxdepth 2 \ + local build=$1 + find "$build" -mindepth 2 -maxdepth 2 \ \( -type d -name data \ -o -type f \( -name '*.img' -o -name '*.iso' -o -name '*.raw' \) \) \ -print -quit @@ -51,7 +48,8 @@ while IFS= read -r version; do compacted=$((compacted + 1)) echo "compacted $version" -done < <(find "$root" -mindepth 1 -maxdepth 1 -type d \ - ! -name "$current" -mmin "+$quiet_minutes" -printf '%f\n' | sort) +# --g: order by the count, never by mtime +done < <(find "$root" -mindepth 1 -maxdepth 1 -type d -name '*-*-g*' -printf '%f\n' \ + | sort -t- -k2,2n | head -n "-$keep") echo "compacted $compacted of $pending older build(s), $((pending - compacted)) left for the next run" -- 2.54.0 From 7be0d8cf077dd9aa0b395689be6b8ccaa7f8ac59 Mon Sep 17 00:00:00 2001 From: Burer Date: Fri, 25 Sep 2026 14:56:34 +0300 Subject: [PATCH 3/5] ci: rehearse with DRY_RUN, zip in parallel, yield to the site - DRY_RUN=1 walks the same selection and changes nothing - compress one image per core: zip is single threaded and a build holds nine, which took a build from 105s to 54s on the storage server - run zip under nice/ionice so compaction does not disturb the NFS export --- .gitea/utils/compact-builds.sh | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/.gitea/utils/compact-builds.sh b/.gitea/utils/compact-builds.sh index 0e973dbec..f7cd9cccd 100755 --- a/.gitea/utils/compact-builds.sh +++ b/.gitea/utils/compact-builds.sh @@ -10,6 +10,7 @@ # ///data/ -> removed # # Usage: compact-builds.sh [keep] [max-per-run] +# DRY_RUN=1 walks the same selection and changes nothing set -euo pipefail @@ -18,6 +19,17 @@ root=${1:?storage root required} keep=${2:-5} # a build is hundreds of MiB to re-compress: cap the first pass over the backlog max=${3:-10} +dry=${DRY_RUN:-0} +verb=compacted +[ "$dry" = 1 ] && verb="would compact" + +# zip -m drops the source only once the archive is written; the box serves the +# site over NFS, so compaction yields to it +zip_image() { + ( cd "$(dirname "$1")" \ + && nice -n 10 ionice -c3 zip -9 -q -m "$(basename "$1").zip" "$(basename "$1")" ) +} +export -f zip_image leftovers() { local build=$1 @@ -37,19 +49,23 @@ while IFS= read -r version; do pending=$((pending + 1)) [ "$compacted" -lt "$max" ] || continue + if [ "$dry" = 1 ]; then + compacted=$((compacted + 1)) + echo "$verb $version" + continue + fi + find "$dir" -mindepth 2 -maxdepth 2 -type d -name data -exec rm -rf {} + - # zip -m drops the source only once the archive is written - while IFS= read -r -d '' image; do - ( cd "$(dirname "$image")" \ - && zip -9 -q -m "$(basename "$image").zip" "$(basename "$image")" ) - done < <(find "$dir" -mindepth 2 -maxdepth 2 -type f \ - \( -name '*.img' -o -name '*.iso' -o -name '*.raw' \) -print0) + # zip is single threaded and a build holds nine images: one per core + find "$dir" -mindepth 2 -maxdepth 2 -type f \ + \( -name '*.img' -o -name '*.iso' -o -name '*.raw' \) -print0 \ + | xargs -0 -r -P "$(nproc)" -I{} bash -c 'zip_image "$@"' _ {} compacted=$((compacted + 1)) - echo "compacted $version" + echo "$verb $version" # --g: order by the count, never by mtime done < <(find "$root" -mindepth 1 -maxdepth 1 -type d -name '*-*-g*' -printf '%f\n' \ | sort -t- -k2,2n | head -n "-$keep") -echo "compacted $compacted of $pending older build(s), $((pending - compacted)) left for the next run" +echo "$verb $compacted of $pending older build(s), $((pending - compacted)) left for the next run" -- 2.54.0 From 1629406ba128d6b0dcbcaf521eb3c9778a085dc9 Mon Sep 17 00:00:00 2001 From: Burer Date: Fri, 25 Sep 2026 15:14:41 +0300 Subject: [PATCH 4/5] ci: archive a build before dropping its unpacked tree A zip that fails midway used to leave the build stripped of its tree and still holding bare images. Archiving first makes a failure leave it merely uncompacted, and the next run selects it again. --- .gitea/utils/compact-builds.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitea/utils/compact-builds.sh b/.gitea/utils/compact-builds.sh index f7cd9cccd..0649f0e6a 100755 --- a/.gitea/utils/compact-builds.sh +++ b/.gitea/utils/compact-builds.sh @@ -55,13 +55,15 @@ while IFS= read -r version; do continue fi - find "$dir" -mindepth 2 -maxdepth 2 -type d -name data -exec rm -rf {} + - - # zip is single threaded and a build holds nine images: one per core + # zip is single threaded and a build holds nine images: one per core. + # Archive before dropping the tree, so a failure here leaves the build + # merely uncompacted and the next run picks it up again find "$dir" -mindepth 2 -maxdepth 2 -type f \ \( -name '*.img' -o -name '*.iso' -o -name '*.raw' \) -print0 \ | xargs -0 -r -P "$(nproc)" -I{} bash -c 'zip_image "$@"' _ {} + find "$dir" -mindepth 2 -maxdepth 2 -type d -name data -exec rm -rf {} + + compacted=$((compacted + 1)) echo "$verb $version" # --g: order by the count, never by mtime -- 2.54.0 From d863551e39de606cc59e9861a69a70c27a97ccba Mon Sep 17 00:00:00 2001 From: Burer Date: Sat, 26 Sep 2026 17:09:55 +0300 Subject: [PATCH 5/5] ci: compress old builds with zstd instead of zip zstd -12 beats zip -9 on both counts on the storage server: a build goes to 290 MB instead of 314 MB, and takes 42s instead of 54s. These are archives, not everyday downloads, so the less familiar extension costs little. zstd appends .zst to the file it is given, so the subshell that zip needed to keep paths out of the archive is gone. Each zstd is held to one thread because the files are already compressed one per core. --- .gitea/ansible/playbooks/deploy.yml | 14 +++++++------- .gitea/utils/compact-builds.sh | 27 ++++++++++++++------------- 2 files changed, 21 insertions(+), 20 deletions(-) diff --git a/.gitea/ansible/playbooks/deploy.yml b/.gitea/ansible/playbooks/deploy.yml index d98d3f14d..487b4e658 100644 --- a/.gitea/ansible/playbooks/deploy.yml +++ b/.gitea/ansible/playbooks/deploy.yml @@ -55,18 +55,18 @@ dest: "{{ storage_path }}/../" mode: preserve - # older builds shrink to zipped images and lose their unpacked tree; the newest few keep both - - name: Looking for zip on the storage server - ansible.builtin.shell: command -v zip - register: zip_present + # older builds shrink to compressed images and lose their unpacked tree; the newest few keep both + - name: Looking for zstd on the storage server + ansible.builtin.shell: command -v zstd + register: zstd_present changed_when: false failed_when: false - - name: Requiring zip for compaction + - name: Requiring zstd for compaction ansible.builtin.assert: - that: zip_present.rc == 0 + that: zstd_present.rc == 0 fail_msg: >- - zip is not installed on the storage server, so older builds cannot + zstd is not installed on the storage server, so older builds cannot be compacted. Install it and re-run this build. - name: Compacting older builds diff --git a/.gitea/utils/compact-builds.sh b/.gitea/utils/compact-builds.sh index 0649f0e6a..2e6844fbb 100755 --- a/.gitea/utils/compact-builds.sh +++ b/.gitea/utils/compact-builds.sh @@ -2,11 +2,12 @@ # SPDX-License-Identifier: GPL-2.0-only # SPDX-FileCopyrightText: 2026 KolibriOS team -# Shrink older builds on the storage server: zip their images and drop their -# unpacked tree, which is rebuildable from the commit. sha256sums.txt keeps -# naming the bare images, so a compacted build has to be unzipped to verify. +# Shrink older builds on the storage server: compress their images and drop +# their unpacked tree, which is rebuildable from the commit. sha256sums.txt +# keeps naming the bare images, so a compacted build has to be decompressed +# before it can be verified. # -# Layout: ///kolibrios--.{img,iso,raw} -> .zip +# Layout: ///kolibrios--.{img,iso,raw} -> .zst # ///data/ -> removed # # Usage: compact-builds.sh [keep] [max-per-run] @@ -23,13 +24,12 @@ dry=${DRY_RUN:-0} verb=compacted [ "$dry" = 1 ] && verb="would compact" -# zip -m drops the source only once the archive is written; the box serves the +# --rm drops the source only once the archive is written; the box serves the # site over NFS, so compaction yields to it -zip_image() { - ( cd "$(dirname "$1")" \ - && nice -n 10 ionice -c3 zip -9 -q -m "$(basename "$1").zip" "$(basename "$1")" ) +compress_image() { + nice -n 10 ionice -c3 zstd -12 -T1 -q --rm "$1" } -export -f zip_image +export -f compress_image leftovers() { local build=$1 @@ -55,12 +55,13 @@ while IFS= read -r version; do continue fi - # zip is single threaded and a build holds nine images: one per core. - # Archive before dropping the tree, so a failure here leaves the build - # merely uncompacted and the next run picks it up again + # a build holds nine images: one per core, each zstd kept to one thread so + # the two levels of parallelism do not fight. Archive before dropping the + # tree, so a failure here leaves the build merely uncompacted and the next + # run picks it up again find "$dir" -mindepth 2 -maxdepth 2 -type f \ \( -name '*.img' -o -name '*.iso' -o -name '*.raw' \) -print0 \ - | xargs -0 -r -P "$(nproc)" -I{} bash -c 'zip_image "$@"' _ {} + | xargs -0 -r -P "$(nproc)" -I{} bash -c 'compress_image "$@"' _ {} find "$dir" -mindepth 2 -maxdepth 2 -type d -name data -exec rm -rf {} + -- 2.54.0