nightwatcher: add purging old nightlies

[Frenzie]

For reference, here's the old purge script (2024 version; I lost the poc 2026 update):

Details

#!/bin/bash

# Script to purge older nightly artifacts
# NB This is called with a cron job to avoid filling up
# Use `sudo crontab -e` to edit the job

# Where is our stuff stored?
STORAGE_DIRS=(
        "/ops/prod/build/download/nightly"
        "/ops/prod/build/ota"
)
# Which targets do we want to purge?
PURGE_TARGETS=(
        "android-arm"
        "android-x86"
        "appimage-x86_64-linux-gnu"
        "cervantes"
        "pocketbook"
        "kobo"
        "kindle-legacy"
        "kindle"
        "kindlepw2"
        "remarkable"
        "sony-prstux"
        "android-arm-linux-androideabi"
        "android-i686-linux-android"
        "cervantes-arm-cervantes-linux-gnueabi"
        "pocketbook-arm-obreey-linux-gnueabi"
        "kobo-arm-kobo-linux-gnueabihf"
        "kindle-legacy-arm-kindle-linux-gnueabi"
        "kindle-arm-kindle5-linux-gnueabi"
        "kindlepw2-arm-kindlepw2-linux-gnueabi"
        "sony-prstux-arm-linux-gnueabihf"
)

# How many things do we want to keep (per target where applicable)
KEEP_OTA_AMOUNT="7"
KEEP_STABLE_AMOUNT="2"

###
ko_purge() {
        # Is this a dry-run?
        if (( $# >= 1 )) ; then
                my_cmd="echo"
                dry_run="true"
        else
                my_cmd="sudo rm -rf"
                dry_run="false"
        fi

        for my_dir in "${STORAGE_DIRS[@]}" ; do
                if [[ "${my_dir}" == */nightly ]] ; then
                        # Stable
                        [[ "${dry_run}" == "true" ]] && echo "=== List of nightly directories to purge of stable releases:"
                        while IFS= read -r -d '' file ; do
                                ${my_cmd} "${file}"
                        done < <(find "${my_dir}" -maxdepth 1 -type d -regex '.*?/v[0-9\.]*$' -print0 | sort -z -V | head -z -n -${KEEP_STABLE_AMOUNT})
                        [[ "${dry_run}" == "true" ]] && echo "=== End of list"
                        # OTA
                        [[ "${dry_run}" == "true" ]] && echo "=== List of nightly directories to purge of OTA releases:"
                        while IFS= read -r -d '' file ; do
                                ${my_cmd} "${file}"
                        done < <(find "${my_dir}" -maxdepth 1 -type d -regex '.*?/v[0-9\.]*-.*?' -print0 | sort -z -V | head -z -n -${KEEP_OTA_AMOUNT})
                        [[ "${dry_run}" == "true" ]] && echo "=== End of list"
                else
                        # Start by purging broken symlniks
                        [[ "${dry_run}" == "true" ]] && echo "=== List of broken OTA symlinks to purge:"
                        while IFS= read -r -d '' file ; do
                                if [[ ! -f "$(readlink ${file} | sed -e 's#^/data/release_download/#/ops/prod/build/download/#')" ]] ; then
                                        ${my_cmd} "${file}"
                                fi
                        done < <(find "${my_dir}" -maxdepth 1 -type l -print0)
                        [[ "${dry_run}" == "true" ]] && echo "=== End of list"

                        for my_target in "${PURGE_TARGETS[@]}" ; do
                                # Stable
                                [[ "${dry_run}" == "true" ]] && echo "=== List of files to purge for '${my_target}' stable releases:"
                                while IFS= read -r -d '' file ; do
                                        ${my_cmd} "${file}"
                                    done < <(find "${my_dir}" -maxdepth 1 \( -type l -o -type f \) -regextype posix-egrep -regex ".*?/koreader-${my_target}-v[0-9\\.]*\\.(targz|apk|AppImage)" -print0 | sort -z -V | head -z -n -${KEEP_STABLE_AMOUNT})
                                [[ "${dry_run}" == "true" ]] && echo "=== End of list"
                                # OTA
                                [[ "${dry_run}" == "true" ]] && echo "=== List of files to purge for '${my_target}' OTA releasess:"
                                while IFS= read -r -d '' file ; do
                                        ${my_cmd} "${file}"
                                    done < <(find "${my_dir}" -maxdepth 1 \( -type l -o -type f \) -regextype posix-egrep -regex ".*?/koreader-${my_target}-v[0-9\\.]*-.*?\\.(targz|apk|AppImage)" -print0 | sort -z -V | head -z -n -${KEEP_OTA_AMOUNT})
                                [[ "${dry_run}" == "true" ]] && echo "=== End of list"
                        done
                fi
        done
}


###
echo "=== Disk usage before purge:"
df -h
echo ""

ko_purge "dry_run"
echo ""

read -r -p "[*] Press enter to purge with above list. Ctrl-C to cancel."

ko_purge
echo ""

echo "=== Disk usage after purge:"
df -h

exit 0

---

This change is 

[benoit-pierre]

We should get rid of Manifest.stable & Manifest.nightly (there unused, and would become outdated after a purge):

 docker/ubuntu/nightswatcher/nightswatcher.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git i/docker/ubuntu/nightswatcher/nightswatcher.py w/docker/ubuntu/nightswatcher/nightswatcher.py
index b82c124..8758716 100644
--- i/docker/ubuntu/nightswatcher/nightswatcher.py
+++ w/docker/ubuntu/nightswatcher/nightswatcher.py
@@ -180,8 +180,6 @@ class Manifest:
         self.nightlies_dir = Path(nightlies_dir)
         self.stables_dir = Path(stables_dir)
         self.ota = {}
-        self.stable = {}
-        self.nightly = {}
         self.by_sha256 = defaultdict(set)
 
     def ensure_dirs(self):
@@ -189,10 +187,12 @@ class Manifest:
             d.mkdir(parents=True, exist_ok=True)
 
     def initial_update(self):
+        stable = {}
+        nightly = {}
         for manifest, directory in (
             (self.ota, self.ota_dir),
-            (self.nightly, self.nightlies_dir),
-            (self.stable, self.stables_dir),
+            (nightly, self.nightlies_dir),
+            (stable, self.stables_dir),
         ):
             for dirpath, _dirnames, filenames in directory.walk():
                 for name in filenames:
@@ -205,8 +205,8 @@ class Manifest:
                     manifest[path.name] = sha256
                     self.by_sha256[sha256].add(realpath)
         logger.info('ota: %u files', len(self.ota))
-        logger.info('stable: %u files', len(self.stable))
-        logger.info('nightly: %u files', len(self.nightly))
+        logger.info('stable: %u files', len(stable))
+        logger.info('nightly: %u files', len(nightly))
 
     def update_asset(self, asset):
         logger.info('Updating asset: %s', asset.name)