From d98ebcfca34d5a9adc0e4c2d1ad6bfdeaabbdd6d Mon Sep 17 00:00:00 2001 From: Andreas Baumann Date: Sat, 5 Feb 2022 16:13:09 +0100 Subject: much more cleanup --- build-all | 216 ++++++++++++++++++++++++++++++++++++--------------------- build-torrent | 3 +- update-website | 28 +++----- 3 files changed, 151 insertions(+), 96 deletions(-) diff --git a/build-all b/build-all index bfa82aa..b0a19d6 100755 --- a/build-all +++ b/build-all @@ -21,14 +21,16 @@ ARCH="i686" DESC="i686 only" #DESC="dual bootable" DATE=$(date +%Y.%m.%d) +#DATE="2022.02.01" ISO="archlinux32-${DATE}-${ARCH}.iso" SIGNATURE="${ISO}.sig" TORRENT="${ISO}.torrent" -TORRENT_SERVER="localhost" -#TORRENT_SERVER="archlinux32.org" +TORRENT_SERVER="archlinux32.org" +TORRENT_USER="hefur" WEBSITE_DIR="/srv/http/archlinux32.org" MIRROR_DIR="/srv/http/mirror/mirror.archlinux32.org" ARCHISOS_DIR="${MIRROR_DIR}/archisos/" +ARCHIVE_DIR="/srv/http/mirror/archive.archlinux32.org/iso" MIRROR_USER='mirror' MIRROR_GROUP='mirror' #Andreas Baumann (sign) @@ -50,6 +52,16 @@ popd() { export pushd popd +usage() { + >&2 echo "" + >&2 echo "build-all: do all release engineering for Archlinux32 ISOs and downloads" + >&2 echo "" + >&2 echo "possible options:" + >&2 echo " -h|--help: show this help and exit." + >&2 echo " -f|--force: overwrite already existing artifacts (use with care!)." + [ -z "$1" ] && exit 1 || exit "$1" +} + # fail on first error set -e @@ -63,114 +75,162 @@ cleanup() { } trap cleanup EXIT +eval set -- "$( + getopt -o hf \ + --long force \ + --long help \ + -n "$(basename "$0")" -- "$@" || \ + echo usage +)" + +force=0 + +while [ $# -gt 0 ]; do + case "$1" in + '--force'|'-f') + force=1 + ;; + '--help'|'-h') + usage 0 + ;; + '--') + shift + break + ;; + *) + >&2 printf 'Whoops, option "%s" is not yet implemented!\n' "$1" + exit 42 + ;; + esac + shift +done + +if [ $# -gt 0 ]; then + >&2 echo 'Too many arguments.' + exit 2 +fi + # build the ISO -echo "Building ISO.." -"${base_dir}/build-iso" --config="${CONFIG}" --arch "${ARCH}" \ - --output-dir="${ARCHISOS_DIR}" --iso "${ISO}" +pushd "${ARCHISOS_DIR}" +if [ $force = 1 ]; then + rm -f "${ISO}" +fi +if [ ! -f "${ISO}" ]; then + echo "Building ISO.." + "${base_dir}/build-iso" --config="${CONFIG}" --arch "${ARCH}" \ + --output-dir="${ARCHISOS_DIR}" --iso "${ISO}" +else + echo "Refusing to overwrite existing ISO '${ISO}'." +fi +popd + +if [ ! -f "${ARCHISOS_DIR}/${ISO}" ]; then + >&2 echo "The ISO file '${ISO}' does not exist, it makes no sense to continue." + exit 1 +fi # sign ISO -echo "Signing ISO.." pushd "${ARCHISOS_DIR}" -rm -rf "${ISO}.sig" -gpg --local-user "${SIGN_KEY}" --batch --no-tty --detach-sign "${ISO}" > sha512sums -[ ! -f md5sums ] || sed -i "/${ISO}$/d" md5sums -md5sum "${ISO}" >> md5sums -sort -k2,2 sha512sums --output sha512sums -sort -k2,2 md5sums --output md5sums +if [ $(grep "${ISO}$" sha512sums | wc -l) = 0 ]; then + [ ! -f sha512sums ] || sed -i "/${ISO}$/d" sha512sums + sha512sum "${ISO}" >> sha512sums + sort -k2,2 sha512sums --output sha512sums +else + echo "SHA512 checksum already done." +fi +if [ $(grep "${ISO}$" md5sums | wc -l) = 0 ]; then + [ ! -f md5sums ] || sed -i "/${ISO}$/d" md5sums + md5sum "${ISO}" >> md5sums + sort -k2,2 md5sums --output md5sums +else + echo "MD5 checksum already done." +fi popd # check sanity of ISO echo "Checking ISO.." -"${base_dir}/check-iso" --output-dir "${ARCHISOS_DIR}" --check --expected-sign-key "${SIGN_KEY}" +"${base_dir}/check-iso" \ + --output-dir "${ARCHISOS_DIR}" --iso="${ISO}" --check --expected-sign-key "${SIGN_KEY}" if [ $? = 1 ]; then - >&2 echo "The ISO file '${iso}' is not sane. Not continuing!" - #~ rm -f "${iso}" + >&2 echo "The ISO file '${iso}' is not sane. Not continuing! You should maybe delete this ISO file.." exit 1 fi -# date used for torrents and commit message -date=$(echo "${ISO}" \ - | sed 's/^.*-\([^-]\+\)-[^-]\+$/\1/' \ - | sort -u -) - -# generate torrents (currently broken) +# generate torrents echo "Generating torrents.." -#~ bash -x "${base_dir}/al32-mktorrent.sh" -d "${date}" -t "hefur@${TORRENT_SERVER}:" "${ARCH}" +pushd "${ARCHISOS_DIR}" +if [ $force = 1 ]; then + rm -f "${ISO}.torrent" +fi +if [ ! -f "${ISO}.torrent" ]; then + "${base_dir}/build-torrent" \ + --arch="${ARCH}" \ + --date="${DATE}" \ + --mirror-dir="${MIRROR_DIR}" \ + --torrent-server="${TORRENT_SERVER}" \ + --torrent-user="${TORRENT_USER}" +else + echo "The torrent file '${ISO}.torrent' already exists, not generating a new one." +fi -# update web page +# get the magnet link and add it to the RSS feed file +echo "Generating magnet link and publish it in the RSS feed file.." +magnet_link="$("${base_dir}"/build-torrent --arch="${ARCH}" --date="${DATE}" --mirror-dir="${MIRROR_DIR}" --magnet-link)" +if [ $force = 1 ]; then + rm -f "feed_${ARCH}.rss" +fi +if [ ! -f "feed_${ARCH}.rss" ]; then + feed_url="https://pool.mirror.archlinux32.org/archisos/feed_${ARCH}.rss" + "${base_dir}/magnet2feed.py" \ + "${magnet_link}" "${ARCH}" "${DATE}" "${feed_url}" +else + echo "The RSS feed file 'feed_${ARCH}.rss' already exists, not generating a new one." +fi + +# update web page for isos and download mirrors echo "Updating website.." -git -C "${WEBSITE_DIR}" reset --hard -git -C "${WEBSITE_DIR}" pull --ff-only +#~ git -C "${WEBSITE_DIR}" reset --hard +#~ git -C "${WEBSITE_DIR}" pull --ff-only "${base_dir}/update-website" \ --website-dir="${WEBSITE_DIR}" \ --mirror-dir="${MIRROR_DIR}" \ --iso="${ISO}" \ --update-iso \ - --update-mirrors - -#~ git -C "${WEBSITE_DIR}" commit 'download/index.html' -m 'download/index.html: new isos ('"${date}"')' + --update-mirrors \ + --torrent-seed="https://pool.mirror.archlinux32.org/archisos/archlinux32-${DATE}-${ARCH}.iso.torrent" \ + --magnet-link="${magnet_link}" +#~ git -C "${WEBSITE_DIR}" commit 'download/index.html' -m 'download/index.html: new isos ('"${DATE}"')' #~ git -C "${WEBSITE_DIR}" push # when did we last updates the date +%s > "${ARCHISOS_DIR}/lastupdate" +# cleanup mirror and move old ISOs to the archive +echo "Cleaning up old ISOs and pushing them into the archive" +#~ "${base_dir}/cleanup-isos" \ + #~ --mirror-dir="/srv/http/mirror/mirror.archlinux32.org" \ + #~ --archive-dir="/srv/http/mirror/archive.archlinux32.org/iso" + # set permissions correctly of all generated or changed fles echo "Fixing/setting permissions.." -#chown "${MIRROR_USER}:${MIRROR_GROUP}" \ -# "${ARCHISOS_DIR}/${ISO}" "${ARCHISOS_DIR}/${SIGNATURE}" "${ARCHISOS_DIR}/${TORRENT}" \ -# "${ARCHISOS_DIR}/sha512sums" "${ARCHISOS_DIR}/md5sums" "${ARCHISOS_DIR}/lastupdate" +chown "${MIRROR_USER}:${MIRROR_GROUP}" \ + "${ARCHISOS_DIR}/${ISO}" "${ARCHISOS_DIR}/${SIGNATURE}" "${ARCHISOS_DIR}/${TORRENT}" \ + "${ARCHISOS_DIR}/sha512sums" "${ARCHISOS_DIR}/md5sums" "${ARCHISOS_DIR}/lastupdate" echo "Finished." exit 0 - -#-- -# TODO from here: - -archive='/mnt/archlinux32archive' -destination='/mnt/archlinux32/archisos' - - mv $( - printf '%s.torrent\n' ${isos} - ) feed_dual.rss feed_i686.rss "${destination}/" - -# torrent generation - -# ATM no torrents -# ATM no dual -# --torrent-seed-dual "https://pool.mirror.archlinux32.org/archisos/archlinux32-${date}-dual.iso.torrent" \ -# --torrent-seed-i686 "https://pool.mirror.archlinux32.org/archisos/archlinux32-${date}-i686.iso.torrent" \ -# - -# this seems to be cleanup of old isos (which must be moved to the archive -# should be in a separate script too like cleanup-isos, can then be called in build-all - - find "${destination}" \( -name 'archlinux32-*' -o -name 'archlinux-*' \) -not -name 'archlinux32-'"${date}"'-*' \ - | while read -r to_delete; do - if diff -q "${to_delete}" "${archive}/iso/${to_delete#${destination}/}" >/dev/null; then - rm "${to_delete}" - printf '%s\n' "${to_delete}" \ - | sed ' - s@^.*/@@ - s/\./\\./g - s@.*@/ \0$/d@ - ' - fi - done \ - >> "${tmp_dir}/delete-regex" - sed -i -f "${tmp_dir}/delete-regex" "${destination}/sha512sums" - sed -i -f "${tmp_dir}/delete-regex" "${destination}/md5sums" - - echo '... done.' - ;; - - - diff --git a/build-torrent b/build-torrent index b3a842d..0c11e06 100755 --- a/build-torrent +++ b/build-torrent @@ -141,6 +141,7 @@ mktorrent --announce=http://${torrent_server}:6969/announce \ --output="${archiso_dir}/${torrent_file}" \ "${archiso_dir}/${iso}" -scp "${archiso_dir}/${torrent_file}" "${torrent_user}@${torrent_server}:${hefur_dir}" +# I have to go over port 9586, copying manually ATM +#scp -P "${torrent_ssh_port}" "${archiso_dir}/${torrent_file}" "${torrent_user}@${torrent_server}:${hefur_dir}" exit 0 diff --git a/update-website b/update-website index c653820..2d735a1 100755 --- a/update-website +++ b/update-website @@ -220,24 +220,18 @@ date=$(echo "${iso}" \ # update magnet links if [ -s "${tmp_dir}/magnet.link.set" ]; then - if ! tr '&' '\n' < \ - "${tmp_dir}/magnet.link.set" | \ - grep -qxF "dn=${iso}"; then - sed -i ' - /
  • ()?Magnet link for \S\+ ('"${desc}"')/ s/>Magnet link for \S\+ />Magnet link for '"${date}"' / - ' "${website_dir}/download/index.html" - else - >&2 printf 'Magnet link for %s is not for %s, skipping.\n' "${arch}" "${iso}" - fi + sed -i ' + /
  • Magnet link for \S\+ ('"${desc}"')/ s/>Magnet link for \S\+ />Magnet link for '"${date}"' / + ' "${website_dir}/download/index.html" fi # update torrent links if [ -s "${tmp_dir}/torrent-seed.set" ]; then if grep -qF "archlinux32-${date}-${arch}" "${tmp_dir}/torrent-seed.set"; then sed -i ' - /
  • ()?Torrent for \S\+ ('"${desc["${arch}"]}"')/ s/>Torrent for \S\+ />Torrent for '"${date}"' / + /
  • Torrent for \S\+ ('"${desc}"')/ s/>Torrent for \S\+ />Torrent for '"${date}"' / ' "${website_dir}/download/index.html" else >&2 printf 'Torrent seed for %s is not for %s, skipping.\n' "${arch}" "${iso}" @@ -250,11 +244,11 @@ if [ -s "${tmp_dir}/update_iso.set" ]; then >&2 echo "Can't find iso to arch '${mirror_dir}/archisos/${iso}'." exit 1 fi - kernel="$("${base_dir}"/check-iso --output-dir "${mirror_dir}/archisos" --kernel-version)" - size="$("${base_dir}"/check-iso --output-dir "${mirror_dir}/archisos" --size)" - md5="$("${base_dir}"/check-iso --output-dir "${mirror_dir}/archisos" --md5sum)" - sha512="$("${base_dir}"/check-iso --output-dir "${mirror_dir}/archisos" --sha512sum)" - sign_key="$("${base_dir}"/check-iso --output-dir "${mirror_dir}/archisos" --sign-key)" + kernel="$("${base_dir}"/check-iso --iso="${iso}" --output-dir "${mirror_dir}/archisos" --kernel-version)" + size="$("${base_dir}"/check-iso --iso="${iso}" --output-dir "${mirror_dir}/archisos" --size)" + md5="$("${base_dir}"/check-iso --iso="${iso}" --output-dir "${mirror_dir}/archisos" --md5sum)" + sha512="$("${base_dir}"/check-iso --iso="${iso}" --output-dir "${mirror_dir}/archisos" --sha512sum)" + sign_key="$("${base_dir}"/check-iso --iso="${iso}" --output-dir "${mirror_dir}/archisos" --sign-key)" fi # adapt release info on the top -- cgit v1.2.3-70-g09d2