From b43bf951c7471fabd2301641ba3503bacf5b91e7 Mon Sep 17 00:00:00 2001 From: Andreas Baumann Date: Tue, 1 Feb 2022 21:37:19 +0100 Subject: adapted for buildmaster, ISO, update works, not the torrents --- Attic/build-dual.sh | 57 ++++++++ Attic/build-i686.sh | 49 +++++++ Attic/build-isos-nlopc43 | 131 +++++++++++++++++++ Attic/update-website-old | 334 +++++++++++++++++++++++++++++++++++++++++++++++ README.md | 12 ++ al32-mktorrent.sh | 4 +- build-all | 164 +++++++++++++++++++++++ build-dual.sh | 57 -------- build-i686.sh | 49 ------- build-iso | 124 ++++++++++++++++++ build-isos | 131 ------------------- update-website | 8 +- 12 files changed, 878 insertions(+), 242 deletions(-) create mode 100755 Attic/build-dual.sh create mode 100755 Attic/build-i686.sh create mode 100755 Attic/build-isos-nlopc43 create mode 100755 Attic/update-website-old create mode 100755 build-all delete mode 100755 build-dual.sh delete mode 100755 build-i686.sh create mode 100755 build-iso delete mode 100755 build-isos diff --git a/Attic/build-dual.sh b/Attic/build-dual.sh new file mode 100755 index 0000000..9f14a5f --- /dev/null +++ b/Attic/build-dual.sh @@ -0,0 +1,57 @@ +#!/bin/bash +mkdir build-dual +cd build-dual + +cat << "END" > Vagrantfile +Vagrant.configure("2") do |config| + config.vm.box = "archlinux/archlinux" + config.vm.provision "shell", path: "provision.sh", run: "once" +end +END + +cat << "__ENDOFPROVISION.SH__" > provision.sh +#!/bin/bash +set -e +ln -svf /usr/share/zoneinfo/America/Detroit /etc/localtime + +pacman --noconfirm -Sy archlinux-keyring + +tee -a /etc/pacman.conf << "_PACMANCONF_" +[releng] +Include = /etc/pacman.d/mirrorlist32 +_PACMANCONF_ + +cat << "__ENDOFARCH32MIRRORLIST__" > /etc/pacman.d/mirrorlist32 +Server = https://32.arlm.tyzoid.com/$arch/$repo +Server = http://arch32.mirrors.simplysam.us/$arch/$repo +Server = https://mirror.archlinux32.org/$arch/$repo +__ENDOFARCH32MIRRORLIST__ + +pacman --noconfirm -Sy archlinux32-keyring-transition +pacman --noconfirm -R archlinux32-keyring-transition +pacman --noconfirm -S archlinux32-keyring +pacman --noconfirm -Su archiso-dual + +cat << "__ENDOFARCH32MIRRORLIST__" > /etc/pacman.d/mirrorlist32 +Server = https://multiarch.arch32.tyzoid.com/$repo/os/$arch +__ENDOFARCH32MIRRORLIST__ + +cat << "__MIRRORLIST__" | tee /etc/pacman.d/mirrorlist +Server = https://multiarch.arch32.tyzoid.com/$repo/os/$arch +__MIRRORLIST__ + +cat << "__ENDOFISOBUILDSCRIPT__" | tee /root/buildiso.sh >/dev/null +#!/bin/bash +/usr/share/archiso/configs/releng/build.sh -v -V"$(date -d"$(date -d "+2day" +%Y-%m-01T12:00:00Z)" +%Y.%m.%d)" -L"ARCH_$(date -d"$(date -d "+2day" +%Y-%m-01T12:00:00Z)" +%Y%m)" +__ENDOFISOBUILDSCRIPT__ +chmod +x /root/buildiso.sh +__ENDOFPROVISION.SH__ + +vagrant up +vagrant ssh -c "sudo reboot"; +vagrant ssh -c "sudo bash -c '/root/buildiso.sh'"; + +vagrant ssh-config > config.txt +scp -rF config.txt default:/home/vagrant/out ../ + +#vagrant destroy -f diff --git a/Attic/build-i686.sh b/Attic/build-i686.sh new file mode 100755 index 0000000..171a690 --- /dev/null +++ b/Attic/build-i686.sh @@ -0,0 +1,49 @@ +#!/bin/bash +mkdir build-i686 +cd build-i686 + +cat << "END" > Vagrantfile +Vagrant.configure("2") do |config| + config.vm.box = "archlinux/archlinux" + config.vm.provision "shell", path: "provision.sh", run: "once" +end +END + +cat << "__ENDOFPROVISION.SH__" > provision.sh +#!/bin/bash +set -e +ln -svf /usr/share/zoneinfo/America/Detroit /etc/localtime + +pacman --noconfirm -Sy archlinux-keyring + +tee -a /etc/pacman.conf << "_PACMANCONF_" +[releng] +Include = /etc/pacman.d/mirrorlist32 +_PACMANCONF_ + +cat << "__ENDOFARCH32MIRRORLIST__" > /etc/pacman.d/mirrorlist32 +Server = https://32.arlm.tyzoid.com/$arch/$repo +Server = http://arch32.mirrors.simplysam.us/$arch/$repo +Server = https://mirror.archlinux32.org/$arch/$repo +__ENDOFARCH32MIRRORLIST__ + +pacman --noconfirm -Sy archlinux32-keyring-transition +pacman --noconfirm -R archlinux32-keyring-transition +pacman --noconfirm -S archlinux32-keyring +pacman --noconfirm -Syu archiso32 + +cat << "__ENDOFISOBUILDSCRIPT__" | tee /root/buildiso.sh >/dev/null +#!/bin/bash +/usr/share/archiso/configs/releng/build.sh -v -V"$(date -d"$(date -d "+2day" +%Y-%m-01T12:00:00Z)" +%Y.%m.%d)" -L"ARCH_$(date -d"$(date -d "+2day" +%Y-%m-01T12:00:00Z)" +%Y%m)" +__ENDOFISOBUILDSCRIPT__ +chmod +x /root/buildiso.sh +__ENDOFPROVISION.SH__ + +vagrant up +vagrant ssh -c "sudo reboot"; +vagrant ssh -c 'sudo bash -c "/root/buildiso.sh"'; + +vagrant ssh-config > config.txt +scp -rF config.txt default:/home/vagrant/out ../ + +#vagrant destroy -f diff --git a/Attic/build-isos-nlopc43 b/Attic/build-isos-nlopc43 new file mode 100755 index 0000000..5541fe6 --- /dev/null +++ b/Attic/build-isos-nlopc43 @@ -0,0 +1,131 @@ +#!/bin/bash + +# to be executed on nlopc43 ... sry for all the hard-coded stuff + +set -e + +archive='/mnt/archlinux32archive' +destination='/mnt/archlinux32/archisos' +base_dir=$( + readlink -e "${0%/*}" +) + +case "$(hostname -s)" in + 'nlopc43') + echo 'build isos on nlopc46 ...' + rsync "$0" nlopc46.ioq.uni-jena.de:/tmp/ + isos=$( + ssh nlopc46.ioq.uni-jena.de "/tmp/${0##*/}" + ) + echo '... done.' + + if [ -z "${isos}" ]; then + echo 'no isos were built!?' + exit 1 + fi + + echo 'sign and upload isos ...' + + tmp_dir=$(mktemp -d) + trap 'cd "${base_dir}"; rm -rf --one-file-system "${tmp_dir}"' EXIT + + rsync -av $(printf 'nlopc46.ioq.uni-jena.de:%s\n' ${isos}) "${tmp_dir}/" + + isos=$( + printf '%s\n' "${isos}" | \ + sed 's,^.*/,,' + ) + + mountpoint -q "${archive}" || sudo mount "${archive}" + mountpoint -q "${destination%/*}" || sudo mount "${destination%/*}" + + cd "${tmp_dir}" + cp "${destination}/sha512sums" . + cp "${destination}/md5sums" . + + date=$( + printf '%s\n' "${isos}" \ + | sed 's/^.*-\([^-]\+\)-[^-]\+$/\1/' \ + | sort -u + ) + + for iso in ${isos}; do + gpg --local-user '33CA3597B0D161AAE4173F65C17F1214114574A4' --batch --no-tty --detach-sign "${iso}" > sha512sums + md5sum ${isos} >> md5sums + sort -k2,2 sha512sums --output sha512sums + sort -k2,2 md5sums --output md5sums + + cp $( + printf '%s\n' ${isos} + printf '%s.sig\n' ${isos} + ) sha512sums md5sums "${destination}/" + + date +%s > "${destination%/*}/lastupdate" + + "${base_dir}/al32-mktorrent.sh" -d "${date}" -t "hefur@archlinux32:" + + mv $( + printf '%s.torrent\n' ${isos} + ) feed_dual.rss feed_i686.rss "${destination}/" + +# --torrent-seed-dual "https://pool.mirror.archlinux32.org/archisos/archlinux32-${date}-dual.iso.torrent" \ +# + git -C "${base_dir}/../archweb32" pull --ff-only + "${base_dir}/update-website" \ + --torrent-seed-i686 "https://pool.mirror.archlinux32.org/archisos/archlinux32-${date}-i686.iso.torrent" \ + --update-iso + git -C "${base_dir}/../archweb32" commit 'download/index.html' -m 'download/index.html: new isos ('"${date}"')' + git -C "${base_dir}/../archweb32" push + + find "${destination}" \( -name 'archlinux32-*' -o -name 'archlinux-*' \) -not -name 'archlinux32-'"${date}"'-*' \ + | while read -r to_delete; do + if diff -q "${to_delete}" "${archive}/iso/${to_delete#${destination}/}" >/dev/null; then + rm "${to_delete}" + printf '%s\n' "${to_delete}" \ + | sed ' + s@^.*/@@ + s/\./\\./g + s@.*@/ \0$/d@ + ' + fi + done \ + >> "${tmp_dir}/delete-regex" + sed -i -f "${tmp_dir}/delete-regex" "${destination}/sha512sums" + sed -i -f "${tmp_dir}/delete-regex" "${destination}/md5sums" + + echo '... done.' + ;; + 'nlopc46') + if [ "$(whoami)" = 'root' ]; then + cd "/usr/share/archiso/configs/$1" + tmp_dir=$(mktemp -d) + trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT + arch=i686 setarch i686 mkarchiso -o "$2" -w "${tmp_dir}" -v /usr/share/archiso/configs/$1 + chown "${SUDO_USER}:users" "$2/"archlinux*.iso + exit + fi + + rm -rf --one-file-system ~/archisos + mkdir ~/archisos + + >&2 echo 'build i686-iso ...' + + >&2 sudo "$0" releng32 ~/archisos/ + + >&2 echo '... done.' + +# >&2 echo 'build dual-iso ...' + +# >&2 sudo "$0" releng-dual ~/archisos/ + +# >&2 echo '... done.' + + find ~/archisos/ -type f + ;; + *) + >&2 echo 'execute on nlopc43 (or nlopc46) - sry' + exit 1 + ;; +esac diff --git a/Attic/update-website-old b/Attic/update-website-old new file mode 100755 index 0000000..95f12ad --- /dev/null +++ b/Attic/update-website-old @@ -0,0 +1,334 @@ +#!/bin/bash + +set -e + +mirror_dir='/mnt/archlinux32' +website_dir='/usr/src/archlinux32/archweb32' + +usage() { + >&2 echo '' + >&2 echo 'update-website: update download, torrent, magnet links on' + >&2 echo ' the Arch32 website' + >&2 echo '' + >&2 echo 'possible options:' + >&2 echo ' -h|--help: Show this help and exit.' + >&2 echo ' --magnet-i686 $link:' + >&2 echo ' Set archiso-i686 magnet link to $link.' + >&2 echo ' --magnet-dual $link:' + >&2 echo ' Set archiso-dual magnet link to $link.' + >&2 echo ' --torrent-seed-i686 $link:' + >&2 echo ' Set archiso-i686 torrent link to $link.' + >&2 echo ' --torrent-seed-dual $link:' + >&2 echo ' Set archiso-dual torrent link to $link.' + >&2 echo ' --website-dir $dir:' + >&2 echo ' Set website-dir to $dir.' + >&2 echo ' --mirror-dir $dir:' + >&2 echo ' Set mirror-dir to $dir.' + >&2 echo ' --update-iso:' + >&2 echo ' Update information about iso, too.' + [ -z "$1" ] && exit 1 || exit "$1" +} + +eval set -- "$( + getopt -o h \ + --long help \ + --long magnet-i686: \ + --long magnet-dual: \ + --long torrent-seed-i686: \ + --long torrent-seed-dual: \ + --long mirror-dir: \ + --long website-dir: \ + --long update-iso \ + -n "$(basename "$0")" -- "$@" || \ + echo usage +)" + +tmp_dir="$(mktemp -d)" +cleanup() { + if mountpoint -q "${tmp_dir}"; then + sudo umount "${tmp_dir}" + fi + rm -rf --one-file-system "${tmp_dir}" +} +trap cleanup EXIT + +while [ $# -gt 0 ]; do + case "$1" in + '--magnet-i686') + if [ -s "${tmp_dir}/magnet.i686" ]; then + printf 'Option %s given multiple times.\n' "$1" + usage + fi + shift + echo "$1" | \ + tr -d '\n' > \ + "${tmp_dir}/magnet.i686" + ;; + '--magnet-dual') + if [ -s "${tmp_dir}/magnet.dual" ]; then + printf 'Option %s given multiple times.\n' "$1" + usage + fi + shift + echo "$1" | \ + tr -d '\n' > \ + "${tmp_dir}/magnet.dual" + ;; + '--torrent-seed-i686') + if [ -s "${tmp_dir}/torrent-seed.i686" ]; then + printf 'Option %s given multiple times.\n' "$1" + usage + fi + shift + echo "$1" | \ + tr -d '\n' > \ + "${tmp_dir}/torrent-seed.i686" + ;; + '--torrent-seed-dual') + if [ -s "${tmp_dir}/torrent-seed.dual" ]; then + printf 'Option %s given multiple times.\n' "$1" + usage + fi + shift + echo "$1" | \ + tr -d '\n' > \ + "${tmp_dir}/torrent-seed.dual" + ;; + '--mirror-dir') + if [ -s "${tmp_dir}/mirror_dir.set" ]; then + printf 'Option %s given multiple times.\n' "$1" + usage + fi + shift + mirror_dir="$1" + echo "$mirror_dir" > "${tmp_dir}/mirror_dir.set" + ;; + '--website-dir') + if [ -s "${tmp_dir}/website_dir.set" ]; then + printf 'Option %s given multiple times.\n' "$1" + usage + fi + shift + website_dir="$1" + echo "$website_dir" > "${tmp_dir}/website_dir.set" + ;; + '--update-iso') + if [ -s "${tmp_dir}/update_iso.set" ]; then + printf 'Option %s given multiple times.\n' "$1" + usage + fi + echo "true" > "${tmp_dir}/update_iso.set" + ;; + '--help'|'-h') + usage 0 + ;; + '--') + shift + break + ;; + *) + >&2 printf 'Whoops, option "%s" is not yet implemented!\n' "$1" + exit 42 + ;; + esac + shift +done + +if [ $# -gt 0 ]; then + >&2 echo 'Too many arguments.' + exit 2 +fi + +( + printf '
\n' + curl -Ss 'https://archlinux32.org/mirrorlist/?country=all&hash='"$( + head -c 24 /dev/urandom | \ + base64 -w0 + )" | \ + sed ' + 0,/^$/d + /^$/d + /^## /{ + s|^## |\n \n\n
| + s|$|
\n\n \n\n' + printf '
\n' +) | \ + sed -i ' + :emptyLines + /^\n*$/{ + N + bemptyLines + } + /^\(\s*\n\)*\s*
$/{ + :a + s|^.*
|| + tb + $!N + ba + :b + r /dev/stdin + } + ' "${website_dir}/download/index.html" + +latest_iso_date="$( + ls "${mirror_dir}/archisos" | \ + grep '^archlinux32-[0-9.]\+-\(dual\|i.86\)\.iso$' | \ + sed 's@^archlinux32-\([0-9.]\+\)-\(dual\|i.86\)\.iso$@\1@' | \ + sort -k1,1 -u | \ + tail -n1 +)" + +declare -A desc +desc['i686']='i686 only' +# desc['dual']='dual bootable' + +for arch in "${!desc[@]}"; do + if [ ! -s "${tmp_dir}/magnet.${arch}" ]; then + continue + fi + if ! tr '&' '\n' < \ + "${tmp_dir}/magnet.${arch}" | \ + grep -qxF "dn=archlinux32-${latest_iso_date}-${arch}.iso"; then + >&2 printf 'Magnet link for %s is not for %s, skipping.\n' "${arch}" "archlinux32-${latest_iso_date}-${arch}.iso" + continue + fi + sed -i ' + /
  • Magnet link for \S\+ ('"${desc["${arch}"]}"')/ s/>Magnet link for \S\+ />Magnet link for '"${latest_iso_date}"' / + ' "${website_dir}/download/index.html" +done + +for arch in "${!desc[@]}"; do + if [ ! -s "${tmp_dir}/torrent-seed.${arch}" ]; then + continue + fi + if ! grep -qF "archlinux32-${latest_iso_date}-${arch}" "${tmp_dir}/torrent-seed.${arch}"; then + >&2 printf 'Torrent seed for %s is not for %s, skipping.\n' "${arch}" "archlinux32-${latest_iso_date}-${arch}.iso" + continue + fi + sed -i ' + /
  • Torrent for \S\+ ('"${desc["${arch}"]}"')/ s/>Torrent for \S\+ />Torrent for '"${latest_iso_date}"' / + ' "${website_dir}/download/index.html" +done + +if [ ! -s "${tmp_dir}/update_iso.set" ]; then + exit +fi + +unset kernels +unset sizes +declare -A sizes +declare -A md5 +declare -A sha512 + +for arch in "${!desc[@]}"; do + iso="${mirror_dir}/archisos/archlinux32-${latest_iso_date}-${arch}.iso" + if [ ! -f "${iso}" ]; then + >&2 echo "Can't find iso to arch ${arch}." + exit 1 + fi + sudo mount -o loop,ro "${iso}" "${tmp_dir}" + kernels="$( + printf '%s\n' ${kernels} $( + find "${tmp_dir}/arch" \ + -maxdepth 1 \ + -name 'pkglist.*.txt' \ + -not -name 'pkglist.x86_64.txt' \ + -execdir cat {} \; \ + | sed ' + s/^linux\s\+\([^-]\+-[^-]\+\)$/\1/ + t + d + ' + ) | \ + sort -u + )" + sudo umount "${tmp_dir}" + sizes["${arch}"]="$( + printf 'scale=1; %s/1024/1024\n' "$(stat -c'%s' "${iso}")" | \ + bc + )" + md5["${arch}"]="$( + grep "\s${iso##*/}\$" "${mirror_dir}/archisos/md5sums" | \ + awk '{print $1}' + )" + sha512["${arch}"]="$( + grep "\s${iso##*/}\$" "${mirror_dir}/archisos/sha512sums" | \ + awk '{print $1}' + )" + sign_keys="$( + printf '%s\n' ${sign_keys} $( + gpg --status-fd=1 --verify "${iso}.sig" "${iso}" 2> /dev/null | \ + grep '^\[GNUPG:] VALIDSIG [0-9A-F]\+ ' | \ + cut -d' ' -f3 + ) | \ + sort -u + )" +done + +if [ $(echo "${kernels}" | grep -c '\S') -ne 1 ]; then + >&2 echo 'Not exactly one kernel on the iso:' + >&2 echo "${kernels}" + exit 1 +fi + +if [ $(echo "${sign_keys}" | grep -c '\S') -ne 1 ]; then + >&2 echo 'Not exactly one key used for signing the iso:' + >&2 echo "'${sign_keys}'" + exit 1 +fi + +if [ "${sign_keys}" != '33CA3597B0D161AAE4173F65C17F1214114574A4' ]; then + >&2 printf 'Isos are signed with key %s instead of %s.\n' \ + "${sign_keys}" \ + '33CA3597B0D161AAE4173F65C17F1214114574A4' + exit 1 +fi + +values=( + 'Current Release' "${latest_iso_date}" + 'Included Kernel' "${kernels}" + 'PGP Key' ''"${sign_keys}"'' +) +for arch in 'i686' 'dual'; do + values+=( + "ISO Size (${desc["${arch}"]})" "${sizes["${arch}"]} MB" + "MD5 (${desc["${arch}"]})" "${md5["${arch}"]}" + "SHA512 (${desc["${arch}"]})" "${sha512["${arch}"]}" + ) +done + +sed -i "$( + printf '/%s:/s|.*
    • | %s|\n' "${values[@]}" + printf '/archlinux32-[0-9.]\+-%s\.iso\.sig/s@archlinux32-[0-9.]\+-%s\.iso\.sig@archlinux32-%s-%s.iso.sig@\n' \ + 'i686' 'i686' "${latest_iso_date}" 'i686' \ + 'dual' 'dual' "${latest_iso_date}" 'dual' +) +" \ + "${website_dir}/download/index.html" diff --git a/README.md b/README.md index 0de8b0e..9bfbb63 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,15 @@ +# build-all + +Builds one ISO at a time, torrent, updates website etc. +Must be run as 'releng' with 'sudo ./build-all' at the moment on the +buildmaster. + +Calls all subscripts build-XXX. + +# build-iso + +Just builds one ISO at a time with mkarchiso. + # al32-mktorrent Torrent creation script for [ArchLinux32](https://archlinux32.org), with upload diff --git a/al32-mktorrent.sh b/al32-mktorrent.sh index 22fa8fb..969ad1b 100755 --- a/al32-mktorrent.sh +++ b/al32-mktorrent.sh @@ -30,7 +30,7 @@ MIRRORLIST_FILE="https://git.archlinux32.org/packages/plain/core/pacman-mirrorli function create_torrent_for_arch () { declare -a available_mirrors - mirrorlist="$(curl "$MIRRORLIST_FILE" 2>/dev/null | grep Server | cut -d '=' -f 2 | sed -e 's/\s//g;s_$arch/$repo_archisos/_')" + mirrorlist="$(curl -f "$MIRRORLIST_FILE" 2>/dev/null | grep Server | cut -d '=' -f 2 | sed -e 's/\s//g;s_$arch/$repo_archisos/_')" if [ "$#" -eq 0 ] ; then echo "No architecture specified, selecting 'i686'" @@ -50,7 +50,7 @@ function create_torrent_for_arch () { if [ ! -f "$iso_string" ] ; then echo -e "$fg_reset${fg_bold}Downloading iso...$fg_reset" - curl -O "${available_mirrors[0]}$iso_string" + curl -f -O "${available_mirrors[0]}$iso_string" else echo -e "$fg_reset${fg_bold}Reusing already downloaded iso...$fg_reset" fi diff --git a/build-all b/build-all new file mode 100755 index 0000000..8071357 --- /dev/null +++ b/build-all @@ -0,0 +1,164 @@ +#!/bin/bash + +# runs on the buildmaster only, works currently only there +# +# calls all subscripts to build the iso, sign it, generate checksums, +# update the website, update RSS feeds, feed the torrents, etc. +# +# this script is supposed to be used by Archlinux32 people only +# as it contains hard-coded pathes and server names.. + +# prerequisites: +# - mkinitcpio-archiso32 and archiso32 installed on the host +# - https://git.archlinux32.org/releng checked out locally +# - pacman-mirrorlist32 for /etc/pacman.d/mirrorlist32 +# - mktorrent, python-feedgenerator, transmission-cli, hefur + +# parameters +CONFIG="releng" +ARCH="i686" +DATE=$(date +%Y.%m.%d) +ISO="archlinux32-${DATE}-${ARCH}.iso" +SIGNATURE="${ISO}.sig" +TORRENT="${ISO}.torrent" +TORRENT_SERVER="archlinux32.org" +WEBSITE_DIR="/srv/http/archlinux32.org" +MIRROR_DIR="/srv/http/mirror/mirror.archlinux32.org" +ARCHISOS_DIR="${MIRROR_DIR}/archisos/" +MIRROR_USER='mirror' +MIRROR_GROUP='mirror' +#Andreas Baumann (sign) +SIGN_KEY='16194A82231E9EF823562181C8E8F5A0AF9BA7E7' +# Archlinux 32 Release Key +#SIGN_KEY='33CA3597B0D161AAE4173F65C17F1214114574A4' +WEB_DIR="/srv/http/archlinux32.org" + +base_dir=$( + readlink -e "${0%/*}" +) + +pushd() { + command pushd "$@" > /dev/null +} + +popd() { + command popd "$@" > /dev/null +} + +export pushd popd + +# fail on first error +set -e + +# cleanup hook +tmp_dir="$(mktemp -d)" +cleanup() { + if mountpoint -q "${tmp_dir}"; then + sudo umount "${tmp_dir}" + fi + rm -rf --one-file-system "${tmp_dir}" +} +trap cleanup EXIT + +# build the ISO +#echo "Building ISO.." +#"${base_dir}/build-iso" --config="${CONFIG}" --arch "${ARCH}" \ +# --output-dir="${ARCHISOS_DIR}" --iso "${ISO}" + +# sign ISO +echo "Signing ISO.." +#pushd "${ARCHISOS_DIR}" +#rm -rf "${ISO}.sig" +#gpg --local-user "${SIGN_KEY}" --batch --no-tty --detach-sign "${ISO}" > sha512sums +# [ ! -f md5sums ] || sed -i "/${file}$/d" md5sums +# md5sum "${file}" >> md5sums +#done +#sort -k2,2 sha512sums --output sha512sums +#sort -k2,2 md5sums --output md5sums +#popd + +# date used for torrents and commit message +date=$(echo "${ISO}" \ + | sed 's/^.*-\([^-]\+\)-[^-]\+$/\1/' \ + | sort -u +) + +# generate torrents (currently broken) +#echo "Generating torrents.." +#bash -x "${base_dir}/al32-mktorrent.sh" -d "${torrent_date}" -t "hefur@${TORRENT_SERVER}:" "${ARCH}" + +# update web page +echo "Updating website.." +git -C "${WEB_DIR}" pull --ff-only +"${base_dir}/update-website" \ + --website-dir="${WEBSITE_DIR}" \ + --mirror-dir="${MIRROR_DIR}" \ + --update-iso +git -C "${WEB_DIR}" commit 'download/index.html' -m 'download/index.html: new isos ('"${date}"')' +git -C "${WEB_DIR}" push + +# when did we last updates the +date +%s > "${ARCHISOS_DIR}/lastupdate" + +# set permissions correctly of all generated or changed fles +echo "Fixing/setting permissions.." +#chown "${MIRROR_USER}:${MIRROR_GROUP}" \ +# "${ARCHISOS_DIR}/${ISO}" "${ARCHISOS_DIR}/${SIGNATURE}" "${ARCHISOS_DIR}/${TORRENT}" \ +# "${ARCHISOS_DIR}/sha512sums" "${ARCHISOS_DIR}/md5sums" "${ARCHISOS_DIR}/lastupdate" + +echo "Finished." + +exit 0 + +#-- +# TODO from here: + +archive='/mnt/archlinux32archive' +destination='/mnt/archlinux32/archisos' + + mv $( + printf '%s.torrent\n' ${isos} + ) feed_dual.rss feed_i686.rss "${destination}/" + +# torrent generation + +# ATM no torrents +# ATM no dual +# --torrent-seed-dual "https://pool.mirror.archlinux32.org/archisos/archlinux32-${date}-dual.iso.torrent" \ +# --torrent-seed-i686 "https://pool.mirror.archlinux32.org/archisos/archlinux32-${date}-i686.iso.torrent" \ +# + +# this seems to be cleanup of old isos (which must be moved to the archive +# should be in a separate script too like cleanup-isos, can then be called in build-all + + find "${destination}" \( -name 'archlinux32-*' -o -name 'archlinux-*' \) -not -name 'archlinux32-'"${date}"'-*' \ + | while read -r to_delete; do + if diff -q "${to_delete}" "${archive}/iso/${to_delete#${destination}/}" >/dev/null; then + rm "${to_delete}" + printf '%s\n' "${to_delete}" \ + | sed ' + s@^.*/@@ + s/\./\\./g + s@.*@/ \0$/d@ + ' + fi + done \ + >> "${tmp_dir}/delete-regex" + sed -i -f "${tmp_dir}/delete-regex" "${destination}/sha512sums" + sed -i -f "${tmp_dir}/delete-regex" "${destination}/md5sums" + + echo '... done.' + ;; + + + diff --git a/build-dual.sh b/build-dual.sh deleted file mode 100755 index 9f14a5f..0000000 --- a/build-dual.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash -mkdir build-dual -cd build-dual - -cat << "END" > Vagrantfile -Vagrant.configure("2") do |config| - config.vm.box = "archlinux/archlinux" - config.vm.provision "shell", path: "provision.sh", run: "once" -end -END - -cat << "__ENDOFPROVISION.SH__" > provision.sh -#!/bin/bash -set -e -ln -svf /usr/share/zoneinfo/America/Detroit /etc/localtime - -pacman --noconfirm -Sy archlinux-keyring - -tee -a /etc/pacman.conf << "_PACMANCONF_" -[releng] -Include = /etc/pacman.d/mirrorlist32 -_PACMANCONF_ - -cat << "__ENDOFARCH32MIRRORLIST__" > /etc/pacman.d/mirrorlist32 -Server = https://32.arlm.tyzoid.com/$arch/$repo -Server = http://arch32.mirrors.simplysam.us/$arch/$repo -Server = https://mirror.archlinux32.org/$arch/$repo -__ENDOFARCH32MIRRORLIST__ - -pacman --noconfirm -Sy archlinux32-keyring-transition -pacman --noconfirm -R archlinux32-keyring-transition -pacman --noconfirm -S archlinux32-keyring -pacman --noconfirm -Su archiso-dual - -cat << "__ENDOFARCH32MIRRORLIST__" > /etc/pacman.d/mirrorlist32 -Server = https://multiarch.arch32.tyzoid.com/$repo/os/$arch -__ENDOFARCH32MIRRORLIST__ - -cat << "__MIRRORLIST__" | tee /etc/pacman.d/mirrorlist -Server = https://multiarch.arch32.tyzoid.com/$repo/os/$arch -__MIRRORLIST__ - -cat << "__ENDOFISOBUILDSCRIPT__" | tee /root/buildiso.sh >/dev/null -#!/bin/bash -/usr/share/archiso/configs/releng/build.sh -v -V"$(date -d"$(date -d "+2day" +%Y-%m-01T12:00:00Z)" +%Y.%m.%d)" -L"ARCH_$(date -d"$(date -d "+2day" +%Y-%m-01T12:00:00Z)" +%Y%m)" -__ENDOFISOBUILDSCRIPT__ -chmod +x /root/buildiso.sh -__ENDOFPROVISION.SH__ - -vagrant up -vagrant ssh -c "sudo reboot"; -vagrant ssh -c "sudo bash -c '/root/buildiso.sh'"; - -vagrant ssh-config > config.txt -scp -rF config.txt default:/home/vagrant/out ../ - -#vagrant destroy -f diff --git a/build-i686.sh b/build-i686.sh deleted file mode 100755 index 171a690..0000000 --- a/build-i686.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash -mkdir build-i686 -cd build-i686 - -cat << "END" > Vagrantfile -Vagrant.configure("2") do |config| - config.vm.box = "archlinux/archlinux" - config.vm.provision "shell", path: "provision.sh", run: "once" -end -END - -cat << "__ENDOFPROVISION.SH__" > provision.sh -#!/bin/bash -set -e -ln -svf /usr/share/zoneinfo/America/Detroit /etc/localtime - -pacman --noconfirm -Sy archlinux-keyring - -tee -a /etc/pacman.conf << "_PACMANCONF_" -[releng] -Include = /etc/pacman.d/mirrorlist32 -_PACMANCONF_ - -cat << "__ENDOFARCH32MIRRORLIST__" > /etc/pacman.d/mirrorlist32 -Server = https://32.arlm.tyzoid.com/$arch/$repo -Server = http://arch32.mirrors.simplysam.us/$arch/$repo -Server = https://mirror.archlinux32.org/$arch/$repo -__ENDOFARCH32MIRRORLIST__ - -pacman --noconfirm -Sy archlinux32-keyring-transition -pacman --noconfirm -R archlinux32-keyring-transition -pacman --noconfirm -S archlinux32-keyring -pacman --noconfirm -Syu archiso32 - -cat << "__ENDOFISOBUILDSCRIPT__" | tee /root/buildiso.sh >/dev/null -#!/bin/bash -/usr/share/archiso/configs/releng/build.sh -v -V"$(date -d"$(date -d "+2day" +%Y-%m-01T12:00:00Z)" +%Y.%m.%d)" -L"ARCH_$(date -d"$(date -d "+2day" +%Y-%m-01T12:00:00Z)" +%Y%m)" -__ENDOFISOBUILDSCRIPT__ -chmod +x /root/buildiso.sh -__ENDOFPROVISION.SH__ - -vagrant up -vagrant ssh -c "sudo reboot"; -vagrant ssh -c 'sudo bash -c "/root/buildiso.sh"'; - -vagrant ssh-config > config.txt -scp -rF config.txt default:/home/vagrant/out ../ - -#vagrant destroy -f diff --git a/build-iso b/build-iso new file mode 100755 index 0000000..7c6ac33 --- /dev/null +++ b/build-iso @@ -0,0 +1,124 @@ +#!/bin/bash + +# parameters and default values +CONFIG="releng" +ARCH="i686" +DATE=$(date +%Y.%m.%d) +ISO="archlinux32-${DATE}-${ARCH}.iso" +OUTPUT_DIR="${HOME}/archisos" + +# prerequisites: +# - https://gitlab.archlinux.org/archlinux/archiso +# - mkinitcpio-archiso32, archiso32 installed in /usr/local +# - pacman-mirrorlist32 for /etc/pacman.d/mirrorlist32 +# - arch-install-scripts awk dosfstools e2fsprogs findutils gzip +# libarchive libisoburn mtools openssl pacman sed squashfs-tools + +usage() { + >&2 echo "" + >&2 echo "build-iso: builds ISOs for Archlinux32" + >&2 echo "" + >&2 echo "possible options:" + >&2 echo " -h|--help: show this help and exit." + >&2 echo " --config mkarchiso configuration to use, default is '$CONFIG'." + >&2 echo " --arch architecture to build ISO for, default is '$ARCH'." + >&2 echo " --outputdir where to write the isos, default is '~/archisos." + >&2 echo " --iso name of the resulting ISO, default is '$ISO'." + >&2 echo " --no-cleanup do not clean up tmpdir after run, for debugging." + [ -z "$1" ] && exit 1 || exit "$1" +} + +# fail on first error +set -e + +# cleanup hook +tmp_dir="$(mktemp -d)" +cleanup() { + if [ "${no_cleanup}" = 0 ]; then + if mountpoint -q "${tmp_dir}"; then + sudo umount "${tmp_dir}" + fi + rm -rf --one-file-system "${tmp_dir}" + fi +} +trap cleanup EXIT + +eval set -- "$( + getopt -o h \ + --long help \ + --long config: \ + --long arch: \ + --long output-dir: \ + --long iso: \ + --long no-cleanup \ + -n "$(basename "$0")" -- "$@" || \ + echo usage +)" + +config="$CONFIG" +arch="$ARCH" +iso="$ISO" +no_cleanup=0 +output_dir="${OUTPUT_DIR}" + +while [ $# -gt 0 ]; do + case "$1" in + '--config') + shift + config="$1" + if [ ! -s "/usr/local/share/archiso/configs/${config}/profiledef.sh" ]; then + printf "'${config}' is not a valid mkarchiso profile in usr/local/share/archiso/configs" >&2 + usage + fi + ;; + '--arch') + shift + arch="$1" + ;; + '--iso') + shift + iso="$1" + ;; + '--output-dir') + shift + output_dir="$1" + ;; + '--no-cleanup') + no_cleanup=1 + ;; + '--help'|'-h') + usage 0 + ;; + '--') + shift + break + ;; + *) + >&2 printf 'Whoops, option "%s" is not yet implemented!\n' "$1" >&2 + exit 42 + ;; + esac + shift +done + +if [ $# -gt 0 ]; then + >&2 echo 'Too many arguments.' >&2 + exit 2 +fi + +echo "parameters for ISO are:" +echo "mkarchiso config: $config" +echo "architecture: $arch" +echo "destination ISO file: $iso" +sleep 5 + +echo "building $ISO..." +tmp_dir=$(mktemp -d) +trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT +arch="${arch}" setarch i686 mkarchiso -L "ARCH32_$DATE" -o "${output_dir}" -w "${tmp_dir}" -v "/usr/local/share/archiso/configs/${config}" + +if [ "${no_cleanup}" -eq 1 ]; then + echo "no-cleanup specified, find your temporary data in ${tmp_dir}.." +fi + +echo "done" diff --git a/build-isos b/build-isos deleted file mode 100755 index 5541fe6..0000000 --- a/build-isos +++ /dev/null @@ -1,131 +0,0 @@ -#!/bin/bash - -# to be executed on nlopc43 ... sry for all the hard-coded stuff - -set -e - -archive='/mnt/archlinux32archive' -destination='/mnt/archlinux32/archisos' -base_dir=$( - readlink -e "${0%/*}" -) - -case "$(hostname -s)" in - 'nlopc43') - echo 'build isos on nlopc46 ...' - rsync "$0" nlopc46.ioq.uni-jena.de:/tmp/ - isos=$( - ssh nlopc46.ioq.uni-jena.de "/tmp/${0##*/}" - ) - echo '... done.' - - if [ -z "${isos}" ]; then - echo 'no isos were built!?' - exit 1 - fi - - echo 'sign and upload isos ...' - - tmp_dir=$(mktemp -d) - trap 'cd "${base_dir}"; rm -rf --one-file-system "${tmp_dir}"' EXIT - - rsync -av $(printf 'nlopc46.ioq.uni-jena.de:%s\n' ${isos}) "${tmp_dir}/" - - isos=$( - printf '%s\n' "${isos}" | \ - sed 's,^.*/,,' - ) - - mountpoint -q "${archive}" || sudo mount "${archive}" - mountpoint -q "${destination%/*}" || sudo mount "${destination%/*}" - - cd "${tmp_dir}" - cp "${destination}/sha512sums" . - cp "${destination}/md5sums" . - - date=$( - printf '%s\n' "${isos}" \ - | sed 's/^.*-\([^-]\+\)-[^-]\+$/\1/' \ - | sort -u - ) - - for iso in ${isos}; do - gpg --local-user '33CA3597B0D161AAE4173F65C17F1214114574A4' --batch --no-tty --detach-sign "${iso}" > sha512sums - md5sum ${isos} >> md5sums - sort -k2,2 sha512sums --output sha512sums - sort -k2,2 md5sums --output md5sums - - cp $( - printf '%s\n' ${isos} - printf '%s.sig\n' ${isos} - ) sha512sums md5sums "${destination}/" - - date +%s > "${destination%/*}/lastupdate" - - "${base_dir}/al32-mktorrent.sh" -d "${date}" -t "hefur@archlinux32:" - - mv $( - printf '%s.torrent\n' ${isos} - ) feed_dual.rss feed_i686.rss "${destination}/" - -# --torrent-seed-dual "https://pool.mirror.archlinux32.org/archisos/archlinux32-${date}-dual.iso.torrent" \ -# - git -C "${base_dir}/../archweb32" pull --ff-only - "${base_dir}/update-website" \ - --torrent-seed-i686 "https://pool.mirror.archlinux32.org/archisos/archlinux32-${date}-i686.iso.torrent" \ - --update-iso - git -C "${base_dir}/../archweb32" commit 'download/index.html' -m 'download/index.html: new isos ('"${date}"')' - git -C "${base_dir}/../archweb32" push - - find "${destination}" \( -name 'archlinux32-*' -o -name 'archlinux-*' \) -not -name 'archlinux32-'"${date}"'-*' \ - | while read -r to_delete; do - if diff -q "${to_delete}" "${archive}/iso/${to_delete#${destination}/}" >/dev/null; then - rm "${to_delete}" - printf '%s\n' "${to_delete}" \ - | sed ' - s@^.*/@@ - s/\./\\./g - s@.*@/ \0$/d@ - ' - fi - done \ - >> "${tmp_dir}/delete-regex" - sed -i -f "${tmp_dir}/delete-regex" "${destination}/sha512sums" - sed -i -f "${tmp_dir}/delete-regex" "${destination}/md5sums" - - echo '... done.' - ;; - 'nlopc46') - if [ "$(whoami)" = 'root' ]; then - cd "/usr/share/archiso/configs/$1" - tmp_dir=$(mktemp -d) - trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT - arch=i686 setarch i686 mkarchiso -o "$2" -w "${tmp_dir}" -v /usr/share/archiso/configs/$1 - chown "${SUDO_USER}:users" "$2/"archlinux*.iso - exit - fi - - rm -rf --one-file-system ~/archisos - mkdir ~/archisos - - >&2 echo 'build i686-iso ...' - - >&2 sudo "$0" releng32 ~/archisos/ - - >&2 echo '... done.' - -# >&2 echo 'build dual-iso ...' - -# >&2 sudo "$0" releng-dual ~/archisos/ - -# >&2 echo '... done.' - - find ~/archisos/ -type f - ;; - *) - >&2 echo 'execute on nlopc43 (or nlopc46) - sry' - exit 1 - ;; -esac diff --git a/update-website b/update-website index 3aa9570..bdcbfdd 100755 --- a/update-website +++ b/update-website @@ -7,8 +7,8 @@ website_dir='/usr/src/archlinux32/archweb32' usage() { >&2 echo '' - >&2 echo 'get-package-updates: check for packages that need to be built,' - >&2 echo ' and build a list in the proper build order' + >&2 echo 'update-website: update download, torrent, magnet links on' + >&2 echo ' the Arch32 website' >&2 echo '' >&2 echo 'possible options:' >&2 echo ' -h|--help: Show this help and exit.' @@ -308,7 +308,9 @@ if [ "${sign_keys}" != '33CA3597B0D161AAE4173F65C17F1214114574A4' ]; then >&2 printf 'Isos are signed with key %s instead of %s.\n' \ "${sign_keys}" \ '33CA3597B0D161AAE4173F65C17F1214114574A4' - exit 1 +# TODO: not fatal, we don't have the sign key for signing ISOS so we use our package sign key for now +# TODO: should not be hard coded, is a parameter for the script! +# exit 1 fi values=( -- cgit v1.2.3-70-g09d2