path: root/bin/build-packages

#!/bin/sh

# build packages one by one, then upload the binary package to the repository server

# shellcheck disable=SC2119,SC2120

# shellcheck source=../lib/load-configuration
. "${0%/*}/../lib/load-configuration"

# TODO: report back memory and hdd stats to the build master on successful build

# TODO: some build failures should skip straws - e.g. if dependencies are unavailable,
# we should not try to rebuild with a cleaned chroot

# TODO: some build environments should be archived when the build fails - but:
# which/how?

# TODO: it might be worth trying to build with makedepends installed from
# [testing] and only depends installed from [staging]

# TODO: the following things need proper locking and/or proper handling of
# external locks:
# - `git` on the package source
# - `archbuild` (e.g. /var/lib/archbuilds/staging-i686-build/root.lock)
# - `find ... -exec umount` on the build chroot

# TODO: releave some locking conditions - meta goal: be able to run multiple
# build slaves on one host from within one archlinux32/builder repository

# TODO: remove hard-coded package suffixes

# TODO: save build-logs in case of success, too - only the build is
# guaranteed to have been successful, maybe the package is broken.

# shellcheck disable=SC2016
usage() {
  >&2 echo ''
  >&2 echo 'build-packages: build package(s) on the build-list'
  >&2 echo ''
  >&2 echo 'possible options:'
  >&2 echo '  -d|--diff source-directory:'
  >&2 echo '               Apply the diff between $source-directory/PKGBUILD and'
  >&2 echo '               $source-directory/../../trunk/PKGBUILD before building.'
  >&2 echo '               This option is intended for building from trunk where minor'
  >&2 echo '               fixes did not get released yet. Hence, it conflicts intentionally'
  >&2 echo '               with -n, -l, -t or -x and it requires -p.'
  >&2 echo '  -h|--help:   Show this help and exit.'
  >&2 echo '  -l|--local pkgname.git-revision.git-mod-revision.repository.architecture:'
  >&2 echo '               Build the given package without asking / reporting to the'
  >&2 echo '               build master. Cannot be combined with -n, -p, -t or -x.'
  >&2 echo '             where:'
  >&2 echo '               - git-revision: packages-HEAD or community-HEAD or a valid commit'
  >&2 echo '               - git-mod-revision: work-tree or a valid commit'
  >&2 echo '             example:'
  >&2 echo '               build-package -l   which.packages-HEAD.work-tree.core.i686'
  >&2 echo '  -n count:    Build $count packages (if available), then exit.'
  >&2 echo '               $count=0 is interpreted as infinity.'
  >&2 echo '               The default is $count=1 or 0 iff -t or -x is given.'
  >&2 echo '               Cannot be combined with -l.'
  >&2 echo '  -p|--prefer pkgname:'
  >&2 echo '               Ask the build master to get an assignment for the given'
  >&2 echo '               package but built what is offered anyway. Cannot be combined'
  >&2 echo '               with -l.'
  >&2 echo '  -s|--straw $straw:'
  >&2 echo '               Use this straw instead of the preconfigured ones. -- May be'
  >&2 echo '               given multiple times to allow using multiple straws.'
  >&2 echo '  -t seconds:  Do not request new assignment(s) $seconds seconds after start.'
  >&2 echo '               Cannot be combined with -l.'
  >&2 echo '  -x:          If package build fails, do not request new assignment(s).'
  >&2 echo '               Cannot be combined with -l.'
  >&2 echo ''
  >&2 echo 'known straws (separated by and enclosed in ":", sets of straws separated by " "):'
  >&2 echo '  :clean_chroot:'
  >&2 echo '    clean the chroot before building'
  >&2 echo '  :haskell_without_check:'
  >&2 echo '    try with :without_check: iff this is a haskell-*, python-* or python2-* package'
  >&2 echo '  :mirrored_source:'
  >&2 echo '    compile from source tarbal from sources.archlinux.org if possible'
  >&2 echo '  :mirrored_source_by_hash:'
  >&2 echo '    download sources from sources.archlinux32.org by hash if possible'
  >&2 echo '  :on_x86_64:'
  >&2 echo '    build any package with x86_64 arch'
  >&2 echo '  :without_check:'
  >&2 echo '    run makepkg with "--no-check"'
  >&2 echo '  :without_systemd_nspawn:'
  >&2 echo '    do not invoke archbuild'"'"'s systemd-nspawn fanciness [RISKY]'
  >&2 echo '  :with_build_support:'
  >&2 echo '    allow using [build-support]'
  >&2 echo '  :with_/dev/fuse:'
  >&2 echo '    mount /dev/fuse into the chroot'
  [ -z "$1" ] && exit 1 || exit "$1"
}

eval set -- "$(
  getopt -o d:hl:n:p:s:t:ux \
    --long diff: \
    --long help \
    --long local: \
    --long prefer: \
    --long straw: \
    --long upload \
    -n "$(basename "$0")" -- "$@" || \
  echo usage
)"

unset count
unset diff_source_dir
unset forced_package
unset forced_straws
unset prefered_package
exit_after_failure=false
timeout=0

while true
do
  case "$1" in
    -d|--diff)
      shift
      if [ -n "${diff_source_dir}" ]; then
        >&2 echo 'Option -d, --diff can be given only once.'
        usage
      fi
      diff_source_dir="$1"
    ;;
    -h|--help)
      usage 0
    ;;
    -l|--local)
      shift
      if [ -n "${forced_package}" ]; then
        >&2 echo 'Option -l, --local can be given only once.'
        usage
      fi
      forced_package="$1"
    ;;
    -n)
      shift
      count="$1"
      [ "${count}" -eq 0 ] && \
        count=-1
    ;;
    -p|--prefer)
      shift
      if [ -n "${prefered_package}" ]; then
        >&2 echo 'Option -p, --prefer can be given only once.'
        usage
      fi
      prefered_package="$1"
    ;;
    -s|--straw)
      shift
      forced_straws="${forced_straws} $1"
    ;;
    -t)
      shift
      timeout="$1"
    ;;
    -x)
      exit_after_failure=true
    ;;
    --)
      shift
      break
    ;;
    *)
      >&2 echo 'Whoops, forgot to implement option "'"$1"'" internally.'
      exit 42
    ;;
  esac
  shift
done

if [ $# -ne 0 ]; then
  >&2 echo 'Too many arguments.'
  usage
fi

if [ -n "${diff_source_dir}" ]; then
  if [ -n "${count}" ] || \
    [ -n "${forced_package}" ] || \
    [ "${timeout}" -ne 0 ] || \
    ${exit_after_failure} || \
    [ -z "${prefered_package}" ]; then
    >&2 echo 'Conflicting flags.'
    usage
  fi
fi

if [ -n "${forced_package}" ]; then
  if [ -n "${count}" ] || \
    [ "${timeout}" -ne 0 ] || \
    [ -n "${prefered_package}" ] || \
    ${exit_after_failure}; then
    >&2 echo 'Conflicting flags.'
    usage
  fi
  upload_to_build_master=false
else
  upload_to_build_master=true
fi

if [ -z "${count}" ]; then
  if [ "${timeout}" -ne 0 ] || ${exit_after_failure}; then
    count=-1
  else
    count=1
  fi
fi

if [ -n "${forced_straws}" ]; then
  straws_that_might_repair_failing_builds="${forced_straws# }"
fi

if [ "${timeout}" -ne 0 ]; then
  timeout=$((timeout+$(date +%s)))
fi

checksum=$(
  calculate_script_checksum
)

# When this script or a script in lib/ got updated, we do not request
# any new assignments. This script should rather exit and be restarted.
while [ "${count}" -ne 0 ] && \
  [ "$(calculate_script_checksum)" = "${checksum}" ]; do

  if [ "${timeout}" -ne 0 ] && [ "${timeout}" -lt "$(date +%s)" ];
  then
    break
  fi

  err=0
  case "$(uname -m)" in
    'i486')
      my_arch='i486'
    ;;
    'i686'|'x86_64')
      if grep '^flags\s*:' '/proc/cpuinfo' | \
        grep -qwF 'sse2'; then
        my_arch='pentium4'
      else
        my_arch='i686'
      fi
    ;;
    *)
      >&2 printf 'Sorry, architecture %s does not work (yet) as a build slave.\n' \
        "$(uname -m)"
      exit 2
    ;;
  esac
  if [ -z "${forced_package}" ]; then
    package=$(
      # shellcheck disable=SC2029
      ssh -o PasswordAuthentication=No \
        -i "${master_build_server_identity}" \
        -p "${master_build_server_port}" \
        "${master_build_server_user}@${master_build_server}" \
        'get-assignment' "${my_arch}" "${prefered_package}"
    ) || err=$?
    expected_packages=$(
      printf '%s\n' "${package}" | \
        sed '1d'
    )
    package=$(
      printf '%s\n' "${package}" | \
        sed -n '1p'
    )
  else
    package=$(
      echo "${forced_package}" | \
        sed '
          s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)$| \1 \2 \3 0 \4|
        '
    )
    expected_packages=''
  fi

  case ${err} in

    #  0: ok, I gave you an assignment
    0)
      [ ${count} -gt 0 ] && \
        count=$((count-1))
      arch="${package##* }"
      package="${package% *}"
      sub_pkgrel="${package##* }"
      package="${package% *}"
      repository="${package##* }"
      package="${package% *}"
      mod_git_revision="${package##* }"
      package="${package% *}"
      git_revision="${package##* }"
      package="${package% *}"
      if [ "${arch}" = 'any' ]; then
        arch="${my_arch}"
        assignment_was_any=true
      else
        assignment_was_any=false
      fi

      if [ -n "${diff_source_dir}" ] &&
        [ "${prefered_package}" != "${package}" ]; then
        >&2 echo 'The prefered package was not handed out.'
        >&2 echo 'Because -d was given, I will abort.'
        # shellcheck disable=SC2029
        ssh -o PasswordAuthentication=No \
          -i "${master_build_server_identity}" \
          -p "${master_build_server_port}" \
          "${master_build_server_user}@${master_build_server}" \
          'return-assignment' 'ABORT'
        exit 2
      fi

      if [ "${git_revision##*-}" = 'HEAD' ]; then
        git_revision=$(
          repo_name="${git_revision%-*}"
          eval l_repo_path='"${repo_paths__'"${repo_name}"'}"'
          # shellcheck disable=SC2154
          if [ -z "${l_repo_path}" ]; then
            >&2 printf 'Unknown git repository "%s".\n' "${l_repo_name}"
            exit 2
          fi
          git -C "${repo_path}" rev-parse HEAD
        )
      fi

      if [ "${mod_git_revision}" = 'work-tree' ]; then
        mod_git_revision=$(
          # we can't just create an empty index-file with mktemp, because git doesn't like it
          find . \
            -mindepth 1 \
            -maxdepth 1 \
            -name 'tmp.build-packages.git.*' \
            -exec rm -rf --one-file-system {} \;
          tmp_subdir=$(mktemp -d 'tmp.build-packages.git.XXXXXXXXXX' --tmpdir)
          trap 'rm -rf --one-file-system "${tmp_subdir}"' EXIT
          export GIT_INDEX_FILE="${tmp_subdir}/index.new"
          git -C "${repo_paths__archlinux32}" add -A
          git -C "${repo_paths__archlinux32}" write-tree
        )
      fi

      # Update git repositories (official packages, community packages and the repository of package customizations).

      for repo_name in ${repo_names}; do
        eval repo_path='"${repo_paths__'"${repo_name}"'}"'
        if [ -d "${repo_path}/.git" ]; then
          git -C "${repo_path}" remote update
        else
          git -C "${repo_path}" fetch origin master:master
        fi || \
          true
      done

      # trigger update of mirror (if configured)
      if [ -n "${mirror_update_command}" ]; then
        ${mirror_update_command}
      fi

      bail_out() {
        err=$?
        if [ -n "$1" ]; then
          err="$1"
        fi
        cd "${base_dir}"
        recursively_umount_and_rm "${tmp_dir}"
        if [ -z "${forced_package}" ]; then
          flock -u 9 || true
        fi
        exit "${err}"
      }
      find "${work_dir}" \
        -mindepth 1 \
        -maxdepth 1 \
        -name 'tmp.build-packages.??????' \
        -printf '%p\n' | \
        while read -r old_tmp_dir; do
          find "${old_tmp_dir}" -xdev -not -type l -exec chmod 777 {} \;
          rm -rf --one-file-system "${old_tmp_dir}"
        done
      tmp_dir=$(mktemp -d "${work_dir}/tmp.build-packages.XXXXXX")
      trap bail_out EXIT

      if ! git_repo=$(find_repository_with_commit "${git_revision}") || \
        [ -z "${git_repo}" ] || \
        ! find_pkgbuilds "${package}" "${repository}" "${git_repo}" "${git_revision}" "${mod_git_revision}" || \
        ! extract_source_directory "${git_repo}" "${git_revision}" "${mod_git_revision}" "${tmp_dir}" "${sub_pkgrel}" || \
        ! apply_trunk_patch "${tmp_dir}" "${diff_source_dir}"; then
        # report local failure (probably a missing commit) to build-master
        # shellcheck disable=SC2029
        ssh -o PasswordAuthentication=No \
          -i "${master_build_server_identity}" \
          -p "${master_build_server_port}" \
          "${master_build_server_user}@${master_build_server}" \
          'return-assignment' 'ABORT'

        recursively_umount_and_rm "${tmp_dir}"
        trap - EXIT
        continue
      fi

      if [ -d "${tmp_dir}/keys/pgp" ]; then
        find "${tmp_dir}/keys/pgp" -type f -exec gpg --import {} +
      fi

      cd "${tmp_dir}"
      echo 'nothing' > "${tmp_dir}/.ping-build-master"
      if [ -z "${forced_package}" ]; then
        # we get a lock on "${work_dir}/ping-build-master.lock",
        # if we release that lock, ping-to-master should stop _immediately_
        exec 9> "${work_dir}/ping-build-master.lock"
        if ! verbose_flock -n 9; then
          >&2 echo 'ERROR: Cannot lock ping-to-master - this should not happen.'
          exit 2
        fi
        "${base_dir}/bin/ping-to-master" "$$" "${tmp_dir}" &
      fi
      success=false

      if printf '%s\n' "${package}" | \
        grep -q '^\(haskell\|python2\?\)-'; then
        straws_that_might_repair_failing_builds=$(
          # shellcheck disable=SC2086
          printf '%s\n' ${straws_that_might_repair_failing_builds} | \
            sed '
              s/^\(.*\):haskell_without_check:\(.*\)$/\1:\2\n\1:without_check:\2/
            '
        )
      fi

      for straw in ${straws_that_might_repair_failing_builds}; do

        if ${success}; then
          break
        fi

        echo 'preparing' > "${tmp_dir}/.ping-build-master"

        outerParameters="-r ${archbuild_chroots}"
        if echo "${straw}" | \
          grep -qF ':clean_chroot:'; then
          outerParameters="-c ${outerParameters}"
        fi

        innerParameters='--nocolor'
        if echo "${straw}" | \
          grep -qF ':without_check:'; then
          innerParameters="${innerParameters} --nocheck"
        fi

        if echo "${straw}" | \
          grep -qF ':with_/dev/fuse:'; then
          middleParameters='-d /dev/fuse'
        else
          middleParameters=''
        fi

        if echo "${straw}" | \
          grep -qF ':with_build_support:'; then
          build_command='staging-with-build-support-'"${arch}"'-build'
        elif echo "${straw}" | \
          grep -qF ':without_systemd_nspawn:'; then
          if [ -z "${prefered_package}" ]; then
            >&2 echo 'straw :without_systemd_nspawn: only allowed with -p'
            exit 2
          fi
          if [ "${prefered_package}" != "${package}" ]; then
            >&2 echo 'The prefered package was not handed out.'
            >&2 echo 'Because straw :without_systemd_nspawn: is active, I will abort.'
            # shellcheck disable=SC2029
            ssh -o PasswordAuthentication=No \
              -i "${master_build_server_identity}" \
              -p "${master_build_server_port}" \
              "${master_build_server_user}@${master_build_server}" \
              'return-assignment' 'ABORT'
            exit 2
          fi
          # we're not interested in what this cpu /can/ do, but what
          # architecture the installed packages actually are
          if ! pacman -Qi pacman | \
            sed 's/^Architecture\s*:\s*//;t;d' | \
            grep -qxF "${arch}"; then
            >&2 echo 'straw :without_systemd_nspawn: requires running build-packages on the'
            >&2 echo 'architecture for which the package should be built.'
            >&2 printf '"%s" != "%s"\n' \
              "$(
                pacman -Qi pacman | \
                  sed 's/^Architecture\s*:\s*//;t;d'
              )" \
              "${arch}"
            exit 2
          fi
          build_command='makepkg'
          outerParameters="${innerParameters} -fcrs --asdeps --noconfirm --holdver"
          middleParameters=''
          innerParameters=''
        elif echo "${straw}" | \
          grep -qF ':on_x86_64:'; then
          if [ -z "${prefered_package}" ]; then
            >&2 echo 'straw :on_x86_64: only allowed with -p'
            exit 2
          fi
          if [ "${prefered_package}" != "${package}" ]; then
            >&2 echo 'The prefered package was not handed out.'
            >&2 echo 'Because straw :on_x86_64: is active, I will abort.'
            # shellcheck disable=SC2029
            ssh -o PasswordAuthentication=No \
              -i "${master_build_server_identity}" \
              -p "${master_build_server_port}" \
              "${master_build_server_user}@${master_build_server}" \
              'return-assignment' 'ABORT'
            exit 2
          fi
          if ! ${assignment_was_any}; then
            >&2 printf 'Can only build "any" packages with :on_x86_64:, but got a "%s" package.\n' "${arch}"
            exit 2
          fi
          build_command='staging-x86_64-build'
        else
          build_command='staging-'"${arch}"'-build'
        fi

        find . -maxdepth 1 -type f \( \
          -name '*.pkg.tar.xz' \
          -o -name '*.pkg.tar.xz.sig' \
          -o -name '*.pkg.tar.zst' \
          -o -name '*.pkg.tar.zst.sig' \
          -o -name '*.pkg.tar' \
        \) \
          -exec rm {} \;

        echo 'checking-source' > "${tmp_dir}/.ping-build-master"
        success=false
        verifysource_trial=0
        while [ ${verifysource_trial} -lt 5 ]; do
          verifysource_trial=$((verifysource_trial + 1))
          log_file="${tmp_dir}/$(
              date -u --iso-8601=seconds | \
                cut -d+ -f1
            ).build-log"
          if CARCH="${arch}" makepkg --verifysource 2> "${log_file}"; then
            success=true
            rm "${log_file}"
            break
          fi

          # receive specific missing keys
          if [ ${verifysource_trial} -eq 1 ]; then
            missing_keys=$(
              sed -n '
                s/^.* FAILED (unknown public key \([0-9A-F]\{16\}\)).*$/0x\1/
                T
                p
              ' "${log_file}"
            )
            if [ -n "${missing_keys}" ]; then
              if printf '%s\n' "${missing_keys}" \
                | sed '
                  s@^@https://archlinux32.org/keys.php?k=@
                ' \
                | xargs curl -Ss \
                | gpg --import; then
                continue
              fi
              if gpg --recv-keys "${missing_keys}"; then
                continue
              fi
            fi
            verifysource_trial=$((verifysource_trial + 1))
          fi

          # download the repository key from github
          if [ ${verifysource_trial} -eq 2 ]; then
            if grep -q ' FAILED (unknown public key \([0-9A-F]\{16\}\))' -- "${log_file}"; then
              # TODO: get the name of the key file from its finger print or
              # some other information inside the repository
              if makepkg --printsrcinfo | \
                  sed -n '
                    s,^\ssource = git+\(https://github\.com/[^/[:space:]]\+\)/[^/]*$,\1.gpg,
                    T
                    p
                  ' | \
                  xargs -rn1 curl -s | \
                  gpg --import; then
                continue
              fi
            fi
            verifysource_trial=$((verifysource_trial + 1))
          fi

          # try to download source from sources.archlinux.org/sources/$repo/$source
          if [ ${verifysource_trial} -eq 3 ]; then
            if echo "${straw}" | \
              grep -qF ':mirrored_source:'; then
              source_name=$(
                makepkg --printsrcinfo | \
                  sed -n '
                    /^\s*\(epoch\|pkg\(base\|ver\|rel\)\) = /{s|^\s\+||;p}
                    /^pkgname = /q
                  ' | \
                  sed '
                    s|^pkgbase = \(.*\)$|0 \1-|
                    s|^epoch = \(.*\)$|1 \1:|
                    s|^pkgver = \(.*\)$|2 \1-|
                    s|^pkgrel = \([^.]*\)\(\..*\)\?$|3 \1.src.tar.gz|
                  ' | \
                  sort -k1n,1 | \
                  sed '
                    s|^[0-9] ||
                    :a
                      N
                      s|\n[0-9] \(\S\+\)$|\1|
                    ta
                  '
              )
              if wget -q --timeout=15 -nc -nd "https://sources.archlinux.org/sources/${git_repo}/${source_name}"; then
                # shellcheck disable=SC2046
                tar -zst --overwrite \
                  -f "${source_name}" \
                  --exclude PKGBUILD \
                  $(
                    if [ -n "${PKGBUILD_mod}" ]; then
                      git -C "${repo_paths__archlinux32}/${PKGBUILD_mod%/*}" archive "${mod_git_revision}" -- . | \
                        tar -t | \
                        sed 's/^/--exclude /'
                    fi
                  ) \
                  --strip-components=1 \
                  || true
                continue
              fi
            fi
            verifysource_trial=$((verifysource_trial + 1))
          fi

          # try to download source from sources.archlinux32.org by its hash
          if [ ${verifysource_trial} -eq 4 ]; then
            if echo "${straw}" | \
              grep -qF ':mirrored_source_by_hash:'; then
              if download_sources_by_hash; then
                continue
              fi
            fi
            verifysource_trial=$((verifysource_trial + 1))
          fi
        done

        if ${success}; then
          echo 'building' > "${tmp_dir}/.ping-build-master"
          >&2 printf '%s: building package "%s" (revisions %s %s, repository %s, straw %s) for %s ...' \
            "$(date +'%Y-%m-%d %T')" \
            "${package}" \
            "${git_revision}" \
            "${mod_git_revision}" \
            "${repository}" \
            "${straw}" \
            "${arch}"
          log_file="${tmp_dir}/$(
            date -u --iso-8601=seconds | \
              cut -d+ -f1
          ).build-log"
          # by piping the log, we don't see anything in the terminal,
          # but all ways to duplicate the logs seem pretty elaborate
          # shellcheck disable=SC2024,SC2086
          if ! PKGEXT='.pkg.tar' "${build_command}" ${outerParameters} -- ${middleParameters} -- ${innerParameters} > \
            "${log_file}" 2>&1; then
            success=false
            build_dir="/var/lib/archbuild/${build_command%-build}/$(whoami)"
            if [ -d "${build_dir}" ]; then
              sed '
                s/^Full log written to\s\+\(\S\+\)\s*$/\1/
                t
                /# An error report file with more information is saved as:/ {
                  N
                  s/^.*\n#\s\+//
                  t
                  d
                }
                s/^See also "\(\S\+\)"\.\s*$/\1/
                t
                s/^See \(\S\+\)\s*$/\1/
                t
                s/^See ['"'"'`]\([^'"'"'`]\+\)['"'"'`]\(\s.*\)\?$/\1/
                t
                s/^A full log can be found at\s\+\(\S\+\)\s*$/\1/
                t
                d
              ' "${log_file}" | \
                while read -r extra_log_file; do
                  if ! grep -HF '' "${build_dir}/${extra_log_file}"; then
                    find "${build_dir}" \
                      -xdev \
                      -type f -name "${extra_log_file##*/}" \
                      -exec grep -HF '' {} + \
                      || true
                  fi
                done | \
                sponge -a "${log_file}"
            fi
            printf 'used straw: %s\n' "${straw}" >> "${log_file}"
          fi
        fi

        if ${success}; then
          # build successful
          echo 'compressing' > "${tmp_dir}/.ping-build-master"
          >&2 printf ' ok, compressing package(s) ...'
          # We requested to build uncompressed packages - those will be
          # compressed now (which can be offloaded to a different
          # machine, if need arises in the future). Note, that it is no
          # error, if the above build command already produced a
          # compressed package.
          # Since --ultra -20 seems too much for our i486 build vms, we
          # silently fall back to lower compression levels if the higher
          # level fails.

          # beware: We must remove *.pkg.tar.zst after failed
          # compression attempts, otherwise future compression attempts
          # will refuse to overwrite them.

          # shellcheck disable=SC2086
          for options in '-T0 --ultra -20' '--ultra -20' $(seq -19 -3); do
            find . -maxdepth 1 -type f -name '*.pkg.tar' \
              -not -execdir zstd --rm ${options} '{}' -o '{}.zst' \; \
              -exec rm -f '{}.zst' \;
          done
          # package files, that still exist, threw an error upon compression
          errors=$(
            find . -maxdepth 1 -type f -name '*.pkg.tar'
          )
          if [ -n "${errors}" ]; then
            success=false
            {
              printf 'Building (and packaging) was successfull, but compressing the following packages failed:\n'
              printf '%s\n' "${errors}"
            } \
            | tee -a "${log_file}"
          fi
        fi

        if ${success}; then
          # compressing successful
          echo 'post-build' > "${tmp_dir}/.ping-build-master"
          >&2 printf ' ok.\n'
          find . -maxdepth 1 -type f -name '*.pkg.tar.zst' \
            -exec sh -c '
              pkg="${1%.pkg.tar.zst}"
              if [ -f "${pkg}.pkg.tar-namcap.log" ]; then
                mv "${pkg}.pkg.tar-namcap.log" "${pkg}.pkg.tar.zst-namcap.log"
              elif [ ! -f "${pkg}.pkg.tar.zst-namcap.log" ]; then
                namcap "$1" > "${pkg}.pkg.tar.zst-namcap.log"
              fi
            ' '_' '{}' \;
          tar_content_dir=$(mktemp -d "${tmp_dir}/tar-content.XXXXXX")
          # remove unexpected packages
          if [ -n "${expected_packages}" ]; then
            {
              find . -maxdepth 1 -type f -name '*.pkg.tar.zst' -printf '%f\n'
              printf '%s\n' "${expected_packages}" | \
                sed 'p'
            } | \
              sort | \
              uniq -u | \
              {
                removed_something_unexpected=false
                while read -r unexpected_package; do
                  >&2 printf 'removing unexpected build artifact "%s"\n' \
                    "${unexpected_package}"
                  rm "${unexpected_package}"*
                  removed_something_unexpected=true
                done
                if "${removed_something_unexpected}"; then
                  >&2 printf 'I was only expecting:\n%s\n' \
                    "${expected_packages}"
                fi
              }
          fi
          >&2 printf 'signing package(s)\n'
          mv "${log_file}" "${tar_content_dir}/"
          find . -maxdepth 1 -type f -name '*.pkg.tar.zst' \
            -execdir gpg --local-user="${package_key}" --detach-sign '{}' \; \
            -execdir mv '{}' '{}.sig' '{}-namcap.log' "${tar_content_dir}/" \; \
            -printf '%f\n' | \
            sponge | \
            while read -r pkg_file; do
              {
                pacman -Spdd --print-format '%l' --noconfirm "${pkg_file%-*-*-*}" 2>/dev/null | \
                  sed '
                    s|/[^/]\+\.pkg\.tar\.zst$||
                  '
                # shellcheck disable=SC2016
                curl -Ss 'https://www.archlinux.org/mirrorlist/?country=all&protocol=https&tier=1&use_mirror_status=on' | \
                  sed -n '
                    s/^#Server = //
                    T
                    s/\$repo/'"${repository}"'/g
                    s/\$arch/x86_64/g
                    p
                  '
              } | \
                sed '
                  s|$|/'"${pkg_file}"'|
                  s/\.[0-9]\+\(-[^-]\+\)$/\1/
                  s/-'"${arch}"'\+\(\.pkg\.tar\.zst\)$/-x86_64\1/
                  p
                  s/\.zst$/.xz/
                ' | \
                shuf -n 100 | \
                while read -r url; do
                  >&2 printf 'downloading "%s" ...' "${url}"
                  if wget -q --timeout=15 -nd "${url}"; then
                    >&2 printf ' ok.\n'
                    break;
                  fi
                  >&2 printf ' failed. Next ...\n'
                done
            done
          >&2 printf 'searching for provided libraries\n'
          find "${tar_content_dir}" -maxdepth 1 \
            -name '*.pkg.tar.zst' \
            -printf '%p\n' | \
            while read -r pkgfile; do
              pacman -Qqlp "${pkgfile}" | \
                sed -n '
                  s,^.*/,,
                  /\.so\(\..\+\)\?$/ {
                    :a
                      p
                      s/\(\.so\(\..\+\)\?\)\.[^.]\+$/\1/
                      t a
                  }
                ' | \
                sort -u > \
                "${pkgfile}.so.provides"
            done
          >&2 printf 'searching for required and more provided libraries\n'
          package_content_dir=$(mktemp -d "${tmp_dir}/package-content.XXXXXX")
          find "${tar_content_dir}" -maxdepth 1 \
            -name '*.pkg.tar.zst' | \
            while read -r pkgfile; do
              if printf '%s\n' "${pkgfile}" | \
                grep -vq -- '-any\.pkg\.tar\.zst$'; then
                # we do not check "any" packages for linked libraries
                # (why do they have them in the first place?)
                mkdir "${package_content_dir}/${pkgfile##*/}"
                tar -C "${package_content_dir}/${pkgfile##*/}" -xf "${pkgfile}" --zstd
                # we rely on "${checksum}" not appearing in any objdump output
                : "${checksum?umm, checksum is unset - this will break below sed-fu}"
                # TODO: symbols may be in object files _inside_archives_
                # In that case, we're interested in the object file's name,
                # not the archive's file name
                find "${package_content_dir}/${pkgfile##*/}" \
                  -name 'opt' -prune , \
                  -type f \
                  -exec objdump -x '{}' \; 2>/dev/null | \
                  sed '
                    /^architecture:.* i386:x86-64, /,/^architecture:.* i386:x86-32, / d
                    \@^\s*RPATH\s\+/usr/lib/perl\([0-9]\+\)/\1\.[0-9.]\+/@ {
                      s@^\s*RPATH\s\+/usr/lib/perl\([0-9]\+\)/\(\1\(\.[0-9.]\+\)\?\)\.\([0-9]\+\)/.*$@\2 \4@
                      w '"${pkgfile}"'.needed-perl-version
                      d
                    }
                    /\sNEEDED\s/ {
                      s/^\s*\S\+\s\+\(\S\+\)\(\s.*\)\?$/\1/
                      /\.c32$/d
                      s,^.*/,,
                      t
                    }
                    /^Version References:$/,/^$/ {
                      /^\s*required from / {
                        s/^\s*required from \(\S\+\):\s*$/'"${checksum}"'\1/
                        T end
                        h
                        d
                      }
                      s/^\s*\(0x[0-9a-fA-F]\+\)\s\+0x[0-9a-fA-F]\+\s\+[0-9]\+\s\+\(\S\+\)$/\2-\1/
                      T end
                      G
                      s/^\(\S\+\)\n'"${checksum}"'\(\S\+\)$/\2-\1/
                      t
                    }
                    :end
                    d
                  ' | \
                  sort -u > \
                  "${pkgfile}.so.needs"
                sed -i '
                  /^\(libav.*\.so\)\.[0-9]\+$/ {
                    :a
                    N
                    s/^\(libav.*\.so\)\(\.[0-9]\+\)\?\n\1\.[0-9]\+$/\1/
                    t a
                    P
                    D
                  }
                ' "${pkgfile}.so.needs"
                sed '
                  s/^installed = \(qt[0-9]\+-\S\+\)-\([0-9.]\+\)\(-[^-]\+\)\{2\}$/\1 \2/
                  t
                  d
                ' "${package_content_dir}/${pkgfile##*/}/.BUILDINFO" | \
                  sort -u | \
                  sort -k1,1 > \
                  "${tmp_dir}/installed-qt-versions"
                sed '
                  s/^depend = \(qt[0-9]\+-\S\+\)$/\1/
                  t
                  d
                ' "${package_content_dir}/${pkgfile##*/}/.PKGINFO" | \
                  sort -u | \
                  join -1 1 -2 1 -o 2.1,2.2 - "${tmp_dir}/installed-qt-versions" | \
                  tr ' ' '=' >> \
                  "${pkgfile}.so.needs"
                find "${package_content_dir}/${pkgfile##*/}" \
                  -name 'opt' -prune , \
                  \( -type f -o -type l \) \
                  -printf "${checksum}"'%f\n' \
                  -exec objdump -x '{}' \; 2>/dev/null | \
                  sed '
                    /^architecture:.* i386:x86-64, /,/^architecture:.* i386:x86-32, / d
                    /^'"${checksum}"'/ {
                      h
                      d
                    }
                    /\sSONAME\s/ {
                      s/^\s*\S\+\s\+\(\S\+\)\s.*$/\1/
                      t
                    }
                    /^Version definitions:$/,/^$/ {
                      s/^[0-9]\+\s\+0x[0-9a-fA-F]\+\s\+\(0x[0-9a-fA-F]\+\)\s\+\(\S\+\)$/\2-\1/
                      T end
                      G
                      s/^\(\S\+\)\n'"${checksum}"'\(\S\+\)$/\2-\1/
                      t
                    }
                    :end
                    d
                  ' | \
                  sort -u >> \
                  "${pkgfile}.so.provides"
                find "${package_content_dir:?}/${pkgfile##*/}" -xdev -not -type l -exec chmod 777 '{}' \;
                rm -rf --one-file-system "${package_content_dir:?}/${pkgfile##*/}"
              fi
              {
                tar -tf "${pkgfile}" --zstd 2>/dev/null | \
                  sed -n '
                    s,^usr/lib/python\(2\?\)\([013-9.][0-9.]*\)/$,python\1 \1\2,
                    t print
                    s,^usr/lib/perl[^/]\+/\([0-9.]\+\)/$,perl \1,
                    t print
                    s,^usr/lib/ruby/\([0-9.]\+\)/$,ruby \1,
                    t print
                    b
                    :print
                    s/\.\([^.]\+\)$/ \1/
                    p
                  '
                if [ -f "${pkgfile}.needed-perl-versions" ]; then
                  sed '
                    s/^/perl /
                  ' "${pkgfile}.needed-perl-versions"
                  rm "${pkgfile}.needed-perl-versions"
                fi
              } | \
                while read -r name major_version minor_version; do
                  printf '%s>=%s.%s\n%s<%s.%s\n' \
                    "${name}" \
                    "${major_version}" \
                    "${minor_version}" \
                    "${name}" \
                    "${major_version}" \
                    "$((minor_version+1))"
                done >> \
                "${pkgfile}.so.needs"
              # TODO: dependency-adding from make_source_info() should
              # be replicated here, somehow ...
            done
          >&2 printf 'running namcap ...'
          if [ "${repository}" = 'multilib' ]; then
            x86_64_build_command='multilib-build'
          else
            x86_64_build_command='extra-x86_64-build'
          fi
          # this is a little hack: makepkg receives '--version', but namcap is run nevertheless
          # (and it only works with devtools32, because they are running namcap on *.pkg.tar.zst in the base directory, too)
          sudo "${x86_64_build_command}" -- -- --version > /dev/null 2>&1 || \
            sudo "${x86_64_build_command}" -c -- -- --version > /dev/null 2>&1 || \
            true
          >&2 printf ' ok.\n'
          >&2 printf 'smoothen namcap log ...'
          # now we generate diffs from the namcap.logs
          find . "${tar_content_dir}/" -maxdepth 1 -type f -name '*.pkg.tar.zst-namcap.log' -printf '%p\n' | \
            while read -r log; do
              smoothen_namcap_log "${log}"
            done
          find "${tar_content_dir}/" -maxdepth 1 -type f -name '*.pkg.tar.zst-namcap.log' -printf '%f\n' | \
            sed '
              s|\(^.*\)-'"${arch}"'\(\.pkg\.tar\.zst-namcap\.log\)$|\0 \1-x86_64\2|
              s|^.*-any\.pkg\.tar\.zst-namcap\.log$|\0 \0|
            ' | \
            while read -r log x86_64_log; do
              if [ -f "${x86_64_log}" ]; then
                diff -u "${x86_64_log}" "${tar_content_dir}/${log}" | \
                  sed '
                    1,3d
                    /^[^+-]/d
                  ' | \
                  sponge "${tar_content_dir}/${log}"
              else
                sed -i 's|^|*|' "${tar_content_dir}/${log}"
              fi
            done
          >&2 printf ' ok.\n'
          echo 'uploading' > "${tmp_dir}/.ping-build-master"
          if ${upload_to_build_master}; then
            find "${tar_content_dir}/" -maxdepth 1 \
              \( \
                -name '*.pkg.tar.zst-namcap.log' -o \
                -name '*.pkg.tar.zst.so.needs' -o \
                -name '*.pkg.tar.zst.so.provides' -o \
                -name '*.build-log' \
              \) \
              -execdir gzip '{}' \;
          else
            find "${tar_content_dir}/" -maxdepth 1 \
              -name '*.pkg.tar.zst-namcap.log' \
              -execdir grep -HF '' '{}' \;
          fi
          # shellcheck disable=SC2046
          tar -cf 'package.tar' -C "${tar_content_dir}" -- $(
            find "${tar_content_dir}/" -maxdepth 1 \
              \( \
                -name '*.pkg.tar.zst' -o \
                -name '*.pkg.tar.zst.sig' -o \
                -name '*.pkg.tar.zst-namcap.log.gz' -o \
                -name '*.pkg.tar.zst.so.needs.gz' -o \
                -name '*.pkg.tar.zst.so.provides.gz' \
              \) \
              -printf '%f\n'
            ) || \
            {
              find "${tar_content_dir}/" -maxdepth 1 >&2
              tar -cf 'package.tar' -T /dev/null
            }
          if ${upload_to_build_master}; then
            # shellcheck disable=SC2046
            rsync $(
              find "${tar_content_dir}/" -maxdepth 1 \
                \( \
                  -name '*.pkg.tar.zst' -o \
                  -name '*.pkg.tar.zst.sig' \
                \) \
                -printf '%p\n'
            ) "rsync://mirror.archlinux32.org/transfer32/" || true
          fi
          while ${upload_to_build_master}; do
            err=0
            # shellcheck disable=SC2029
            ssh -o PasswordAuthentication=No \
              -i "${master_build_server_identity}" \
              -p "${master_build_server_port}" \
              "${master_build_server_user}@${master_build_server}" \
              'return-assignment' "${package}" "${git_revision}" "${mod_git_revision}" "${repository}" "${arch}" "${sub_pkgrel}" \
              < 'package.tar' || \
              err=$?
            case ${err} in
              0)
                # upload successful
                break
              ;;
              1)
                >&2 echo '"return-assignment" was running already.'
                wait_some_time 30
              ;;
              2)
                >&2 echo 'I was too slow, the package is outdated. I will continue ...'
                break
              ;;
              3)
                >&2 echo "'return-assignment' reports a signature error."
                exit 1
              ;;
              4)
                >&2 echo "'return-assignment' reports too many or missing packages."
                exit 1
              ;;
              5)
                >&2 echo "'return-assignment' was called with wrong arguments."
                exit 1
              ;;
              *)
                >&2 echo "unknown return code ${err} from 'return-assignment'"
                exit ${err}
            esac
          done
          break
        fi
        echo 'failure' > "${tmp_dir}/.ping-build-master"
        >&2 printf ' failed.\n'
      done

      if ! ${success}; then
        for log in *'.build-log'; do
          if [ -f "${log}" ]; then
            if ${upload_to_build_master}; then
              printf '%s@%s\n' \
                "$(whoami)" \
                "$(uname -n)" >> \
                "${log}"
              gzip "${log}"
            else
              grep -HF '' "${log}"
            fi
          fi
        done
        if ${upload_to_build_master} && \
          tar -cf 'build-logs.gz.tar' \
          -- *'.build-log.gz'; then
          while true; do
            err=0
            # shellcheck disable=SC2029
            ssh -o PasswordAuthentication=No \
              -i "${master_build_server_identity}" \
              -p "${master_build_server_port}" \
              "${master_build_server_user}@${master_build_server}" \
              'return-assignment' "${package}" "${git_revision}" "${mod_git_revision}" "${repository}" "${arch}" 'ERROR' \
              < 'build-logs.gz.tar' || \
              err=$?
            case ${err} in
              0)
                # upload successful
                break
              ;;
              1)
                >&2 echo '"return-assignment" was running already.'
                wait_some_time 30
              ;;
              2)
                >&2 echo 'I was too slow, the package is outdated. I will continue ...'
                break
              ;;
              5)
                >&2 echo '"return-assignment" was called with wrong arguments.'
                exit 1
              ;;
              *)
                >&2 echo "unknown return code ${err} from 'return-assignment'"
                exit ${err}
            esac
          done
        fi
        if ${exit_after_failure}; then
          >&2 echo 'Build failed, exiting now'
          exit 0
        fi
      fi

      # clean up tmp_dir
      cd "${base_dir}"
      recursively_umount_and_rm "${tmp_dir}"
      if [ -z "${forced_package}" ]; then
        flock -u 9 || true
      fi
      trap - EXIT

      continue

    ;;

    1)

      >&2 echo 'get-assignment told me:'
      >&2 echo ' come back (shortly) later - I was running already'
      wait_some_time 30
      continue

    ;;

    2)

      >&2 echo 'get-assignment told me:'
      >&2 echo ' 2: come back later - there are still packages to be built,'
      >&2 echo ' but currently none has all its dependencies ready'
      wait_some_time 30
      continue

    ;;

    3)

      >&2 echo 'get-assignment told me:'
      >&2 echo ' 3: come back after the next run of get-package-updates or when'
      >&2 echo ' some currently building packages are returned - currently'
      >&2 echo ' there are no pending buildable packages'
      exit 0

    ;;

    4)

      >&2 echo 'get-assignment told me:'
      >&2 echo ' 4: wrong number of arguments'
      exit 5

    ;;

    *)

      >&2 echo "ERROR: Unknown exit code ${err} from 'get-assignment'."
      exit ${err}

    ;;

  esac

done

# remove the slave switch file if it exists (and if we are runnig on a slave)  
if ! ${i_am_the_master}; then
  SLAVE=$(whoami)
  [ -f "/tmp/do-not-run-build-slave.$SLAVE" ] && rm "/tmp/do-not-run-build-slave.$SLAVE"
fi

>&2 echo 'Done.'