bin/common-functions -> lib/common-functions, bin/mysql-functions -> lib/mysql-functions

1 files changed, 0 insertions, 1378 deletions

diff --git a/bin/common-functions b/bin/common-functions
deleted file mode 100755
index a84271b..0000000
--- a/bin/common-functions
+++ /dev/null
@@ -1,1378 +0,0 @@
-#!/bin/sh
-
-# contains functions used by more than one script
-
-# shellcheck disable=SC2039
-
-# TODO: include link depenendencies in run-depends metadata
-
-# TODO: have full information (currently in files) in database
-
-# TODO: remove state files / metadata files
-
-if [ -z "${base_dir}" ]; then
-  # just to make shellcheck happy
-  . 'conf/default.conf'
-fi
-
-# find_pkgbuilds package repository git_repository git_revision mod_git_revision
-# find the PKGBUILD and modification of $package from $repository
-# sets $PKGBUILD and $PKGBUILD_mod
-
-find_pkgbuilds() {
-
-  local package="$1"
-  local repository="$2"
-  local git_repository="$3"
-  local git_revision="$4"
-  local mod_git_revision="$5"
-
-  local repo_path
-  eval 'repo_path="${repo_paths__'"${git_repository}"'}"'
-
-  PKGBUILD=$(
-    git -C "${repo_path}" archive "${git_revision}" -- "${package}/repos/" 2> /dev/null | \
-      tar -t 2> /dev/null | \
-      grep "$(printf '^%s-.*/PKGBUILD' "$(str_to_regex "${package}/repos/${repository}")")" | \
-      grep -v -- '-i686/PKGBUILD$' | \
-      grep -v -- '[-/]\(staging\|testing\|unstable\)-[^/]\+/PKGBUILD$' | \
-      sort | \
-      tail -n1
-  )
-
-  PKGBUILD_mod=$(
-    git -C "${repo_paths__archlinux32}" archive "${mod_git_revision}" 2> /dev/null | \
-      tar -t "${repository}/${package}/PKGBUILD" 2> /dev/null
-  ) || true
-
-  if [ -z "${PKGBUILD}" ] && \
-    [ -z "${PKGBUILD_mod}" ]; then
-    >&2 printf 'Neither PKGBUILD nor modification of PKGBUILD found for package "%s" from %s (%s), revisions %s and %s.\n' \
-      "${package}" \
-      "${repository}" \
-      "${git_repository}" \
-      "${git_revision}" \
-      "${mod_git_revision}"
-    return 1
-  fi
-
-}
-
-# find_repository_with_commit commit
-# find the repository which has $commit
-
-find_repository_with_commit() {
-
-  local repository
-
-  for repository in ${repo_names}; do
-    # shellcheck disable=SC2016
-    if [ "$(eval git -C "$(printf '"${repo_paths__%s}"' "${repository}")" cat-file -t '"$1"' 2> /dev/null)" = "commit" ]; then
-      echo "${repository}"
-      return 0
-    fi
-  done
-  >&2 printf 'find_repository_with_commit: Cannot find repository with commit "%s"\n' "$1"
-  exit 1
-
-}
-
-# find_git_repository_to_package_repository repository
-# find the git repository which tracks the package repository $repository
-
-find_git_repository_to_package_repository() {
-
-  local repository
-  local package_repository
-  local repo_path
-
-  package_repository="$1"
-
-  if [ "$1" = 'build-support' ]; then
-    echo 'packages'
-    return 0
-  fi
-
-  for repository in ${repo_names}; do
-    if [ "${repository}" = "archlinux32" ]; then
-      continue
-    fi
-    eval 'repo_path="${repo_paths__'"${repository}"'}"'
-    if git -C "${repo_path}" archive "$(cat "${work_dir}/${repository}.revision")" -- | \
-      tar -t --wildcards '*/repos' | \
-      grep '^\([^/]\+/\)\{3\}PKGBUILD$' | \
-      cut -d/ -f3 | \
-      sed 's|-[^-]\+$||' | \
-      sort -u | \
-      grep -qxF "${package_repository}"; then
-      echo "${repository}"
-      return 0
-    fi
-  done
-  >&2 echo "can't find git repository with package repository '$1'"
-  exit 1
-
-}
-
-# generate_package_metadata $package $git_revision $mod_git_revision $repository
-# or
-# generate_package_metadata $package.$git_revision.$mod_git_revision.$repository
-# generate the meta data files of a package (dependencies, built packages, ...)
-
-generate_package_metadata() {
-
-  local package="$1"
-  local git_revision="$2"
-  local mod_git_revision="$3"
-  local repository="$4"
-  local file_prefix
-  local file
-  local PKGBUILD
-
-  if [ $# -eq 1 ]; then
-    # second form
-    repository="${package##*.}"
-    package="${package%.*}"
-    mod_git_revision="${package##*.}"
-    package="${package%.*}"
-    git_revision="${package##*.}"
-    package="${package%.*}"
-  fi
-
-  file_prefix="${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.${repository}"
-
-  if [ -e "${file_prefix}.builds" ] && \
-    [ -e "${file_prefix}.build-depends" ] && \
-    [ -e "${file_prefix}.run-depends" ] && \
-    [ -e "${file_prefix}.groups" ] && \
-    [ -e "${file_prefix}.packages" ]; then
-    return 0
-  fi
-
-  if ! make_source_info "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${file_prefix}.SRCINFO"; then
-    printf '"make_source_info %s %s %s %s %s" failed.\n' "${package}" "${repository}" "${git_revision}" "${mod_git_revision}" "${file_prefix}.SRCINFO"
-    exit 1
-  fi
-  if [ ! -s "${file_prefix}.SRCINFO" ]; then
-    >&2 printf '"%s" not created by "make_source_info" - eh, what?' "${file_prefix}.SRCINFO"
-    exit 1
-  fi
-
-  # otherwise this just calls for trouble
-  sed -i '
-    /^[^=]*=\s*$/d
-    s/_i686\(\s*=\)/\1/
-  ' "${file_prefix}.SRCINFO"
-
-  # extract "groups" = groups \cup provides
-  grep "$(printf '^\t\\(groups\\|provides\\) = ')" "${file_prefix}.SRCINFO" | \
-    cut -d= -f2 | \
-    sed 's|^\s\+||; s|[<>]$||' | \
-    sort -u > \
-    "${file_prefix}.groups"
-
-  # extract "packages" = pkgname
-  grep '^pkgname = ' "${file_prefix}.SRCINFO" | \
-    cut -d= -f2 | \
-    sed 's|^\s\+||; s|[<>]$||' | \
-    sort -u > \
-    "${file_prefix}.packages"
-
-  # extract "builds" = provides \cup pkgname \cup groups
-  cat "${file_prefix}.groups" "${file_prefix}.packages" | \
-    sort -u > \
-    "${file_prefix}.builds"
-
-  # extract "build-depends" = makedepends \cup depends \cup \{ base, base-devel \} \setminus "builds"
-  {
-    {
-      printf 'all_depend = %s\n' 'base' 'base-devel'
-      sed -n "$(
-        printf '/^\t%s = /p\n' \
-          'depends' \
-          'makedepends'
-      )" "${file_prefix}.SRCINFO"
-    } | \
-      cut -d= -f2 | \
-      sed 's|^\s\+||; s|[<>]$||' | \
-      sort -u
-    sed 'p' "${file_prefix}.builds"
-  } | \
-    sort | \
-    uniq -u > \
-    "${file_prefix}.build-depends"
-
-  # extract "run-depends" = depends \cup \{ base \} \setminus "builds"
-  {
-    {
-      printf 'all_depend = %s\n' 'base'
-      sed -n "$(printf '/^\tdepends = /p')" "${file_prefix}.SRCINFO"
-    } | \
-      cut -d= -f2 | \
-      sed 's|^\s\+||; s|[<>]$||' | \
-      sort -u
-    sed 'p' "${file_prefix}.builds"
-  } | \
-    sort | \
-    uniq -u > \
-    "${file_prefix}.run-depends"
-
-  rm "${file_prefix}.SRCINFO"
-
-}
-
-# delete_old_metadata
-# delete old (=unneeded) meta data of packages
-
-delete_old_metadata() {
-
-  local current_metadata
-
-  current_metadata=$(
-    find "${work_dir}/package-infos" -mindepth 1 -maxdepth 1 -printf '%f\n' | \
-      sed '
-        s|\.[^.]\+$||
-        s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)$| \1 \2 \3|
-      ' | \
-      sort -u
-  )
-
-  ( # the new shell is intentional
-    # what we have
-    echo "${current_metadata}"
-
-    # package-states should stay
-    find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -printf '%f\n' | \
-      sed '
-        s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3|
-      ' | \
-      sort -u | \
-      sed 'p'
-
-    # build-list items should stay
-    sed 'p' "${work_dir}/build-list"
-
-    tmp_dir=$(mktemp -d 'tmp.common-functions.delete_old_metadata.XXXXXXXXXX' --tmpdir)
-    trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
-
-    echo "${current_metadata}" | \
-      sort -k1,1 > \
-      "${tmp_dir}/current-metadata"
-
-    # the newest of the following should stay:
-    {
-      # deletion-list items
-      cat "${work_dir}/deletion-list"
-      # all packages in the repos
-      for repo in ${repo_names}; do
-        eval 'git -C "${repo_paths__'"${repo}"'}" archive '"$(cat "${work_dir}/${repo}.revision")" | \
-          tar -t | \
-          sed '
-            s|/$||
-            /\//d
-          '
-      done
-    } | \
-      sort -u | \
-      join -j 1 -o 2.2,2.3,2.4,2.1 - "${tmp_dir}/current-metadata" | \
-      sort -k4,4 > \
-      "${tmp_dir}/find-newest-revisions"
-
-    uniq -uf3 < \
-      "${tmp_dir}/find-newest-revisions" | \
-      awk '{print $4 " " $1 " " $2 " " $3}' | \
-      sed 'p'
-
-    uniq -Df3 < \
-      "${tmp_dir}/find-newest-revisions" | \
-      uniq --group=append -f3 | \
-      {
-        revs=''
-        mod_revs=''
-        opkg=''
-        orepo=''
-        while read -r rev mod_rev repo pkg; do
-
-          if [ -z "${rev}" ] && \
-            [ -z "${mod_rev}" ] && \
-            [ -z "${repo}" ] && \
-            [ -z "${pkg}" ]; then
-
-            printf '%s %s %s %s\n' \
-              "$(
-                printf '%s\n' ${revs} | \
-                  find_newest_of_git_revisions
-              )" \
-              "$(
-                printf '%s\n' ${mod_revs} | \
-                  find_newest_of_git_revisions
-              )" \
-              "${orepo}" \
-              "${opkg}"
-
-            revs=''
-            mod_revs=''
-            orepo=''
-            opkg=''
-            continue
-          fi
-          revs=$(
-            # shellcheck disable=SC2086
-            printf '%s\n' ${revs} ${rev} | \
-              sort -u
-          )
-          mod_revs=$(
-            # shellcheck disable=SC2086
-            printf '%s\n' ${mod_revs} ${mod_rev} | \
-              sort -u
-          )
-          orepo="${repo}"
-          opkg="${pkg}"
-        done
-      } | \
-      awk '{print $4 " " $1 " " $2 " " $3}' | \
-      sed 'p'
-  ) | \
-    sort | \
-    uniq -u | \
-    while read -r pkg rev mod_rev repo; do
-      rm -f "${work_dir}/package-infos/${pkg}.${rev}.${mod_rev}.${repo}."*
-    done
-}
-
-# repository_of_package $package.$repo_revision.$mod_repo_revision.$repository
-# print which (stable) repository a package belongs to
-
-repository_of_package() {
-  local package="$1"
-  local repository="${package##*.}"
-  package="${package%.*}"
-  local a32_rev="${package##*.}"
-  package="${package%.*.*}"
-
-  case "${repository}" in
-    'multilib')
-      if git -C "${repo_paths__archlinux32}" archive --format=tar "${a32_rev}" -- 'extra-from-multilib' | \
-        tar -Ox | \
-        grep -qFx "${package%.*.*.*}"; then
-        echo 'extra'
-      else
-        echo 'community'
-      fi
-      ;;
-    *)
-      echo "${repository}"
-  esac
-}
-
-# official_or_community $package.$repo_revision.$mod_repo_revision.$repository $ending
-# print wether the specified package is an official package (print
-# $ending) or a community package (print 'community-$ending') or a
-# build-support package (print 'build-support')
-
-official_or_community() {
-  local prepo
-  prepo=$(repository_of_package "$1")
-
-  if [ "${prepo}" = 'community' ]; then
-    echo 'community-'"$2"
-  elif [ "${prepo}" = 'build-support' ]; then
-    echo 'build-support'
-  else
-    echo "$2"
-  fi
-}
-
-# ls_master_mirror $path
-# list content of $path on the master mirror (via rsync)
-
-ls_master_mirror() {
-
-  local path="$1"
-
-  ${master_mirror_rsync_command} \
-    "${master_mirror_rsync_directory}/${path}/" | \
-    grep -v '\s\.$' | \
-    awk '{print $5}'
-
-}
-
-# TODO: the actions of remove_old_package_versions should be done
-# on basis of the information in the database
-
-# remove_old_package_versions $arch $repository $package_file
-
-# removes all older (not-newer) versions of $package_file
-# in all repositories not-older (newer) than $repository
-
-# TODO: should remove all other version (also newer) from
-# some repositories :-/
-
-# A package is considered not newer if
-#  a) its version is not newer
-# A package is considered older if
-#  b) its version is older or
-#  c) if it's "not newer" and its architecture is 'any' and different or
-#  d) if it's "not newer" and the other architecture is 'any' and different
-
-# this ensures an any package may replace arch-specific packages of the same version and vice versa
-
-remove_old_package_versions() {
-
-  local arch="$1"
-  local repository="$2"
-  local package="$3"
-
-  local pkgname
-  local epoch
-  local pkgver
-  local pkgrel
-  local sub_pkgrel
-  pkgname="${package%-*}"
-  pkgrel="${pkgname##*-}"
-  sub_pkgrel="${pkgrel##*.}"
-  if [ "${sub_pkgrel}" = "${pkgrel}" ]; then
-    sub_pkgrel='0'
-  else
-    pkgrel="${pkgrel%.*}"
-  fi
-  pkgname="${pkgname%-*}"
-  pkgver="${pkgname##*-}"
-  epoch="${pkgver%%:*}"
-  if [ "${epoch}" = "${pkgver}" ]; then
-    epoch='0'
-  else
-    pkgver="${pkgver#*:}"
-  fi
-  pkgname="${pkgname%-*}"
-
-  # shellcheck disable=SC2016
-  {
-    printf 'SELECT "bogus",CONCAT(from_base64("%s"),"/",from_base64("%s")),1,from_base64("%s");\n' \
-      "$(
-        printf '%s' "${repository}" | \
-          base64 -w0
-      )" \
-      "$(
-        printf '%s' "${package}" | \
-          base64 -w0
-      )" \
-      "$(
-        printf '%s' "${package}" | \
-          sed '
-            s/^.*-\([^-]\+-[^-]\+\)-[^-]\+$/\1/
-          ' | \
-          base64 -w0
-      )"
-    printf 'SELECT '
-    printf '`binary_packages`.`id`,'
-    printf 'CONCAT(`repositories`.`name`,"/",'
-    mysql_package_name_query
-    printf '),'
-    # should we delete packages of identical version?
-    printf 'IF((`more_stable_repos`.`id`!=`repositories`.`id`) AND (`more_stable_repos`.`stability`=`repositories`.`stability`),2,0),'
-    printf 'CONCAT('
-      printf 'IF(`binary_packages`.`epoch`=0,"",CONCAT(`binary_packages`.`epoch`,":")),'
-      printf '`binary_packages`.`pkgver`,"-",'
-      printf '`binary_packages`.`pkgrel`,".",'
-      printf '`binary_packages`.`sub_pkgrel`'
-    printf ')'
-    printf ' FROM `binary_packages`'
-    mysql_join_binary_packages_repositories
-    mysql_join_binary_packages_architectures
-    printf ' JOIN `repository_stability_relations` ON `repository_stability_relations`.`less_stable`=`repositories`.`stability`'
-    printf ' JOIN `repositories` AS `more_stable_repos` ON `repository_stability_relations`.`more_stable`=`more_stable_repos`.`stability`'
-    # name must match
-    printf ' WHERE `binary_packages`.`pkgname`=from_base64("%s")' \
-      "$(printf '%s' "${package%-*-*-*}" | base64 -w0)"
-    # repository, where package should be deleted, should be less stable
-    printf ' AND `more_stable_repos`.`name`=from_base64("%s")' \
-      "$(printf '%s' "${repository}" | base64 -w0)"
-    printf ';\n'
-  } | \
-    ${mysql_command} --raw --batch | \
-    sed '
-      /^\S\+\sCONCAT(/d
-      y/\t/ /
-    ' | \
-    expand_version 4 | \
-    sort -k4V,4 -k3r,3 | \
-    shrink_version 4 | \
-    sed -n '
-      /^bogus /q
-      p
-    ' | \
-    cut -d' ' -f1,2 >&2
-
-  # repositories in which older packages should be deleted
-  local delete_older_repositories
-  # repositories in which not-newer packages should be deleted
-  local delete_not_newer_repositories
-
-  if echo "${standalone_package_repositories}" | \
-    grep -qxF "${repository}"; then
-
-    delete_older_repositories="${repository}"
-    delete_not_newer_repositories=''
-
-  elif echo "${staging_package_repositories}" | \
-    grep -qxF "${repository}"; then
-
-    delete_older_repositories="${repository}"
-    delete_not_newer_repositories=$(
-      echo "${staging_package_repositories}" | \
-        grep -vxF "${repository}"
-    ) || true
-
-  elif echo "${testing_package_repositories}" | \
-    grep -qxF "${repository}"; then
-
-    delete_older_repositories=$(
-      printf '%s\n' "${staging_package_repositories}" "${repository}"
-    )
-    delete_not_newer_repositories=$(
-      echo "${testing_package_repositories}" | \
-        grep -vxF "${repository}"
-    ) || true
-
-  elif echo "${stable_package_repositories}" | \
-    grep -qxF "${repository}"; then
-
-    delete_older_repositories=$(
-      printf '%s\n' "${staging_package_repositories}" "${testing_package_repositories}" "${repository}"
-    )
-    delete_not_newer_repositories=$(
-      echo "${stable_package_repositories}" | \
-        grep -vxF "${repository}"
-    ) || true
-
-  else
-
-    >&2 printf 'remove_old_package_versions: Unknown repository "%s".\n' "${repository}"
-    return 1
-
-  fi
-
-  ( # the new shell is intentional
-    tmp_dir=$(mktemp -d 'tmp.common-functions.remove_old_package_versions.XXXXXXXXXX' --tmpdir)
-    trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
-
-    {
-      # the architecture of the package (any vs. i686)
-      package_arch="${package##*-}"
-      package_arch="${package_arch%%.*}"
-      if [ "${package_arch}" = 'any' ]; then
-        package_arch_regex_inverter='!'
-      else
-        unset package_arch_regex_inverter
-      fi
-
-      for repo in ${delete_older_repositories}; do
-        ls_master_mirror "${arch}/${repo}" | \
-          sed -n '
-            /\.pkg\.tar\.xz$/!d
-            s|-\([^-]\+-[^-]\+\)-\([^-]\+\)$| \1 \2|
-            /^'"$(str_to_regex "${package%-*-*-*}")"' / {
-              s|^|2 '"${arch} ${repo}"' |
-              / any\.pkg\.tar\.xz$/'"${package_arch_regex_inverter}"'{
-                s|^2|0|
-              }
-              p
-            }
-          '
-      done
-      for repo in ${delete_not_newer_repositories}; do
-        ls_master_mirror "${arch}/${repo}" | \
-          sed -n '
-            /\.pkg\.tar\.xz$/!d
-            s|-\([^-]\+-[^-]\+\)-\([^-]\+\)$| \1 \2|
-            /^'"$(str_to_regex "${package%-*-*-*}")"' / {
-              s|^|0 '"${arch} ${repo}"' |
-              p
-            }
-          '
-      done
-      echo "${package%-*}" | \
-        sed 's|^.*-\([^-]\+-[^-]\+\)$|1 %cut% %it% %here% \1|'
-
-      # the generated list contains the following columns:
-      # $delete-if-newer-vs-not-older $arch-directory $repo-directory $pkgname $pkgver-$pkgrel $pkg-arch.pkg.tar.xz
-    } | \
-      expand_version 5 | \
-      sort -k5V,5 -k1n,1 | \
-      shrink_version 5 | \
-      sed -n '
-        /^1 %cut% %it% %here% /q
-        s/^[02] //
-        s/ \(\S\+\)$/-\1/
-        p
-      ' | \
-      sort -u > \
-      "${tmp_dir}/packages-to-delete"
-    # this file contains a list of packages to be deleted, one on each line:
-    # $architecture-directory $repository-directory $package-name $pkgver-$pkgrel-$package-architecture.pkg.tar.xz
-
-    cut -d' ' -f1,2 < \
-      "${tmp_dir}/packages-to-delete" | \
-      grep -vxF "${arch} ${repository}" | \
-      sort -u > \
-      "${tmp_dir}/repositories-to-modify"
-
-    # fetch all databases being modified
-    while read -r del_arch del_repo; do
-      mkdir -p "${tmp_dir}/${del_arch}/${del_repo}"
-      ${master_mirror_rsync_command} \
-        "${master_mirror_rsync_directory}/${del_arch}/${del_repo}/${del_repo}.db."* \
-        "${master_mirror_rsync_directory}/${del_arch}/${del_repo}/${del_repo}.files."* \
-        "${tmp_dir}/${del_arch}/${del_repo}/"
-    done < \
-      "${tmp_dir}/repositories-to-modify"
-
-    while read -r del_arch del_repo del_package _; do
-      if [ "${del_arch}/${del_repo}" = "${arch}/${repository}" ]; then
-        # we do not repo-remove the package in the target repository
-        continue
-      fi
-      repo-remove -q "${tmp_dir}/${del_arch}/${del_repo}/${del_repo}.db.tar.gz" \
-        "${del_package}"
-    done < \
-      "${tmp_dir}/packages-to-delete"
-
-    # upload modified databases
-    while read -r del_arch del_repo; do
-      ${master_mirror_rsync_command} \
-        "${tmp_dir}/${del_arch}/${del_repo}/${del_repo}.db."* \
-        "${tmp_dir}/${del_arch}/${del_repo}/${del_repo}.files."* \
-        "${master_mirror_rsync_directory}/${del_arch}/${del_repo}/"
-    done < \
-      "${tmp_dir}/repositories-to-modify"
-
-    # shellcheck disable=SC2016
-    sed '
-      s/\.pkg\.tar\.xz$//
-      s/^\S\+ //
-      s/-\([^-. ]\+\)\(-[^- ]\+\)$/-\1.0\2/
-      s/ \([^-: ]\+\(-[^- ]\+\)\{2\}\)$/ 0:\1/
-      s/ \([^-.]\+\):\([^-:]\+\)-\([^-.]\+\)\.\([^-.]\+\)-\([^-]\+\)$/ \1 \2 \3 \4 \5/
-    ' "${tmp_dir}/packages-to-delete" | \
-      while read -r repo pkgname epoch pkgver pkgrel sub_pkgrel arch; do
-        printf 'DELETE FROM `binary_packages` WHERE'
-        printf ' `binary_packages`.`%s`=(SELECT `%s`.`id` FROM `%s` WHERE `%s`.`name`=from_base64("%s")) AND' \
-          'architecture' 'architectures' 'architectures' 'architectures' "$(printf '%s' "${arch}" | base64 -w0)" \
-          'repository'   'repositories'  'repositories'  'repositories'  "$(printf '%s' "${repo}" | base64 -w0)"
-        printf ' `binary_packages`.`%s`=from_base64("%s") AND' \
-          'pkgname' "$(printf '%s' "${pkgname}" | base64 -w0)" \
-          'epoch' "$(printf '%s' "${epoch}" | base64 -w0)" \
-          'pkgver' "$(printf '%s' "${pkgver}" | base64 -w0)" \
-          'pkgrel' "$(printf '%s' "${pkgrel}" | base64 -w0)" \
-          'sub_pkgrel' "$(printf '%s' "${sub_pkgrel}" | base64 -w0)" | \
-          sed 's/ AND$//'
-        printf ';\n'
-      done | \
-      ${mysql_command}
-
-    sed '
-      s| \(\S\+\)$|-\1|
-      y| |/|
-      s|^|rm "|
-      s|$|"|
-      p
-      s|"$|.sig"|
-    ' "${tmp_dir}/packages-to-delete" | \
-      ${master_mirror_sftp_command}
-  )
-
-}
-
-# wait_some_time $minimum $diff
-# wait between minimum and minimum+diff seconds (diff defaults to 30)
-
-wait_some_time() {
-  local minimum=$1
-  local diff=$2
-  local random
-
-  if [ -z "${diff}" ]; then
-    diff=30
-  fi
-
-  random=$(
-    dd if='/dev/urandom' count=1 2> /dev/null | \
-      cksum | \
-      cut -d' ' -f1
-  )
-
-  sleep $((minimum + random % diff))
-}
-
-# str_to_regex $string
-# escape dots for use in regex
-
-str_to_regex() {
-  echo "$1" | \
-    sed '
-      s|[.[]|\\\0|g
-    '
-}
-
-# make_source_info $package $repository $git_revision $mod_git_revision $output
-# create .SRCINFO from PKGBUILD within git repositories, output to $output
-
-make_source_info() {
-
-  local package="$1"
-  local repository="$2"
-  local git_revision="$3"
-  local mod_git_revision="$4"
-  local output="$5"
-
-  local git_repo
-  local PKGBUILD
-  local PKGBUILD_mod
-
-  git_repo=$(find_repository_with_commit "${git_revision}")
-
-  if [ -z "${git_repo}" ]; then
-    return 1
-  fi
-
-  find_pkgbuilds "${package}" "${repository}" "${git_repo}" "${git_revision}" "${mod_git_revision}"
-
-  ( # the new shell is intentional
-
-    tmp_dir=$(mktemp -d "${work_dir}/tmp.make_source_info.XXXXXX")
-    trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT
-
-    extract_source_directory "${git_repo}" "${git_revision}" "${mod_git_revision}" "${tmp_dir}" '0'
-
-    {
-      cd "${tmp_dir}"
-      makepkg --printsrcinfo
-      cd ..
-    } |
-    if [ "${package%-i18n}-i18n" = "${package}" ]; then
-      sed '
-        1 a \\tdepends = '"${package%-i18n}"'
-      '
-    else
-      cat
-    fi > \
-      "${output}"
-
-  )
-
-}
-
-# recursively_umount_and_rm $dir
-# umount all mountpoints in $dir which are also in $dir's
-#  filesystem, possibly also $dir itself and then
-#  rm -rf --one-file-system $dir
-
-recursively_umount_and_rm() {
-  local dir="$1"
-
-  if [ -z "${dir}" ]; then
-    >&2 echo 'ERROR: recursively_umount_and_rm requires an argument'
-    exit 42
-  fi
-
-  find "${dir}" \
-    -xdev -depth -type d \
-    -exec 'mountpoint' '-q' '{}' ';' \
-    -exec 'sudo' 'umount' '-l' '{}' ';'
-  rm -rf --one-file-system "${dir}"
-}
-
-# mangle_pkgbuild $PKGBUILD [$sub_pkgrel]
-#  mangle $arch in PKBUILDs to contain i486, i586, i686
-#  append $sub_pkgrel to the pkgrel
-
-mangle_pkgbuild() {
-  local PKGBUILD="$1"
-  local sub_pkgrel="$2"
-
-  if [ -n "${sub_pkgrel}" ]; then
-    sub_pkgrel=".${sub_pkgrel}"
-  fi
-
-  sed -i '
-    /^arch=[^#]*any/!{
-      /^arch=(/s/(/(i686 /
-    }
-    s/^\(\s*pkgrel=\)['"'"'"]\?\([0-9.]\+\)['"'"'"]\?\s*\(#.*\)\?$/\1"\2'"${sub_pkgrel}"'"/
-  ' "${PKGBUILD}"
-}
-
-# find_newest_of_git_revisions
-# find newest git revision of the ones provided at stdin
-# (assuming linear history)
-
-find_newest_of_git_revisions() {
-  local revisions
-  local repo
-  revisions=$(cat)
-
-  if [ "$(
-    echo "${revisions}" | \
-      wc -l
-    )" -eq 1 ]; then
-
-    echo "${revisions}"
-    return
-
-  fi
-
-  repo=$(
-    find_repository_with_commit \
-      "$(
-        echo "${revisions}" | \
-          grep -xm1 '[0-9a-f]\{40\}'
-      )"
-  )
-
-  eval 'repo="${repo_paths__'"${repo}"'}"'
-
-  echo "${revisions}" | \
-    xargs -rn1 git -C "${repo}" rev-parse | \
-    {
-      newest=''
-      while read -r current; do
-        if [ -z "${newest}" ] || \
-          git -C "${repo}" merge-base --is-ancestor "${newest}" "${current}"; then
-            newest="${current}"
-          fi
-      done
-      echo "${newest}"
-    }
-}
-
-# find_package_repository_to_package $package $git_repository $git_commit
-# find the package repository a package from a given git repository
-# belongs to
-
-find_package_repository_to_package() {
-
-  local package="$1"
-  local git_repository="$2"
-  local git_commit="$3"
-  local repo_path
-  local repo
-
-  eval 'repo_path="${repo_paths__'"${git_repository}"'}"'
-
-  if [ "${git_repository}" = 'archlinux32' ]; then
-    repo=$(
-      git -C "${repo_path}" archive "${git_commit}" -- | \
-        tar -t --wildcards "*/${package}/" | \
-        cut -d/ -f1 | \
-        sort -u
-    )
-  else
-    repo=$(
-      git -C "${repo_path}" archive "${git_commit}" -- "${package}/repos" 2> /dev/null | \
-        tar -t | \
-        cut -d/ -f3 | \
-        grep -vxF '' | \
-        grep -v 'staging\|testing\|-unstable' | \
-        grep -v -- '-i686$' | \
-        sed 's|-[^-]\+$||' | \
-        sort -u
-    )
-  fi
-
-  if [ -z "${repo}" ]; then
-    return 1
-  fi
-
-  if [ "$(
-    echo "${repo}" | \
-      wc -l
-    )" -ne 1 ]; then
-    return 1
-  fi
-
-  echo "${repo}"
-
-}
-
-# extract_source_directory $git_repo $rev $mod_rev $output $sub_pkgrel
-# extract files found in the svn/git source directories
-# $PKGBUILD and $PKGBUILD_mod are expected to be set correctly
-
-extract_source_directory() {
-
-  local git_repo="$1"
-  local rev="$2"
-  local mod_rev="$3"
-  local output="$4"
-  local sub_pkgrel="$5"
-
-  if [ -n "${PKGBUILD}" ]; then
-    eval 'git -C "${repo_paths__'"${git_repo}"'}" archive "${rev}" -- "${PKGBUILD%/*}"' | \
-      tar -x --strip-components=3 -C "${output}"
-  fi
-
-  if [ -n "${PKGBUILD_mod}" ]; then
-    git -C "${repo_paths__archlinux32}" archive "${mod_rev}" -- "${PKGBUILD_mod%/*}" | \
-      tar -x --overwrite --exclude 'PKGBUILD' --strip-components=2 -C "${output}" 2> /dev/null || \
-      true
-    git -C "${repo_paths__archlinux32}" archive "${mod_rev}" -- "${PKGBUILD_mod}" | \
-      tar -Ox "${PKGBUILD_mod}" >> \
-      "${output}/PKGBUILD"
-  fi
-
-  mangle_pkgbuild "${output}/PKGBUILD" "${sub_pkgrel}"
-
-  # shellcheck disable=SC2016
-  sed -i '/^\$Id\$$/d' "${output}/PKGBUILD"
-
-}
-
-# find_dependencies_on_build_list $package $git_revision $mod_git_revision $repository
-# return a list of packages on the build list which are (run- / build- / check-time)
-# dependencies of the given package
-
-find_dependencies_on_build_list() {
-
-  local package="$1"
-  local git_revision="$2"
-  local mod_git_revision="$3"
-  local repository="$4"
-
-  generate_package_metadata "${package}" "${git_revision}" "${mod_git_revision}" "${repository}"
-
-  {
-    cat "${work_dir}/package-infos/${package}.${git_revision}.${mod_git_revision}.${repository}.build-depends"
-    awk '{print $1 "." $2 "." $3 "." $4}' < \
-      "${work_dir}/build-list" | \
-      sed '
-        s|^|'"${work_dir}"'/package-infos/|
-        s|$|\.builds|
-      ' | \
-      xargs -r cat | \
-      sort -u
-  } | \
-    sort | \
-    uniq -d
-
-}
-
-# download_sources_by_hash $package $repository $git_revision $git_mod_revision
-# try to download all sources by their hash into the current directory
-# returns 0 if any source was downloaded and 1 otherwise
-
-download_sources_by_hash() {
-
-  local package="$1"
-  local repository="$2"
-  local git_revision="$3"
-  local git_mod_revision="$4"
-
-  local return_value=1
-  local tmp_dir
-  local sum_type
-  local arch_suffix
-
-  tmp_dir=$(mktemp -d 'tmp.common-functions.download_sources_by_hash.XXXXXXXXXX' --tmpdir)
-
-  if ! make_source_info "${package}" "${repository}" "${git_revision}" "${git_mod_revision}" "${tmp_dir}/.SRCINFO"; then
-    >&2 echo 'download_sources_by_hash: make_source_info failed.'
-    rm -rf --one-file-system "${tmp_dir}"
-    return 1
-  fi
-
-  if ! [ -s "${tmp_dir}/.SRCINFO" ]; then
-    >&2 echo 'download_sources_by_hash: ".SRCINFO" has not been created by make_source_info.'
-    rm -rf --one-file-system "${tmp_dir}"
-    return 1
-  fi
-
-  for arch_suffix in '' '_i686'; do
-    for sum_type in 'sha256sum' 'sha512sum'; do
-      grep '^\s*'"${sum_type}s${arch_suffix}"' = ' "${tmp_dir}/.SRCINFO" | \
-        sed 's|^.* = ||' | \
-        cat -n > \
-        "${tmp_dir}/sums"
-      grep '^\s*source'"${arch_suffix}"' = ' "${tmp_dir}/.SRCINFO" | \
-        sed '
-          s|^.* = ||
-          s|::.*$||
-          s|.*/||
-        ' | \
-        cat -n > \
-        "${tmp_dir}/urls"
-      if [ "$(wc -l < "${tmp_dir}/sums")" -eq "$(wc -l < "${tmp_dir}/urls")" ]; then
-        join -1 1 -2 1 -o 1.2,2.2 "${tmp_dir}/sums" "${tmp_dir}/urls" > \
-          "${tmp_dir}/joined"
-        while read -r sum file; do
-          if [ "${sum}" = 'SKIP' ]; then
-            continue
-          fi
-          if echo "${sum}  ${file}" | \
-            ${sum_type} -c > /dev/null 2>&1; then
-            # the correct source is already there
-            continue
-          fi
-          if wget -O "${tmp_dir}/transfer" "${source_by_hash_mirror}${sum}"; then
-            mv "${tmp_dir}/transfer" "${file}"
-            return_value=0
-          fi
-        done < \
-          "${tmp_dir}/joined"
-      fi
-    done
-  done
-
-  rm -rf --one-file-system "${tmp_dir}"
-  return ${return_value}
-
-}
-
-# expand_version $column_num
-# add "0:" to version in $colum_num-th column if no ":" is there (epoch)
-# add "+0" to version in $colum_num-th column if no "+" is there (git count/hash)
-
-expand_version() {
-  local column_num
-  column_num="$1"
-
-  sed '
-    /^\(\S\+\s\+\)\{'"$((column_num-1))"'\}\S*+/! s/^\(\(\S\+\s\+\)\{'"$((column_num-1))"'\}\S*\)-/\1+0-/
-    /^\(\S\+\s\+\)\{'"$((column_num-1))"'\}\S*:/! s/^\(\(\S\+\s\+\)\{'"$((column_num-1))"'\}\)/\10:/
-  '
-}
-
-# shrink_version $column_num
-# remove "0:" from version in $colum_num-th column (epoch)
-# remove "+0" from version in $colum_num-th column (git count/hash)
-
-shrink_version() {
-  local column_num
-  column_num="$1"
-
-  sed '
-    s/^\(\(\S\+\s\+\)\{'"$((column_num-1))"'\}\S*\)+0-/\1-/
-    s/^\(\(\S\+\s\+\)\{'"$((column_num-1))"'\}\)0:/\1/
-  '
-}
-
-# find_biggest_subset_of_packages $omega $keep $all_builds $all_depends [ $force ]
-
-# Return (to stdout) the biggest subset A of the packages in $omega whose
-# runtime dependencies in $omega \cup $keep are also in A
-
-# $all_builds and $all_depends either point to an empty file - then they will get
-# filled with cached data for subsequent calls - or to the same files of a previous
-# call
-
-# If non-empty, $force contains packages which are assumed to match the above
-# condition without checking.
-
-# The arguments are names of files with one $package.$revision.$mod_revision.$repository
-# per line.
-
-find_biggest_subset_of_packages() {
-
-  ( # the new shell is intentional
-    omega="$1"
-    keep="$2"
-    all_builds="$3"
-    all_depends="$4"
-    if [ $# -eq 4 ]; then
-      force='/dev/null'
-    elif [ $# -eq 5 ]; then
-      force="$5"
-    else
-      >&2 printf 'find_biggest_subset_of_packages: Wrong number of arguments: %s given, 4 or 5 expected.' "$#"
-      return 2
-    fi
-
-    if [ ! -s "${all_builds}" ]; then
-      find "${work_dir}/package-infos/" -maxdepth 1 -name '*.builds' \
-        -exec sed '
-          s|^|{} |
-          s|^\S\+/||
-          s|\.builds | |
-        ' {} \; | \
-        sort -k2,2 > \
-        "${all_builds}"
-    fi
-
-    if [ ! -s "${all_depends}" ]; then
-      find "${work_dir}/package-infos/" -maxdepth 1 -name '*.run-depends' \
-        -exec sed '
-          s|^|{} |
-          s|^\S\+/||
-          s|\.run-depends | |
-        ' {} \; | \
-        grep -v ' base$' | \
-        sort -k2,2 > \
-        "${all_depends}"
-    fi
-
-    sort -u "${omega}" | \
-      sponge "${omega}"
-
-    temp_dir=$(mktemp -d 'tmp.common-functions.find_biggest_subset_of_packages.XXXXXXXXXX' --tmpdir)
-    trap 'rm -rf --one-file-system "${temp_dir}"' EXIT
-
-    {
-      sort -u "${keep}"
-      cat "${force}" "${force}"
-    } | \
-      sort | \
-      uniq -u > \
-      "${temp_dir}/keep.new"
-    touch "${temp_dir}/keep"
-
-    while [ -s "${temp_dir}/keep.new" ]; do
-      cat "${temp_dir}/keep.new" "${temp_dir}/keep" | \
-        sort -u | \
-        sponge "${temp_dir}/keep"
-
-      {
-        # we append all packages which are run-dependencies of keep-packages
-        # to the keep-list
-        sed '
-          s|^|'"${work_dir}"'/package-infos/|
-          s|$|.run-depends|
-        ' "${temp_dir}/keep" | \
-          xargs -r grep -HF '' | \
-          sed '
-            s|^.*/||
-            s|\.run-depends:| |
-          ' | \
-          sort -u | \
-          sort -k2,2 | \
-          uniq -f1 | \
-          join -1 2 -2 2 -o 2.1 - "${all_builds}"
-
-        # we append all packages with run-dependencies on the keep-list
-        # to the keep-list
-        sed '
-          s|^|'"${work_dir}"'/package-infos/|
-          s|$|.builds|
-        ' "${temp_dir}/keep" | \
-          xargs -r grep -HF '' | \
-          sed '
-            s|^.*/||
-            s|\.builds:| |
-          ' | \
-          sort -u | \
-          sort -k2,2 | \
-          uniq -f1 | \
-          join -1 2 -2 2 -o 2.1 - "${all_depends}"
-      } | \
-        sort -u | \
-        join -1 1 -2 1 -o 2.1 - "${omega}" | \
-        sort -u > \
-        "${temp_dir}/keep.new"
-
-      # "new" is only what has not been there before and what is not forced
-      cat "${temp_dir}/keep" "${temp_dir}/keep" "${force}" "${force}" "${temp_dir}/keep.new" | \
-        sort | \
-        uniq -u | \
-        sponge "${temp_dir}/keep.new"
-    done
-
-    cat "${omega}" "${temp_dir}/keep" "${temp_dir}/keep" | \
-      sort | \
-      uniq -u
-
-  )
-
-}
-
-# sort_square_bracket_content $file
-# sort the content of [] in $file, print to stdout
-
-sort_square_bracket_content() {
-  local file
-  local line
-  local token
-  local token_list
-  local rest
-  file="$1"
-
-  while read -r line; do
-    printf '%s ' "${line}" | \
-      tr ' ' '\n' | \
-      while read -r token; do
-        if echo "${token}" | \
-          grep -qF '['; then
-          printf '%s[' "${token%[*}"
-          token="${token##*[}"
-          token_list="${token%,}"
-          while ! echo "${token_list}" | \
-            grep -qF ']'; do
-            read -r token
-            token_list=$(
-              printf '%s\n' \
-                "${token_list}" \
-                "${token%,}"
-            )
-          done
-          rest="]${token_list#*]}"
-          token_list="${token_list%%]*}"
-          token=$(
-            printf '%s' "${token_list}" | \
-              sort | \
-              sed 's|$|,|'
-            printf '%s' "${rest}"
-          )
-        fi
-        printf '%s\n' "${token}"
-      done | \
-      tr '\n' ' ' | \
-      sed '
-        s|, ]|]|g
-        s| $||
-      '
-    printf '\n'
-  done < \
-    "${file}"
-}
-
-# smoothen_namcap_log $file
-# remove unneccesary differences from namcap-logs:
-#   - remove architecture specific information
-#   - sort lines
-#   - sort content of square brackets
-
-smoothen_namcap_log() {
-  local file
-  file="$1"
-  # shellcheck disable=SC2016
-  sort_square_bracket_content "${file}" | \
-    sed '
-      # normalize architecture specific information
-      s|i[34567]86|$ARCH|g
-      s|x86\([-_]64\)\?|$ARCH|g
-      # remove haskell hashes
-      s|\('"'"'[^'"'"']*-[0-9.]\+\)-[a-zA-Z0-9]\{1,22\}\(-ghc[^'"'"']*'"'"'\)|\1\2|g
-    ' | \
-    sort | \
-    sponge "${file}"
-}
-
-# print_list_of_archaic_packages $source1 $source2 ...
-# print a list of packages which have not been touched for a while,
-# but which are still in the pipeline, e.g. in $source1, $source2 or ...
-
-print_list_of_archaic_packages() {
-  for source in "$@"; do
-    case "${source}" in
-      'testing')
-        # packages remaining longer than $max_package_age_testing days in testing will be marked tested if no bug for them exists on FS32
-        find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.testing' -mtime "+${max_package_age_testing}" \
-          -exec head -n1 {} \; | \
-          "${base_dir}/bin/modify-package-state" -n --tested /dev/stdin
-        # packages remaining longer than $max_package_age_broken_testing days in testing (w/o being tested!) will be considered outdated
-        # and no longer block other packages from being moved
-        find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.testing' -mtime "+${max_package_age_broken_testing}" -printf '%f\n' | \
-          sed '
-            s|\.testing$||
-          '
-      ;;
-      'build-list')
-        while read -r pkg rev mod_rev repo; do
-          git_repo=$(
-            find_repository_with_commit "${rev}"
-          )
-          eval repo_path='"${repo_paths__'"${git_repo}"'}"'
-          commit_date=$(
-            git -C "${repo_path}" show -s --format=%ct "${rev}"
-          )
-          mod_commit_date=$(
-            git -C "${repo_paths__archlinux32}" show -s --format=%ct "${mod_rev}"
-          )
-          if [ "${mod_commit_date}" -gt "${commit_date}" ]; then
-            commit_date="${mod_commit_date}"
-          fi
-          # packages remaining longer than $max_package_age_build_list days on the build list
-          if [ "$((commit_date + 24*60*60*max_package_age_build_list))" -lt "$(date '+%s')" ]; then
-            printf '%s %s %s %s\n' \
-              "${pkg}" \
-              "${rev}" \
-              "${mod_rev}" \
-              "${repo}"
-          fi
-        done < \
-          "${work_dir}/build-list"
-      ;;
-      'staging')
-        # packages remaining longer than $max_package_age_staging days in staging
-        find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 -name '*.done' -mtime "+${max_package_age_staging}" -printf '%f\n' | \
-          sed '
-            s|\.done$||
-          '
-      ;;
-      *)
-        >&2 printf 'unknown archaic-source "%s" - skipped.\n' "${source}"
-      ;;
-    esac
-  done | \
-    sort -u
-}
-
-# modification_revision_link "${mod_rev}" "${repo}" "${pkg}"
-# print the given modification revision possibly with a html link to github
-
-modification_revision_link() {
-  local mod_rev="$1"
-  local repo="$2"
-  local pkg="$3"
-
-  if git -C "${repo_paths__archlinux32}" archive "${mod_rev}" -- "${repo}/${pkg}/PKGBUILD" > /dev/null 2>&1; then
-    printf '<a href="https://github.com/archlinux32/packages/tree/%s/%s/%s">%s</a>\n' \
-      "${mod_rev}" \
-      "${repo}" \
-      "${pkg}" \
-      "${mod_rev}"
-  else
-    printf '%s\n' \
-      "${mod_rev}"
-  fi
-}
-
-# trigger_mirror_refreshs
-# trigger a refresh of capable tier 1 mirrors (as backup for master mirror)
-
-trigger_mirror_refreshs() {
-  local tmp_file
-
-  tmp_file=$(mktemp "tmp.common-functions.trigger_mirror_refreshs.XXXXXXXXXX" --tmpdir)
-  date '+%s' > \
-    "${tmp_file}"
-  ${master_mirror_rsync_command} \
-    "${tmp_file}" \
-    "${master_mirror_rsync_directory}/lastupdate"
-  rm "${tmp_file}"
-  for trigger_url in ${mirror_refresh_trigger_urls}; do
-    screen -S trigger-mirror-update -d -m curl -L "${trigger_url}"
-  done
-}
-
-# extract_pkgname_epoch_pkgver_pkgrel_sub_pkgrel_arch_from_package_name
-extract_pkgname_epoch_pkgver_pkgrel_sub_pkgrel_arch_from_package_name() {
-  pkgname="$1"
-  pkgname="${pkgname%.pkg.tar.xz}"
-  arch="${pkgname##*-}"
-  pkgname="${pkgname%-*}"
-  sub_pkgrel="${pkgname##*-}"
-  pkgname="${pkgname%-*}"
-  pkgrel="${sub_pkgrel%.*}"
-  if [ "${pkgrel}" = "${sub_pkgrel}" ]; then
-    sub_pkgrel='0'
-  else
-    sub_pkgrel="${sub_pkgrel##*.}"
-  fi
-  epoch="${pkgname##*-}"
-  pkgname="${pkgname%-*}"
-  pkgver="${epoch#*:}"
-  if [ "${pkgver}" = "${epoch}" ]; then
-    epoch='0'
-  else
-    epoch="${epoch%%:*}"
-  fi
-}