path: root/bin/db-update

#!/bin/sh

# move binary packages from staging to testing (if possible [1]) and
# additionally all packages specified on the command line from testing
# to the respective stable repository

# 1] Condition for moving a package A from staging to testing is that:
#   a) nothing on the build-list run-depends on A and
#   b) no done package B which is not being moved run-depends on A

# TODO: correctly handle if multiple versions of a single package are marked
# as "done" or "testing"

# TODO: be (even?) more atomic

# TODO: separate locks for staging, testing (and stable)

# TODO: correctly handle if a package moved from community to official or
# vice versa (delete residues)

# shellcheck disable=SC2039
# shellcheck source=conf/default.conf
. "${0%/*}/../conf/default.conf"

# shellcheck disable=SC2016
usage() {
  >&2 echo ''
  >&2 echo 'db-update [options] [packages]:'
  >&2 echo ' move possible packages from staging to testing.'
  >&2 echo ' move packages on the command line from testing to stable.'
  >&2 echo ''
  >&2 echo 'possible options:'
  >&2 echo '  -b|--block:       If necessary, wait for lock blocking.'
  >&2 echo '  -f|--from $file:  Read packages to move from testing to'
  >&2 echo '                    stable from $file (- is stdin).'
  >&2 echo '  -h|--help:        Show this help and exit.'
  >&2 echo '  -n|--no-action:   Only print what would be moved.'
  >&2 echo '  -u|--unstage $package:'
  >&2 echo '                    Unstage package $package, even if it'
  >&2 echo '                    would not be unstaged otherwise.'
  [ -z "$1" ] && exit 1 || exit "$1"
}

# move_packages file with one "$package $from_repository $to_repository" per line
# the existence of a directory $tmp_dir is assumed

move_packages() {

  if [ -z "${tmp_dir}" ] || [ ! -d "${tmp_dir}" ]; then
    >&2 echo 'move_packages: No tmp_dir provided.'
    exit 2
  fi

  local package
  local from_repo
  local to_repo
  local from_ending
  local to_ending
  local repo
  local part
  local dummynator
  local file

  if [ -e "${tmp_dir:?}/tmp" ]; then
    rm -rf --one-file-system "${tmp_dir:?}/tmp"
  fi
  mkdir "${tmp_dir}/tmp"

  touch "${tmp_dir}/tmp/repos"
  touch "${tmp_dir}/tmp/packages"
  touch "${tmp_dir}/tmp/master-mirror-listing"
  mkdir "${tmp_dir}/tmp/transit"

  if ${no_action}; then
    dummynator='echo'
  else
    dummynator=''
  fi

  ls_master_mirror 'i686' | \
    while read -r repo; do
      ls_master_mirror "i686/${repo}" | \
        sed "s|^|i686/${repo}/|" >> \
        "${tmp_dir}/tmp/master-mirror-listing"
    done

  while read -r package from_repo to_repo; do
    if [ -z "${package}" ]; then
      continue
    fi

    if ${no_action}; then
      printf \
        'move "%s" from "%s" to "%s"\n' \
        "${package}" \
        "${from_repo}" \
        "${to_repo}"
    fi

    echo "${package}" >> \
      "${tmp_dir}/tmp/packages"

    if echo "${from_repo}" | \
      grep -q 'staging$' && \
      echo "${to_repo}" | \
        grep -q 'testing$'; then
      from_ending='done'
      to_ending='testing'
    elif echo "${from_repo}" | \
      grep -q 'testing$' && \
      ! echo "${to_repo}" | \
        grep -q 'testing$\|staging$'; then
      from_ending='testing'
      to_ending=''
    else
      >&2 printf 'move_packages: Cannot move package "%s" from "%s" to "%s".\n' "${package}" "${from_repo}" "${to_repo}"
      exit 2
    fi

    echo "${from_repo}" > \
      "${tmp_dir}/tmp/${package}.from_repo"
    echo "${to_repo}" > \
      "${tmp_dir}/tmp/${package}.to_repo"
    echo "${from_ending}" > \
      "${tmp_dir}/tmp/${package}.from_ending"
    echo "${to_ending}" > \
      "${tmp_dir}/tmp/${package}.to_ending"

    if [ ! -f "${work_dir}/package-states/${package}.${from_ending}" ]; then
      >&2 printf 'move_packages: Cannot find package state file "%s"\n' "${package}.${from_ending}"
      exit 2
    fi

    cp \
      "${work_dir}/package-states/${package}.${from_ending}" \
      "${tmp_dir}/tmp/${package}.parts"

    sed \
      's|\(-[^-]\+\)\{3\}\.pkg\.tar\.xz$||' \
      "${tmp_dir}/tmp/${package}.parts" > \
      "${tmp_dir}/tmp/${package}.parts_names"

    sed \
      'p;s|$|.sig|' \
      "${tmp_dir}/tmp/${package}.parts" > \
      "${tmp_dir}/tmp/${package}.parts_and_signatures"

    while read -r part; do
      if ! grep -qxF "i686/${from_repo}/${part}" "${tmp_dir}/tmp/master-mirror-listing"; then
        >&2 printf \
          'move_packages: Cannot find file "%s", part of package "%s".\n' \
          "i686/${from_repo}/${part}" \
          "${package}"
        exit 2
      fi
    done < \
      "${tmp_dir}/tmp/${package}.parts"

    mkdir -p "${tmp_dir}/tmp/${from_repo}"
    mkdir -p "${tmp_dir}/tmp/${to_repo}"

    repos=$(
      # shellcheck disable=SC2046
      printf '%s\n' "${from_repo}" "${to_repo}" $(cat "${tmp_dir}/tmp/repos") | \
        sort -u
    )
    echo "${repos}" > \
      "${tmp_dir}/tmp/repos"

  done < \
    "$1"

  if ${no_action}; then
    find "${tmp_dir}/tmp" -type f | \
      while read -r file; do
        if [ "${file%.pkg.tar.xz}.pkg.tar.xz" = "${file}" ] ||
          [ "${file%.pkg.tar.xz.sig}.pkg.tar.xz.sig" = "${file}" ]; then
          echo "'${file}'"
        else
          echo "${file}:"
          sed 's|^|<<|;s|$|>>|' "${file}"
        fi
        echo
      done
  fi

  # receive the *.db.tar.gz's and *.files.tar.gz's

  while read -r repo; do

    ${master_mirror_rsync_command} \
      "${master_mirror_rsync_directory}/i686/${repo}/${repo}.db."* \
      "${master_mirror_rsync_directory}/i686/${repo}/${repo}.files."* \
      "${tmp_dir}/tmp/${repo}/"

    # add and remove the packages locally

    if grep -qxF "${repo}" "${tmp_dir}/tmp/"*".from_repo"; then

      # shellcheck disable=SC2046
      repo-remove -q \
        "${tmp_dir}/tmp/${repo}/${repo}.db.tar.gz" \
        $(
          grep -lxF "${repo}" "${tmp_dir}/tmp/"*".from_repo" | \
            sed '
              s|\.from_repo$|.parts_names|
            ' | \
            xargs -rn1 cat
        )
    fi

    if grep -qxF "${repo}" "${tmp_dir}/tmp/"*".to_repo"; then
      grep -lxF "${repo}" "${tmp_dir}/tmp/"*".to_repo" | \
        sed '
          s|\.to_repo$||
        ' | \
        while read -r package; do
          while read -r part; do
            ${master_mirror_rsync_command} \
              "${master_mirror_rsync_directory}/i686/$(cat "${package}.from_repo")/${part}" \
              "${master_mirror_rsync_directory}/i686/$(cat "${package}.from_repo")/${part}.sig" \
              "${tmp_dir}/tmp/transit/"
            repo-add -q \
              "${tmp_dir}/tmp/${repo}/${repo}.db.tar.gz" \
              "${tmp_dir}/tmp/transit/${part}"
            rm \
              "${tmp_dir}/tmp/transit/${part}" \
              "${tmp_dir}/tmp/transit/${part}.sig"
          done < \
            "${package}.parts"
        done
    fi

  done < "${tmp_dir}/tmp/repos"

  if ${no_action}; then
    find "${tmp_dir}/tmp" -type f
  fi

  # move the packages remotely via sftp

  (
    while read -r package; do

      if [ -z "${package}" ]; then
        continue
      fi

      while read -r part; do
        if [ -z "${part}" ]; then
          continue
        fi
        printf \
          'rename "%s" "%s"\n' \
          "i686/$(cat "${tmp_dir}/tmp/${package}.from_repo")/${part}" \
          "i686/$(cat "${tmp_dir}/tmp/${package}.to_repo")/${part}"
      done < \
        "${tmp_dir}/tmp/${package}.parts_and_signatures"

    done < \
      "${tmp_dir}/tmp/packages"
    echo 'quit'
  ) | \
    if ${no_action}; then
      sed 's|^|sftp: |'
    else
      ${master_mirror_sftp_command}
    fi

  # and push our local *.db.tar.gz via rsync

  while read -r repo; do

    # shellcheck disable=SC2086
    ${dummynator} ${master_mirror_rsync_command} \
      "${tmp_dir}/tmp/${repo}/${repo}.db."* \
      "${tmp_dir}/tmp/${repo}/${repo}.files."* \
      "${master_mirror_rsync_directory}/i686/${repo}/"

  done < \
    "${tmp_dir}/tmp/repos"

  while read -r package; do

    # then we can safely remove old versions

    while read -r part; do
      ${dummynator} remove_old_package_versions 'i686' "$(cat "${tmp_dir}/tmp/${package}.to_repo")" "${part}"
    done < \
      "${tmp_dir}/tmp/${package}.parts"

    # and update the state files

    from_ending=$(
      cat "${tmp_dir}/tmp/${package}.from_ending"
    )
    to_ending=$(
      cat "${tmp_dir}/tmp/${package}.to_ending"
    )

    if [ -z "${to_ending}" ]; then
      ${dummynator} rm \
        "${work_dir}/package-states/${package}.${from_ending}"
    else
      # remove old state files of $package with ending $to_ending
      find "${work_dir}/package-states" -maxdepth 1 | \
        grep "/$(str_to_regex "${package%.*.*.*}")\(\.[^.]\+\)\{3\}\.${to_ending}\$" | \
        xargs -rn1 ${dummynator} rm
      ${dummynator} mv \
        "${work_dir}/package-states/${package}.${from_ending}" \
        "${work_dir}/package-states/${package}.${to_ending}"
    fi

  done < \
    "${tmp_dir}/tmp/packages"

  if ! ${no_action}; then
    date '+%s' > \
      "${tmp_dir}/tmp/lastupdate"
    # shellcheck disable=SC2086
    ${dummynator} ${master_mirror_rsync_command} \
      "${tmp_dir}/tmp/lastupdate" \
      "${master_mirror_rsync_directory}/lastupdate"
  fi

  rm -rf --one-file-system "${tmp_dir:?}/tmp"

}

eval set -- "$(
  getopt -o bf:hnu: \
    --long block \
    --long from: \
    --long help \
    --long no-action \
    --long unstage: \
    -n "$(basename "$0")" -- "$@" || \
  echo usage
)"

block_flag='-n'
no_action=false

while true
do
  case "$1" in
    -b|--block)
      block_flag=''
    ;;
    -f|--from)
      shift
      if [ "x$1" = "x-" ]; then
        packages_to_stabilize=$(cat)
      else
        packages_to_stabilize=$(cat "$1")
      fi
    ;;
    -h|--help)
      usage 0
    ;;
    -n|--no-action)
      no_action=true
    ;;
    -u|--unstage)
      shift
      packages_to_force_unstage="${packages_to_force_unstage} $1"
    ;;
    --)
      shift
      break
    ;;
    *)
      >&2 echo 'Whoops, forgot to implement option "'"$1"'" internally.'
      exit 42
    ;;
  esac
  shift
done

packages_to_stabilize=$(
  # shellcheck disable=SC2086
  printf '%s\n' \
    ${packages_to_stabilize} \
    "${@}"
)

if [ -s "${work_dir}/build-master-sanity" ]; then
  >&2 echo 'Build master is not sane.'
  exit 1
fi

tmp_dir=$(mktemp -d "${work_dir}/tmp.XXXXXX")
trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT

packages_to_stabilize=$(
  (
    # shellcheck disable=SC2086
    printf '%s\n' ${packages_to_stabilize} | \
      sort -u | \
      sed '
        /\.pkg\.tar\.xz$/{
          w '"${tmp_dir}/packages-to-stabilize"'
          d
        }
      '
    find "${work_dir}/package-states" -maxdepth 1 -type f -name '*.testing' -exec \
      grep -HF '' {} \; | \
      sed '
        s|^.*/||
        s|^\([^:]\+\)\.testing:|\1 |
      ' | \
      sort -k2,2 | \
      join -1 2 -2 1 -o 1.1 - "${tmp_dir}/packages-to-stabilize"
  ) | \
    sort -u
)

for package in ${packages_to_stabilize}; do
  # some sanity checks
  if [ ! -f "${work_dir}/package-states/${package}.testing" ]; then
    >&2 echo "Package '${package}' is not in testing!"
    exit 2
  fi
done

for package in ${packages_to_force_unstage}; do
  # some sanity checks
  if [ ! -f "${work_dir}/package-states/${package}.done" ]; then
    >&2 echo "Package '${package}' is not in staging!"
    exit 2
  fi
done

# Create a lock file and a trap.

exec 9> "${build_list_lock_file}"
if ! flock ${block_flag} 9; then
  >&2 echo 'come back (shortly) later - I cannot lock build list.'
  exit 1
fi

exec 8> "${package_database_lock_file}"
if ! flock ${block_flag} 8; then
  >&2 echo 'come back (shortly) later - I cannot lock package database.'
  exit 1
fi

clean_up_lock_file() {
  rm -f "${package_database_lock_file}" "${build_list_lock_file}"
  rm -rf --one-file-system "${tmp_dir}"
}

trap clean_up_lock_file EXIT

# sanity check

for ending in 'done' 'testing'; do
  if [ "${ending}" = 'testing' ] && \
    [ -z "${packages_to_stabilize}" ]; then
    # if nothing is to be untested, we don't care about duplicate
    # testing packages (and maybe an unstaging fixes this anyway)
    continue
  fi
  errors=$(
    find "${work_dir}/package-states" -name "*.${ending}" -printf '%f\n' | \
      sed 's|\(\.[^.]\+\)\{4\}$||' | \
      sort | \
      uniq -d
  )
  if [ -n "${errors}" ]; then
    >&2 echo 'Removing duplicates not yet implemented:'
    >&2 echo "${errors}"
    exit 42
  fi
done

# packages which are done

find "${work_dir}/package-states" -maxdepth 1 -type f -name '*.done' -printf '%f\n' | \
  sed '
    s|\.done$||
  ' | \
  sort -u > \
  "${tmp_dir}/done-packages"

# packages still on the build-list

grep -vxF 'break_loops' "${work_dir}/build-list" | \
  tr ' ' '.' | \
  sort -u > \
  "${tmp_dir}/build-list-packages"

find "${work_dir}/package-infos" -name '*.groups' \
  -exec grep -qxF 'base' {} \; \
  -printf '%f\n' | \
  sed '
    s|\.groups$||
  ' | \
  sort -u > \
  "${tmp_dir}/base-packages"

# shellcheck disable=SC2086
printf '%s\n' ${packages_to_force_unstage} > \
  "${tmp_dir}/force-unstage-packages"

# calculate what packages should be unstaged:

find_biggest_subset_of_packages "${tmp_dir}/done-packages" "${tmp_dir}/build-list-packages" "${tmp_dir}/all-run-depends" "${tmp_dir}/force-unstage-packages" > \
  "${tmp_dir}/unstage-packages"

# no base / base-devel packages on the build list anymore?
if [ -z "$(
    join -j 1 \
      "${tmp_dir}/base-packages" \
      "${tmp_dir}/build-list-packages"
  )" ]; then

  # we pretend, the group "base" does not exist, so we only fetch 'direct' dependencies on base-packages

  mv "${tmp_dir}/all-run-depends" "${tmp_dir}/really-all-run-depends"
  grep -v ' base$' "${tmp_dir}/really-all-run-depends" > \
    "${tmp_dir}/all-run-depends" || \
    true

  find_biggest_subset_of_packages "${tmp_dir}/done-packages" "${tmp_dir}/build-list-packages" "${tmp_dir}/all-run-depends" "${tmp_dir}/force-unstage-packages" > \
    "${tmp_dir}/unstage-packages"

  mv "${tmp_dir}/really-all-run-depends" "${tmp_dir}/all-run-depends"
fi

# calculate what packages should be stabilized

cat "${tmp_dir}/done-packages" "${tmp_dir}/build-list-packages" | \
  sort -u > \
  "${tmp_dir}/keep-packages"

# shellcheck disable=SC2086
printf '%s\n' ${packages_to_stabilize} > \
  "${tmp_dir}/stabilize-packages"

find_biggest_subset_of_packages "${tmp_dir}/stabilize-packages" "${tmp_dir}/keep-packages" "${tmp_dir}/all-run-depends" | \
  sponge "${tmp_dir}/stabilize-packages"

# unlock build list

rm -f "${build_list_lock_file}"
flock -u 9

clean_up_lock_file() {
  rm -rf --one-file-system "${tmp_dir}"
  rm -f "${package_database_lock_file}"
}

# testing -> stable

while read -r package; do
  if [ -z "${package}" ]; then
    continue
  fi
  printf '%s %s %s\n' \
    "${package}" \
    "$(official_or_community "${package}" 'testing')" \
    "$(repository_of_package "${package}")"
done < \
  "${tmp_dir}/stabilize-packages" | \
  sponge "${tmp_dir}/stabilize-packages"

# staging -> testing

while read -r package; do
  if [ -z "${package}" ]; then
    continue
  fi
  printf '%s %s %s\n' \
    "${package}" \
    "$(official_or_community "${package}" 'staging')" \
    "$(official_or_community "${package}" 'testing')"
done < \
  "${tmp_dir}/unstage-packages" | \
  sponge "${tmp_dir}/unstage-packages"

# move packages in packages_to_stabilize from *testing/ to the stable repos
if [ -s "${tmp_dir}/stabilize-packages" ]; then
  move_packages "${tmp_dir}/stabilize-packages"
fi

# move packages from *staging to *testing
if [ -s "${tmp_dir}/unstage-packages" ]; then
  move_packages "${tmp_dir}/unstage-packages"
fi

clean_up_lock_file