path: root/bin/db-update

#!/bin/sh

# move binary packages from staging to testing (if possible [1]) and
# additionally all packages specified on the command line from testing
# to the respective stable repository

# 1] Condition for moving a package A from staging to testing is that:
#   a) nothing on the build-list depends on A and
#   b) no done package B which is not being moved depends on A

# TODO:

#  the meta data used to evaluate condition "b" might be for a different
#    version

#  correctly handle if multiple versions of a single package are marked
#    as "done" or "testing"

#  be more atomic

#  do not download and afterwards upload packages which are just being
#    moved

. "${0%/*}/../conf/default.conf"

usage() {
  >&2 echo ''
  >&2 echo 'db-update [options] [packages]:'
  >&2 echo ' move possible packages from staging to testing.'
  >&2 echo ' move packages on the command line from testing to stable.'
  >&2 echo ''
  >&2 echo 'possible options:'
  >&2 echo '  -b|--block:       If necessary, wait for lock blocking.'
  >&2 echo '  -f|--from $file:  Read packages to move from testing to'
  >&2 echo '                    stable from $file (- is stdin).'
  >&2 echo '  -h|--help:        Show this help and exit.'
  [ -z "$1" ] && exit 1 || exit $1
}

# move_package $package $from_repository $to_repository
# the existence of a directory $tmp_dir is assumed

move_package() {

  if [ -z "${tmp_dir}" ] || [ ! -d "${tmp_dir}" ]; then
    >&2 echo 'move_package: No tmp_dir provided.'
    exit 2
  fi

  local package="$1"
  local from_repo="$2"
  local to_repo="$3"
  local from_ending
  local to_ending
  local part

  if echo "${from_repo}" | \
    grep -q 'staging$' && \
    echo "${to_repo}" | \
      grep -q 'testing$'; then
    from_ending='done'
    to_ending='testing'
  elif echo "${from_repo}" | \
    grep -q 'testing$' && \
    ! echo "${to_repo}" | \
      grep -q 'testing$\|staging$'; then
    from_ending='testing'
    to_ending=''
  else
    >&2 printf 'move_package: Cannot move package from "%s" to "%s".\n' "${from_repo}" "${to_repo}"
    exit 2
  fi

  if [ ! -f "${work_dir}/package-states/${package}.${from_ending}" ]; then
    >&2 printf 'move_package: Cannot find package state file "%s"\n' "${package}.${from_ending}"
    exit 2
  fi

  rm -rf --one-file-system "${tmp_dir}/"*

  cp \
    "${work_dir}/package-states/${package}.${from_ending}" \
    "${tmp_dir}/parts"
  sed \
    's|\(-[^-]\+\)\{3\}\.pkg\.tar\.xz$||' \
    "${tmp_dir}/parts" > \
    "${tmp_dir}/parts_names"
  sed \
    'p;s|$|.sig|' \
    "${tmp_dir}/parts" > \
    "${tmp_dir}/parts_and_signatures"

  mkdir "${tmp_dir}/from"
  mkdir "${tmp_dir}/to"

  ${master_mirror_command} \
    --files-from="${tmp_dir}/parts_and_signatures" \
    "${master_mirror_directory}/i686/${from_repo}/" \
    "${tmp_dir}/from/"

  ${master_mirror_command} \
    "${master_mirror_directory}/i686/${from_repo}/${from_repo}.db."* \
    "${tmp_dir}/from"

  ${master_mirror_command} \
    "${master_mirror_directory}/i686/${to_repo}/${to_repo}.db."* \
    "${tmp_dir}/to"

  repo-remove \
    "${tmp_dir}/from/${from_repo}.db.tar.gz" \
    $(cat "${tmp_dir}/parts_names")

  while read -r part; do
    mv \
      "${tmp_dir}/from/${part}" \
      "${tmp_dir}/to/"
  done < \
    "${tmp_dir}/parts_and_signatures"

  repo-add \
    "${tmp_dir}/to/${to_repo}.db.tar.gz" \
    $(
      sed \
        "s|^|${tmp_dir}/to/|" \
        "${tmp_dir}/parts"
    )

  ${master_mirror_command} \
    --files-from="${tmp_dir}/parts_and_signatures" \
    "${tmp_dir}/to/" \
    "${master_mirror_directory}/i686/${to_repo}/"

  ${master_mirror_command} \
    "${tmp_dir}/to/${to_repo}.db."* \
    "${master_mirror_directory}/i686/${to_repo}/"

  ${master_mirror_command} \
    "${tmp_dir}/from/${from_repo}.db."* \
    "${master_mirror_directory}/i686/${from_repo}/"

  while read -r part; do
    remove_old_package_versions "i686/${to_repo}" "${part}"
    # the next line will remove _all_ versions of the package $part from $from_repo
    remove_old_package_versions "i686/${from_repo}" "${part%.pkg.tar.xz}0.pkg.tar.xz"
  done < \
    "${tmp_dir}/parts"

  if [ -z "${to_ending}" ]; then
    rm \
      "${work_dir}/package-states/${package}.${from_ending}"
  else
    # remove old state files of $package with ending $to_ending
    ls "${work_dir}/package-states" | \
      grep "^$(str_to_regex "${package}")\(\.[^.]\+\)\{3\}\.${to_ending}\$" | \
      sed "s|^|${work_dir}/package-states/|" | \
      xargs -rn1 rm
    mv \
      "${work_dir}/package-states/${package}.${from_ending}" \
      "${work_dir}/package-states/${package}.${to_ending}"
  fi

  updated_package_database=true

  rm -rf --one-file-system "${tmp_dir}/"*

}

eval set -- "$(
  getopt -o bf:h \
    --long block \
    --long from: \
    --long help \
    -n "$(basename "$0")" -- "$@" || \
  echo usage
)"

block_flag='-n'

while true
do
  case "$1" in
    -b|--block)
      block_flag=''
    ;;
    -f|--from)
      shift
      if [ "x$1" = "x-" ]; then
        packages_to_stabilize=$(cat)
      else
        packages_to_stabilize=$(cat "$1")
      fi
    ;;
    -h|--help)
      usage 0
    ;;
    --)
      shift
      break
    ;;
    *)
      >&2 echo 'Whoops, forgot to implement option "'"$1"'" internally.'
      exit 42
    ;;
  esac
  shift
done

packages_to_stabilize=${packages_to_stabilize} ${@}

for package in ${packages_to_stabilize}; do
  # some sanity checks
  if [ ! -f "${work_dir}/package-states/${package}.testing" ]; then
    >&2 echo "Package '${package}' is not in testing!"
    exit 2
  fi
done

updated_package_database=false

# Create a lock file and a trap.

exec 9> "${build_list_lock_file}"
if ! flock ${block_flag} 9; then
  >&2 echo 'come back (shortly) later - I cannot lock build list.'
  exit 1
fi

exec 8> "${package_database_lock_file}"
if ! flock ${block_flag} 8; then
  >&2 echo 'come back (shortly) later - I cannot lock package database.'
  exit 1
fi

tmp_dir="$(mktemp -d)"

clean_up_lock_file() {
  rm -f "${package_database_lock_file}" "${build_list_lock_file}"
  rm -rf --one-file-system "${tmp_dir}"
}

trap clean_up_lock_file EXIT

# sanity check

for ending in 'done' 'testing'; do
  if [ "${ending}" = 'testing' ] && \
    [ -z "${packages_to_stabilize}" ]; then
    # if nothing is to be untested, we don't care about duplicate
    # testing packages (and maybe a unstaging fixes this anyway)
    continue
  fi
  if [ -n "$(
    ls "${work_dir}/package-states" | \
      grep "\.${ending}\$" | \
      sed 's|\(\.[^.]\+\)\{4\}$||' | \
      sort | \
      uniq -d
    )" ]; then
    >&2 echo 'Removing duplicates not yet implemented:'
    ls "${work_dir}/package-states" | \
      grep "\.${ending}\$" | \
      sed 's|\(\.[^.]\+\)\{4\}$||' | \
      sort | \
      uniq -d
    exit 42
  fi
done

# packages which can't be un-staged because they're still dependencies
# of any job on the build-list

grep -vxF 'break_loops' "${work_dir}/build-list" | \
  while read -r pkg pkg_rev mod_rev repo; do
    generate_package_metadata "${pkg}" "${pkg_rev}" "${mod_rev}" "${repo}"
    cat "${work_dir}/package-infos/${pkg}.${pkg_rev}.${mod_rev}.depends"
  done | \
  sort -u > \
  "${tmp_dir}/keep_packages"

# packages which are done

ls "${work_dir}/package-states" | \
  grep '\.done$' | \
  sed 's|^\(.*\)\(\(\.[^.]\+\)\{3\}\)\.done$|\1 \1\2|' | \
  sort > \
  "${tmp_dir}/done_packages"

# remove packages not yet done from keep-packages list

keep_packages="$(
  join -1 1 -2 1 -o 2.2 \
    "${tmp_dir}/keep_packages" \
    "${tmp_dir}/done_packages"
)"
printf '%s\n' "${keep_packages}" | \
  grep -vxF '' > \
  "${tmp_dir}/keep_packages" || \
  true

# find all dependencies of the unstageable packages
mv \
  "${tmp_dir}/keep_packages" \
  "${tmp_dir}/new_keep_packages"
keep_packages=''

while [ -s "${tmp_dir}/new_keep_packages" ]; do

  while read -r package; do
    generate_package_metadata "${package}"
  done < "${tmp_dir}/new_keep_packages"

  keep_packages="$(
    (
      printf '%s\n' "${keep_packages}"
      cat "${tmp_dir}/new_keep_packages"
    ) | \
      sort -u
  )"

  new_keep_packages="$(
    while read -r package; do
      cat "${work_dir}/package-infos/${package%.*}.depends"
    done < \
      "${tmp_dir}/new_keep_packages" | \
      sort -u
  )"
  printf '%s\n' "${new_keep_packages}" > \
    "${tmp_dir}/new_keep_packages"

  new_keep_packages="$(
    join -1 1 -2 1 -o 2.2 \
      "${tmp_dir}/new_keep_packages" \
      "${tmp_dir}/done_packages"
  )"

  # "new" is only what has not been there before
  printf '%s\n' "${keep_packages}" "${keep_packages}" "${new_keep_packages}" | \
    sort | \
    uniq -u > \
    "${tmp_dir}/new_keep_packages"

done

done_packages="$(cat "${tmp_dir}/done_packages")"

# if build list is empty, remember all entries of 'deletion-list'
if grep -qvxF 'break_loops' "${work_dir}/build-list"; then
  delete_packages=''
else
  delete_packages="$(
    cat "${work_dir}/deletion-list"
  )"
fi

if [ -z "${delete_packages}" ]; then

  # unlock build list

  rm -f "${build_list_lock_file}"
  flock -u 9

  clean_up_lock_file() {
    rm -rf --one-file-system "${tmp_dir}"
    rm -f "${package_database_lock_file}"
  }

fi

# calculate unstageable packages from keep_packages and done_packages

done_packages="$(
  echo "${done_packages}" | \
    cut -d' ' -f2
)"

done_packages="$(
  (
    printf '%s\n' "${done_packages}" "${keep_packages}"
  ) | \
    sort | \
    uniq -u
)"

unset keep_packages

for package in ${done_packages}; do

  if [ -z "${package}" ]; then
    continue
  fi

  is_community="$(official_or_community "${package}")"

  move_package "${package}" "${is_community}staging" "${is_community}testing"

done

# move packages in packages_to_stabilize from *testing/ to the stable repos
for package in ${packages_to_stabilize}; do

  if [ -z "${package}" ]; then
    continue
  fi

  move_package "${package}" "$(official_or_community ${package})testing" "${package##*.}"

done

if [ -n "${delete_packages}" ]; then
  echo "there is something to delete:"
  echo "${delete_packages}"
fi

if ${updated_package_database}; then
  date '+%s' > \
    "${master_mirror_directory}/lastupdate"
fi