path: root/bin/seed-build-list

#!/bin/sh

# seed the build list from differences between an x86_64 and our master mirror

# shellcheck source=conf/default.conf
. "${0%/*}/../conf/default.conf"

# TODO: read information from database

# shellcheck disable=SC2016
usage() {
  >&2 echo ''
  >&2 echo 'seed-build-list [options]:'
  >&2 echo '  seed the build list from'
  >&2 echo '    - differences between an x86_64 and our master mirror'
  >&2 echo '    - a list of packages which need to be rebuilt'
  >&2 echo ''
  >&2 echo 'possible options:'
  >&2 echo '  -h|--help:'
  >&2 echo '    Show this help and exit.'
  >&2 echo '  -i|--ignore $package:'
  >&2 echo '    Do not update $package.'
  >&2 echo '  -m|--mirror $url:'
  >&2 echo '    Get x86_64 packages from mirror at $url.'
  >&2 echo '  -n|--no-action:'
  >&2 echo '    Do not actually update build-list, just print it.'
  >&2 echo '  -p|--package $pkg_regex:'
  >&2 echo '    Update packages matching $pkg_regex.'
  [ -z "$1" ] && exit 1 || exit "$1"
}

tmp_dir=$(mktemp -d 'tmp.seed-build-list.XXXXXXXXXX' --tmpdir)
# shellcheck disable=SC2064
trap "rm -rf --one-file-system '${tmp_dir:?}'" EXIT

eval set -- "$(
  getopt -o hi:m:np: \
    --long help \
    --long ignore: \
    --long mirror: \
    --long no-action \
    --long package: \
    -n "$(basename "$0")" -- "$@" || \
    echo usage
  )"

touch "${tmp_dir}/mirrors"
touch "${tmp_dir}/delta-packages"
touch "${tmp_dir}/package-regexes"
touch "${tmp_dir}/ignore-packages"

update=true

while true
do
  case "$1" in
    -h|--help)
      usage 0
    ;;
    -i|--ignore)
      shift
      echo "$1" >> \
        "${tmp_dir}/ignore-packages"
    ;;
    -m|--mirror)
      shift
      echo "$1" >> \
        "${tmp_dir}/mirrors"
    ;;
    -n|--no-action)
      update=false
    ;;
    -p|--package)
      shift
      echo "$1" >> \
        "${tmp_dir}/package-regexes"
    ;;
    --)
      shift
      break
    ;;
    *)
      >&2 echo 'Whoops, forgot to implement option "'"$1"'" internally.'
      exit 42
    ;;
  esac
  shift
done

if [ $# -ne 0 ]; then
  usage 1
fi

if [ ! -s "${tmp_dir}/mirrors" ] && \
  [ ! -s "${tmp_dir}/package-regexes" ]; then
  # nothing to do
  exit 0
fi

repos="${stable_package_repositories}"

find "${work_dir}/package-infos" -maxdepth 1 -name '*.packages' -printf '%f\n' | \
  sed 's|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+$| \1 \2 \3|' | \
  sort -k1,1 > \
  "${tmp_dir}/known-packages"

mod_repo_rev=$(cat "${work_dir}/archlinux32.revision")
{
  for repo in ${repo_names}; do
    if [ "${repo_names}" = 'archlinux32' ]; then
      continue
    fi
    eval 'repo_path="${repo_paths__'"${repo}"'}"'
    repo_rev=$(cat "${work_dir}/${repo}.revision")
    git -C "${repo_path}" archive "$(cat "${work_dir}/${repo}.revision")" | \
      tar -t | \
      grep '^[^/]\+/repos/[^/]\+/PKGBUILD$' | \
      grep -v -- '-i686/PKGBUILD$' | \
      grep -v -- '[-/]\(staging\|testing\|unstable\)-[^/]\+/PKGBUILD$' | \
      sed '
        s|^\([^/]\+\)/repos/\([^/]\+\)-[^/-]\+/PKGBUILD$|'"${repo_rev}"' \1 \2|
      '
  done | \
    sort -u | \
    sort -k1,1
  awk '{print "nothing " $1 " " $4}' "${tmp_dir}/known-packages" | \
    sort -u | \
    sed 'p'
} | \
  sort -k2,3 | \
  uniq -uf1 | \
  while read -r repo_rev pkg prepo; do
    generate_package_metadata "${pkg}" "${repo_rev}" "${mod_repo_rev}" "${prepo}"
  done

# shellcheck disable=SC2016
{
  printf 'SELECT '
  printf '`package_sources`.`%s`,' \
    'pkgbase' 'git_revision' 'mod_git_revision'
  printf '`upstream_repositories`.`name`,`binary_packages`.`pkgname`'
  printf ' FROM `binary_packages`'
  printf ' JOIN `%s` ON `%s`.`id`=`%s`.`%s`' \
    'build_assignments'     'build_assignments'     'binary_packages'   'build_assignment' \
    'package_sources'       'package_sources'       'build_assignments' 'package_source' \
    'upstream_repositories' 'upstream_repositories' 'package_sources'   'upstream_package_repository'
} | \
  ${mysql_command} --raw --batch | \
  sed '
    1d
    y/\t/ /
  ' > /dev/null # not yet working (the query works, but the entries are missing)
find "${work_dir}/package-infos" -maxdepth 1 -name '*.packages' -exec grep -HF '' "{}" \; | \
  sed '
    s|^.*/||
    s|\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.[^.]\+:| \1 \2 \3 |
  ' | \
  sort -k5,5 > \
  "${tmp_dir}/known-packages"

# generate delta-packages from package-regexes
while read -r pkg_regex; do
  if [ -z "${pkg_regex}" ]; then
    continue
  fi
  awk '{print $5}' "${tmp_dir}/known-packages" | \
    grep "${pkg_regex}" || \
    true
done < \
  "${tmp_dir}/package-regexes" >> \
  "${tmp_dir}/delta-packages"

# genereate delta_packages from mirror delta

while read -r mirror; do
  if [ -z "${mirror}" ]; then
    continue
  fi
  {
    # theirs
    for repo in ${repos}; do
      curl -sS "${mirror}/${repo}/os/x86_64/${repo}.db.tar.gz" | \
        tar -tz
    done | \
      grep '/$' | \
      sed '
        s|/$||
        s|^\(.*\)-\([^-]\+-[^-]\+\)|theirs \1-\2 \2 \1|
      '

    # ours
    for repo in $(ls_master_mirror 'i686'); do
      ls_master_mirror "i686/${repo}" | \
        grep '\.pkg\.tar\.xz$' | \
        sed 's|-[^-]\+$||'
    done | \
      sed '
        s|^\(.*\)-\([^-]\+-[^-]\+\)|ours \1-\2 \2 \1|
      '
  } | \
    expand_version 3 | \
    sort -k4,4 -k3Vr,3 -k1,1 | \
    shrink_version 3 | \
    uniq -f3 | \
    grep '^theirs ' | \
    awk '{print $4}' | \
    sort -k1,1 >> \
    "${tmp_dir}/delta-packages"
done < \
  "${tmp_dir}/mirrors"

sort -k1,1 -u "${tmp_dir}/delta-packages" | \
  sponge "${tmp_dir}/delta-packages"

cat \
  "${tmp_dir}/delta-packages" \
  "${tmp_dir}/ignore-packages" \
  "${tmp_dir}/ignore-packages" | \
  sort | \
  uniq -u | \
  sponge "${tmp_dir}/delta-packages"

join -1 1 -2 5 -o 2.1,2.2,2.3,2.4,2.5 "${tmp_dir}/delta-packages" "${tmp_dir}/known-packages" >> \
  "${tmp_dir}/append-packages"

{
  awk '{print $5}' "${tmp_dir}/append-packages"
  cat "${tmp_dir}/delta-packages"
} | \
  sort | \
  uniq -u | \
  sponge "${tmp_dir}/delta-packages"

if [ -s "${tmp_dir}/delta-packages" ]; then
  echo 'There are still packages newer for x86_64 which I cannot identify :-/'
  cat "${tmp_dir}/delta-packages"
  exit 2
fi

rm "${tmp_dir}/delta-packages"

# convert append-packages to build-list.new (add git hashes)

cut -d' ' -f1,4 < \
  "${tmp_dir}/append-packages" | \
  sort -u | \
  while read -r pkg repo; do
    git_repo=$(find_git_repository_to_package_repository "${repo}")
    printf '%s %s %s %s\n' \
      "${pkg}" \
      "$(cat "${work_dir}/${git_repo}.revision")" \
      "$(cat "${work_dir}/archlinux32.revision")" \
      "${repo}"
  done | \
  sort -u > \
  "${tmp_dir}/build-list.append"

# Create a lock file for build list.

if ${update}; then
  # always block if locked
  exec 9> "${build_list_lock_file}"
  flock 9

  exec 8> "${sanity_check_lock_file}"
  flock -s 8
fi

{
  awk '{print $2 " " $3 " " $4 " " $1}' "${tmp_dir}/build-list.append"

  # ignore packages on the build-list
  awk '{print $2 " " $3 " " $4 " " $1}' < \
    "${work_dir}/build-list" | \
    sed 'p'

  # ignore packages on the deletion-list
  awk '{print "X X X " $1}' "${work_dir}/deletion-list" | \
    sed 'p'

  # ignore packages on the blacklist
  git -C "${repo_paths__archlinux32}" archive "$(cat "${work_dir}/archlinux32.revision")" -- blacklist | \
    tar -Ox blacklist | \
    sed '
      s/\s*#.*$//
      /^\s*$/d
    ' | \
    awk '{print "X X X " $1}' | \
    sed 'p'

  # ignore explicitely ignored packages
  awk '{print "X X X " $1}' "${tmp_dir}/ignore-packages" | \
    sed 'p'
} | \
  sort -k4,4 | \
  uniq -uf3 > \
  "${tmp_dir}/build-list.new"

if ${update}; then
  awk '{print $4 " " $1 " " $2 " " $3}' \
    "${tmp_dir}/build-list.new" | \
    tee -a "${work_dir}/build-list"

  while read -r git_revision mod_git_revision repository pkgbase; do
    mysql_generate_package_metadata "${pkgbase}" "${git_revision}" "${mod_git_revision}" "${repository}"
  done < \
    "${tmp_dir}/build-list.new"

  # Remove the lock file

  rm -f "${build_list_lock_file}"
else
  awk '{print $4 " " $1 " " $2 " " $3}' \
    "${tmp_dir}/build-list.new"
fi