path: root/bin/seed-build-list

#!/bin/sh

# shellcheck disable=SC2119,SC2120

# shellcheck source=../lib/load-configuration
. "${0%/*}/../lib/load-configuration"

# TODO: upon un-deleting a package, actually _remove_ it from the
# deletion-list

# shellcheck disable=SC2016
usage() {
  >&2 echo ''
  >&2 echo 'seed-build-list [options]:'
  >&2 echo '  seed the build list from different sources'
  >&2 echo ''
  >&2 echo 'possible options:'
  >&2 echo '  -a|--auto:'
  >&2 echo '    Automatically reschedule packages which have run-time'
  >&2 echo '    dependencies that are not available from any real'
  >&2 echo '    repository and which cannot be replaced by another,'
  >&2 echo '    less stable version of the same package.'
  >&2 echo '  -h|--help:'
  >&2 echo '    Show this help and exit.'
  >&2 echo '  -i|--ignore $pkgbase:'
  >&2 echo '    Do not update the given package.'
  >&2 echo '  -m|--mirror $url:'
  >&2 echo '    Schedule all packages, that are newer on the given'
  >&2 echo '    x86_64 mirror - except packages in'
  >&2 echo '    conf/seed-ignore-packages.'
  >&2 echo '  -n|--no-action:'
  >&2 echo '    Do not actually update build-list, just print it.'
  >&2 echo '  -p|--package $pkg_regex:'
  >&2 echo '    Reschedule packages with matching pkgname or pkgbase.'
  >&2 echo '    Note, that these packages must be known to the database.'
  >&2 echo '  -u|--undelete $url: '
  >&2 echo '    Schedule all former deletion-list packages which do not'
  >&2 echo '    belong on the deletion-list anymore (e.g. can be built)'
  >&2 echo '    and which are available on the given x86_64 mirror.'
  >&2 echo '  -w|--wait:'
  >&2 echo '    Wait for lock if necessary.'
  [ -z "$1" ] && exit 1 || exit "$1"
}

tmp_dir=$(mktemp -d 'tmp.seed-build-list.XXXXXXXXXX' --tmpdir)
# shellcheck disable=SC2064
trap "rm -rf --one-file-system '${tmp_dir:?}'" EXIT

eval set -- "$(
  getopt -o ahi:m:np:u:w \
    --long auto \
    --long help \
    --long ignore: \
    --long mirror: \
    --long no-action \
    --long package: \
    --long undelete: \
    --long wait \
    -n "$(basename "$0")" -- "$@" || \
    echo usage
  )"

auto=false
update=true
wait_for_lock='-n'

if [ -r "${base_dir}/conf/seed-ignore-packages" ]; then
  sed 's/^/2\t/' "${base_dir}/conf/seed-ignore-packages" >> \
    "${tmp_dir}/ignore-packages"
fi

while true
do
  case "$1" in
    -a|--auto)
      auto=true
    ;;
    -h|--help)
      usage 0
    ;;
    -i|--ignore)
      shift
      printf '3\t%s\n' "$1" >> \
        "${tmp_dir}/ignore-packages"
    ;;
    -m|--mirror)
      shift
      printf '%s\n' "$1" >> \
        "${tmp_dir}/mirrors"
    ;;
    -n|--no-action)
      update=false
    ;;
    -p|--package)
      shift
      printf '%s\n' "$1" >> \
        "${tmp_dir}/package-regexes"
    ;;
    -u|--undelete)
      shift
      printf '%s\n' "$1" >> \
        "${tmp_dir}/undelete-mirrors"
    ;;
    -w|--wait)
      wait_for_lock=''
    ;;
    --)
      shift
      break
    ;;
    *)
      >&2 printf 'Whoops, forgot to implement option "%s" internally.\n' \
        "$1"
      exit 42
    ;;
  esac
  shift
done

if [ $# -ne 0 ]; then
  usage 1
fi

if [ ! -s "${tmp_dir}/mirrors" ] && \
  [ ! -s "${tmp_dir}/package-regexes" ] && \
  [ ! -s "${tmp_dir}/undelete-mirrors" ] && \
  ! ${auto}; then
  # nothing to do
  >&2 echo 'No options given to do anything.'
  usage 0
  exit 0
fi

# get locks
if ${update}; then
  exec 9> "${sanity_check_lock_file}"
  if ! verbose_flock -s ${wait_for_lock} 9; then
    >&2 echo 'Cannot get sanity-check lock.'
    exit 1
  fi

  exec 8> "${build_list_lock_file}"
  if ! verbose_flock ${wait_for_lock} 8; then
    >&2 echo 'Cannot get build-list lock.'
    exit 1
  fi
fi

# shellcheck disable=SC2016
repos=$(
  {
    printf 'SELECT DISTINCT `repositories`.`name`'
    printf ' FROM `repositories`'
    # shellcheck disable=SC2154
    printf ' WHERE `repositories`.`stability`=%s;\n' \
      "${repository_stability_ids__stable}"
  } | \
    mysql_run_query
  printf '%s\n' 'multilib'
)

# TODO: translate lib32-* packages instead of ignoring them for -u and -m

# harvest pkgnames from mirror delta
if [ -s "${tmp_dir}/mirrors" ]; then
  {
    # theirs
    while read -r mirror; do
      if [ -z "${mirror}" ]; then
        continue
      fi
      for repo in ${repos}; do
        curl -sS "${mirror}/${repo}/os/x86_64/${repo}.db.tar.gz" | \
          tar -Oxz --wildcards '*/desc' | \
          sed '
            /^%FILENAME%$/!d
            N
            s/^.*\n//
            /^lib32-/d
            s/-x86_64\(\.pkg\.tar\.xz\)$/-i686\1/
            s/^\(.*\)-\([^-]\+-[^-]\+\)-\([^-]\+\)$/theirs \2 \3 \1/
          '
      done
    done < \
      "${tmp_dir}/mirrors"
    # ours
    # shellcheck disable=SC2016
    {
      printf 'SELECT '
      mysql_package_name_query
      printf ' FROM `binary_packages`'
      mysql_join_binary_packages_architectures
      mysql_join_binary_packages_binary_packages_in_repositories
      printf ' WHERE NOT `binary_packages_in_repositories`.`is_to_be_deleted`'
    } | \
      mysql_run_query | \
      sed '
        s/^\(.*\)-\([^-]\+-[^-]\+\)-\([^-]\+\)$/ours \2 \3 \1/
      '
  } | \
    expand_version 2 | \
    sort -k3,4 -k2Vr,2 -k1,1 | \
    shrink_version 2 | \
    uniq -f2 | \
    sed -n '
      s/^theirs \(\S\+ \)\{2\}//
      T
      p
    ' | \
    if [ -r "${base_dir}/conf/seed-ignore-packages" ]; then
      grep -vxF "$(cat "${base_dir}/conf/seed-ignore-packages")"
    else
      cat
    fi | \
    sed 's/^/1\t/' | \
    sort -u >> \
    "${tmp_dir}/pkgnames"
fi

# harvest pkgnames from undelete-mirrors
if [ -s "${tmp_dir}/undelete-mirrors" ]; then
  while read -r mirror; do
    if [ -z "${mirror}" ]; then
      continue
    fi
    for repo in ${repos}; do
      curl -sS "${mirror}/${repo}/os/x86_64/${repo}.db.tar.gz" | \
        tar -Oxz --wildcards '*/desc' | \
        sed '
          /^%FILENAME%$/!d
          N
          s/^.*\n//
          s/-x86_64\(\.pkg\.tar\.xz\)$/-i686\1/
          /^lib32-/d
          s/^\(.*\)\(-[^-]\+\)\{3\}$/\1/
        '
    done
  done < \
    "${tmp_dir}/undelete-mirrors" | \
    if [ -r "${base_dir}/conf/seed-ignore-packages" ]; then
      grep -vxF "$(cat "${base_dir}/conf/seed-ignore-packages")"
    else
      cat
    fi | \
    sed 's/^/2\t/' | \
    sort -u >> \
    "${tmp_dir}/pkgnames"
fi

# harvest from the package-regexes
if [ -s "${tmp_dir}/package-regexes" ]; then
  # shellcheck disable=SC2016
  {
    printf 'CREATE TEMPORARY TABLE `names` (`name` VARCHAR(64));\n'
    printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `names` (`name`);\n' \
      "${tmp_dir}/package-regexes"

    printf 'SELECT DISTINCT "pkgbase",`package_sources`.`pkgbase`,`upstream_repositories`.`name`'
    printf ' FROM `names`'
    printf ' JOIN `package_sources` ON `package_sources`.`pkgbase` REGEXP `names`.`name`'
    mysql_join_package_sources_upstream_repositories
    printf ';\n'

    printf 'SELECT DISTINCT "pkgname",`binary_packages`.`pkgname`'
    printf ' FROM `names`'
    printf ' JOIN `binary_packages` ON `binary_packages`.`pkgname` REGEXP `names`.`name`;\n'
  } | \
    mysql_run_query | \
    sed -n '
      /^pkgbase\s/ {
        s/^\S\+\s/3\t/
        w /dev/stdout
        d
      }
      /^pkgname\s/ {
        s/^\S\+\s/3\t/
        w /dev/stderr
        d
      }
    ' \
    >>"${tmp_dir}/pkgbases" \
    2>>"${tmp_dir}/pkgnames"
fi

# shellcheck disable=SC2016
{
  # create pkgbases to given pkgnames
  if [ -s "${tmp_dir}/pkgnames" ]; then
    printf 'CREATE TEMPORARY TABLE `pkgnames` ('
      printf '`priority` SMALLINT,'
      printf '`pkgname` VARCHAR(64),'
      printf 'UNIQUE KEY `pkgname`(`pkgname`)'
    printf ');\n'
    printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `pkgnames` (`priority`,`pkgname`);\n' \
      "${tmp_dir}/pkgnames"

    printf 'SELECT DISTINCT "pkgbase",`pkgnames`.`priority`,`package_sources`.`pkgbase`,`upstream_repositories`.`name`'
    printf ' FROM `pkgnames`'
    printf ' JOIN `binary_packages`'
    printf ' ON `binary_packages`.`pkgname`=`pkgnames`.`pkgname`'
    mysql_join_binary_packages_build_assignments
    mysql_join_build_assignments_package_sources
    mysql_join_package_sources_upstream_repositories
    printf ';\n'

    printf 'SELECT DISTINCT "pkgname",`pkgnames`.`priority`,`pkgnames`.`pkgname`'
    printf ' FROM `package_sources`'
    mysql_join_package_sources_upstream_repositories
    mysql_join_package_sources_build_assignments
    mysql_join_build_assignments_binary_packages
    printf ' RIGHT JOIN `pkgnames`'
    printf ' ON `pkgnames`.`pkgname`=`binary_packages`.`pkgname`'
    printf ' WHERE `package_sources`.`pkgbase` IS NULL;\n'

    printf 'DROP TEMPORARY TABLE `pkgnames`;\n'
  fi
  # auto-detect pkgbases
  if ${auto}; then
    printf 'SELECT DISTINCT "pkgbase",1,`package_sources`.`pkgbase`,`upstream_repositories`.`name` FROM ('
      printf 'SELECT DISTINCT `binary_packages`.`id`,'
      printf '`binary_packages`.`build_assignment`'
      printf ' FROM ('
        printf 'SELECT `binary_packages`.`pkgname`,'
        printf 'MAX(`repositories`.`stability`) AS `stability`'
        printf ' FROM `binary_packages`'
        mysql_join_binary_packages_binary_packages_in_repositories
        mysql_join_binary_packages_in_repositories_repositories
        printf ' AND ('
          printf '`repositories`.`is_on_master_mirror`'
          # we need to take into account build-list items here, because
          # they may make re-scheduling unnecessary
          printf ' OR `repositories`.`name`="build-list"'
        printf ') AND `repositories`.`name`!="build-support"'
        printf ' GROUP BY `binary_packages`.`pkgname`'
      printf ') AS `least_stable`'
      printf ' JOIN `binary_packages`'
      printf ' ON `binary_packages`.`pkgname`=`least_stable`.`pkgname`'
      mysql_join_binary_packages_binary_packages_in_repositories
      mysql_join_binary_packages_in_repositories_repositories
      printf ' AND `repositories`.`stability`=`least_stable`.`stability`'
      # now we can drop build-list items
      printf ' AND `repositories`.`is_on_master_mirror`'
    printf ') AS `binary_packages`'
    mysql_join_binary_packages_build_assignments
    mysql_join_build_assignments_package_sources
    mysql_join_package_sources_upstream_repositories
    mysql_join_binary_packages_dependencies
    mysql_join_dependencies_dependency_types
    printf ' AND `dependency_types`.`relevant_for_binary_packages`'
    mysql_join_dependencies_install_target_providers '' 'itp_dummy'
    # some dependencies are not provided by the least stable packages
    printf ' WHERE NOT EXISTS ('
      printf 'SELECT 1 FROM `install_target_providers`'
      mysql_join_install_target_providers_binary_packages '' 'subst_bp'
      mysql_join_binary_packages_binary_packages_in_repositories 'subst_bp' 'subst_bir'
      mysql_join_binary_packages_in_repositories_repositories 'subst_bir' 'subst_r'
      printf ' AND `subst_r`.`name` NOT IN ("build-support","deletion-list","to-be-decided")'
      printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
      # this is the least stable, built install_target_provider with that name
      printf ' AND NOT EXISTS ('
        printf 'SELECT 1 FROM `binary_packages` AS `ss_bp`'
        mysql_join_binary_packages_binary_packages_in_repositories 'ss_bp' 'ss_bir'
        mysql_join_binary_packages_in_repositories_repositories 'ss_bir' 'ss_r'
        printf ' AND `ss_r`.`name` NOT IN ("build-list","build-support","deletion-list","to-be-decided")'
        printf ' JOIN `repository_stability_relations` AS `ss_rsr`'
        printf ' ON `ss_rsr`.`less_stable`=`ss_r`.`stability`'
        printf ' WHERE `ss_bp`.`pkgname`=`subst_bp`.`pkgname`'
        printf ' AND `ss_bp`.`id`!=`subst_bp`.`id`'
        printf ' AND `ss_rsr`.`more_stable`=`subst_r`.`stability`'
      printf ')'
    printf ')'
    printf ';\n'
  fi
} | \
  mysql_run_query | \
  sed -n '
    /^pkgbase\s/ {
      s/^\S\+\s//
      w /dev/stderr
      d
    }
    /^pkgname\s/ {
      s/^\S\+\s//
      w /dev/stdout
      d
    }
  ' \
  2>>"${tmp_dir}/pkgbases" \
  | sponge "${tmp_dir}/pkgnames"

if [ ! -s "${tmp_dir}/pkgbases" ] && \
  [ ! -s "${tmp_dir}/pkgnames" ]; then
  >&2 echo 'No packages matched.'
  exit
fi

for s in "${tmp_dir}/pkgbases" "${tmp_dir}/pkgnames"; do
  if [ -s "${s}" ]; then
    sort -u "${s}" | \
      sponge "${s}"
  fi
done

# pkgnames -> pkgbases (with help from upstream)
if [ -s "${tmp_dir}/pkgnames" ]; then
  while read -r priority pkgname; do
    content=$(
      curl -Ss 'https://www.archlinux.org/packages/search/json/?name='"${pkgname}" | \
        tr ',' '\n'
    )
    repo=$(
      printf '%s\n' "${content}" | \
        sed -n '
          s/^\s*"repo"\s*:\s*"//
          T
          s/".*$//
          T
          p
        '
    )
    pkgbase=$(
      printf '%s\n' "${content}" | \
        sed -n '
          s/^\s*"pkgbase"\s*:\s*"//
          T
          s/".*$//
          T
          p
        '
    )
    if [ -z "${pkgbase}" ] || [ -z "${repo}" ]; then
      printf '%s\t%s\n' "${priority}" "${pkgname}"
      continue
    fi
    printf '%s\t%s\t%s\n' \
      "${priority}" \
      "${pkgbase}" \
      "${repo}" >> \
      "${tmp_dir}/pkgbases"
  done < \
    "${tmp_dir}/pkgnames" | \
    sponge "${tmp_dir}/pkgnames"
fi

if [ -s "${tmp_dir}/pkgnames" ]; then
  >&2 echo 'Could not find some packages upstream:'
  >&2 cat "${tmp_dir}/pkgnames"
  exit 2
fi

# now we (re)schedule all pkgbases which are:
#  - not explicitely ignored (priority 3)             <- explicitely scheduled
#  - not explicitely/implicitely ignored, but not currently available
#    (priority 2) <- from undelete-mirror-delta
#  - neither ignored nor to-be-deleted (priority 1)   <- from mirror-delta or auto-rebuild

# shellcheck disable=SC2016
{
  printf 'CREATE TEMPORARY TABLE `ignore_packages` ('
    printf '`priority` SMALLINT,'
    printf '`pkgbase` VARCHAR(64),'
    printf 'UNIQUE KEY `content`(`priority`,`pkgbase`)'
  printf ');\n'
  printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `ignore_packages`(`priority`,`pkgbase`);\n' \
    "${tmp_dir}/ignore-packages"

  printf 'CREATE TEMPORARY TABLE `pkgbases` ('
    printf '`priority` SMALLINT,'
    printf '`pkgbase` VARCHAR(64),'
    printf '`repo` VARCHAR(64),'
    printf 'UNIQUE KEY `pkgbase`(`pkgbase`)'
  printf ');\n'
  printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `pkgbases`(`priority`,`pkgbase`,`repo`);\n' \
    "${tmp_dir}/pkgbases"

  printf 'INSERT IGNORE INTO `ignore_packages`(`priority`,`pkgbase`)'
  printf ' SELECT 2,`package_sources`.`pkgbase`'
  printf ' FROM `package_sources`'
  mysql_join_package_sources_build_assignments
  mysql_join_build_assignments_binary_packages
  printf ' WHERE `build_assignments`.`is_black_listed` IS NOT NULL'
  printf ' OR `package_sources`.`pkgbase` LIKE "lib32-%%"'
  printf ' OR EXISTS ('
    printf 'SELECT 1 FROM `dependencies`'
    mysql_join_dependencies_dependency_types
    printf ' AND ('
      printf '`dependency_types`.`relevant_for_building`'
      printf ' OR `dependency_types`.`relevant_for_binary_packages`'
    printf ')'
    mysql_join_dependencies_install_target_providers '' 'it_dummy'
    printf ' WHERE `dependencies`.`dependent`=`binary_packages`.`id`'
    printf ' AND NOT EXISTS ('
      printf 'SELECT 1 FROM `install_target_providers`'
      mysql_join_install_target_providers_binary_packages '' 'it_bp'
      mysql_join_binary_packages_binary_packages_in_repositories 'it_bp' 'it_bpir'
      printf ' WHERE NOT `it_bpir`.`is_to_be_deleted`'
      printf ' AND `install_target_providers`.`install_target`=`dependencies`.`depending_on`'
    printf ')'
  printf ');\n'

  printf 'INSERT IGNORE INTO `ignore_packages`(`priority`,`pkgbase`)'
  printf ' SELECT 1,`package_sources`.`pkgbase`'
  printf ' FROM `package_sources`'
  mysql_join_package_sources_build_assignments
  mysql_join_build_assignments_binary_packages
  mysql_join_binary_packages_binary_packages_in_repositories
  printf ' WHERE `binary_packages_in_repositories`.`is_to_be_deleted`'
  printf ';\n'

  printf 'SELECT '
  printf '`pkgbases`.`pkgbase`,'
  printf '`git_repositories`.`head`,'
  printf '('
    printf 'SELECT `al32`.`head` FROM `git_repositories` AS `al32`'
    printf ' WHERE `al32`.`name`="archlinux32"'
  printf '),'
  printf '`pkgbases`.`repo`'
  printf ' FROM `pkgbases`'
  printf ' JOIN `upstream_repositories`'
  printf ' ON `upstream_repositories`.`name`=`pkgbases`.`repo`'
  mysql_join_upstream_repositories_git_repositories
  printf ' WHERE NOT EXISTS ('
    printf 'SELECT 1 FROM `ignore_packages`'
    printf ' WHERE `ignore_packages`.`pkgbase`=`pkgbases`.`pkgbase`'
    printf ' AND `ignore_packages`.`priority`>=`pkgbases`.`priority`'
  printf ') AND ('
    printf '`pkgbases`.`priority`!=2'
    printf ' OR NOT EXISTS ('
      printf 'SELECT 1 FROM `package_sources`'
      mysql_join_package_sources_build_assignments
      mysql_join_build_assignments_binary_packages
      mysql_join_binary_packages_binary_packages_in_repositories
      printf ' AND NOT `binary_packages_in_repositories`.`is_to_be_deleted`'
      mysql_join_binary_packages_in_repositories_repositories
      printf ' AND ('
        printf '`repositories`.`is_on_master_mirror`'
        printf ' OR `repositories`.`name`="build-list"'
      printf ') WHERE `package_sources`.`pkgbase`=`pkgbases`.`pkgbase`'
    printf ')'
  printf ');\n'
} | \
  mysql_run_query | \
  sort -u | \
  tr '\t' ' ' | \
  if ${update}; then
    while read -r pkgbase git_rev mod_git_rev repo; do
      printf '%s ' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}" | \
        sed 's/ $/\n/'
      mysql_generate_package_metadata "${repository_ids__any_build_list}" "${pkgbase}" "${git_rev}" '' "${mod_git_rev}" "${repo}"
    done

    rm -f "${build_list_lock_file}"
  else
    cat
  fi