bin/get-package-updates: change layout of blacklist

1 files changed, 90 insertions, 173 deletions

diff --git a/bin/get-package-updates b/bin/get-package-updates
index d159ace..3b7f440 100755
--- a/bin/get-package-updates
+++ b/bin/get-package-updates
@@ -40,20 +40,16 @@ usage() {
   >&2 echo '                git repository (e.g. ignore -d for it).'
   >&2 echo '                Requires -d.'
   >&2 echo '  -w|--wait:    If necessary, wait for lock blocking.'
-  >&2 echo '  -x|--test-exclusion $package:'
-  >&2 echo '                Print additionally deleted/excluded packages if'
-  >&2 echo '                "$package" would be black listed.'
   [ -z "$1" ] && exit 1 || exit "$1"
 }
 
 eval set -- "$(
-  getopt -o d:hinrwx: \
+  getopt -o d:hinrw \
     --long date: \
     --long help \
     --long ignore-insanity \
     --long no-pull \
     --long recent-modifications \
-    --long test-exclusion: \
     --long wait \
     -n "$(basename "$0")" -- "$@" || \
   echo usage
@@ -64,7 +60,6 @@ date_time=''
 ignore_insanity=false
 pull=true
 recent_modifications=false
-test_exclusion=''
 
 while true
 do
@@ -88,15 +83,6 @@ do
     -w|--wait)
       block_flag=''
     ;;
-    -x|--test-exclusion)
-      shift
-      if [ -n "${test_exclusion}" ]; then
-        >&2 printf 'I already have --test-exclusion=%s and you gave me another one.\n' "${test_exclusion}"
-        >&2 printf 'But I can only handle one exclusion test at a time.\n'
-        exit 2
-      fi
-      test_exclusion="$1"
-    ;;
     --)
       shift
       break
@@ -134,81 +120,77 @@ fi
 # delete_package arch package
 # mark $arch/$package for deletion
 delete_package() {
-  # TODO: Run the command below unconditionally, but with some changes,
-  # so we can easily revert.
-  if [ -z "${test_exclusion}" ]; then
-    # shellcheck disable=SC2016
-    query_delete_packages=$(
-      printf '`architectures` AS `d_a`'
-      printf ' JOIN `architecture_compatibilities` AS `a_c`'
-      printf ' ON `a_c`.`runs_on`=`d_a`.`id`'
-      printf ' AND `d_a`.`name`=from_base64("%s")' \
-        "$(printf '%s' "$1" | base64 -w0)"
-      printf ' JOIN `build_assignments`'
-      printf ' ON (`a_c`.`built_for`=`build_assignments`.`architecture`'
-      # "any" references all architectures, but this is not represented
-      # in `architecture_compatibilities`: If a package is not buildable
-      # for "any", this means literally, that it is not buildable for
-      # the _least_ architecture (e.g. it is not generic)
-      printf ' OR `d_a`.`name`="any")'
-      mysql_join_build_assignments_package_sources
-      printf ' AND `package_sources`.`pkgbase`=from_base64("%s")' \
-        "$(printf '%s' "$2" | base64 -w0)"
-      mysql_join_build_assignments_binary_packages
-      mysql_join_binary_packages_binary_packages_in_repositories
-    )
-    # shellcheck disable=SC2016
-    {
-      # packages from the build-list/to-be-decided go straight to the deletion-list
-      # this happens in two steps, because we need to create one item per
-      # target architecture
-      printf 'INSERT IGNORE INTO `binary_packages_in_repositories` ('
-        printf '`package`,'
-        printf '`repository`,'
-        printf '`last_moved`,'
-        printf '`is_to_be_deleted`'
-      printf ') SELECT'
-      printf ' `binary_packages`.`id`,'
-      printf '`d_r`.`id`,'
-      printf 'NOW(),'
-      printf '1'
-      printf ' FROM'
-      printf ' %s' "${query_delete_packages}"
-      mysql_join_binary_packages_in_repositories_repositories
-      # shellcheck disable=SC2154
-      printf ' AND `repositories`.`stability` in (%s,%s)' \
-        "${repository_stability_ids__unbuilt}" \
-        "${repository_stability_ids__virtual}"
-      mysql_join_build_assignments_architectures '' 'ba_a'
-      printf ' JOIN `repositories` AS `d_r`'
-      printf ' ON ('
-        # arch-specific build_assignments must match exactly
-        printf '`d_r`.`architecture`=`ba_a`.`id`'
-        # "any" build_assignments build for all architectures
-        printf ' OR `ba_a`.`name`="any"'
-      printf ')'
-      # shellcheck disable=SC2154
-      printf ' AND `d_r`.`stability`=%s;\n' \
-        "${repository_stability_ids__forbidden}"
+  # shellcheck disable=SC2016
+  query_delete_packages=$(
+    printf '`architectures` AS `d_a`'
+    printf ' JOIN `architecture_compatibilities` AS `a_c`'
+    printf ' ON `a_c`.`runs_on`=`d_a`.`id`'
+    printf ' AND `d_a`.`name`=from_base64("%s")' \
+      "$(printf '%s' "$1" | base64 -w0)"
+    printf ' JOIN `build_assignments`'
+    printf ' ON (`a_c`.`built_for`=`build_assignments`.`architecture`'
+    # "any" references all architectures, but this is not represented
+    # in `architecture_compatibilities`: If a package is not buildable
+    # for "any", this means literally, that it is not buildable for
+    # the _least_ architecture (e.g. it is not generic)
+    printf ' OR `d_a`.`name`="any")'
+    mysql_join_build_assignments_package_sources
+    printf ' AND `package_sources`.`pkgbase`=from_base64("%s")' \
+      "$(printf '%s' "$2" | base64 -w0)"
+    mysql_join_build_assignments_binary_packages
+    mysql_join_binary_packages_binary_packages_in_repositories
+  )
+  # shellcheck disable=SC2016
+  {
+    # packages from the build-list/to-be-decided go straight to the deletion-list
+    # this happens in two steps, because we need to create one item per
+    # target architecture
+    printf 'INSERT IGNORE INTO `binary_packages_in_repositories` ('
+      printf '`package`,'
+      printf '`repository`,'
+      printf '`last_moved`,'
+      printf '`is_to_be_deleted`'
+    printf ') SELECT'
+    printf ' `binary_packages`.`id`,'
+    printf '`d_r`.`id`,'
+    printf 'NOW(),'
+    printf '1'
+    printf ' FROM'
+    printf ' %s' "${query_delete_packages}"
+    mysql_join_binary_packages_in_repositories_repositories
+    # shellcheck disable=SC2154
+    printf ' AND `repositories`.`stability` in (%s,%s)' \
+      "${repository_stability_ids__unbuilt}" \
+      "${repository_stability_ids__virtual}"
+    mysql_join_build_assignments_architectures '' 'ba_a'
+    printf ' JOIN `repositories` AS `d_r`'
+    printf ' ON ('
+      # arch-specific build_assignments must match exactly
+      printf '`d_r`.`architecture`=`ba_a`.`id`'
+      # "any" build_assignments build for all architectures
+      printf ' OR `ba_a`.`name`="any"'
+    printf ')'
+    # shellcheck disable=SC2154
+    printf ' AND `d_r`.`stability`=%s;\n' \
+      "${repository_stability_ids__forbidden}"
 
-      printf 'COMMIT;\n'
+    printf 'COMMIT;\n'
 
-      printf 'DELETE `binary_packages_in_repositories`'
-      printf ' FROM %s' "${query_delete_packages}"
-      mysql_join_binary_packages_in_repositories_repositories
-      # shellcheck disable=SC2154
-      printf ' AND `repositories`.`stability` in (%s,%s);\n' \
-        "${repository_stability_ids__unbuilt}" \
-        "${repository_stability_ids__virtual}"
+    printf 'DELETE `binary_packages_in_repositories`'
+    printf ' FROM %s' "${query_delete_packages}"
+    mysql_join_binary_packages_in_repositories_repositories
+    # shellcheck disable=SC2154
+    printf ' AND `repositories`.`stability` in (%s,%s);\n' \
+      "${repository_stability_ids__unbuilt}" \
+      "${repository_stability_ids__virtual}"
 
-      printf 'COMMIT;\n'
+    printf 'COMMIT;\n'
 
-      # other packages are marked as `is_to_be_deleted`
-      printf 'UPDATE %s' "${query_delete_packages}"
-      printf ' SET `binary_packages_in_repositories`.`is_to_be_deleted`=1;\n'
-    } | \
-      mysql_run_query
-  fi
+    # other packages are marked as `is_to_be_deleted`
+    printf 'UPDATE %s' "${query_delete_packages}"
+    printf ' SET `binary_packages_in_repositories`.`is_to_be_deleted`=1;\n'
+  } | \
+    mysql_run_query
 }
 
 something_new=false
@@ -330,12 +312,11 @@ echo 'Check modified packages from the last update, and put them to the build li
     mysql_join_upstream_repositories_git_repositories
     printf ' WHERE `package_sources`.`pkgbase` IN ('
     # shellcheck disable=SC2154
-    git -C "${repo_paths__archlinux32}" diff "${old_repo_revisions__archlinux32}" "${new_repo_revisions__archlinux32}" -- blacklist | \
-      sed -n '
-        s/^-\([^-]\)/\1/
-        T
-        s/\s*#.*$//
-        p
+    git -C "${repo_paths__archlinux32}" diff --no-renames --name-status "${old_repo_revisions__archlinux32}" "${new_repo_revisions__archlinux32}" -- 'blacklist' | \
+      sed '
+        s@^D\tblacklist/[^/]\+/[^/]\+/@@
+        t
+        d
       ' | \
       base64_encode_each | \
       sed '
@@ -444,6 +425,7 @@ echo 'Check modified packages from the last update, and put them to the build li
       fi
     fi
     if [ "${mode}" = 'A' ] || [ "${mode}" = 'M' ]; then
+
       # shellcheck disable=SC2016
       {
         # delete old build assignment and associated binary packages
@@ -476,6 +458,7 @@ echo 'Check modified packages from the last update, and put them to the build li
           )"
       } | \
         mysql_run_query
+      printf '%s\n' "${package}" >> "${tmp_dir}/modified-packages"
       # shellcheck disable=SC2154
       printf '%s ' "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}" >&2
       mysql_generate_package_metadata "${repository_ids__any_to_be_decided}" "${package}" "${git_revision}" "${new_repo_revisions__archlinux32}" "${repository}"
@@ -487,89 +470,23 @@ echo 'Check modified packages from the last update, and put them to the build li
     exit 1
   done
 
-echo 'apply blacklisting'
-echo ' - generate list of packages'
-# ignore blacklisted packages and dependent packages
-# this is the first time when all the information is available and up to date
-
-{
+if [ -s "${tmp_dir}/modified-packages" ]; then
+  echo 'Delete black-listed packages.'
+  sort -u "${tmp_dir}/modified-packages" --output "${tmp_dir}/modified-packages"
+  # extract black-listed packages
   git -C "${repo_paths__archlinux32}" archive "${new_repo_revisions__archlinux32}" -- 'blacklist' | \
-    tar -Ox 'blacklist' | \
+    tar -t 'blacklist' | \
     sed '
-      s/\s*#.*$//
-      /^\S\+\s\+\S\+$/!d
-    '
-  if [ -n "${test_exclusion}" ]; then
-    echo "${test_exclusion}"
-  fi
-  # shellcheck disable=SC2016
-  {
-    printf 'SELECT DISTINCT "any",`package_sources`.`pkgbase`'
-    printf ' FROM `package_sources`'
-    printf ' WHERE `package_sources`.`pkgbase` LIKE "lib32-%%"'
-  } | \
-    mysql_run_query
-} | \
-  tr ' ' '\t' | \
-  sort -u > \
-  "${tmp_dir}/black-listed"
-
-# shellcheck disable=SC2016
-{
-  printf 'CREATE TEMPORARY TABLE `blacklist` ('
-  printf '`arch` VARCHAR(16),'
-  printf '`pkgbase` VARCHAR(64),'
-  printf 'UNIQUE KEY `content`(`arch`,`pkgbase`)'
-  printf ');\n'
-  printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `blacklist` (`arch`,`pkgbase`);\n' \
-    "${tmp_dir}/black-listed"
-  printf 'CALL `blacklist_packages`;\n'
-} | \
-  mysql_run_query | \
-  sort -u | \
-  sponge "${tmp_dir}/black-listed"
-
-deletion_list_count=$(
-  grep -cv '^i486\s' < \
-    "${tmp_dir}/black-listed"
-)
-if [ "${deletion_list_count}" -gt 2000 ]; then
-  >&2 printf 'There are %s > 2000 packages on the deletion-list. This does not seem right.\n' \
-    "${deletion_list_count}"
-  if [ ! -s "${work_dir}/told-irc-about-too-many-deletion-list-packages" ]; then
-    # shellcheck disable=SC2119
-    {
-      printf '\001ACTION refuses to blacklist %s packages.\001\n' "${deletion_list_count}"
-      for repo_name in ${repo_names}; do
-        eval 'old_revision="${old_repo_revisions__'"${repo_name}"'}"'
-        eval 'new_revision="${new_repo_revisions__'"${repo_name}"'}"'
-        # shellcheck disable=SC2154
-        if [ "${old_revision}" != "${new_revision}" ]; then
-          printf '%s: %s -> %s\n' \
-            "${repo_name}" \
-            "${old_revision}" \
-            "${new_revision}"
-        fi
-      done
-    } | \
-      tee "${work_dir}/told-irc-about-too-many-deletion-list-packages" | \
-      irc_say
-  fi
-  exit 3
-fi
-rm -f "${work_dir}/told-irc-about-too-many-deletion-list-packages"
-
-echo ' - delete packages'
-
-while read -r arch package; do
-  delete_package "${arch}" "${package}"
-done < \
-  "${tmp_dir}/black-listed"
-
-if [ -n "${test_exclusion}" ]; then
-  # TODO: reimplement test_exclusion with information from the database
-  >&2 echo 'sry, not yet done.'
-  exit 0
+      s@^blacklist/\([^/]\+\)/[^/]\+/\([^/]\+\)$@\1\t\2@
+      t
+      d
+    ' | \
+    sort -u | \
+    sort -k2,2 | \
+    join -1 1 -2 2 -o 2.1,2.2 "${tmp_dir}/modified-packages" - | \
+    while read -r arch pkgbase; do
+      delete_package "${arch}" "${pkgbase}"
+    done
 fi
 
 echo 'Done - mark decisions as final.'