bin/seed-build-list -> bin/schedule-for-rebuild

* adapt name to what it can do
* building completely new packages is not needed anymore
* building completely new packages was complex anyways

1 files changed, 77 insertions, 184 deletions

diff --git a/bin/seed-build-list b/bin/schedule-for-rebuild
index 343a07b..086f578 100755
--- a/bin/seed-build-list
+++ b/bin/schedule-for-rebuild
@@ -2,12 +2,11 @@
 
 # shellcheck disable=SC2119,SC2120
 
-# seed the build list from various sources
+# schedule selected packages for a rebuild
 #
 # This uses the following schedule:
-# - find scheduleworthy packages by pkgname/pkgbase (diff to mirror,
-#   broken dependencies, explicite regex) and their architecture
-# - find pkgbases of those packages (ask upstream, if necessary)
+# - find scheduleworthy packages by pkgname/pkgbase (broken
+#   dependencies, explicite regex) and their architecture
 # - remove all ignored packages (seed-ignore, explicitely ignored,
 #   blacklist)
 # - run mysql_generate_package_metadata on each of them once (ignoring
@@ -27,12 +26,12 @@
 # shellcheck disable=SC2016
 usage() {
   >&2 echo ''
-  >&2 echo 'seed-build-list [options]:'
-  >&2 echo '  seed the build list from different sources'
+  >&2 echo 'schedule-for-rebuild [options]:'
+  >&2 echo '  schedule selected packages for a rebuild'
   >&2 echo ''
   >&2 echo 'possible options:'
   >&2 echo '  -a|--auto:'
-  >&2 echo '    Automatically reschedule packages which have run-time'
+  >&2 echo '    Automatically schedule packages which have run-time'
   >&2 echo '    dependencies that are not available from any real'
   >&2 echo '    repository and which cannot be replaced by another,'
   >&2 echo '    less stable version of the same package.'
@@ -41,14 +40,13 @@ usage() {
   >&2 echo '  -h|--help:'
   >&2 echo '    Show this help and exit.'
   >&2 echo '  -i|--ignore $pkgbase|$arch/$pkgbase:'
-  >&2 echo '    Do not update the given package (for the given $arch).'
+  >&2 echo '    Never schedule the given package (for the given $arch).'
   >&2 echo '  -j|--jostle'
   >&2 echo '    Give new build assignments the highest priority.'
   >&2 echo '  -n|--no-action:'
-  >&2 echo '    Do not actually update build-list, just print it.'
+  >&2 echo '    Do not actually schedule anything, just print it.'
   >&2 echo '  -p|--package $pkg_regex:'
-  >&2 echo '    Reschedule packages with matching pkgname or pkgbase.'
-  >&2 echo '    Note, that these packages must be known to the database.'
+  >&2 echo '    Schedule packages with matching pkgname or pkgbase.'
   >&2 echo '  -s|--skip-arch $arch:'
   >&2 echo '    Ignore packages, that are outdated on $arch only (-a).'
   >&2 echo '  -w|--wait:'
@@ -207,77 +205,61 @@ if [ -s "${tmp_dir}/package-regexes" ]; then
     printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `names` (`name`);\n' \
       "${tmp_dir}/package-regexes"
 
-    printf 'SELECT DISTINCT "pkgbase",`package_sources`.`pkgbase`,`upstream_repositories`.`name`'
+    printf 'SELECT DISTINCT '
+      printf '"any",'
+      printf '`package_sources`.`pkgbase`,'
+      printf '`package_sources`.`git_revision`,'
+      printf '`upstream_repositories`.`name`'
     printf ' FROM `names`'
     printf ' JOIN `package_sources`'
     printf ' ON `package_sources`.`pkgbase` REGEXP `names`.`name`'
     mysql_join_package_sources_upstream_repositories
     printf ';\n'
 
-    printf 'SELECT DISTINCT "pkgname",`binary_packages`.`pkgname`'
-    printf ' FROM `names`'
-    printf ' JOIN `binary_packages`'
-    printf ' ON `binary_packages`.`pkgname` REGEXP `names`.`name`;\n'
+    printf 'SELECT DISTINCT '
+      printf '"any",'
+      printf '`package_sources`.`pkgbase`,'
+      printf '`package_sources`.`git_revision`,'
+      printf '`upstream_repositories`.`name`'
+    printf ' FROM `binary_packages`'
+    mysql_join_binary_packges_build_assignments
+    mysql_join_build_assignments_package_sources
+    mysql_join_package_sources_upstream_repositories
+    printf ' JOIN `names`'
+    printf ' ON `binary_packages`.`pkgname` REGEXP `names`.`name`'
+    printf ' OR `package_sources`.`pkgbase` REGEXP `names`.`name`'
+    printf ' LEFT JOIN `versions`'
+    printf ' ON `versions`.`epoch`=`binary_packages`.`epoch`'
+    printf ' AND `versions`.`version`=`binary_packages`.`pkgver`'
+    printf ' WHERE NOT EXISTS ('
+      printf 'SELECT 1'
+      printf ' FROM `binary_packages` AS `newer_bp`'
+      printf ' JOIN `versions` AS `newer_v`'
+      printf ' ON `newer_v`.`epoch`=`newer_bp`.`epoch`'
+      printf ' AND `newer_v`.`version`=`newer_bp`.`pkgver`'
+      printf ' WHERE (`newer_v`.`order`>`versions`.`order`'
+        printf ' OR ('
+          printf '`newer_v`.`order`=`versions`.`order`'
+          printf ' AND `newer_bp`.`pkgrel`>`binary_packages`.`pkgrel`'
+          # sub_pkgrel does not need to be compared, as it is only
+          # relevant for binary_packages with the same build_assignment
+        printf ')'
+      printf ')'
+      printf ' AND `binary_packages`.`%s`=`newer_bp`.`%s`' \
+        'pkgname' 'pkgname' \
+        'architecture' 'architecture'
+    printf ');\n'
   } | \
     mysql_run_query | \
     tr '\t' ' ' | \
-    sed -n '
-      /^pkgbase / {
-        s/^\S\+ /any /
-        w /dev/stdout
-        d
-      }
-      /^pkgname / {
-        s/^\S\+ /any /
-        w /dev/stderr
-        d
-      }
-    ' \
-    >>"${tmp_dir}/pkgbases" \
-    2>>"${tmp_dir}/pkgnames"
+    sort -u \
+    >>"${tmp_dir}/pkgbases"
 fi
 
-# shellcheck disable=SC2016
-{
-  # create pkgbases to given pkgnames
-  if [ -s "${tmp_dir}/pkgnames" ]; then
-    printf 'CREATE TEMPORARY TABLE `pkgnames` ('
-      printf '`architecture` VARCHAR(16),'
-      printf '`pkgname` VARCHAR(64),'
-      printf 'UNIQUE KEY `content`(`architecture`,`pkgname`)'
-    printf ');\n'
-    printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `pkgnames` COLUMNS TERMINATED BY " " (`architecture`,`pkgname`);\n' \
-      "${tmp_dir}/pkgnames"
-
-    printf 'SELECT DISTINCT'
-    printf ' "pkgbase",'
-    printf '`pkgnames`.`architecture`,'
-    printf '`package_sources`.`pkgbase`,'
-    printf '`upstream_repositories`.`name`'
-    printf ' FROM `pkgnames`'
-    printf ' JOIN `binary_packages`'
-    printf ' ON `binary_packages`.`pkgname`=`pkgnames`.`pkgname`'
-    mysql_join_binary_packages_build_assignments
-    mysql_join_build_assignments_package_sources
-    mysql_join_package_sources_upstream_repositories
-    printf ';\n'
-
-    printf 'SELECT DISTINCT'
-    printf ' "pkgname",'
-    printf '`pkgnames`.`architecture`,'
-    printf '`pkgnames`.`pkgname`'
-    printf ' FROM `package_sources`'
-    mysql_join_package_sources_upstream_repositories
-    mysql_join_package_sources_build_assignments
-    mysql_join_build_assignments_binary_packages
-    printf ' RIGHT JOIN `pkgnames`'
-    printf ' ON `pkgnames`.`pkgname`=`binary_packages`.`pkgname`'
-    printf ' WHERE `package_sources`.`pkgbase` IS NULL;\n'
-
-    printf 'DROP TEMPORARY TABLE `pkgnames`;\n'
-  fi
-  # auto-detect pkgbases
-  if ${auto}; then
+# auto-detect pkgbases
+if ${auto}; then
+  # shellcheck disable=SC2016
+  {
     printf 'CREATE TEMPORARY TABLE `relevant_archs`(`id` SMALLINT, PRIMARY KEY `id`(`id`));\n'
     printf 'INSERT INTO `relevant_archs`(`id`)'
     printf ' SELECT `architectures`.`id`'
@@ -380,7 +362,11 @@ fi
         "${tmp_dir}/ignore-packages"
     fi
 
-    printf 'SELECT DISTINCT "pkgbase",`ba_a`.`name`,`package_sources`.`pkgbase`,`upstream_repositories`.`name`'
+    printf 'SELECT DISTINCT '
+      printf '`ba_a`.`name`,'
+      printf '`package_sources`.`pkgbase`,'
+      printf '`package_sources`.`git_revision`,'
+      printf '`upstream_repositories`.`name`'
     printf ' FROM `least_stable_bp`'
     printf ' JOIN `binary_packages`'
     printf ' ON `least_stable_bp`.`id`=`binary_packages`.`id`'
@@ -442,133 +428,40 @@ fi
     printf 'DROP TEMPORARY TABLE `least_stable_bp%s`;\n' \
       '' '_copy'
     printf 'DROP TEMPORARY TABLE `ignore_packages`;\n'
-  fi
-} | \
-  mysql_run_query | \
-  tr '\t' ' ' | \
-  sed -n '
-    /^pkgbase / {
-      s/^\S\+ //
-      w /dev/stderr
-      d
-    }
-    /^pkgname / {
-      s/^\S\+ //
-      w /dev/stdout
-      d
-    }
-  ' \
-  2>>"${tmp_dir}/pkgbases" \
-  | sponge "${tmp_dir}/pkgnames"
-
-if [ ! -s "${tmp_dir}/pkgbases" ] && \
-  [ ! -s "${tmp_dir}/pkgnames" ]; then
-  >&2 echo 'No packages matched.'
-  exit
-fi
-
-for s in "${tmp_dir}/pkgbases" "${tmp_dir}/pkgnames"; do
-  if [ -s "${s}" ]; then
-    sort -u "${s}" | \
-      sponge "${s}"
-  fi
-done
-
-# pkgnames -> pkgbases (with help from upstream)
-if [ -s "${tmp_dir}/pkgnames" ]; then
-  while read -r architecture pkgname; do
-    content=$(
-      curl -Ss 'https://www.archlinux.org/packages/search/json/?name='"${pkgname}" | \
-        tr ',' '\n'
-    )
-    repo=$(
-      printf '%s\n' "${content}" | \
-        sed -n '
-          s/^\s*"repo"\s*:\s*"//
-          T
-          s/".*$//
-          T
-          p
-        '
-    )
-    pkgbase=$(
-      printf '%s\n' "${content}" | \
-        sed -n '
-          s/^\s*"pkgbase"\s*:\s*"//
-          T
-          s/".*$//
-          T
-          p
-        '
-    )
-    if [ -z "${pkgbase}" ] || [ -z "${repo}" ]; then
-      printf '%s %s\n' "${architecture}" "${pkgname}"
-      continue
-    fi
-    printf '%s %s %s\n' \
-      "${architecture}" \
-      "${pkgbase}" \
-      "${repo}" >> \
-      "${tmp_dir}/pkgbases"
-  done < \
-    "${tmp_dir}/pkgnames" | \
-    sponge "${tmp_dir}/pkgnames"
+  } \
+  | mysql_run_query \
+  | tr '\t' ' ' \
+  | sort -u \
+  >>"${tmp_dir}/pkgbases"
 fi
 
-if [ -s "${tmp_dir}/pkgnames" ]; then
-  >&2 echo 'Could not find some packages upstream:'
-  >&2 cat "${tmp_dir}/pkgnames"
-  exit 2
+if [ ! -s "${tmp_dir}/pkgbases" ]; then
+  >&2 echo 'No packages matched.'
+  exit
 fi
 
 sort -u "${tmp_dir}/pkgbases" \
-| sort -k3,3 \
 | sponge "${tmp_dir}/pkgbases"
 
 # get the current HEADs
 
-# shellcheck disable=SC2016
-{
-  printf 'SELECT'
-  printf ' `upstream_repositories`.`name`,'
-  printf '`git_repositories`.`head`'
-  printf ' FROM `upstream_repositories`'
-  mysql_join_upstream_repositories_git_repositories
-} \
-| mysql_run_query \
-| tr '\t' ' ' \
-| sort -k1,1 \
-| join -1 3 -2 1 -o 1.1,1.2,1.3,2.2 "${tmp_dir}/pkgbases" - \
-| sponge "${tmp_dir}/pkgbases"
-
 if ${update}; then
   cut -d' ' -f 2,3,4 "${tmp_dir}/pkgbases" \
   | sort -u \
-  | while read -r pkgbase repo git_rev; do
+  | while read -r pkgbase git_rev repo; do
     success=false
     # shellcheck disable=SC2154
-    for gr_r in \
-      "${git_rev}:${repo}" \
-      "${repo_heads__packages}:core" \
-      "${repo_heads__packages}:extra" \
-      "${repo_heads__community}:community"; do
-      printf '%s ' "${pkgbase}" "${gr_r%:*}" "${repo_heads__archlinux32}" "${gr_r#*:}" | \
-        sed 's/ $/\n/'
-      if mysql_generate_package_metadata "${repository_ids__any_build_list}" "${pkgbase}" "${gr_r%:*}" "${repo_heads__archlinux32}" "${gr_r#*:}"; then
-        success=true
-        break
-      fi
-    done
-    if ! ${success} && \
+    printf '%s ' "${pkgbase}" "${git_rev}" "${repo_heads__archlinux32}" "${repo}" \
+    | sed 's/ $/\n/'
+    if ! mysql_generate_package_metadata "${repository_ids__any_build_list}" "${pkgbase}" "${git_rev}" "${repo_heads__archlinux32}" "${repo}" && \
       ! ${ignore_mysql_generate_package_metadata_errors}; then
       exit 2
     fi
   done
 
-  cut -d' ' -f1 < \
-    "${tmp_dir}/pkgbases" | \
-    sort -u | \
-    sponge "${tmp_dir}/pkgbases"
+  cut -d' ' -f2 "${tmp_dir}/pkgbases" \
+  | sort -u \
+  | sponge "${tmp_dir}/pkgbases"
 
   # shellcheck disable=SC2016
   {
@@ -689,8 +582,8 @@ if ${update}; then
         mysql_join_binary_packages_binary_packages_in_repositories 'j_bp' 'j_bpir'
       )"' AND `j_bpir`.`repository`='"${repository_ids__any_build_list}" \
       '`repositories`.`stability`='"${repository_stability_ids__forbidden}"
-  } | \
-    mysql_run_query
+  } \
+  | mysql_run_query
 
   mysql_cleanup
 
@@ -699,8 +592,8 @@ if ${update}; then
   mysql_find_build_assignment_loops
 
   # remove duplicate binary_packages from "build-list"
-  mysql_query_remove_old_binary_packages_from_build_list | \
-    mysql_run_query
+  mysql_query_remove_old_binary_packages_from_build_list \
+  | mysql_run_query
 else
   cat "${tmp_dir}/pkgbases"
 fi