| index : builder | |
| Archlinux32 build system | gitolite user |
| summaryrefslogtreecommitdiff |
| -rwxr-xr-x | bin/get-package-updates | 176 |
diff --git a/bin/get-package-updates b/bin/get-package-updates index 1b3bd68..724e261 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -207,7 +207,13 @@ if ! verbose_flock -s ${block_flag} 8; then exit fi -trap mysql_cleanup EXIT +cleanup() { + mysql_cleanup + rm -rf --one-file-system "${tmp_dir:?}" +} +tmp_dir=$(mktemp -d 'tmp.get-package-updates.XXXXXXXXXX' --tmpdir) + +trap cleanup EXIT # shellcheck disable=SC2119 mysql_cleanup @@ -422,107 +428,81 @@ echo 'apply blacklisting' # ignore blacklisted packages and dependent packages # this is the first time when all the information is available and up to date -# TODO: use 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `%s`' instead of -# "INSERT IGNORE INTO `%s` VALUES ..." - -black_listed='' -black_listed_new=$( +touch "${tmp_dir}/black-listed" +{ + git -C "${repo_paths__archlinux32}" archive "${new_repo_revisions__archlinux32}" -- 'blacklist' | \ + tar -Ox 'blacklist' | \ + sed ' + s/\s*#.*$// + /^\s*$/d + ' + if [ -n "${test_exclusion}" ]; then + echo "${test_exclusion}" + fi + # shellcheck disable=SC2016 { - { - git -C "${repo_paths__archlinux32}" archive "${new_repo_revisions__archlinux32}" -- 'blacklist' | \ - tar -Ox 'blacklist' | \ - sed ' - s/\s*#.*$// - /^\s*$/d - ' - if [ -n "${test_exclusion}" ]; then - echo "${test_exclusion}" - fi - } | \ - base64_encode_each - # shellcheck disable=SC2016 - { - printf 'SELECT DISTINCT replace(to_base64(`package_sources`.`pkgbase`),"\\n","")' - printf ' FROM `package_sources`' - printf ' WHERE `package_sources`.`pkgbase` LIKE "lib32-%%"' - } | \ - mysql_run_query + printf 'SELECT DISTINCT `package_sources`.`pkgbase`' + printf ' FROM `package_sources`' + printf ' WHERE `package_sources`.`pkgbase` LIKE "lib32-%%"' } | \ - sort -u -) + mysql_run_query +} | \ + sort -u > \ + "${tmp_dir}/black-listed-new" -while [ -n "${black_listed_new}" ]; do - black_listed=$( - printf '%s\n' "${black_listed}" "${black_listed_new}" | \ - grep -vxF '' | \ - sort -u - ) - black_listed_new=$( - { - printf '%s\n' "${black_listed}" "${black_listed}" +while [ -s "${tmp_dir}/black-listed-new" ]; do + cat \ + "${tmp_dir}/black-listed-new" \ + "${tmp_dir}/black-listed" | \ + sort -u | \ + sponge "${tmp_dir}/black-listed" + { + sed 'p' "${tmp_dir}/black-listed" - # shellcheck disable=SC2016 - { - printf 'CREATE TEMPORARY TABLE `bl` (`pkgbase` VARCHAR(64));\n' - printf 'INSERT INTO `bl` (`pkgbase`) VALUES ' - printf '%s\n' "${black_listed}" | \ - sort -u | \ - sed ' - s/^/(from_base64("/ - s/$/")),/ - $ s/,$/;/ - ' - printf 'SELECT replace(to_base64(`a_ps`.`pkgbase`),"\\n","")' - printf ' FROM `package_sources` AS `a_ps`' - mysql_join_package_sources_build_assignments 'a_ps' 'a_ba' - mysql_join_build_assignments_binary_packages 'a_ba' 'a_bp' - mysql_join_binary_packages_dependencies 'a_bp' - mysql_join_dependencies_dependency_types - printf ' AND (`dependency_types`.`relevant_for_building`' - printf ' OR `dependency_types`.`relevant_for_binary_packages`)' + # shellcheck disable=SC2016 + { + printf 'CREATE TEMPORARY TABLE `bl` (`pkgbase` VARCHAR(64));\n' + printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `bl` (`pkgbase`);\n' \ + "${tmp_dir}/black-listed" + printf 'SELECT `a_ps`.`pkgbase`' + printf ' FROM `package_sources` AS `a_ps`' + mysql_join_package_sources_build_assignments 'a_ps' 'a_ba' + mysql_join_build_assignments_binary_packages 'a_ba' 'a_bp' + mysql_join_binary_packages_dependencies 'a_bp' + mysql_join_dependencies_dependency_types + printf ' AND (`dependency_types`.`relevant_for_building`' + printf ' OR `dependency_types`.`relevant_for_binary_packages`)' + printf ' WHERE NOT EXISTS (' + printf ' SELECT 1' + printf ' FROM `install_target_providers`' printf ' WHERE NOT EXISTS (' printf ' SELECT 1' - printf ' FROM `install_target_providers`' - printf ' WHERE NOT EXISTS (' - printf ' SELECT 1' - printf ' FROM `bl`' - printf ' JOIN `package_sources` AS `b_ps` ON `bl`.`pkgbase`=`b_ps`.`pkgbase`' - mysql_join_package_sources_build_assignments 'b_ps' 'b_ba' - mysql_join_build_assignments_binary_packages 'b_ba' 'b_bp' - printf ' WHERE `install_target_providers`.`package`=`b_bp`.`id`' - printf ')' - printf ' AND `install_target_providers`.`install_target`=`dependencies`.`depending_on`' - printf ') AND EXISTS (' - # TODO: This should be corrected at the root: automatic install targets, which are bogus should - # not be added in the first place - but how do we detect that? -> add exceptions! - printf ' SELECT 1' - printf ' FROM `install_target_providers`' - printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`' - printf ');\n' - } | \ - mysql_run_query | \ - sort -u + printf ' FROM `bl`' + printf ' JOIN `package_sources` AS `b_ps` ON `bl`.`pkgbase`=`b_ps`.`pkgbase`' + mysql_join_package_sources_build_assignments 'b_ps' 'b_ba' + mysql_join_build_assignments_binary_packages 'b_ba' 'b_bp' + printf ' WHERE `install_target_providers`.`package`=`b_bp`.`id`' + printf ')' + printf ' AND `install_target_providers`.`install_target`=`dependencies`.`depending_on`' + printf ') AND EXISTS (' + # TODO: This should be corrected at the root: automatic install targets, which are bogus should + # not be added in the first place - but how do we detect that? -> add exceptions! + printf ' SELECT 1' + printf ' FROM `install_target_providers`' + printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`' + printf ');\n' } | \ - grep -vxF '' | \ - sort | \ - uniq -u - ) + mysql_run_query | \ + sort -u + } | \ + sort | \ + uniq -u | \ + sponge "${tmp_dir}/black-listed-new" done -black_listed=$( - printf '%s\n' "${black_listed}" | \ - while read -r line; do - printf '%s' \ - "${line}" | \ - base64 -d - printf '\n' - done -) - deletion_list_count=$( - # shellcheck disable=SC2086 - printf '%s\n' ${black_listed} | \ - wc -l + wc -l < \ + "${tmp_dir}/black-listed" ) if [ "${deletion_list_count}" -gt 1000 ]; then >&2 printf 'There are %s > 1000 packages on the deletion-list. This does not seem right.\n' \ @@ -550,14 +530,10 @@ if [ "${deletion_list_count}" -gt 1000 ]; then fi rm -f "${work_dir}/told-irc-about-too-many-deletion-list-packages" -echo "${black_listed}" | \ - while read -r package; do - if [ -z "${package}" ]; then - continue - fi - delete_package "${package}" - done - +while read -r package; do + delete_package "${package}" +done < \ + "${tmp_dir}/black-listed" if [ -n "${test_exclusion}" ]; then # TODO: reimplement test_exclusion with information from the database |