#!/bin/sh # shellcheck disable=SC2119,SC2120 # shellcheck source=../lib/load-configuration . "${0%/*}/../lib/load-configuration" # TODO: allow to filter by architecture # shellcheck disable=SC2016 usage() { >&2 echo '' >&2 echo 'seed-build-list [options]:' >&2 echo ' seed the build list from different sources' >&2 echo '' >&2 echo 'possible options:' >&2 echo ' -a|--auto:' >&2 echo ' Automatically reschedule packages which have run-time' >&2 echo ' dependencies that are not available from any real' >&2 echo ' repository and which cannot be replaced by another,' >&2 echo ' less stable version of the same package.' >&2 echo ' -f|--force' >&2 echo ' Do not exit if mysql_generate_package_metadata() failed.' >&2 echo ' -h|--help:' >&2 echo ' Show this help and exit.' >&2 echo ' -i|--ignore $pkgbase:' >&2 echo ' Do not update the given package.' >&2 echo ' -m|--mirror $url:' >&2 echo ' Schedule all packages, that are newer on the given' >&2 echo ' x86_64 mirror - except packages in' >&2 echo ' conf/seed-ignore-packages.' >&2 echo ' -n|--no-action:' >&2 echo ' Do not actually update build-list, just print it.' >&2 echo ' -p|--package $pkg_regex:' >&2 echo ' Reschedule packages with matching pkgname or pkgbase.' >&2 echo ' Note, that these packages must be known to the database.' >&2 echo ' -u|--undelete $url: ' >&2 echo ' Schedule all former deletion-list packages which do not' >&2 echo ' belong on the deletion-list anymore (e.g. can be built)' >&2 echo ' and which are available on the given x86_64 mirror.' >&2 echo ' -w|--wait:' >&2 echo ' Wait for lock if necessary.' [ -z "$1" ] && exit 1 || exit "$1" } tmp_dir=$(mktemp -d 'tmp.seed-build-list.XXXXXXXXXX' --tmpdir) # shellcheck disable=SC2064 trap "rm -rf --one-file-system '${tmp_dir:?}'" EXIT eval set -- "$( getopt -o afhi:m:np:u:w \ --long auto \ --long force \ --long help \ --long ignore: \ --long mirror: \ --long no-action \ --long package: \ --long undelete: \ --long wait \ -n "$(basename "$0")" -- "$@" || \ echo usage )" auto=false update=true wait_for_lock='-n' ignore_mysql_generate_package_metadata_errors=false if [ -r "${base_dir}/conf/seed-ignore-packages" ]; then sed 's/^/2\t/' "${base_dir}/conf/seed-ignore-packages" >> \ "${tmp_dir}/ignore-packages" fi while true do case "$1" in -a|--auto) auto=true ;; -f|--force) ignore_mysql_generate_package_metadata_errors=true ;; -h|--help) usage 0 ;; -i|--ignore) shift printf '3\t%s\n' "$1" >> \ "${tmp_dir}/ignore-packages" ;; -m|--mirror) shift printf '%s\n' "$1" >> \ "${tmp_dir}/mirrors" ;; -n|--no-action) update=false ;; -p|--package) shift printf '%s\n' "$1" >> \ "${tmp_dir}/package-regexes" ;; -u|--undelete) shift printf '%s\n' "$1" >> \ "${tmp_dir}/undelete-mirrors" ;; -w|--wait) wait_for_lock='' ;; --) shift break ;; *) >&2 printf 'Whoops, forgot to implement option "%s" internally.\n' \ "$1" exit 42 ;; esac shift done if [ $# -ne 0 ]; then usage 1 fi if [ ! -s "${tmp_dir}/mirrors" ] && \ [ ! -s "${tmp_dir}/package-regexes" ] && \ [ ! -s "${tmp_dir}/undelete-mirrors" ] && \ ! ${auto}; then # nothing to do >&2 echo 'No options given to do anything.' usage 0 exit 0 fi # get locks if ${update}; then exec 9> "${sanity_check_lock_file}" if ! verbose_flock -s ${wait_for_lock} 9; then >&2 echo 'Cannot get sanity-check lock.' exit 1 fi exec 8> "${build_list_lock_file}" if ! verbose_flock ${wait_for_lock} 8; then >&2 echo 'Cannot get build-list lock.' exit 1 fi fi # shellcheck disable=SC2016 repos=$( { printf 'SELECT DISTINCT `repositories`.`name`' printf ' FROM `repositories`' # shellcheck disable=SC2154 printf ' WHERE `repositories`.`stability`=%s;\n' \ "${repository_stability_ids__stable}" } | \ mysql_run_query printf '%s\n' 'multilib' ) # TODO: translate lib32-* packages instead of ignoring them for -u and -m # harvest pkgnames from mirror delta if [ -s "${tmp_dir}/mirrors" ]; then { # theirs while read -r mirror; do if [ -z "${mirror}" ]; then continue fi for repo in ${repos}; do curl -sS "${mirror}/${repo}/os/x86_64/${repo}.db.tar.gz" | \ tar -Oxz --wildcards '*/desc' | \ sed ' /^%FILENAME%$/!d N s/^.*\n// /^lib32-/d s/^\(.*-\)x86_64\(\.pkg\.tar\.xz\)$/\1i486\2\n\1i686\2/ ' | \ sed ' s/^\(.*\)-\([^-]\+-[^-]\+\)-\([^-]\+\)$/theirs \2 \3 \1/ ' done done < \ "${tmp_dir}/mirrors" # ours # shellcheck disable=SC2016 { printf 'SELECT ' mysql_package_name_query printf ' FROM `binary_packages`' mysql_join_binary_packages_architectures mysql_join_binary_packages_binary_packages_in_repositories printf ' WHERE NOT `binary_packages_in_repositories`.`is_to_be_deleted`' } | \ mysql_run_query | \ sed ' s/^\(.*\)-\([^-]\+-[^-.]\+\)\(\.[^-.]\+\)\?-\([^-]\+\)$/ours \2 \4 \1/ ' } | \ expand_version 2 | \ sort -k3,4 -k2Vr,2 -k1,1 | \ shrink_version 2 | \ uniq -f2 | \ sed -n ' s/^theirs \(\S\+ \)\{2\}// T p ' | \ if [ -r "${base_dir}/conf/seed-ignore-packages" ]; then grep -vxF "$(cat "${base_dir}/conf/seed-ignore-packages")" else cat fi | \ sed 's/^/1\t/' | \ sort -u >> \ "${tmp_dir}/pkgnames" fi # harvest pkgnames from undelete-mirrors if [ -s "${tmp_dir}/undelete-mirrors" ]; then while read -r mirror; do if [ -z "${mirror}" ]; then continue fi for repo in ${repos}; do curl -sS "${mirror}/${repo}/os/x86_64/${repo}.db.tar.gz" | \ tar -Oxz --wildcards '*/desc' | \ sed ' /^%FILENAME%$/!d N s/^.*\n// /^lib32-/d s/^\(.*-\)x86_64\(\.pkg\.tar\.xz\)$/\1i486\2\n\1i686\2/ ' | \ sed ' s/^\(.*\)\(-[^-]\+\)\{3\}$/\1/ ' done done < \ "${tmp_dir}/undelete-mirrors" | \ if [ -r "${base_dir}/conf/seed-ignore-packages" ]; then grep -vxF "$(cat "${base_dir}/conf/seed-ignore-packages")" else cat fi | \ sed 's/^/2\t/' | \ sort -u >> \ "${tmp_dir}/pkgnames" fi # harvest from the package-regexes if [ -s "${tmp_dir}/package-regexes" ]; then # shellcheck disable=SC2016 { printf 'CREATE TEMPORARY TABLE `names` (`name` VARCHAR(64));\n' printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `names` (`name`);\n' \ "${tmp_dir}/package-regexes" printf 'SELECT DISTINCT "pkgbase",`package_sources`.`pkgbase`,`upstream_repositories`.`name`' printf ' FROM `names`' printf ' JOIN `package_sources` ON `package_sources`.`pkgbase` REGEXP `names`.`name`' mysql_join_package_sources_upstream_repositories printf ';\n' printf 'SELECT DISTINCT "pkgname",`binary_packages`.`pkgname`' printf ' FROM `names`' printf ' JOIN `binary_packages` ON `binary_packages`.`pkgname` REGEXP `names`.`name`;\n' } | \ mysql_run_query | \ sed -n ' /^pkgbase\s/ { s/^\S\+\s/3\t/ w /dev/stdout d } /^pkgname\s/ { s/^\S\+\s/3\t/ w /dev/stderr d } ' \ >>"${tmp_dir}/pkgbases" \ 2>>"${tmp_dir}/pkgnames" fi # shellcheck disable=SC2016 { # create pkgbases to given pkgnames if [ -s "${tmp_dir}/pkgnames" ]; then printf 'CREATE TEMPORARY TABLE `pkgnames` (' printf '`priority` SMALLINT,' printf '`pkgname` VARCHAR(64),' printf 'UNIQUE KEY `pkgname`(`pkgname`)' printf ');\n' printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `pkgnames` (`priority`,`pkgname`);\n' \ "${tmp_dir}/pkgnames" printf 'SELECT DISTINCT "pkgbase",`pkgnames`.`priority`,`package_sources`.`pkgbase`,`upstream_repositories`.`name`' printf ' FROM `pkgnames`' printf ' JOIN `binary_packages`' printf ' ON `binary_packages`.`pkgname`=`pkgnames`.`pkgname`' mysql_join_binary_packages_build_assignments mysql_join_build_assignments_package_sources mysql_join_package_sources_upstream_repositories printf ';\n' printf 'SELECT DISTINCT "pkgname",`pkgnames`.`priority`,`pkgnames`.`pkgname`' printf ' FROM `package_sources`' mysql_join_package_sources_upstream_repositories mysql_join_package_sources_build_assignments mysql_join_build_assignments_binary_packages printf ' RIGHT JOIN `pkgnames`' printf ' ON `pkgnames`.`pkgname`=`binary_packages`.`pkgname`' printf ' WHERE `package_sources`.`pkgbase` IS NULL;\n' printf 'DROP TEMPORARY TABLE `pkgnames`;\n' fi # auto-detect pkgbases if ${auto}; then printf 'SELECT DISTINCT "pkgbase",1,`package_sources`.`pkgbase`,`upstream_repositories`.`name` FROM (' printf 'SELECT DISTINCT `binary_packages`.`id`,' printf '`binary_packages`.`build_assignment`' printf ' FROM (' printf 'SELECT `binary_packages`.`pkgname`,' printf 'MAX(`repositories`.`stability`) AS `stability`' printf ' FROM `binary_packages`' mysql_join_binary_packages_binary_packages_in_repositories mysql_join_binary_packages_in_repositories_repositories printf ' AND (' printf '`repositories`.`is_on_master_mirror`' # we need to take into account build-list items here, because # they may make re-scheduling unnecessary printf ' OR `repositories`.`name`="build-list"' printf ') AND `repositories`.`name`!="build-support"' # TODO: care about i486 here, too - if it is complete enough printf ' AND `binary_packages`.`architecture`!=3' printf ' GROUP BY `binary_packages`.`pkgname`' printf ') AS `least_stable`' printf ' JOIN `binary_packages`' printf ' ON `binary_packages`.`pkgname`=`least_stable`.`pkgname`' mysql_join_binary_packages_binary_packages_in_repositories mysql_join_binary_packages_in_repositories_repositories printf ' AND `repositories`.`stability`=`least_stable`.`stability`' # now we can drop build-list items printf ' AND `repositories`.`is_on_master_mirror`' printf ') AS `binary_packages`' mysql_join_binary_packages_build_assignments mysql_join_build_assignments_package_sources mysql_join_package_sources_upstream_repositories mysql_join_binary_packages_dependencies mysql_join_dependencies_dependency_types printf ' AND `dependency_types`.`relevant_for_binary_packages`' mysql_join_dependencies_install_target_providers_with_versions '' 'itp_dummy' # some dependencies are not provided by the least stable packages printf ' WHERE NOT EXISTS (' printf 'SELECT 1 FROM `install_target_providers`' mysql_join_install_target_providers_binary_packages '' 'subst_bp' mysql_join_binary_packages_binary_packages_in_repositories 'subst_bp' 'subst_bir' mysql_join_binary_packages_in_repositories_repositories 'subst_bir' 'subst_r' printf ' AND `subst_r`.`name` NOT IN ("build-support","deletion-list","to-be-decided")' printf ' WHERE `install_target_providers`.`install_target`=`dependencies`.`depending_on`' # this is the least stable, built install_target_provider with that name printf ' AND NOT EXISTS (' printf 'SELECT 1 FROM `binary_packages` AS `ss_bp`' mysql_join_binary_packages_binary_packages_in_repositories 'ss_bp' 'ss_bir' mysql_join_binary_packages_in_repositories_repositories 'ss_bir' 'ss_r' printf ' AND `ss_r`.`name` NOT IN ("build-list","build-support","deletion-list","to-be-decided")' printf ' JOIN `repository_stability_relations` AS `ss_rsr`' printf ' ON `ss_rsr`.`less_stable`=`ss_r`.`stability`' printf ' WHERE `ss_bp`.`pkgname`=`subst_bp`.`pkgname`' printf ' AND `ss_bp`.`id`!=`subst_bp`.`id`' printf ' AND `ss_rsr`.`more_stable`=`subst_r`.`stability`' printf ')' printf ')' printf ';\n' fi } | \ mysql_run_query | \ sed -n ' /^pkgbase\s/ { s/^\S\+\s// w /dev/stderr d } /^pkgname\s/ { s/^\S\+\s// w /dev/stdout d } ' \ 2>>"${tmp_dir}/pkgbases" \ | sponge "${tmp_dir}/pkgnames" if [ ! -s "${tmp_dir}/pkgbases" ] && \ [ ! -s "${tmp_dir}/pkgnames" ]; then >&2 echo 'No packages matched.' exit fi for s in "${tmp_dir}/pkgbases" "${tmp_dir}/pkgnames"; do if [ -s "${s}" ]; then sort -u "${s}" | \ sponge "${s}" fi done # pkgnames -> pkgbases (with help from upstream) if [ -s "${tmp_dir}/pkgnames" ]; then while read -r priority pkgname; do content=$( curl -Ss 'https://www.archlinux.org/packages/search/json/?name='"${pkgname}" | \ tr ',' '\n' ) repo=$( printf '%s\n' "${content}" | \ sed -n ' s/^\s*"repo"\s*:\s*"// T s/".*$// T p ' ) pkgbase=$( printf '%s\n' "${content}" | \ sed -n ' s/^\s*"pkgbase"\s*:\s*"// T s/".*$// T p ' ) if [ -z "${pkgbase}" ] || [ -z "${repo}" ]; then printf '%s\t%s\n' "${priority}" "${pkgname}" continue fi printf '%s\t%s\t%s\n' \ "${priority}" \ "${pkgbase}" \ "${repo}" >> \ "${tmp_dir}/pkgbases" done < \ "${tmp_dir}/pkgnames" | \ sponge "${tmp_dir}/pkgnames" fi if [ -s "${tmp_dir}/pkgnames" ]; then >&2 echo 'Could not find some packages upstream:' >&2 cat "${tmp_dir}/pkgnames" exit 2 fi # now we (re)schedule all pkgbases which are: # - not explicitely ignored (priority 3) <- explicitely scheduled # - not explicitely/implicitely ignored, but not currently available # (priority 2) <- from undelete-mirror-delta # - neither ignored nor to-be-deleted (priority 1) <- from mirror-delta or auto-rebuild # shellcheck disable=SC2016 { printf 'CREATE TEMPORARY TABLE `ignore_packages` (' printf '`priority` SMALLINT,' printf '`pkgbase` VARCHAR(64),' printf 'UNIQUE KEY `content`(`priority`,`pkgbase`)' printf ');\n' printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `ignore_packages`(`priority`,`pkgbase`);\n' \ "${tmp_dir}/ignore-packages" printf 'CREATE TEMPORARY TABLE `pkgbases` (' printf '`priority` SMALLINT,' printf '`pkgbase` VARCHAR(64),' printf '`repo` VARCHAR(64),' printf 'UNIQUE KEY `pkgbase`(`pkgbase`)' printf ');\n' printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `pkgbases`(`priority`,`pkgbase`,`repo`);\n' \ "${tmp_dir}/pkgbases" printf 'INSERT IGNORE INTO `ignore_packages`(`priority`,`pkgbase`)' printf ' SELECT 2,`package_sources`.`pkgbase`' printf ' FROM `package_sources`' mysql_join_package_sources_build_assignments mysql_join_build_assignments_binary_packages printf ' WHERE `build_assignments`.`is_black_listed` IS NOT NULL' printf ' OR `package_sources`.`pkgbase` LIKE "lib32-%%"' printf ' OR EXISTS (' printf 'SELECT 1 FROM `dependencies`' mysql_join_dependencies_dependency_types printf ' AND (' printf '`dependency_types`.`relevant_for_building`' printf ' OR `dependency_types`.`relevant_for_binary_packages`' printf ')' mysql_join_dependencies_install_target_providers_with_versions '' 'it_dummy' printf ' WHERE `dependencies`.`dependent`=`binary_packages`.`id`' printf ' AND NOT EXISTS (' printf 'SELECT 1 FROM `install_target_providers`' mysql_join_install_target_providers_binary_packages '' 'it_bp' mysql_join_binary_packages_binary_packages_in_repositories 'it_bp' 'it_bpir' printf ' WHERE NOT `it_bpir`.`is_to_be_deleted`' printf ' AND `install_target_providers`.`install_target`=`dependencies`.`depending_on`' printf ')' printf ');\n' printf 'INSERT IGNORE INTO `ignore_packages`(`priority`,`pkgbase`)' printf ' SELECT 1,`package_sources`.`pkgbase`' printf ' FROM `package_sources`' mysql_join_package_sources_build_assignments mysql_join_build_assignments_binary_packages mysql_join_binary_packages_binary_packages_in_repositories printf ' WHERE `binary_packages_in_repositories`.`is_to_be_deleted`' printf ';\n' printf 'SELECT ' printf '`pkgbases`.`pkgbase`,' printf '`git_repositories`.`head`,' printf '(' printf 'SELECT `al32`.`head` FROM `git_repositories` AS `al32`' printf ' WHERE `al32`.`name`="archlinux32"' printf '),' printf '`pkgbases`.`repo`' printf ' FROM `pkgbases`' printf ' JOIN `upstream_repositories`' printf ' ON `upstream_repositories`.`name`=`pkgbases`.`repo`' mysql_join_upstream_repositories_git_repositories printf ' WHERE NOT EXISTS (' printf 'SELECT 1 FROM `ignore_packages`' printf ' WHERE `ignore_packages`.`pkgbase`=`pkgbases`.`pkgbase`' printf ' AND `ignore_packages`.`priority`>=`pkgbases`.`priority`' printf ') AND (' printf '`pkgbases`.`priority`!=2' printf ' OR NOT EXISTS (' printf 'SELECT 1 FROM `package_sources`' mysql_join_package_sources_build_assignments mysql_join_build_assignments_binary_packages mysql_join_binary_packages_binary_packages_in_repositories printf ' AND NOT `binary_packages_in_repositories`.`is_to_be_deleted`' mysql_join_binary_packages_in_repositories_repositories printf ' AND (' printf '`repositories`.`is_on_master_mirror`' printf ' OR `repositories`.`name`="build-list"' printf ') WHERE `package_sources`.`pkgbase`=`pkgbases`.`pkgbase`' printf ')' printf ');\n' } | \ mysql_run_query | \ tr '\t' ' ' | \ sort -u > \ "${tmp_dir}/update-list" if ${update}; then while read -r pkgbase git_rev mod_git_rev repo; do printf '%s ' "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}" | \ sed 's/ $/\n/' if ! mysql_generate_package_metadata "${repository_ids__any_build_list}" "${pkgbase}" "${git_rev}" "${mod_git_rev}" "${repo}" && \ ! ${ignore_mysql_generate_package_metadata_errors}; then exit 2 fi done < \ "${tmp_dir}/update-list" cut -d' ' -f1 < \ "${tmp_dir}/update-list" | \ sort -u | \ sponge "${tmp_dir}/update-list" # shellcheck disable=SC2016 { printf 'CREATE TEMPORARY TABLE `pkgbases` (' printf '`pkgbase` VARCHAR (64),' printf 'UNIQUE KEY `pkgbase`(`pkgbase`)' printf ');\n' printf 'LOAD DATA LOCAL INFILE "%s" INTO TABLE `pkgbases`(`pkgbase`);\n' \ "${tmp_dir}/update-list" printf 'DELETE `d_bpir`' printf ' FROM `pkgbases`' # the package on the deletion-list ... printf ' JOIN `package_sources` AS `d_ps`' printf ' ON `pkgbases`.`pkgbase`=`d_ps`.`pkgbase`' mysql_join_package_sources_build_assignments 'd_ps' 'd_ba' mysql_join_build_assignments_binary_packages 'd_ba' 'd_bp' mysql_join_binary_packages_binary_packages_in_repositories 'd_bp' 'd_bpir' mysql_join_binary_packages_in_repositories_repositories 'd_bpir' 'd_r' # shellcheck disable=SC2154 printf ' AND `d_r`.`stability`=%s' \ "${repository_stability_ids__forbidden}" # ... needs a counter-part on the build-list (which was just created) printf ' JOIN `package_sources` AS `a_ps`' printf ' ON `d_ps`.`pkgbase`=`a_ps`.`pkgbase`' mysql_join_package_sources_build_assignments 'a_ps' 'a_ba' mysql_join_build_assignments_binary_packages 'a_ba' 'a_bp' printf ' JOIN `architecture_compatibilities` AS `ac`' printf ' ON `ac`.`built_for`=`a_bp`.`architecture`' printf ' AND `ac`.`runs_on`=`d_r`.`architecture`' printf ' AND `ac`.`fully_compatible`' mysql_join_binary_packages_binary_packages_in_repositories 'a_bp' 'a_bpir' printf ' AND `a_bpir`.`repository`=%s;\n' \ "${repository_ids__any_build_list}" } | \ mysql_run_query mysql_cleanup mysql_sort_versions # remove duplicate binary_packages from "build-list" mysql_query_remove_old_binary_packages_from_build_list | \ mysql_run_query else cat "${tmp_dir}/update-list" fi