| index : builder | |
| Archlinux32 build system | gitolite user |
| summaryrefslogtreecommitdiff |
| -rwxr-xr-x | bin/common-functions | 94 | ||||
| -rwxr-xr-x | bin/db-update | 282 |
diff --git a/bin/common-functions b/bin/common-functions index 2d99f80..dd3b772 100755 --- a/bin/common-functions +++ b/bin/common-functions @@ -927,3 +927,97 @@ shrink_version() { s/^\(\(\S\+\s\+\)\{'"$((column_num-1))"'\}\S*\)+0-/\1-/ ' } + +# find_biggest_subset_of_packages $omega $keep $all_run_depends [ $force ] + +# Return (to stdout) the biggest subset A of the packages in $omega whose +# runtime dependencies in $omega \cup $keep are also in A + +# $all_builds either points to an empty file - then it will get filled +# with cached data for subsequent calls - or to the same file of a previous +# call + +# If non-empty, $force contains packages which are assumed to match the above +# condition without checking. + +# The arguments are names of files with one $package.$revision.$mod_revision.$repository +# per line. + +find_biggest_subset_of_packages() { + + ( + omega="$1" + keep="$2" + all_builds="$3" + if [ $# -eq 3 ]; then + force='/dev/null' + elif [ $# -eq 4 ]; then + force="$4" + else + >&2 printf 'find_biggest_subset_of_packages: Wrong number of arguments: %s given, 3 or 4 expected.' "$#" + return 2 + fi + + if [ ! -s "${all_run_depends}" ]; then + find "${work_dir}/package-infos/" -maxdepth 1 -name '*.builds' \ + -exec sed ' + s|^|{} | + s|^\S\+/|| + s|\.builds | | + ' {} \; | \ + sort -k2,2 > \ + "${all_builds}" + fi + + sort -u "${omega}" | \ + sponge "${omega}" + + temp_dir=$(mktemp -d) + trap 'rm -rf --one-file-system "${temp_dir}"' EXIT + + { + sort -u "${keep}" + cat "${force}" "${force}" + } | \ + sort | \ + uniq -u > \ + "${temp_dir}/keep.new" + touch "${temp_dir}/keep" + + while [ -s "${temp_dir}/keep.new" ]; do + cat "${temp_dir}/keep.new" "${temp_dir}/keep" | \ + sort -u | \ + sponge "${temp_dir}/keep" + + sed ' + s|^|'"${work_dir}"'/package-infos/| + s|$|.run-depends| + ' "${temp_dir}/keep" | \ + xargs -r grep -HF '' | \ + sed ' + s|^.*/|| + s|\.run-depends:| | + ' | \ + sort -u | \ + sort -k2,2 | \ + uniq -f1 | \ + join -1 2 -2 2 -o 2.1 - "${all_builds}" | \ + sort -u | \ + join -1 1 -2 1 -o 2.1 - "${omega}" | \ + sort -u > \ + "${temp_dir}/keep.new" + + # "new" is only what has not been there before and what is not forced + cat "${temp_dir}/keep" "${temp_dir}/keep" "${force}" "${force}" "${temp_dir}/keep.new" | \ + sort | \ + uniq -u | \ + sponge "${temp_dir}/keep.new" + done + + cat "${omega}" "${temp_dir}/keep" "${temp_dir}/keep" | \ + sort | \ + uniq -u + + ) + +} diff --git a/bin/db-update b/bin/db-update index d2b3980..3926884 100755 --- a/bin/db-update +++ b/bin/db-update @@ -23,9 +23,6 @@ # correctly handle if a package moved from community to official or # vice versa -# only move packages from testing to stable if all dependencies are -# moved, too - # shellcheck disable=SC2039 # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" @@ -49,7 +46,7 @@ usage() { [ -z "$1" ] && exit 1 || exit "$1" } -# move_packages $package/$from_repository/$to_repository ... +# move_packages file with one "$package $from_repository $to_repository" per line # the existence of a directory $tmp_dir is assumed move_packages() { @@ -69,12 +66,15 @@ move_packages() { local dummynator local file - rm -rf --one-file-system "${tmp_dir:?}/"* + if [ -e "${tmp_dir:?}/tmp" ]; then + rm -rf --one-file-system "${tmp_dir:?}/tmp" + fi + mkdir "${tmp_dir}/tmp" - touch "${tmp_dir}/repos" - touch "${tmp_dir}/packages" - touch "${tmp_dir}/master-mirror-listing" - mkdir "${tmp_dir}/transit" + touch "${tmp_dir}/tmp/repos" + touch "${tmp_dir}/tmp/packages" + touch "${tmp_dir}/tmp/master-mirror-listing" + mkdir "${tmp_dir}/tmp/transit" if ${no_action}; then dummynator='echo' @@ -86,19 +86,14 @@ move_packages() { while read -r repo; do ls_master_mirror "i686/${repo}" | \ sed "s|^|i686/${repo}/|" >> \ - "${tmp_dir}/master-mirror-listing" + "${tmp_dir}/tmp/master-mirror-listing" done - for package in "$@"; do + while read -r package from_repo to_repo; do if [ -z "${package}" ]; then continue fi - to_repo="${package##*/}" - package="${package%/*}" - from_repo="${package##*/}" - package="${package%/*}" - if ${no_action}; then printf \ 'move "%s" from "%s" to "%s"\n' \ @@ -108,7 +103,7 @@ move_packages() { fi echo "${package}" >> \ - "${tmp_dir}/packages" + "${tmp_dir}/tmp/packages" if echo "${from_repo}" | \ grep -q 'staging$' && \ @@ -128,13 +123,13 @@ move_packages() { fi echo "${from_repo}" > \ - "${tmp_dir}/${package}.from_repo" + "${tmp_dir}/tmp/${package}.from_repo" echo "${to_repo}" > \ - "${tmp_dir}/${package}.to_repo" + "${tmp_dir}/tmp/${package}.to_repo" echo "${from_ending}" > \ - "${tmp_dir}/${package}.from_ending" + "${tmp_dir}/tmp/${package}.from_ending" echo "${to_ending}" > \ - "${tmp_dir}/${package}.to_ending" + "${tmp_dir}/tmp/${package}.to_ending" if [ ! -f "${work_dir}/package-states/${package}.${from_ending}" ]; then >&2 printf 'move_packages: Cannot find package state file "%s"\n' "${package}.${from_ending}" @@ -143,20 +138,20 @@ move_packages() { cp \ "${work_dir}/package-states/${package}.${from_ending}" \ - "${tmp_dir}/${package}.parts" + "${tmp_dir}/tmp/${package}.parts" sed \ 's|\(-[^-]\+\)\{3\}\.pkg\.tar\.xz$||' \ - "${tmp_dir}/${package}.parts" > \ - "${tmp_dir}/${package}.parts_names" + "${tmp_dir}/tmp/${package}.parts" > \ + "${tmp_dir}/tmp/${package}.parts_names" sed \ 'p;s|$|.sig|' \ - "${tmp_dir}/${package}.parts" > \ - "${tmp_dir}/${package}.parts_and_signatures" + "${tmp_dir}/tmp/${package}.parts" > \ + "${tmp_dir}/tmp/${package}.parts_and_signatures" while read -r part; do - if ! grep -qxF "i686/${from_repo}/${part}" "${tmp_dir}/master-mirror-listing"; then + if ! grep -qxF "i686/${from_repo}/${part}" "${tmp_dir}/tmp/master-mirror-listing"; then >&2 printf \ 'move_packages: Cannot find file "%s", part of package "%s".\n' \ "i686/${from_repo}/${part}" \ @@ -164,23 +159,24 @@ move_packages() { exit 2 fi done < \ - "${tmp_dir}/${package}.parts" + "${tmp_dir}/tmp/${package}.parts" - mkdir -p "${tmp_dir}/${from_repo}" - mkdir -p "${tmp_dir}/${to_repo}" + mkdir -p "${tmp_dir}/tmp/${from_repo}" + mkdir -p "${tmp_dir}/tmp/${to_repo}" repos=$( # shellcheck disable=SC2046 - printf '%s\n' "${from_repo}" "${to_repo}" $(cat "${tmp_dir}/repos") | \ + printf '%s\n' "${from_repo}" "${to_repo}" $(cat "${tmp_dir}/tmp/repos") | \ sort -u ) echo "${repos}" > \ - "${tmp_dir}/repos" + "${tmp_dir}/tmp/repos" - done + done < \ + "$1" if ${no_action}; then - find "${tmp_dir}" -type f | \ + find "${tmp_dir}/tmp" -type f | \ while read -r file; do if [ "${file%.pkg.tar.xz}.pkg.tar.xz" = "${file}" ] || [ "${file%.pkg.tar.xz.sig}.pkg.tar.xz.sig" = "${file}" ]; then @@ -200,17 +196,17 @@ move_packages() { ${master_mirror_rsync_command} \ "${master_mirror_rsync_directory}/i686/${repo}/${repo}.db."* \ "${master_mirror_rsync_directory}/i686/${repo}/${repo}.files."* \ - "${tmp_dir}/${repo}/" + "${tmp_dir}/tmp/${repo}/" # add and remove the packages locally - if grep -qxF "${repo}" "${tmp_dir}/"*".from_repo"; then + if grep -qxF "${repo}" "${tmp_dir}/tmp/"*".from_repo"; then # shellcheck disable=SC2046 repo-remove -q \ - "${tmp_dir}/${repo}/${repo}.db.tar.gz" \ + "${tmp_dir}/tmp/${repo}/${repo}.db.tar.gz" \ $( - grep -lxF "${repo}" "${tmp_dir}/"*".from_repo" | \ + grep -lxF "${repo}" "${tmp_dir}/tmp/"*".from_repo" | \ sed ' s|\.from_repo$|.parts_names| ' | \ @@ -218,8 +214,8 @@ move_packages() { ) fi - if grep -qxF "${repo}" "${tmp_dir}/"*".to_repo"; then - grep -lxF "${repo}" "${tmp_dir}/"*".to_repo" | \ + if grep -qxF "${repo}" "${tmp_dir}/tmp/"*".to_repo"; then + grep -lxF "${repo}" "${tmp_dir}/tmp/"*".to_repo" | \ sed ' s|\.to_repo$|| ' | \ @@ -228,22 +224,22 @@ move_packages() { ${master_mirror_rsync_command} \ "${master_mirror_rsync_directory}/i686/$(cat "${package}.from_repo")/${part}" \ "${master_mirror_rsync_directory}/i686/$(cat "${package}.from_repo")/${part}.sig" \ - "${tmp_dir}/transit/" + "${tmp_dir}/tmp/transit/" repo-add -q \ - "${tmp_dir}/${repo}/${repo}.db.tar.gz" \ - "${tmp_dir}/transit/${part}" + "${tmp_dir}/tmp/${repo}/${repo}.db.tar.gz" \ + "${tmp_dir}/tmp/transit/${part}" rm \ - "${tmp_dir}/transit/${part}" \ - "${tmp_dir}/transit/${part}.sig" + "${tmp_dir}/tmp/transit/${part}" \ + "${tmp_dir}/tmp/transit/${part}.sig" done < \ "${package}.parts" done fi - done < "${tmp_dir}/repos" + done < "${tmp_dir}/tmp/repos" if ${no_action}; then - find "${tmp_dir}" -type f + find "${tmp_dir}/tmp" -type f fi # move the packages remotely via sftp @@ -261,13 +257,13 @@ move_packages() { fi printf \ 'rename "%s" "%s"\n' \ - "i686/$(cat "${tmp_dir}/${package}.from_repo")/${part}" \ - "i686/$(cat "${tmp_dir}/${package}.to_repo")/${part}" + "i686/$(cat "${tmp_dir}/tmp/${package}.from_repo")/${part}" \ + "i686/$(cat "${tmp_dir}/tmp/${package}.to_repo")/${part}" done < \ - "${tmp_dir}/${package}.parts_and_signatures" + "${tmp_dir}/tmp/${package}.parts_and_signatures" done < \ - "${tmp_dir}/packages" + "${tmp_dir}/tmp/packages" echo 'quit' ) | \ if ${no_action}; then @@ -282,29 +278,29 @@ move_packages() { # shellcheck disable=SC2086 ${dummynator} ${master_mirror_rsync_command} \ - "${tmp_dir}/${repo}/${repo}.db."* \ - "${tmp_dir}/${repo}/${repo}.files."* \ + "${tmp_dir}/tmp/${repo}/${repo}.db."* \ + "${tmp_dir}/tmp/${repo}/${repo}.files."* \ "${master_mirror_rsync_directory}/i686/${repo}/" done < \ - "${tmp_dir}/repos" + "${tmp_dir}/tmp/repos" while read -r package; do # then we can safely remove old versions while read -r part; do - ${dummynator} remove_old_package_versions 'i686' "$(cat "${tmp_dir}/${package}.to_repo")" "${part}" + ${dummynator} remove_old_package_versions 'i686' "$(cat "${tmp_dir}/tmp/${package}.to_repo")" "${part}" done < \ - "${tmp_dir}/${package}.parts" + "${tmp_dir}/tmp/${package}.parts" # and update the state files from_ending=$( - cat "${tmp_dir}/${package}.from_ending" + cat "${tmp_dir}/tmp/${package}.from_ending" ) to_ending=$( - cat "${tmp_dir}/${package}.to_ending" + cat "${tmp_dir}/tmp/${package}.to_ending" ) if [ -z "${to_ending}" ]; then @@ -321,18 +317,18 @@ move_packages() { fi done < \ - "${tmp_dir}/packages" + "${tmp_dir}/tmp/packages" if ! ${no_action}; then date '+%s' > \ - "${tmp_dir}/lastupdate" + "${tmp_dir}/tmp/lastupdate" # shellcheck disable=SC2086 ${dummynator} ${master_mirror_rsync_command} \ - "${tmp_dir}/lastupdate" \ + "${tmp_dir}/tmp/lastupdate" \ "${master_mirror_rsync_directory}/lastupdate" fi - rm -rf --one-file-system "${tmp_dir:?}/"* + rm -rf --one-file-system "${tmp_dir:?}/tmp" } @@ -497,10 +493,10 @@ find "${work_dir}/package-states" -maxdepth 1 -type f -name '*.done' -printf '%f grep -vxF 'break_loops' "${work_dir}/build-list" | \ tr ' ' '.' | \ sort -u > \ - "${tmp_dir}/keep-packages" + "${tmp_dir}/build-list-packages" find "${work_dir}/package-infos" -name '*.groups' \ - -exec grep -qx 'base\(-devel\)\?' {} \; \ + -exec grep -qxF 'base' {} \; \ -printf '%f\n' | \ sed ' s|\.groups$|| @@ -508,80 +504,43 @@ find "${work_dir}/package-infos" -name '*.groups' \ sort -u > \ "${tmp_dir}/base-packages" -# no base / base-devel packages on the build list? +# shellcheck disable=SC2086 +printf '%s\n' ${packages_to_force_unstage} > \ + "${tmp_dir}/force-unstage-packages" + +# calculate what packages should be unstaged: + +find_biggest_subset_of_packages "${tmp_dir}/done-packages" "${tmp_dir}/build-list-packages" "${tmp_dir}/all-run-depends" "${tmp_dir}/force-unstage-packages" > \ + "${tmp_dir}/unstage-packages" + +# no base / base-devel packages on the build list anymore? if [ -z "$( join -j 1 \ "${tmp_dir}/base-packages" \ - "${tmp_dir}/keep-packages" + "${tmp_dir}/build-list-packages" )" ]; then - # unstage all base / base-devel packages from staging - cat "${tmp_dir}/base-packages" "${tmp_dir}/base-packages" "${tmp_dir}/keep-packages" | \ - sort | \ - uniq -u | \ - sponge "${tmp_dir}/keep-packages" -fi -# find all dependencies of the unstageable packages -mv \ - "${tmp_dir}/keep-packages" \ - "${tmp_dir}/new-keep-packages" -touch "${tmp_dir}/keep-packages" + # we pretend, the group "base" does not exist, so we only fetch 'direct' dependencies on base-packages -while [ -s "${tmp_dir}/new-keep-packages" ]; do + mv "${tmp_dir}/all-run-depends" "${tmp_dir}/really-all-run-depends" + grep -v ' base$' "${tmp_dir}/really-all-run-depends" > \ + "${tmp_dir}/all-run-depends" || \ + true - cat "${tmp_dir}/new-keep-packages" "${tmp_dir}/keep-packages" | \ - sort -u | \ - sponge "${tmp_dir}/keep-packages" + find_biggest_subset_of_packages "${tmp_dir}/done-packages" "${tmp_dir}/build-list-packages" "${tmp_dir}/all-run-depends" "${tmp_dir}/force-unstage-packages" > \ + "${tmp_dir}/unstage-packages" - sed ' - s|^|'"${work_dir}"'/package-infos/| - s|$|.run-depends| - ' "${tmp_dir}/keep-packages" | \ - xargs -r grep -HF '' | \ - sed ' - s|^.*/|| - s|\.run-depends:| | - ' | \ - sort -u | \ - sort -k2,2 > \ - "${tmp_dir}/keep-packages.run-depends" + mv "${tmp_dir}/really-all-run-depends" "${tmp_dir}/all-run-depends" +fi - sed ' - s|^|'"${work_dir}"'/package-infos/| - s|$|.builds| - ' "${tmp_dir}/done-packages" | \ - xargs -r grep -HF '' | \ - sed ' - s|^.*/|| - s|\.builds:| | - ' | \ - sort -u | \ - sort -k2,2 > \ - "${tmp_dir}/done-packages.builds" - - join -j 2 -o 1.1 \ - "${tmp_dir}/done-packages.builds" \ - "${tmp_dir}/keep-packages.run-depends" | \ - sort -u > \ - "${tmp_dir}/new-keep-packages" - - # "new" is only what has not been there before - cat "${tmp_dir}/keep-packages" "${tmp_dir}/keep-packages" "${tmp_dir}/new-keep-packages" | \ - sort | \ - uniq -u | \ - sponge "${tmp_dir}/new-keep-packages" +# calculate what packages should be stabilized -done +cat "${tmp_dir}/done-packages" "${tmp_dir}/build-list-packages" | \ + sort -u > \ + "${tmp_dir}/keep-packages" -# remove force-unstage packages from keep-packages -{ - cat "${tmp_dir}/keep-packages" - # shellcheck disable=SC2086 - printf '%s\n' ${packages_to_force_unstage} ${packages_to_force_unstage} -} | \ - sort | \ - uniq -u | \ - sponge "${tmp_dir}/keep-packages" +find_biggest_subset_of_packages "${tmp_dir}/stabilize-packages" "${tmp_dir}/keep-packages" "${tmp_dir}/all-run-depends" | \ + sponge "${tmp_dir}/stabilize-packages" # unlock build list @@ -593,55 +552,36 @@ clean_up_lock_file() { rm -f "${package_database_lock_file}" } -# calculate unstageable packages from keep_packages and done_packages +# testing -> stable -done_packages=$( - cat \ - "${tmp_dir}/keep-packages" \ - "${tmp_dir}/keep-packages" \ - "${tmp_dir}/done-packages" | \ - sort | \ - uniq -u -) +while read -r package; do + printf '%s %s %s\n' \ + "${package}" \ + "$(official_or_community "${package}" 'testing')" \ + "$(repository_of_package "${package}")" +done < \ + "${tmp_dir}/stabilize-packages" | \ + sponge "${tmp_dir}/stabilize-packages" -# move packages in packages_to_stabilize from *testing/ to the stable repos +# staging -> testing -# shellcheck disable=SC2046 -move_packages $( +while read -r package; do + printf '%s %s %s\n' \ + "${package}" \ + "$(official_or_community "${package}" 'staging')" \ + "$(official_or_community "${package}" 'testing')" +done < \ + "${tmp_dir}/unstage-packages" | \ + sponge "${tmp_dir}/unstage-packages" - for package in ${packages_to_stabilize}; do - - if [ -z "${package}" ]; then - continue - fi - - printf '%s/%s/%s\n' \ - "${package}" \ - "$(official_or_community "${package}" 'testing')" \ - "$(repository_of_package "${package}")" - - done - -) +# move packages in packages_to_stabilize from *testing/ to the stable repos +if [ -s "${tmp_dir}/stabilize-packages" ]; then + move_packages "${tmp_dir}/stabilize-packages" +fi # move packages from *staging to *testing - -# shellcheck disable=SC2046 -move_packages $( - - for package in ${done_packages}; do - - if [ -z "${package}" ]; then - continue - fi - - printf '%s/%s/%s\n' \ - "${package}" \ - "$(official_or_community "${package}" 'staging')" \ - "$(official_or_community "${package}" 'testing')" - - done - -) +if [ -s "${tmp_dir}/unstage-packages" ]; then + move_packages "${tmp_dir}/unstage-packages" +fi clean_up_lock_file |