| index : builder | |
| Archlinux32 build system | gitolite user |
| summaryrefslogtreecommitdiff |
| -rwxr-xr-x | bin/bootstrap-mysql | 60 | ||||
| -rwxr-xr-x | bin/cleanup | 20 | ||||
| -rwxr-xr-x | bin/db-update | 62 | ||||
| -rwxr-xr-x | bin/get-assignment | 27 | ||||
| -rwxr-xr-x | bin/get-package-updates | 6 | ||||
| -rwxr-xr-x | bin/modify-package-state | 38 | ||||
| -rwxr-xr-x | bin/return-assignment | 38 |
diff --git a/bin/bootstrap-mysql b/bin/bootstrap-mysql index 4c1938f..fc347d7 100755 --- a/bin/bootstrap-mysql +++ b/bin/bootstrap-mysql @@ -3,6 +3,10 @@ # shellcheck source=conf/default.conf . "${0%/*}/../conf/default.conf" +# TODO: this should become some sort of mysql dump to bootstrap the +# tables and stored functions only - when the database is gone, we're +# pretty much screwed anyway :-D + tmp_dir=$(mktemp -d 'tmp.bootstrap-mysql.XXXXXXXXXX' --tmpdir) trap 'rm -rf --one-file-system "${tmp_dir}"' EXIT @@ -792,68 +796,12 @@ fi } | \ mysql_run_query -find "${work_dir}/package-states" \ - -maxdepth 1 \ - -mindepth 1 \ - \( \ - -name '*.done' -o \ - -name '*.testing' -o \ - -name '*.tested' \ - \) \ - -exec sed ' - 1!d - s@^.*-\([^-]\+\)-\([^-.]\+\)\.pkg\.tar\.xz$@{} \1 \2 \0@ - s@^\S*/@@ - s/^\(\S\+\)\.\(done\|testing\|tested\) /\1 / - s/ \([0-9]\+\) / \1.0 / - s/ [0-9]\+\.\([0-9]\+\) / \1 / - ' {} \; | \ - while read -r state_file sub_pkgrel arch package_file; do - printf '%s ' "${state_file}" - repository=$( - grep '^\S\+ '"$(str_to_regex "${package_file}")"'$' "${tmp_dir}/master-mirror-listing" | \ - cut -d' ' -f1 - ) - if [ "$(echo "${repository}" | grep -cx '\S\+')" -ne 1 ]; then - printf 'not found exactly once on the master mirror.\n' - >&2 printf '"%s"\n' "${state_file}" "${sub_pkgrel}" "${arch}" "${package_file}" - continue - fi - mysql_generate_package_metadata "${sub_pkgrel}" "${repository}" "${state_file}" - printf ' ok\n' - done - tr ' ' '.' < \ "${work_dir}/build-list" | \ while read -r state_file; do mysql_generate_package_metadata 'build-list' "${state_file}" done -# shellcheck disable=SC2016 -find "${work_dir}/package-states" -name '*.blocked' -printf '%p %f\n' | \ - sed ' - s|\.\([^. ]\+\)\.\([^. ]\+\)\.\([^. ]\+\)\.blocked$| \1 \2 \3| - ' | \ - while read -r state_file pkgbase git_revision mod_git_revision repository; do - printf 'UPDATE `build_assignments`' - mysql_join_build_assignments_package_sources - mysql_join_package_sources_upstream_repositories - printf ' SET `build_assignments`.`is_blocked`=from_base64("%s")' \ - "$( - tr -d '\n' < \ - "${state_file}" | \ - base64 -w0 - )" - printf ' WHERE' - printf ' `package_sources`.`%s`=from_base64("%s") AND' \ - 'pkgbase' "$(printf '%s' "${pkgbase}" | base64 -w0)" \ - 'git_revision' "$(printf '%s' "${git_revision}" | base64 -w0)" \ - 'mod_git_revision' "$(printf '%s' "${mod_git_revision}" | base64 -w0)" - printf ' `upstream_repositories`.`name`=from_base64("%s");\n' \ - "$(printf '%s' "${repository}" | base64 -w0)" - done | \ - mysql_run_query - grep '^\('"$( # shellcheck disable=SC2086 printf '%s\\|' ${standalone_package_repositories} ${stable_package_repositories} diff --git a/bin/cleanup b/bin/cleanup index a0bf790..7aaa39f 100755 --- a/bin/cleanup +++ b/bin/cleanup @@ -19,26 +19,6 @@ if [ -s "${work_dir}/build-master-sanity" ]; then exit fi -# remove blocked/broken/locked markes of packages not on the buildlist anymore - -{ - find "${work_dir}/package-states" -maxdepth 1 \( -name '*.broken' -o -name '*.locked' -o -name '*.blocked' \) -printf '%f\n' | \ - sed ' - s|^\(.*\)\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)\.\([^.]\+\)$|state \0 \1 \2 \3 \4 \5| - ' - awk '{ - print "order " $1 "." $2 "." $3 "." $4 " " $1 " " $2 " " $3 " " $4 " broken" - print "order " $1 "." $2 "." $3 "." $4 " " $1 " " $2 " " $3 " " $4 " blocked" - print "order " $1 "." $2 "." $3 "." $4 " " $1 " " $2 " " $3 " " $4 " locked" - }' "${work_dir}/build-list" -} | \ - sort -k3 | \ - uniq -uf2 | \ - grep '^state ' | \ - awk '{print $2}' | \ - sed "s|^|${work_dir}/package-states/|" | \ - xargs -rn1 rm - # remove logs where package is not broken/locked anymore { diff --git a/bin/db-update b/bin/db-update index 6ecf9c9..6722f24 100755 --- a/bin/db-update +++ b/bin/db-update @@ -207,12 +207,6 @@ for source_stability in 'testing' 'staging'; do mysql_join_repositories_repository_stabilities printf ' AND `repository_stabilities`.`name`="%s"' \ "${source_stability}" - # TODO: we may relax this condition, if we a) got rid of state files - # or b) improved keeping them in sync (*.testing -> /dev/null is not - # implemented, only .tested -> /dev/null) - if [ ! "${source_stability}" = 'staging' ]; then - printf ' AND `binary_packages`.`is_tested`' - fi mysql_join_binary_packages_build_assignments mysql_join_build_assignments_package_sources mysql_join_package_sources_upstream_repositories @@ -401,62 +395,6 @@ for source_stability in 'testing' 'staging'; do printf ' SET `binary_packages`.`repository`=`moved_binary_packages`.`new_repository`;\n' } | \ mysql_run_query - - sed_apply_rm=$( - while read -r pkg _ _; do - printf '/^%s$/d\n' "$(str_to_regex "${pkg}")" - done < \ - "${tmp_dir}/rm" - ) - - sed_apply_remove_part_of_mv=$( - while read -r pkg _ _; do - printf '/^%s$/d\n' "$(str_to_regex "${pkg}")" - done < \ - "${tmp_dir}/mv" - ) - - sed_apply_move_part_of_mv=$( - while read -r pkg _ _; do - printf '/^%s$/{\n' "$(str_to_regex "${pkg}")" - printf 'w %%s\n' - printf 'd\n' - printf '}\n' - done < \ - "${tmp_dir}/mv" - ) - - find "${work_dir}/package-states/" \ - \( \ - -name '*.done' -o \ - -name '*.testing' -o \ - -name '*.tested' \ - \) \ - -exec sed -i "${sed_apply_rm}" '{}' \; - find "${work_dir}/package-states/" \ - -name '*.tested' \ - -exec sed -i "${sed_apply_remove_part_of_mv}" '{}' \; - find "${work_dir}/package-states/" \ - -name '*.done' \ - -exec grep -qxF "$(cut -d' ' -f1 < "${tmp_dir}/mv")" '{}' \; \ - -printf '%p\n' | \ - while read -r sf; do - sed -i "$( - printf '%s\n' "${sed_apply_move_part_of_mv}" | \ - sed ' - s,%s,'"${sf%.done}.testing"',g - ' - )" "${sf}" - done - find "${work_dir}/package-states/" \ - \( \ - -name '*.done' -o \ - -name '*.testing' -o \ - -name '*.tested' \ - \) \ - -type f \ - -not -exec test -s '{}' \; \ - -delete done trigger_mirror_refreshs diff --git a/bin/get-assignment b/bin/get-assignment index 3357fc6..a82e1f8 100755 --- a/bin/get-assignment +++ b/bin/get-assignment @@ -18,17 +18,8 @@ # TODO: honor manual build order of tool-chain: # toolchain build order: linux-api-headers->glibc->binutils->gcc->binutils->glibc -mkdir -p "${work_dir}/package-states" - hand_out_assignment() { - # we don't care anymore if an older version of this package was - # "locked" or "broken" - find "${work_dir}/package-states" -maxdepth 1 -regextype grep \ - -regex '.*/'"$(str_to_regex "$1")"'\(\.[^.]\+\)\{3\}\.\(locked\|broken\)' \ - -not -regex '.*/'"$(str_to_regex "$1.$2.$3.$4.")"'[^.]\+' \ - -delete - # move that build order to the end of the build-list sed -i ' /^'"$(str_to_regex "$1 $2 $3 $4")"'$/ { @@ -57,21 +48,7 @@ hand_out_assignment() { printf ' AND `upstream_repositories`.`name`=from_base64("%s")' \ "$(printf '%s' "$4" | base64 -w0)" printf ' LIMIT 1;\n' - } | \ - mysql_run_query | \ - tr '\t' ' ' - { - # shellcheck disable=SC2154 - echo "${slave}" - if [ -r "${work_dir}/package-states/$1.$2.$3.$4.locked" ]; then - cat "${work_dir}/package-states/$1.$2.$3.$4.locked" - fi - } | \ - sort -u | \ - sponge "${work_dir}/package-states/$1.$2.$3.$4.locked" - # shellcheck disable=SC2016 - { printf 'UPDATE `build_slaves`' printf ' SET `currently_building` = (' printf ' SELECT `build_assignments`.`id`' @@ -86,6 +63,7 @@ hand_out_assignment() { printf ' `upstream_repositories`.`name` = from_base64("%s")' \ "$(printf '%s' "$4" | base64 -w0)" printf ')' + # shellcheck disable=SC2154 printf ' WHERE `build_slaves`.`name`=from_base64("%s");\n' \ "$(printf '%s' "${slave}" | base64 -w0)" @@ -101,7 +79,8 @@ hand_out_assignment() { printf ' `upstream_repositories`.`name` = from_base64("%s");\n' \ "$(printf '%s' "$4" | base64 -w0)" } | \ - mysql_run_query + mysql_run_query | \ + tr '\t' ' ' exit 0 diff --git a/bin/get-package-updates b/bin/get-package-updates index a36cb8a..3e9edf9 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -111,9 +111,6 @@ delete_package() { mysql_run_query >> \ "${work_dir}/deletion-list.new" sed -i "/^$(str_to_regex "${1}") /d" "${work_dir}/build-list.new" - find "${work_dir}/package-states" -maxdepth 1 -mindepth 1 -regextype grep \ - -regex '.*/'"$(str_to_regex "${1}")"'\(\.[^.]\+\)\{3\}\.\(broken\|blocked\|locked\)' \ - -delete # TODO: Once we want to rely on the database for test_exclusion, we # need to run the command below unconditionally, but with some # changes, so we can easily revert. @@ -317,9 +314,6 @@ done | \ if [ "${mode}" = 'A' ] || [ "${mode}" = 'M' ]; then # new or modified PKGBUILD sed -i "/^$(str_to_regex "${package}") /d" "${work_dir}/build-list.new" - find "${work_dir}/package-states" -regextype grep \ - -regex '.*/'"$(str_to_regex "${package}")"'\(\.[0-9a-f]\{40\}\)\{2\}\.[^.]\+\.\(blocked\|broken\)' \ - -delete # shellcheck disable=SC2154 echo "${package} ${git_revision} ${new_repo_revisions__archlinux32} ${repository}" >> \ "${work_dir}/build-list.new" diff --git a/bin/modify-package-state b/bin/modify-package-state index 76b8c1e..4a2f5ac 100755 --- a/bin/modify-package-state +++ b/bin/modify-package-state @@ -243,46 +243,8 @@ while read -r package reason; do { printf 'UPDATE %s SET %s %s' "${combiner}" "${modifier}" "${selector}" printf ';\n' - # TODO: remove the below part, once the database if fully functional - printf 'UPDATE `binary_packages` AS `a`' - printf ' JOIN `binary_packages` AS `b`' - printf ' ON `a`.`build_assignment`=`b`.`build_assignment`' - mysql_join_binary_packages_repositories 'b' - printf ' SET `b`.`is_tested`=1' - printf ' WHERE `a`.`is_tested`' - printf ' AND NOT `b`.`is_tested`' - printf ' AND NOT `repositories`.`name`="deletion-list";\n' } | \ mysql_run_query - case "${action}" in - 'block') - printf '%s' "${reason}" > \ - "${work_dir}/package-states/${package}.blocked" - ;; - 'unblock') - rm -f "${work_dir}/package-states/${package}.blocked" - ;; - 'faulty') - find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ - -name '*.tested' \ - -exec grep -lxF "${package}" '{}' \; | \ - while read -r sf; do - mv "${sf}" "${sf%ed}ing" - done - ;; - 'tested') - find "${work_dir}/package-states" -mindepth 1 -maxdepth 1 \ - -name '*.testing' \ - -exec grep -lxF "${package}" '{}' \; | \ - while read -r sf; do - mv "${sf}" "${sf%ing}ed" - done - ;; - *) - >&2 printf 'Whooops, action "%s" not implemented yet.\n' "${action}" - exit 42 - ;; - esac printf '%s %s\n' "${action}" "${package}" fi done \ diff --git a/bin/return-assignment b/bin/return-assignment index 02bd491..da9e762 100755 --- a/bin/return-assignment +++ b/bin/return-assignment @@ -89,9 +89,6 @@ if [ "$5" = 'ERROR' ]; then '*.build-log.gz' ) - # shellcheck disable=SC2154 - echo "${slave}" >> \ - "${work_dir}/package-states/$1.$2.$3.$4.broken" # shellcheck disable=SC2016 { if [ -n "${saved_build_logs}" ]; then @@ -162,23 +159,6 @@ if [ "$5" = 'ERROR' ]; then } | \ mysql_run_query - # shellcheck disable=SC2154 - sed -i ' - /^'"$(str_to_regex "${slave}")"'$/d - ' "${work_dir}/package-states/$1.$2.$3.$4.locked" - if [ ! -s "${work_dir}/package-states/$1.$2.$3.$4.locked" ]; then - rm "${work_dir}/package-states/$1.$2.$3.$4.locked" - - # move that build order to the end of the build-list - sed -i ' - /^'"$(str_to_regex "$1 $2 $3 $4")"'$/ { - $ b - d - } - $ a '"$1 $2 $3 $4" \ - "${work_dir}/build-list" - fi - # release lock on build-list - otherwise seed-build-list won't run flock -u 9 @@ -517,24 +497,9 @@ for package in ${packages}; do remove_old_package_versions 'i686' "${destination}" "${package}" done -# remove old state files (these should be only "done" markers, but -# actually we don't care what it is) - as long as it's not "testing" or "tested" -find "${work_dir}/package-states" -maxdepth 1 -regextype grep \ - -not -name '*.testing' \ - -not -name '*.tested' \ - -regex '.*/'"$(str_to_regex "$1")"'\(\.[^.]\+\)\{4\}' \ - -not -regex '.*/'"$(str_to_regex "$1.$2.$3.$4")"'\.[^.]\+' \ - -exec rm '{}' \; - # remove package from build list sed -i "/^$(str_to_regex "$1 $2 $3 $4")\$/d" "${work_dir}/build-list" -# remove package lock file -if ! [ "${destination}" = 'build-support' ]; then - # shellcheck disable=SC2086 - printf '%s\n' ${packages} > \ - "${work_dir}/package-states/$1.$2.$3.$4.done" -fi # shellcheck disable=SC2016 { printf 'UPDATE `build_assignments`' @@ -578,6 +543,3 @@ fi printf 'DROP TABLE `loops_to_delete`;\n' } | \ mysql_run_query -rm -f \ - "${work_dir}/package-states/$1.$2.$3.$4.locked" \ - "${work_dir}/package-states/$1.$2.$3.$4.broken" |