| index : builder | |
| Archlinux32 build system | gitolite user |
| summaryrefslogtreecommitdiff |
| -rwxr-xr-x | bin/build-master-status | 2 | ||||
| -rwxr-xr-x | bin/build-packages | 17 | ||||
| -rwxr-xr-x | bin/common-functions | 49 | ||||
| -rwxr-xr-x | bin/db-update | 56 | ||||
| -rwxr-xr-x | bin/get-assignment | 8 | ||||
| -rwxr-xr-x | bin/get-package-updates | 62 | ||||
| -rwxr-xr-x | bin/return-assignment | 29 | ||||
| -rwxr-xr-x | bin/slave-build-connect | 2 | ||||
| -rwxr-xr-x | bin/why_dont_you_build | 4 |
diff --git a/bin/build-master-status b/bin/build-master-status index ba1dba3..d6c761e 100755 --- a/bin/build-master-status +++ b/bin/build-master-status @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh # report about status of build master diff --git a/bin/build-packages b/bin/build-packages index 4130738..052e4fe 100755 --- a/bin/build-packages +++ b/bin/build-packages @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh # build packages one by one, then upload the binary package to the repository server # Details: @@ -117,15 +117,16 @@ while [ ${count} -ne 0 ]; do # Update git repositories (official packages, community packages and the repository of package customizations). - for repo in "${repo_paths[@]}"; do - git -C "${repo}" clean -df - git -C "${repo}" reset --hard - git -C "${repo}" checkout master - git -C "${repo}" pull || true + for repo_name in ${repo_names}; do + eval repo_path='$repo_paths__'"${repo}" + git -C "${repo_path}" clean -df + git -C "${repo_path}" reset --hard + git -C "${repo_path}" checkout master + git -C "${repo_path}" pull || true done - git -C "${repo_paths["$(find_repository_with_commit "${git_revision}")"]}" checkout "${git_revision}" > /dev/null 2>&1 - git -C "${repo_paths["archlinux32"]}" checkout "${mod_git_revision}" > /dev/null 2>&1 + git -C "$(eval printf '$repo_paths__%s' "$(find_repository_with_commit "${git_revision}")")" checkout "${git_revision}" > /dev/null 2>&1 + git -C "${repo_paths__archlinux32}" checkout "${mod_git_revision}" > /dev/null 2>&1 PKGBUILD="$(find_pkgbuild "${package}" "${repository}")" diff --git a/bin/common-functions b/bin/common-functions index 2f9c115..3280f05 100755 --- a/bin/common-functions +++ b/bin/common-functions @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh # contains functions used by more than one script @@ -12,26 +12,28 @@ find_pkgbuild() { local PKGBUILD='' repo file - if [ -f "${repo_paths["archlinux32"]}/$2/$1/PKGBUILD" ]; then + if [ -f "${repo_paths__archlinux32}/$2/$1/PKGBUILD" ]; then # If this package has some modification, repo="$(find_git_repository_to_package_repository "$2")" - if ! [ -d "${repo_paths["${repo}"]}/$1" ]; then + local package_path="$(eval printf '$repo_paths__%s' "${repo}")/$1" + if ! [ -d "${package_path}" ]; then # create some dummy files if it is also new. - mkdir -p "${repo_paths["${repo}"]}/$1/repos/$2-x86_64" - touch "${repo_paths["${repo}"]}/$1/repos/$2-x86_64/PKGBUILD" + mkdir -p "${package_path}/repos/$2-x86_64" + touch "${package_path}/repos/$2-x86_64/PKGBUILD" fi fi - for repo in "${!repo_paths[@]}"; do + for repo in ${repo_names}; do if [ "${repo}" = "archlinux32" ]; then # this is not a repository of packages continue fi - if ! [ -d "${repo_paths["${repo}"]}/$1" ]; then + local package_path="$(eval printf '$repo_paths__%s' "${repo}")/$1" + if ! [ -d "${package_path}" ]; then continue fi PKGBUILD="$( - ls "${repo_paths["${repo}"]}/$1/repos/$2-"*"/PKGBUILD" 2> /dev/null | \ + ls "${package_path}/repos/$2-"*"/PKGBUILD" 2> /dev/null | \ tr ' ' '\n' | \ grep -v -- '-i686/PKGBUILD$' | \ grep -v -- '-\(staging\|testing\)-[^/]\+/PKGBUILD$' | \ @@ -70,13 +72,13 @@ apply_package_customizations() { if [ ! -f 'PKGBUILD.changes-applied' ]; then # add i686 to the arch list sed '/^arch=[^#]*any/!s|^\(arch=(\)\([^#]*)\)\s*\(#.*\)\?$|\1'"'i686'"' \2|' -i 'PKGBUILD' - if [ -f "${repo_paths["archlinux32"]}/${repo}/${package}/PKGBUILD" ]; then + if [ -f "${repo_paths__archlinux32}/${repo}/${package}/PKGBUILD" ]; then # If this package has modifications (or is new), apply them now: # append PKGBUILD - cat "${repo_paths["archlinux32"]}/${repo}/${package}/PKGBUILD" >> \ + cat "${repo_paths__archlinux32}/${repo}/${package}/PKGBUILD" >> \ 'PKGBUILD' # copy (and overwrite) other files - for file in "${repo_paths["archlinux32"]}/${repo}/${package}/"*; do + for file in "${repo_paths__archlinux32}/${repo}/${package}/"*; do if [ -f "${file}" ] && [ "${file##*/}" != 'PKGBUILD' ]; then cp "${file}" ./ fi @@ -93,8 +95,8 @@ find_repository_with_commit() { local repository - for repository in "${!repo_paths[@]}"; do - if [ "$(git -C "${repo_paths["${repository}"]}" cat-file -t "$1" 2> /dev/null)" == "commit" ]; then + for repository in ${repo_names}; do + if [ "$(git -C "$(eval printf '$repo_paths__%s' "${repository}")" cat-file -t "$1" 2> /dev/null)" = "commit" ]; then echo "${repository}" return 0 fi @@ -111,13 +113,13 @@ find_git_repository_to_package_repository() { local repository - for repository in "${!repo_paths[@]}"; do + for repository in ${repo_names}; do if [ "${repository}" = "archlinux32" ]; then continue fi if [ -n "$( ( - ls "${repo_paths["${repository}"]}/"*"/repos" | \ + ls "$(eval printf '$repo_paths__%s' "${repository}")/"*"/repos" | \ grep -v ':$' | \ sed 's|-[^-]\+$||' | \ sort -u @@ -176,8 +178,8 @@ generate_package_metadata() { return 0 fi - git -C "${repo_paths["$(find_repository_with_commit "${git_revision}")"]}" checkout "${git_revision}" - git -C "${repo_paths["$(find_repository_with_commit "${mod_git_revision}")"]}" checkout "${mod_git_revision}" + git -C "$(eval printf '$repo_paths__%s' "$(find_repository_with_commit "${git_revision}")")" checkout "${git_revision}" + git -C "$(eval printf '$repo_paths__%s' "$(find_repository_with_commit "${mod_git_revision}")")" checkout "${mod_git_revision}" PKGBUILD="$(find_pkgbuild "${package}" "${repository}")" @@ -281,7 +283,7 @@ official_or_community() { echo 'community-' ;; 'multilib') - if ! git -C "${repo_paths['archlinux32']}" archive --format=tar "${a32_rev}" -- 'extra-from-multilib' | \ + if ! git -C "${repo_paths__archlinux32}" archive --format=tar "${a32_rev}" -- 'extra-from-multilib' | \ tar -Ox | \ grep -qFx "${package%.*.*.*}"; then echo 'community-' @@ -299,8 +301,8 @@ remove_old_package_versions() { local pkgname="${package%-*-*-*.pkg.tar.xz}" ls | \ - grep "^${pkgname//./\\.}\(-[^-]\+\)\{3\}\.pkg\.tar\.xz\(\.sig\)\?\$" | \ - grep -v "^${package//./\\.}\(\.sig\)\?\$" | \ + grep "^$(str_to_regex "${pkgname}")\(-[^-]\+\)\{3\}\.pkg\.tar\.xz\(\.sig\)\?\$" | \ + grep -v "^$(str_to_regex "${package}")\(\.sig\)\?\$" | \ xargs -rn1 rm } @@ -324,3 +326,10 @@ wait_some_time() { sleep $((${minimum} + ${random} % ${diff})) } + +# str_to_regex $string +# escape dots for use in regex +str_to_regex() { + echo "$1" | \ + sed 's|\.|\\.|g' +} diff --git a/bin/db-update b/bin/db-update index 57c8111..9b7b3d0 100755 --- a/bin/db-update +++ b/bin/db-update @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh # move binary packages from staging to testing (if possible [1]) and # additionally all packages specified on the command line from testing @@ -59,9 +59,9 @@ do shift done -packages_to_stabilize=("${@}") +packages_to_stabilize=${@} -for package in "${packages_to_stabilize[@]}"; do +for package in ${packages_to_stabilize}; do # some sanity checks if [ ! -f "${work_dir}/package-states/${package}.testing" ]; then >&2 echo "Package '${package}' is not in testing!" @@ -205,28 +205,24 @@ done_packages="$( unset keep_packages for package in ${done_packages}; do - parts=($(cat "${work_dir}/package-states/${package}.done")) - parts_names=( - $( - printf '%s\n' "${parts[@]}" | \ - sed 's|\(-[^-]\+\)\{3\}\.pkg\.tar\.xz$||' - ) + parts=$(cat "${work_dir}/package-states/${package}.done") + parts_names=$( + printf '%s\n' ${parts} | \ + sed 's|\(-[^-]\+\)\{3\}\.pkg\.tar\.xz$||' ) is_community="$(official_or_community "${package}")" - pushd "${master_mirror_directory}/i686/${is_community}staging" > /dev/null + cd "${master_mirror_directory}/i686/${is_community}staging" mkdir -p "../${is_community}testing" - repo-remove "${is_community}staging.db.tar.gz" "${parts_names[@]}" - for part in "${parts[@]}"; do + repo-remove "${is_community}staging.db.tar.gz" ${parts_names} + for part in ${parts}; do mv "${part}" "${part}.sig" "../${is_community}testing/" updated_package_database=true done - popd > /dev/null - pushd "${master_mirror_directory}/i686/${is_community}testing" > /dev/null - repo-add "${is_community}testing.db.tar.gz" "${parts[@]}" - for part in "${parts[@]}"; do + cd "${master_mirror_directory}/i686/${is_community}testing" + repo-add "${is_community}testing.db.tar.gz" ${parts} + for part in ${parts}; do remove_old_package_versions "${part}" done - popd > /dev/null mv \ "${work_dir}/package-states/${package}.done" \ @@ -234,29 +230,25 @@ for package in ${done_packages}; do done # move packages in packages_to_stabilize from *testing/ to the stable repos -for package in "${packages_to_stabilize[@]}"; do - parts=($(cat "${work_dir}/package-states/${package}.testing")) - parts_names=( - $( - printf '%s\n' "${parts[@]}" | \ - sed 's|\(-[^-]\+\)\{3\}\.pkg\.tar\.xz$||' - ) +for package in ${packages_to_stabilize}; do + parts=$(cat "${work_dir}/package-states/${package}.testing") + parts_names=$( + printf '%s\n' ${parts} | \ + sed 's|\(-[^-]\+\)\{3\}\.pkg\.tar\.xz$||' ) source="$(official_or_community ${package})testing" destination="${package##*.}" - pushd "${master_mirror_directory}/i686/${source}" > /dev/null - repo-remove "${source}.db.tar.gz" "${parts_names[@]}" - for part in "${parts[@]}"; do + cd "${master_mirror_directory}/i686/${source}" + repo-remove "${source}.db.tar.gz" ${parts_names} + for part in ${parts}; do mv "${part}" "${part}.sig" "../${destination}/" updated_package_database=true done - popd > /dev/null - pushd "${master_mirror_directory}/i686/${destination}" > /dev/null - repo-add "${destination}.db.tar.gz" "${parts[@]}" - for part in "${parts[@]}"; do + cd "${master_mirror_directory}/i686/${destination}" + repo-add "${destination}.db.tar.gz" ${parts} + for part in ${parts}; do remove_old_package_versions "${part}" done - popd > /dev/null rm "${work_dir}/package-states/${package}.testing" done diff --git a/bin/get-assignment b/bin/get-assignment index 7b0a5da..8adcdbf 100755 --- a/bin/get-assignment +++ b/bin/get-assignment @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh # receive one package to be built from the build-list whose dependencies # are already satisfied or which breaks a dependency cycle @@ -31,7 +31,7 @@ hand_out_assignment() { # we don't care anymore if an older version of this package was # "locked" or "broken" (we keep only marker for older "done" packages) ls "${work_dir}/package-states" | \ - grep "^${1//./\\.}\(\.[^.]\+\)\{3\}\.\(locked\|broken\)\$" | \ + grep "^$(str_to_regex "${1}")\(\.[^.]\+\)\{3\}\.\(locked\|broken\)\$" | \ sed "s|^|${work_dir}/package-states/|" | \ xargs -rn1 rm -f @@ -39,7 +39,7 @@ hand_out_assignment() { echo "${slave}" > "${work_dir}/package-states/$1.$2.$3.$4.locked" # lock every loop this package breaks - grep "^${1//./\\.}\$" "${work_dir}/build-list.loops/"loop_* | \ + grep -xF "${1}" "${work_dir}/build-list.loops/"loop_* | \ cut -d: -f1 | \ tee -a "${work_dir}/package-states/$1.$2.$3.$4.locked" | \ sed 's|$|.locked|' | \ @@ -153,7 +153,7 @@ for package in $( sort -k1nr,1 | \ awk '{print $2}' ); do - if assignment="$(grep "^${package//./\\.} " "${work_dir}/build-list")"; then + if assignment="$(grep "^$(str_to_regex "${package}") " "${work_dir}/build-list")"; then hand_out_assignment ${assignment} fi done diff --git a/bin/get-package-updates b/bin/get-package-updates index 09f2958..b78187e 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh # check for packages that need to be built, and build a list in the proper build order # Details: @@ -71,36 +71,35 @@ fi delete_package() { echo "$1" >> \ "${work_dir}/deletion-list.new" - sed -i "/^${1//./\\.} /d" "${work_dir}/build-list.new" + sed -i "/^$(str_to_regex "${1}") /d" "${work_dir}/build-list.new" } # Update git repositories (official packages, community packages and the repository of package customizations). -for repo in "${repo_paths[@]}"; do +for repo in ${repo_names}; do + eval repo_path='$repo_paths__'"${repo}" # TODO: # this is somewhat redundant and slow -- improve it! - git -C "${repo}" checkout -f master - git -C "${repo}" clean -xdf - git -C "${repo}" fetch + git -C "${repo_path}" checkout -f master + git -C "${repo_path}" clean -xdf + git -C "${repo_path}" fetch if ${pull}; then - git -C "${repo}" reset --hard origin/master + git -C "${repo_path}" reset --hard origin/master fi done # Read previous git revision numbers from files. -declare -A old_repo_revisions -declare -A new_repo_revisions - -for repo in "${!repo_paths[@]}"; do - old_repo_revisions["${repo}"]="$( +for repo in ${repo_names}; do + eval "old_repo_revisions__${repo}='$( cat "${work_dir}/${repo}.revision" 2> /dev/null || \ echo NONE - )" - new_repo_revisions["${repo}"]="$( - git -C "${repo_paths["${repo}"]}" rev-parse HEAD | \ + )'" + eval repo_path='$repo_paths__'"${repo}" + eval "new_repo_revisions__${repo}='$( + git -C "${repo_path}" rev-parse HEAD | \ tee "${work_dir}/${repo}.revision.new" - )" + )'" done # Create a lock file for build list. @@ -125,15 +124,18 @@ cp \ "${work_dir}/deletion-list" \ "${work_dir}/deletion-list.new" -for repo in "${!repo_paths[@]}"; do +for repo in ${repo_names}; do + eval repo_path='$repo_paths__'"${repo}" + eval old_repo_revision='$old_repo_revisions__'"${repo}" + eval new_repo_revision='$new_repo_revisions__'"${repo}" ( # if old revision unknown, mimic "git diff"-output - if [ "${old_repo_revisions["${repo}"]}" = "NONE" ]; then - git -C "${repo_paths["${repo}"]}" archive --format=tar HEAD | \ + if [ "${old_repo_revision}" = "NONE" ]; then + git -C "${repo_path}" archive --format=tar HEAD | \ tar -t | \ sed 's|^|A\t|' else - git -C "${repo_paths["${repo}"]}" diff --no-renames --name-status "${old_repo_revisions["${repo}"]}" HEAD + git -C "${repo_path}" diff --no-renames --name-status "${old_repo_revision}" HEAD fi ) | \ # only track changes in PKGBUILDs @@ -143,10 +145,12 @@ for repo in "${!repo_paths[@]}"; do # to the one of an original source repository sed 's|^\(.\t\)\([^/]\+\)/\([^/]\+\)/\(.\+\)$|\2 \1\3/repos/\2-x86_64/\4|' | \ while read -r pkg_repo rest; do - echo "${new_repo_revisions["$(find_git_repository_to_package_repository "${pkg_repo}")"]} ${rest}" + printf '%s %s\n' \ + "$(eval printf '$new_repo_revisions__%s' "$(find_git_repository_to_package_repository "${pkg_repo}")")" \ + "${rest}" done else - sed "s|^|${new_repo_revisions["${repo}"]} |" + sed "s|^|${new_repo_revision} |" fi | \ grep '^\S\+ .\s[^/]\+/repos/[^/]\+/PKGBUILD$' | \ # ignore i686 @@ -161,10 +165,10 @@ done | \ # new or modified PKGBUILD "A"|"M") - sed -i "/^${package//./\\.} /d" "${work_dir}/build-list.new" - echo "${package} ${git_revision} ${new_repo_revisions["archlinux32"]} ${repository}" >> \ + sed -i "/^$(str_to_regex "${package}") /d" "${work_dir}/build-list.new" + echo "${package} ${git_revision} ${new_repo_revisions__archlinux32} ${repository}" >> \ "${work_dir}/build-list.new" - sed -i "/^${package//./\\.}\$/d" "${work_dir}/deletion-list.new" + sed -i "/^$(str_to_regex "${package}")\$/d" "${work_dir}/deletion-list.new" ;; # deleted PKGBUILD @@ -205,7 +209,7 @@ echo 'apply blacklisting' black_listed='' black_listed_new="$( - cat "${repo_paths["archlinux32"]}/blacklist" + cat "${repo_paths__archlinux32}/blacklist" ls "${work_dir}/package-infos/" | \ grep '^lib32-.*\.depends' | \ sed ' @@ -324,11 +328,9 @@ fi rm -rf --one-file-system "${work_dir}/build-list.loops" ( - echo "build-list.loops" "build-list.new" "build-list" "deletion-list" - echo "${!repo_paths[@]}" | \ - sed 's@\( \|$\)@.revision\1@g' + printf '%s\n' "build-list.loops" "build-list.new" "build-list" "deletion-list" + printf '%s.revision\n' ${repo_names} ) | \ - tr ' ' '\n' | \ while read -r file; do mv "${work_dir}/${file}.new" "${work_dir}/${file}" done diff --git a/bin/return-assignment b/bin/return-assignment index 715a896..af6b728 100755 --- a/bin/return-assignment +++ b/bin/return-assignment @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh # report back on a build assignment # either on success via: @@ -58,7 +58,7 @@ if [ "$5" = 'ERROR' ]; then grep '\.locked$' | \ sed 's@^.*/\([^/]\+\)\.\([0-9a-f]\{40\}\.\)\{2\}[^.]\+\.locked$@\1@' )" - grep "^${1//./\\.}\$" "${work_dir}/build-list.loops/loop_"* 2> /dev/null | \ + grep -xF "${1}" "${work_dir}/build-list.loops/loop_"* 2> /dev/null | \ cut -d: -f1 | \ while read -r loop; do if [ -z "$( @@ -79,20 +79,20 @@ fi # the build was successful on the build slave -if ! grep -q "^${1//./\\.} $2 $3 $4\$" "${work_dir}/build-list" || +if ! grep -qxF "$1 $2 $3 $4" "${work_dir}/build-list" || ! [ -f "${work_dir}/package-states/$1.$2.$3.$4.locked" ]; then >&2 echo 'Sorry, the sent package is outdated.' exit 2 fi clean_up_tmp_dir() { - popd > /dev/null + cd "${base_dir}" rm -rf --one-file-system "${tmp_dir}" clean_up_lock_file } tmp_dir="$(mktemp -d)" -pushd "${tmp_dir}" > /dev/null +cd "${tmp_dir}" trap clean_up_tmp_dir EXIT # extract package(s) @@ -115,11 +115,10 @@ if [ -n "${signature_errors}" ]; then fi # check if the sent packages are the expected ones -packages=(*.pkg.tar.xz) +packages=*.pkg.tar.xz package_errors="$( ( - echo "${packages[@]}" | \ - tr ' ' '\n' | \ + printf '%s\n' ${packages} | \ sed ' s@\(-[^-]\+\)\{2\}-\([^-]\+\)\.pkg\.tar\.xz@ \2@ / any$/{s|any$|i686|} @@ -147,11 +146,11 @@ mv \ ( cd "${master_mirror_directory}/i686/${destination}" - repo-add "${destination}.db.tar.gz" "${packages[@]}" - # repo-add -v -s -k "${repo_key}" "${destination}.db.tar.gz" "${packages[@]}" + repo-add "${destination}.db.tar.gz" ${packages} + # repo-add -v -s -k "${repo_key}" "${destination}.db.tar.gz" ${packages} # remove old versions of same packages - for package in "${packages[@]}"; do + for package in ${packages}; do remove_old_package_versions "${package}" done ) @@ -159,8 +158,8 @@ mv \ # remove old state files (these should be only "done" markers, but # actually we don't care what it is) ls "${work_dir}/package-states" | \ - grep "^${1//./\\.}\(\.[^.]\+\)\{4\}\$" | \ - grep -v "^${1//./\\.}\.$2\.$3\.$4\.[^.]\+\$" | \ + grep "^$(str_to_regex "$1")\(\.[^.]\+\)\{4\}\$" | \ + grep -v "^$(str_to_regex "$1.$2.$3.$4")\.[^.]\+\$" | \ grep -v '\.testing$' | \ sed "s|^|${work_dir}/package-states/|" | \ xargs -rn1 rm -f @@ -172,9 +171,9 @@ grep -xF "$1" "${work_dir}/build-list.loops/loop_"* 2> /dev/null | \ xargs -rn1 rm -f # remove package from build list -sed -i "/^${1//./\\.} $2 $3 $4\$/d" "${work_dir}/build-list" +sed -i "/^$(str_to_regex "$1 $2 $3 $4")\$/d" "${work_dir}/build-list" # remove package lock file -printf '%s\n' "${packages[@]}" > \ +printf '%s\n' ${packages} > \ "${work_dir}/package-states/$1.$2.$3.$4.done" rm "${work_dir}/package-states/$1.$2.$3.$4.locked" diff --git a/bin/slave-build-connect b/bin/slave-build-connect index 930fdd9..b0bbfd5 100755 --- a/bin/slave-build-connect +++ b/bin/slave-build-connect @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/sh . "${0%/*}/../conf/default.conf" diff --git a/bin/why_dont_you_build b/bin/why_dont_you_build index 714816a..15a4ca5 100755 --- a/bin/why_dont_you_build +++ b/bin/why_dont_you_build @@ -1,11 +1,11 @@ -#!/bin/bash +#!/bin/sh # investigate, why a certain package is not being built . "${0%/*}/../conf/default.conf" for pkg in "$@"; do - grep "^${pkg//./\\.} " "${work_dir}/build-list" | \ + grep "^$(str_to_regex "${pkg}") " "${work_dir}/build-list" | \ while read -r package git_revision mod_git_revision repository; do if [ -f "${work_dir}/${package}.${git_revision}.${mod_git_revision}.${repository}.done" ] || |