| index : builder | |
| Archlinux32 build system | gitolite user |
| summaryrefslogtreecommitdiff |
| -rwxr-xr-x | bin/get-package-updates | 302 | ||||
| -rw-r--r-- | conf/.gitignore | 1 | ||||
| -rwxr-xr-x | conf/default.conf | 41 |
diff --git a/bin/get-package-updates b/bin/get-package-updates index 572053f..50c8216 100755 --- a/bin/get-package-updates +++ b/bin/get-package-updates @@ -5,24 +5,13 @@ # https://github.com/archlinux32/builder/wiki/Build-system#get-package-updates # TODO: -# use tsort and mksrcinfo - -set -e - -lock_file="/tmp/get-package-updates.lock" - -# TODO: # include repository of package customizations -declare -A repo_paths -repo_paths["packages"]="/usr/src/archlinux/packages" -repo_paths["community"]="/usr/src/archlinux/community" - -base_dir="$(dirname "$(readlink -f "$0")")/.." +. "${0%/*}/../conf/default.conf" # find the PKGBUILD of a given package in a given repository # TODO: -# include repository of package customizations +# _properly_ include repository of package customizations find_pkgbuild() { @@ -31,6 +20,7 @@ find_pkgbuild() { ls "${prefix}/$1/repos/$2-"*"/PKGBUILD" 2> /dev/null && break done | \ tr ' ' '\n' | \ + grep -v -- '-i686/PKGBUILD$' | \ grep -v -- '-\(staging\|testing\)-[^/]\+/PKGBUILD$' | \ sort | \ tail -n1 @@ -52,9 +42,16 @@ done declare -A old_repo_revisions for repo in "${!repo_paths[@]}"; do - old_repo_revisions["${repo}"]="$(cat ${base_dir}/work/${repo}.revision || echo NONE)" + old_repo_revisions["${repo}"]="$( + cat "${work_dir}/${repo}.revision" 2> /dev/null || \ + echo NONE + )" + git -C "${repo_paths["${repo}"]}" rev-parse HEAD > \ + "${work_dir}/${repo}.revision.new" done +echo 'Check modified packages from the last update, and put them to the build list.' + # Check modified packages from the last update, and put them to the build list. # If a package is updated, but already on the rebuild list, then just update the git revision number. # If a package is deleted, remove from the rebuild list, and add it to the deletion list. @@ -63,38 +60,43 @@ done # TODO: # ignore packages from the blacklist -for repo in "${!repo_paths[@]}"; do - current_HEAD="$( - git -C "${repo_paths["${repo}"]}" rev-parse HEAD - )" - ( - if [ "${old_repo_revisions["${repo}"]}" == "NONE" ]; then - git -C "${repo_paths["${repo}"]}" archive --format=tar HEAD | \ - tar -t | \ - sed 's|^|A\t|' - else - git -C "${repo_paths["${repo}"]}" diff --name-status "${old_repo_revisions["${repo}"]}" HEAD - fi - ) | \ - grep '^.\s[^/]\+/repos/[^/]\+/PKGBUILD$' | \ - sed 's|^\(.\)\t\([^/]\+\)/repos/\([^/]\+\)-[^/-]\+/PKGBUILD$|\1 \2 '"${current_HEAD}"' \3|' | \ - grep -v '\(staging\|testing\)$' -done | \ - sort -u | \ +cp "${work_dir}/build-list"{,.new} +cp "${work_dir}/deletion-list"{,.new} + +( + for repo in "${!repo_paths[@]}"; do + current_HEAD="$(cat "${work_dir}/${repo}.revision.new")" + ( + if [ "${old_repo_revisions["${repo}"]}" == "NONE" ]; then + git -C "${repo_paths["${repo}"]}" archive --format=tar HEAD | \ + tar -t | \ + sed 's|^|A\t|' + else + git -C "${repo_paths["${repo}"]}" diff --name-status "${old_repo_revisions["${repo}"]}" HEAD + fi + ) | \ + grep '^.\s[^/]\+/repos/[^/]\+/PKGBUILD$' | \ + grep -v -- '-i686/PKGBUILD$' | \ + sed 's|^\(.\)\t\([^/]\+\)/repos/\([^/]\+\)-[^/-]\+/PKGBUILD$|\1 \2 '"${current_HEAD}"' \3|' | \ + grep -v '\(staging\|testing\)$' + done | \ + sort -u + sed "s|^\(.*\)\$|D \1 $(cat "${work_dir}/archlinux32.revision.new") archlinux32|" "${repo_paths["archlinux32"]}/blacklist" +) | \ while read -r mode package git_revision repository; do case "${mode}" in "A"|"M") - sed -i "/^${package} /d" "${base_dir}/work/build-list" + sed -i "/^${package} /d" "${work_dir}/build-list" echo "${package} ${git_revision} ${repository}" >> \ - "${base_dir}/work/build-list" - sed -i "/^${package}\$/d" "${base_dir}/work/deletion-list" + "${work_dir}/build-list.new" + sed -i "/^${package}\$/d" "${work_dir}/deletion-list.new" ;; "D") echo "${package}" >> \ - "${base_dir}/work/deletion-list" - sed -i "/^${package} /d" "${base_dir}/work/build-list" + "${work_dir}/deletion-list.new" + sed -i "/^${package} /d" "${work_dir}/build-list.new" ;; *) @@ -105,176 +107,106 @@ done | \ esac done +echo 'Extract dependencies of packages.' + # Put the list in the proper build order. +# First, we extract the dependencies of each package. +# TODO: +# package modifications need to be included here + +mkdir -p "${work_dir}/package-infos" -declare -A packages_built -declare -A packages_needed -declare -A packages_needed_for_make +rm -f "${work_dir}/build-order" -while read -r package dummy repository; do +while read -r package git_rev repository; do PKGBUILD="$(find_pkgbuild "${package}" "${repository}")" if [ ! -r "${PKGBUILD}" ]; then - echo "'${package}'" "'${repository}'" "'${PKGBUILD}'" + echo "can't find PKGBUILD to package '${package}' from repository '${repository}': '${PKGBUILD}'" exit 1 fi - content="$( - sed 's|#.*$||;s|^\s\+||;s|\s\+$||;/^$/d' "${PKGBUILD}" | \ - sed ' - :begin; - $!N; - s@^\([^=]\+=([^)]*\)\n\(.*\)$@\1 \2@; - tbegin; - P; - D - ' - )" - - packages_built["${package}"]="$( - echo "${content}" | \ - grep '^provides=\|^pkgname=' | \ - sed 's|^[^=]\+=||; s|=.*$||' | \ - tr "' ()\"" '\n' | \ - sort -u | \ - sed '/^$/d' - )" + if [ ! -e "${work_dir}/package-infos/${package}.${git_rev}.builds" ] || \ + [ ! -e "${work_dir}/package-infos/${package}.${git_rev}.needs" ]; then - packages_needed["${package}"]="$( ( - echo "${content}" | \ - grep '^depends=' | \ - sed 's|^[^=]\+=||; s|>\?=.*$||' | \ - tr "' ()\"" '\n' | \ - sort -u | \ - sed '/^$/d' - echo "${packages_built["${package}"]}" | \ - sed 'p' - ) | \ - sort | \ - uniq -u - )" + cd "${PKGBUILD%/*}" + mksrcinfo -o "${work_dir}/package-infos/${package}.${git_rev}.SRCINFO" + ) + + grep '^\('$'\t''provides\|pkgname\) = ' "${work_dir}/package-infos/${package}.${git_rev}.SRCINFO" | \ + cut -d= -f2 | \ + sed 's|^\s\+||; s|[<>]$||' | \ + sort -u > \ + "${work_dir}/package-infos/${package}.${git_rev}.builds" - packages_needed_for_make["${package}"]="$( ( - echo "${content}" | \ - grep '^makedepends=' | \ - sed 's|^[^=]\+=||; s|>\?=.*$||' | \ - tr "' ()\"" '\n' | \ - sort -u | \ - sed '/^$/d' - echo "${packages_built["${package}"]}" | \ + ( + sed -n '/^pkgname = /q;/^'$'\t''depends = /p' "${work_dir}/package-infos/${package}.${git_rev}.SRCINFO" + grep '^'$'\t''\(makedepends\|checkdepends\) = ' "${work_dir}/package-infos/${package}.${git_rev}.SRCINFO" + ) | \ + cut -d= -f2 | \ + sed 's|^\s\+||; s|[<>]$||' | \ + sort -u + cat "${work_dir}/package-infos/${package}.${git_rev}.builds" | \ sed 'p' ) | \ sort | \ - uniq -u - )" - -done < "${base_dir}/work/build-list" - -rm -f "${base_dir}/work/build-list.new" + uniq -u > \ + "${work_dir}/package-infos/${package}.${git_rev}.needs" -while [ ${#packages_built[@]} -gt 0 ]; do + rm "${work_dir}/package-infos/${package}.${git_rev}.SRCINFO" - found_something=false - - for package in "${!packages_built[@]}"; do - - if [ -z "$( - ( - echo "${packages_needed["${package}"]}" - echo "${packages_needed_for_make["${package}"]}" - echo "${!packages_built[@]}" | \ - tr ' ' '\n' - ) | \ - sort | \ - uniq -d - )" ]; then - grep "^${package} " "${base_dir}/work/build-list" >> \ - "${base_dir}/work/build-list.new" - unset packages_built["${package}"] - unset packages_needed["${package}"] - unset packages_needed_for_make["${package}"] - found_something=true - fi - done - - if ${found_something}; then - continue - fi - - for package in "${!packages_built[@]}"; do - - if [ -z "$( - ( - echo "${packages_needed_for_make["${package}"]}" - echo "${!packages_built[@]}" | \ - tr ' ' '\n' - ) | \ - sort | \ - uniq -d - )" ]; then - grep "^${package} " "${base_dir}/work/build-list" >> \ - "${base_dir}/work/build-list.new" - unset packages_built["${package}"] - unset packages_needed["${package}"] - unset packages_needed_for_make["${package}"] - found_something=true - break - fi - done - - if ${found_something}; then - continue - fi - - for package in "${!packages_built[@]}"; do - - if [ -z "$( - ( - echo "${packages_needed["${package}"]}" - echo "${!packages_built[@]}" | \ - tr ' ' '\n' - ) | \ - sort | \ - uniq -d - )" ]; then - grep "^${package} " "${base_dir}/work/build-list" >> \ - "${base_dir}/work/build-list.new" - unset packages_built["${package}"] - unset packages_needed["${package}"] - unset packages_needed_for_make["${package}"] - found_something=true - break - fi - done - - if ${found_something}; then - continue fi - >&2 echo 'uaagh: there seem to be cyclic dependencies - I give up.' - >&2 echo "packages left: ${!packages_built[@]}" - - for pkg in "${!packages_built[@]}"; do - echo "${pkg}:" - echo ${packages_needed["${pkg}"]} + while read target; do + echo "${target} ${target}" >> \ + "${work_dir}/build-order" + while read dependency; do + echo "${target} ${dependency}" >> \ + "${work_dir}/build-order" + done < "${work_dir}/package-infos/${package}.${git_rev}.needs" + done < "${work_dir}/package-infos/${package}.${git_rev}.builds" + +done < "${work_dir}/build-list.new" + +echo 'Now actually sort it.' + +( + tsort "${work_dir}/build-order" 2> "${work_dir}/tsort.error" | \ + nl -ba | \ + awk '{print $1 " not-git whatever " $2}' + awk '{print "0 " $2 " " $3 " " $1}' < \ + "${work_dir}/build-list.new" +) | \ + sort -k4,4 -k1nr | \ + uniq -f3 -D | \ + sed 'N;s|\n| |' | \ + sort -k1n,1 | \ + awk '{print $4 " " $6 " " $7}' > \ + "${work_dir}/build-list.new.new" + +if [ -s "${work_dir}/tsort.error" ]; then + >&2 echo 'WARNING: There is a dependency cycle!' + >&2 cat "${work_dir}/tsort.error" + >&2 echo + >&2 echo 'I will continue anyway.' +else + rm "${work_dir}/tsort.error" +fi + +# Move the .new-files to the actual files + +( + echo "build-list.new" "build-list" "deletion-list" + echo "${!repo_paths[@]}" | \ + sed 's@\( \|$\)@.revision\1@g' +) | \ + tr ' ' '\n' | \ + while read -r file; do + mv "${work_dir}/${file}.new" "${work_dir}/${file}" done -# TODO: Break circular dependencies - - exit 1 -done - -mv "${base_dir}/work/build-list.new" "${base_dir}/work/build-list" - -# Write the current git revision numbers to files. - -for repo in "${!repo_paths[@]}"; do - git -C "${repo_paths["${repo}"]}" rev-parse HEAD > \ - "${base_dir}/work/${repo}.revision" -done - # Remove the lock file rm -f "${lock_file}" diff --git a/conf/.gitignore b/conf/.gitignore new file mode 100644 index 0000000..f21e1b2 --- /dev/null +++ b/conf/.gitignore @@ -0,0 +1 @@ +local.conf diff --git a/conf/default.conf b/conf/default.conf new file mode 100755 index 0000000..1517272 --- /dev/null +++ b/conf/default.conf @@ -0,0 +1,41 @@ +#!/bin/bash + +# set up some common variables + +set -e + +base_dir="$(dirname "$(readlink -f "$0")")/.." + +work_dir="${base_dir}/work" + +declare -A repo_paths +repo_paths["packages"]="${work_dir}/repos/packages" +repo_paths["community"]="${work_dir}/repos/community" +repo_paths["archlinux32"]="${work_dir}/repos/packages32" + +lock_file="/tmp/${0##*/}.lock" + +# possibly pull in custom modifications + +[ -r "${base_dir}/conf/local.conf" ] && . "${base_dir}/conf/local.conf" + +# check / set up environment + +mkdir -p "${work_dir}" +touch "${work_dir}/build-list" +touch "${work_dir}/deletion-list" + +for repo in "${!repo_paths[@]}"; do + + mkdir -p "${repo_paths["${repo}"]%/*}" + + if ! git -C "${repo_paths["${repo}"]}" status &> /dev/null; then + if [ "${repo}" == "packages32" ]; then + repo_source='git@github.com:archlinux32/packages.git' + else + repo_source="git://git.archlinux.org/svntogit/${repo}.git" + fi + git clone "${repo_source}" "${repo_paths["${repo}"]}" + fi + +done |