path: root/bin/build-packages

#!/bin/sh

# build packages one by one, then upload the binary package to the repository server
# Details:
#  https://github.com/archlinux32/builder/wiki/Build-system#build-packages

# TODOs:

#  use different build commands for different repositories - do we need
#   this actually?

#  handle if build fails due to "local issues" (e.g. unclean
#   build environment, wrong mirror, ...)

#  force different cache for builds (since we don't want to build
#   against an empty i686 cache on a x86_64 host)

#  remove explicit installation of archlinux32-keyring inside the build
#   environment - this should be possible after the next pacman upgrade

#  avoid any-packages from x86_64 mirrors in /var/cache/pacman/pkg of
#   build slave

#  maybe use 'git archive' instead of 'git checkout' to get current
#   port -- this way we could run more than one build job per repo,
#   and we could get rid of the 'git clean', ... mess

. "${0%/*}/../conf/default.conf"

usage() {
  >&2 echo ''
  >&2 echo 'build-packages: build package(s) on the build-list'
  >&2 echo ''
  >&2 echo 'possible options:'
  >&2 echo '  -h|--help:   Show this help and exit.'
  >&2 echo '  -n count:    Build $count packages (if available), then exit.'
  >&2 echo '               $count=0 is interpreted as infinity.'
  >&2 echo '               The default is $count=1 or 0 iff -t or -x is given.'
  >&2 echo '  -t seconds:  Do not request new assignment(s) $seconds seconds after start.'
  >&2 echo '  -x:          If package build fails, do not request new assignment(s).'
  [ -z "$1" ] && exit 1 || exit $1
}

eval set -- "$(
  getopt -o hn:t:x \
    --long help \
    -n "$(basename "$0")" -- "$@" || \
  echo usage
)"

unset count
exit_after_failure=false
timeout=0

while true
do
  case "$1" in
    -h|--help)
      usage 0
    ;;
    -n)
      shift
      count="$1"
      [ ${count} -eq 0 ] && \
        count=-1
    ;;
    -t)
      shift
      timeout="$1"
    ;;
    -x)
      exit_after_failure=true
    ;;
    --)
      shift
      break
    ;;
    *)
      >&2 echo 'Whoops, forgot to implement option "'"$1"'" internally.'
      exit 42
    ;;
  esac
  shift
done

if [ $# -ne 0 ]; then
  >&2 echo 'Too many arguments.'
  usage
fi

if [ -z "${count}" ]; then
  if [ ${timeout} -ne 0 ] || ${exit_after_failure}; then
    count=-1
  else
    count=1
  fi
fi

if [ ${timeout} -ne 0 ]; then
  timeout=$((${timeout}+$(date +%s)))
fi

while [ ${count} -ne 0 ]; do

  if [ ${timeout} -ne 0 ] && [ ${timeout} -lt $(date +%s) ];
  then
    break
  fi

  err=0
  package="$(
    ssh \
      -i "${master_build_server_identity}" \
      -p "${master_build_server_port}" \
      "${master_build_server_user}@${master_build_server}" \
      'get-assignment'
  )" || err=$?

  case ${err} in

    #  0: ok, I gave you an assignment
    0)
      [ ${count} -gt 0 ] && \
        count=$((${count}-1))
      repository="${package##* }"
      package="${package% *}"
      mod_git_revision="${package##* }"
      package="${package% *}"
      git_revision="${package##* }"
      package="${package% *}"

      # Update git repositories (official packages, community packages and the repository of package customizations).

      for repo_name in ${repo_names}; do
        eval repo_path='$repo_paths__'"${repo_name}"
        git -C "${repo_path}" clean -df
        git -C "${repo_path}" reset --hard
        git -C "${repo_path}" checkout master
        git -C "${repo_path}" pull || true
      done

      eval git -C "$(printf '$repo_paths__%s' "$(find_repository_with_commit "${git_revision}")")" checkout "${git_revision}" > /dev/null 2>&1
      git -C "${repo_paths__archlinux32}" checkout "${mod_git_revision}" > /dev/null 2>&1

      PKGBUILD="$(find_pkgbuild "${package}" "${repository}")"

      if [ ! -r "${PKGBUILD}" ]; then
        echo "can't find PKGBUILD to package '${package}' from repository '${repository}': '${PKGBUILD}'"
        exit 1
      fi

      cd "${PKGBUILD%/*}"
      apply_package_customizations
      success=false
      for mirrored_source in false true; do

        if ${mirrored_source}; then
          # maybe a missing source is/was the problem?
          # try to download them from sources.archlinux.org/sources/$repo/$source
          source_name="$(
            makepkg --printsrcinfo | \
              sed -n '
                /^\s*\(epoch\|pkg\(base\|ver\|rel\)\) = /{s|^\s\+||;p}
                /^pkgname = /q
              ' | \
              sed '
                s|^pkgbase = \(.*\)$|0 \1-|
                s|^epoch = \(.*\)$|1 \1:|
                s|^pkgver = \(.*\)$|2 \1-|
                s|^pkgrel = \(.*\)$|3 \1.src.tar.gz|
              ' | \
              sort -k1n,1 | \
              sed '
                s|^[0-9] ||
                :a
                  N
                  s|\n[0-9] ||
                ta
              '
          )"
          git_repo="$(find_git_repository_to_package_repository "${repository}")"
          if ! wget -nc -nd "https://sources.archlinux.org/sources/${git_repo}/${source_name}"; then
            # we can't improve anything
            break
          fi
          tar -xzkf "${source_name}" --exclude PKGBUILD --strip-components=1 || true
        fi

        for parameters in '' '-c'; do
          rm -f *".pkg.tar.xz" *".pkg.tar.xz.sig"

          if staging-i686-build ${parameters} -- archlinux32-keyring 2>&1 > \
            "$(
              date -u --iso-8601=seconds | \
                cut -d+ -f1
            ).build-log"; then
            # build successful
            ls -1 *.pkg.tar.xz | \
              xargs -rn1 gpg --local-user="${package_key}" --detach-sign
            tar -cf 'package.tar' $(
                ls -1 | \
                  grep '\.pkg\.tar\.xz\(\.sig\)\?$' | \
                  grep -v -- '-debug-'
              )
            while true; do
              err=0
              ssh \
                -i "${master_build_server_identity}" \
                -p "${master_build_server_port}" \
                "${master_build_server_user}@${master_build_server}" \
                'return-assignment' "${package}" "${git_revision}" "${mod_git_revision}" "${repository}" \
                < 'package.tar' || \
                err=$?
              case ${err} in
                0)
                  # upload successful
                  break
                ;;
                1)
                  # 'return-assignment' was running already
                  wait_some_time 15
                ;;
                2)
                  >&2 echo 'I was too slow, the package is outdated. I will continue ...'
                  break
                ;;
                3)
                  >&2 echo "'return-assignment' reports a signature error."
                  exit 1
                ;;
                4)
                  >&2 echo "'return-assignment' reports too many or missing packages."
                  exit 1
                ;;
                *)
                  >&2 echo "unknown return code ${err} from 'return-assignment'"
                  exit 1
              esac
            done
            success=true
            break
          fi
        done
        if ${success}; then
          break
        fi
      done

      if ! ${success}; then
        for log in *.'.build-log'; do
          gzip "${log}"
        done
        tar -cf 'build-logs.gz.tar' \
          *'.build-log.gz'
        while ! ssh \
          -i "${master_build_server_identity}" \
          -p "${master_build_server_port}" \
          "${master_build_server_user}@${master_build_server}" \
          'return-assignment' "${package}" "${git_revision}" "${mod_git_revision}" "${repository}" 'ERROR' < \
          'build-logs.gz.tar'; do
          wait_some_time 15
        done
        if ${exit_after_failure}; then
          >&2 echo 'Build failed, exiting now'
          exit
        fi
      fi

      continue

    ;;

    #  1: come back (shortly) later - I was running already
    1)
      wait_some_time 15
      continue
    ;;

    #  2: come back later - there are still packages to be built,
    #     but currently none has all its dependencies ready
    2)
      wait_some_time 60
      continue
    ;;

    #  3: come back after the next run of get-package-updates - currently
    #     there are no pending packages
    3)

      >&2 echo 'Done. No more packages left to build.'
      exit 0

    ;;

    #  4: come back, when you've done your work - you hit the limit on
    #     maximum allowed parallel jobs per ip
    4)

      >&2 echo 'ERROR: There are too many parallel builds running on this machine.'
      exit 1

    ;;

    *)

      >&2 echo "ERROR: Unknown exit code ${err} from 'get-assignment'."
      exit 1

    ;;

  esac

done

>&2 echo 'Done.'