path: root/bin/nit-picker

#!/bin/sh

# shellcheck disable=SC2119

# shellcheck source=../lib/load-configuration
. "${0%/*}/../lib/load-configuration"

# this script shall host all the tests that are too slow to be run on
# the buildmaster (thus, it runs asynchronously and must therefor be
# resilient against transient states of the buildmaster):
# - check sanity of git
# TODO:
# - check for differences of dependencies between mysql and git
# - check for differences of dependencies between mysql and packages
# - check for installability of packages
# - It would be also nice to have the possibility to fix broken
#   dependencies in the database automatically.


if [ $# -ge 1 ] && [ "x$1" = 'x-n' ]; then
  >&2 echo 'not joining irc'
  irc=false
  shift
else
  irc=true
  # shellcheck disable=SC2016
  if [ $# -ne 0 ]; then
    >&2 echo 'usage: nit-picker [-n [$single_test]]'
    >&2 echo '  -n: do not join irc'
    >&2 echo '  $single_test: only execute the given test'
    exit 1
  fi
fi

clean_up() {
  rm -rf --one-file-system "${tmp_dir}"
}

tmp_dir=$(mktemp -d 'tmp.nit-picker.XXXXXXXXXX' --tmpdir)
trap 'clean_up' EXIT

if ${irc}; then
  if pgrep -x ii; then
    >&2 echo 'ii is already running - this will not work'
    exit 1
  fi

  rm -rf --one-file-system "${irc_dir}"

  ii -s irc.freenode.net -n nit-picker -f nit-picker >/dev/null 2>&1 &
  ii_pid=$!

  clean_up() {
    rm -rf --one-file-system "${tmp_dir}"
    kill "${ii_pid}"
  }

  # wait for nickserv complaint
  while ! grep -qF 'This nickname is registered. Please choose a different nickname' "${irc_dir}/nickserv/out"; do
    sleep 1
  done

  # register
  printf 'identify %s\n' "${irc_password}" | \
    sponge "${irc_dir}/nickserv/in"

  # wait for registering to succeed
  while ! grep -qF 'You are now identified for' "${irc_dir}/nickserv/out"; do
    sleep 1
  done

  # join channel
  echo '/j #archlinux32' | \
    sponge "${irc_dir}/in"

  while [ ! -f "${irc_dir}/#archlinux32/out" ]; do
    sleep 1
  done
else
  irc_say() {
    sed 's/^/irc: /'
  }
fi

mysql_load_min_and_max_versions

build_duration_check=0

while pgrep -x ii >/dev/null \
|| ! ${irc}; do
  if [ "$(date +%s)" -gt "$((build_duration_check + 60*60*24))" ]; then
    check_build_duration=true
    build_duration_check=$(date +%s)
  else
    check_build_duration=false
  fi
  if [ $# -eq 0 ]; then
    # shellcheck disable=SC2016
    {
      printf 'SELECT DISTINCT'
      printf ' "commit",'
      printf '`git_repositories`.`name`,'
      printf '`git_repositories`.`head`,'
      printf '`package_sources`.`git_revision`'
      printf ' FROM `package_sources`'
      mysql_join_package_sources_upstream_repositories
      mysql_join_upstream_repositories_git_repositories
      printf ';\n'

      printf 'SELECT DISTINCT'
      printf ' "commit",'
      printf '"archlinux32",'
      # shellcheck disable=SC2154
      printf '"%s",' \
        "${repo_heads__archlinux32}"
      printf '`package_sources`.`mod_git_revision`'
      printf ' FROM `package_sources`'
      printf ';\n'

      printf 'SELECT DISTINCT'
      printf ' "binary-signature",'
      mysql_package_name_query
      printf ' FROM `binary_packages`'
      mysql_join_binary_packages_architectures
      mysql_join_binary_packages_binary_packages_in_repositories
      mysql_join_binary_packages_in_repositories_repositories
      printf ' WHERE `repositories`.`is_on_master_mirror`'
      printf ';\n'

      if "${check_build_duration}"; then
        printf 'SELECT'
        printf ' "build-duration",'
        printf '`build_slaves`.`name`'
        printf ' FROM `build_slaves`'
        printf ';\n'
      fi

      printf 'SELECT DISTINCT'
      printf ' "binary-dependencies",'
      mysql_package_name_query
      printf ' FROM `binary_packages`'
      mysql_join_binary_packages_architectures
      mysql_join_binary_packages_binary_packages_in_repositories
      mysql_join_binary_packages_in_repositories_repositories
      printf ' WHERE `repositories`.`is_on_master_mirror`'
      printf ';\n'
    } | \
      mysql_run_query | \
      tr '\t' ' ' | \
      shuf
  else
    printf '%s\n' "$*"
  fi | \
    while read -r action parameters; do
      if ${irc} && ! pgrep -x ii >/dev/null; then
        break
      fi
      case "${action}" in
        'commit') # check whether a given commit is present in the git repo
          git_repo="${parameters%% *}"
          git_rev="${parameters#${git_repo} }"
          git_head="${git_rev%% *}"
          git_rev="${git_rev#${git_head} }"
          # shellcheck disable=SC2016
          eval "$(
            printf 'git_dir="${repo_paths__%s}"\n' \
              "${git_repo}"
          )"
          # shellcheck disable=SC2154
          if ! git -C "${git_dir}" cat-file -t "${git_rev}" 2> /dev/null | \
            grep -qxF 'commit'; then
            git -C "${git_dir}" fetch --all -p >/dev/null 2>&1
            if ! git -C "${git_dir}" cat-file -t "${git_rev}" 2> /dev/null | \
              grep -qxF 'commit'; then
              printf 'commit %s is missing from repository %s\n' \
                "${git_rev}" \
                "${git_repo}" \
              | irc_say
            fi
          fi
          # shellcheck disable=SC2154
          if ! git -C "${git_dir}" cat-file -t "${git_head}" 2> /dev/null | \
            grep -qxF 'commit'; then
            git -C "${git_dir}" fetch --all -p >/dev/null 2>&1
            if ! git -C "${git_dir}" cat-file -t "${git_head}" 2> /dev/null | \
              grep -qxF 'commit'; then
              printf 'commit %s is missing from repository %s\n' \
                "${git_head}" \
                "${git_repo}" \
              | irc_say
            fi
          fi
          # shellcheck disable=SC2154
          if ! git -C "${git_dir}" merge-base --is-ancestor "${git_rev}" "${git_head}" 2> /dev/null; then
            current_git_head=$(
              # shellcheck disable=SC2016
              {
                printf 'SELECT DISTINCT'
                printf ' `git_repositories`.`head`'
                printf ' FROM `git_repositories`'
                printf ' WHERE `git_repositories`.`name`=from_base64("%s");\n' \
                  "$(
                    printf '%s' "${git_repo}" \
                    | base64 -w0
                  )"
              } \
              | mysql_run_query
            )
            if ! git -C "${git_dir}" merge-base --is-ancestor "${git_rev}" "${current_git_head}" 2> /dev/null; then
              git -C "${git_dir}" fetch --all -p >/dev/null 2>&1
              if ! git -C "${git_dir}" merge-base --is-ancestor "${git_rev}" "${current_git_head}" 2> /dev/null; then
                printf 'commit %s is not an ancestor of HEAD %s in repository %s\n' \
                  "${git_rev}" \
                  "${current_git_head}" \
                  "${git_repo}" \
                | irc_say
              fi
            fi
          fi
        ;;
        'binary-dependencies')
          if ! ${master_mirror_rsync_command} \
            "${master_mirror_rsync_directory}/pool/${parameters}" \
            "${tmp_dir}/"; then
            rm -f "${tmp_dir}/${parameters}"
            continue
          fi
          extract_dependencies_from_package \
            "${tmp_dir}/${parameters}" \
          > "${tmp_dir}/pkg-deps"
          # shellcheck disable=SC2016
          {
            printf 'SELECT'
            printf ' `dependency_types`.`name`,'
            printf '`install_targets`.`name`,'
            printf '`dependencies`.`version_relation`,'
            printf '`versions`.`epoch`,'
            printf '`versions`.`version`'
            printf ' FROM `binary_packages`'
            mysql_join_binary_packages_dependencies
            mysql_join_binary_packages_architectures
            mysql_join_dependencies_dependency_types
            mysql_join_dependencies_versions
            mysql_join_dependencies_install_targets
            printf ' WHERE '
            mysql_package_name_query
            printf '="%s"' \
              "${parameters}"
            printf ' AND `dependency_types`.`name` IN ("run","make","check")'
            printf ';\n'
          } \
          | mysql_run_query \
          | tr '\t' ' ' \
          | sort -u \
          > "${tmp_dir}/db-deps"
          if ! diff -q "${tmp_dir}/db-deps" "${tmp_dir}/pkg-deps"; then
            build_date=$(
              bsdtar -Oxf "${tmp_dir}/${parameters}" '.PKGINFO' \
              | sed '
                s/^builddate = //
                t
                d
              '
            )
            build_date=$(
              date -I -d@"${build_date}"
            )
            printf 'dependencies of %s (built on %s) differ between the package and our database\n' \
              "${parameters}" \
              "${build_date}" \
            | irc_say 'deep42thought'
            if ! ${irc}; then
              diff -u --color "${tmp_dir}/db-deps" "${tmp_dir}/pkg-deps" || true
            fi
            if [ $# -eq 0 ]; then
              sleep 60
            fi
          fi
          rm \
            "${tmp_dir}/${parameters}" \
            "${tmp_dir}/db-deps" \
            "${tmp_dir}/pkg-deps"
        ;;
        'binary-signature')
          if ! ${master_mirror_rsync_command} \
            "${master_mirror_rsync_directory}/pool/${parameters}" \
            "${master_mirror_rsync_directory}/pool/${parameters}.sig" \
            "${tmp_dir}/"; then
            rm -f "${tmp_dir}/${parameters}" "${tmp_dir}/${parameters}.sig"
            continue
          fi
          unset error_message
          if ! gpg_output=$(
            gpg --batch --status-fd 1 -q --homedir /etc/pacman.d/gnupg \
              --verify "${tmp_dir}/${parameters}.sig" "${tmp_dir}/${parameters}" \
              2>/dev/null
          ); then
            sleep 1
            if ! gpg_output=$(
              gpg --batch --status-fd 1 -q --homedir /etc/pacman.d/gnupg \
                --verify "${tmp_dir}/${parameters}.sig" "${tmp_dir}/${parameters}" \
                2>/dev/null
            ); then
              error_message="package ${parameters} has an invalid signature."
            fi
          fi
          if [ -z "${error_message}" ]; then
            gpg_key=$(
              printf '%s\n' "${gpg_output}" \
              | sed '
                s/^\[GNUPG:] KEY_CONSIDERED \([0-9A-F]\{40\}\) 0$/\1/
                t
                d
              ' \
              | sort -u
            )
            if [ -z "${gpg_key}" ]; then
              error_message="cannot find pgp_key of package ${parameters}."
            fi
          fi
          if [ -z "${error_message}" ]; then
            for expiration in $(
              gpg --batch --homedir /etc/pacman.d/gnupg --with-colons --list-keys "0x${gpg_key}" \
              2>/dev/null \
              | grep '^\(sub\|pub\):' \
              | cut -d: -f7
            ); do
              expiration_days=$(((expiration - $(date +%s))/24/60/60))
              if [ ${expiration_days} -lt 100 ]; then
                error_message=$(
                  printf 'signing key %s (from %s) for package %s expires on %s (in %s < 100 days).\n' \
                  "${gpg_key}" \
                  "$(
                    gpg --batch --homedir /etc/pacman.d/gnupg --with-colons --list-keys "0x${gpg_key}" \
                    2>/dev/null \
                    | grep '^\(uid\):' \
                    | cut -d: -f10
                  )" \
                  "${parameters}" \
                  "$(date -I -d@"${expiration}")" \
                  "${expiration_days}"
                )
                break
              fi
            done
          fi
          if [ -n "${error_message}" ]; then
            printf '%s\n' "${error_message}" \
            | irc_say
            if [ $# -eq 0 ]; then
              sleep 60
            fi
          fi
          rm \
            "${tmp_dir}/${parameters}" \
            "${tmp_dir}/${parameters}.sig"
        ;;
        'build-duration')
          infos=$(
            # shellcheck disable=SC2016
            {
              printf 'SELECT'
              printf '`persons`.`name`,'
              printf '`build_slaves`.`name`,'
              printf '`architectures`.`name`,'
              printf '`package_sources`.`pkgbase`,'
              printf 'UNIX_TIMESTAMP(NOW())-UNIX_TIMESTAMP('
                printf 'MAX('
                  printf 'IF('
                    printf '`ssh_log`.`action`="get-assignment",'
                    printf '`ssh_log`.`date`,'
                    printf 'NULL'
                  printf ')'
                printf ')'
              printf ')'
              printf ' FROM `build_slaves`'
              mysql_join_build_slaves_build_assignments
              mysql_join_build_assignments_architectures
              mysql_join_build_assignments_package_sources
              mysql_join_build_slaves_ssh_keys
              mysql_join_ssh_keys_persons
              mysql_join_build_slaves_ssh_log
              printf ' WHERE `ssh_log`.`date`>=ADDDATE(NOW(),"-7 00:00:00")'
              printf ' AND `build_slaves`.`name`=from_base64("%s")' \
                "$(
                  printf '%s' "${parameters}" \
                  | base64 -w0
                )"
              printf ' GROUP BY `build_slaves`.`id`'
              printf ';\n'
            } \
            | mysql_run_query \
            | tr '\t' ' '
          )
          if [ -z "${infos}" ]; then
            continue
          fi
          if [ "${infos##* }" -lt $((60*60*24)) ]; then
            continue
          fi
          printf '%s %s\n' \
            "${infos% *}" \
            "$(
              date -d"@${infos##* }" +'%-j %H:%M:%S'
            )" \
          | awk '{print $1 ": your slave " $2 " builds " $3 "/" $4 " for more than a day, now (" ($5-1) " day(s) " $6 ")"}' \
          | irc_say
          if [ $# -eq 0 ]; then
            sleep 60
          fi
        ;;
        *)
          >&2 printf 'action "%s" is not yet implemented ...\n' "${action}"
        ;;
      esac
    done
  if [ $# -ge 1 ]; then
    break
  fi
  sleep 120
done