From a44310b220fab8ddc602ffd757c293288bb2cd3b Mon Sep 17 00:00:00 2001 From: David Runge Date: Thu, 1 Jul 2021 00:34:21 +0200 Subject: Rename build script .gitlab/ci/build_archiso.sh: Rename .gitlab/ci/build-inside-vm.sh to .gitlab/ci/build_archiso.sh. .gitlab-ci.yml: Rename BUILD_SCRIPT to build_archiso.sh. --- .gitlab-ci.yml | 2 +- .gitlab/ci/build-inside-vm.sh | 270 ------------------------------------------ .gitlab/ci/build_archiso.sh | 270 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 271 insertions(+), 271 deletions(-) delete mode 100755 .gitlab/ci/build-inside-vm.sh create mode 100755 .gitlab/ci/build_archiso.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2d615e7..3f77b7e 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,7 +6,7 @@ default: image: "archlinux:latest" variables: - BUILD_SCRIPT: ./.gitlab/ci/build-inside-vm.sh + BUILD_SCRIPT: ./.gitlab/ci/build_archiso.sh QEMU_BUILD_TIMEOUT: 2400 QEMU_VM_MEMORY: 3072 ARCHISO_COW_SPACE_SIZE: 2g diff --git a/.gitlab/ci/build-inside-vm.sh b/.gitlab/ci/build-inside-vm.sh deleted file mode 100755 index 6424b32..0000000 --- a/.gitlab/ci/build-inside-vm.sh +++ /dev/null @@ -1,270 +0,0 @@ -#!/usr/bin/env bash -# -# This script is run within a virtual environment to build the available archiso profiles and their available build -# modes and create checksum files for the resulting images. -# The script needs to be run as root and assumes $PWD to be the root of the repository. -# -# Dependencies: -# * all archiso dependencies -# * coreutils -# * gnupg -# * openssl -# * zsync -# -# $1: profile -# $2: buildmode - -set -euo pipefail -shopt -s extglob - -readonly orig_pwd="${PWD}" -readonly output="${orig_pwd}/output" -readonly tmpdir_base="${orig_pwd}/tmp" -readonly profile="${1}" -readonly buildmode="${2}" -readonly install_dir="arch" - -tmpdir="" -tmpdir="$(mktemp --dry-run --directory --tmpdir="${tmpdir_base}")" -gnupg_homedir="" -codesigning_dir="" -codesigning_cert="" -codesigning_key="" -pgp_key_id="" - -print_section_start() { - # gitlab collapsible sections start: https://docs.gitlab.com/ee/ci/jobs/#custom-collapsible-sections - local _section _title - _section="${1}" - _title="${2}" - - printf "\e[0Ksection_start:%(%s)T:%s\r\e[0K%s\n" '-1' "${_section}" "${_title}" -} - -print_section_end() { - # gitlab collapsible sections end: https://docs.gitlab.com/ee/ci/jobs/#custom-collapsible-sections - local _section - _section="${1}" - - printf "\e[0Ksection_end:%(%s)T:%s\r\e[0K\n" '-1' "${_section}" -} - -cleanup() { - # clean up temporary directories - print_section_start "cleanup" "Cleaning up temporary directory" - - if [ -n "${tmpdir_base:-}" ]; then - rm -fr "${tmpdir_base}" - fi - - print_section_end "cleanup" -} - -create_checksums() { - # create checksums for files - # $@: files - local _file_path _file_name _current_pwd - _current_pwd="${PWD}" - - print_section_start "checksums" "Creating checksums" - - for _file_path in "$@"; do - cd "$(dirname "${_file_path}")" - _file_name="$(basename "${_file_path}")" - b2sum "${_file_name}" > "${_file_name}.b2" - md5sum "${_file_name}" > "${_file_name}.md5" - sha1sum "${_file_name}" > "${_file_name}.sha1" - sha256sum "${_file_name}" > "${_file_name}.sha256" - sha512sum "${_file_name}" > "${_file_name}.sha512" - ls -lah "${_file_name}."{b2,md5,sha{1,256,512}} - cat "${_file_name}."{b2,md5,sha{1,256,512}} - done - cd "${_current_pwd}" - - print_section_end "checksums" -} - -create_zsync_delta() { - # create zsync control files for files - # $@: files - local _file - - print_section_start "zsync_delta" "Creating zsync delta" - - for _file in "$@"; do - if [[ "${buildmode}" == "bootstrap" ]]; then - # zsyncmake fails on 'too long between blocks' with default block size on bootstrap image - zsyncmake -v -b 512 -C -u "${_file##*/}" -o "${_file}".zsync "${_file}" - else - zsyncmake -v -C -u "${_file##*/}" -o "${_file}".zsync "${_file}" - fi - done - - print_section_end "zsync_delta" -} - -create_metrics() { - local _metrics="${output}/${profile}/metrics.txt" - # create metrics - print_section_start "metrics" "Creating metrics" - - { - # create metrics based on buildmode - case "${buildmode}" in - iso) - printf 'image_size_mebibytes{image="%s"} %s\n' \ - "${profile}" \ - "$(du -m -- "${output}/${profile}/"*.iso | cut -f1)" - printf 'package_count{image="%s"} %s\n' \ - "${profile}" \ - "$(sort -u -- "${tmpdir}/${profile}/iso/"*/pkglist.*.txt | wc -l)" - if [[ -e "${tmpdir}/${profile}/efiboot.img" ]]; then - printf 'eltorito_efi_image_size_mebibytes{image="%s"} %s\n' \ - "${profile}" \ - "$(du -m -- "${tmpdir}/${profile}/efiboot.img" | cut -f1)" - fi - # shellcheck disable=SC2046 - # shellcheck disable=SC2183 - printf 'initramfs_size_mebibytes{image="%s",initramfs="%s"} %s\n' \ - $(du -m -- "${tmpdir}/${profile}/iso/"*/boot/**/initramfs*.img | \ - awk -v profile="${profile}" \ - 'function basename(file) { - sub(".*/", "", file) - return file - } - { print profile, basename($2), $1 }' - ) - ;; - netboot) - printf 'netboot_size_mebibytes{image="%s"} %s\n' \ - "${profile}" \ - "$(du -m -- "${output}/${profile}/${install_dir}/" | tail -n1 | cut -f1)" - printf 'netboot_package_count{image="%s"} %s\n' \ - "${profile}" \ - "$(sort -u -- "${tmpdir}/${profile}/iso/"*/pkglist.*.txt | wc -l)" - ;; - bootstrap) - printf 'bootstrap_size_mebibytes{image="%s"} %s\n' \ - "${profile}" \ - "$(du -m -- "${output}/${profile}/"*.tar*(.gz|.xz|.zst) | cut -f1)" - printf 'bootstrap_package_count{image="%s"} %s\n' \ - "${profile}" \ - "$(sort -u -- "${tmpdir}/${profile}/"*/bootstrap/root.*/pkglist.*.txt | wc -l)" - ;; - esac - } > "${_metrics}" - ls -lah "${_metrics}" - cat "${_metrics}" - - print_section_end "metrics" -} - -create_ephemeral_pgp_key() { - # create an ephemeral PGP key for signing the rootfs image - print_section_start "ephemeral_pgp_key" "Creating ephemeral PGP key" - - gnupg_homedir="$tmpdir/.gnupg" - mkdir -p "${gnupg_homedir}" - chmod 700 "${gnupg_homedir}" - - cat << __EOF__ > "${gnupg_homedir}"/gpg.conf -quiet -batch -no-tty -no-permission-warning -export-options no-export-attributes,export-clean -list-options no-show-keyring -armor -no-emit-version -__EOF__ - - gpg --homedir "${gnupg_homedir}" --gen-key <> "${codesigning_conf}" - openssl req \ - -newkey rsa:4096 \ - -keyout "${codesigning_key}" \ - -nodes \ - -sha256 \ - -x509 \ - -days 365 \ - -out "${codesigning_cert}" \ - -config "${codesigning_conf}" \ - -subj "${codesigning_subj}" \ - -extensions codesigning - - print_section_end "ephemeral_codesigning_key" -} - -run_mkarchiso() { - # run mkarchiso - create_ephemeral_pgp_key - create_ephemeral_codesigning_key - - print_section_start "mkarchiso" "Running mkarchiso" - mkdir -p "${output}/${profile}" "${tmpdir}/${profile}" - GNUPGHOME="${gnupg_homedir}" ./archiso/mkarchiso \ - -D "${install_dir}" \ - -c "${codesigning_cert} ${codesigning_key}" \ - -g "${pgp_key_id}" \ - -o "${output}/${profile}" \ - -w "${tmpdir}/${profile}" \ - -m "${buildmode}" \ - -v "configs/${profile}" - - print_section_end "mkarchiso" - - if [[ "${buildmode}" =~ "iso" ]]; then - create_zsync_delta "${output}/${profile}/"*.iso - create_checksums "${output}/${profile}/"*.iso - fi - if [[ "${buildmode}" == "bootstrap" ]]; then - create_zsync_delta "${output}/${profile}/"*.tar*(.gz|.xz|.zst) - create_checksums "${output}/${profile}/"*.tar*(.gz|.xz|.zst) - fi - create_metrics - - print_section_start "ownership" "Setting ownership on output" - - if [[ -n "${SUDO_UID:-}" ]] && [[ -n "${SUDO_GID:-}" ]]; then - chown -Rv "${SUDO_UID}:${SUDO_GID}" -- "${output}" - fi - print_section_end "ownership" -} - -trap cleanup EXIT - -run_mkarchiso diff --git a/.gitlab/ci/build_archiso.sh b/.gitlab/ci/build_archiso.sh new file mode 100755 index 0000000..6424b32 --- /dev/null +++ b/.gitlab/ci/build_archiso.sh @@ -0,0 +1,270 @@ +#!/usr/bin/env bash +# +# This script is run within a virtual environment to build the available archiso profiles and their available build +# modes and create checksum files for the resulting images. +# The script needs to be run as root and assumes $PWD to be the root of the repository. +# +# Dependencies: +# * all archiso dependencies +# * coreutils +# * gnupg +# * openssl +# * zsync +# +# $1: profile +# $2: buildmode + +set -euo pipefail +shopt -s extglob + +readonly orig_pwd="${PWD}" +readonly output="${orig_pwd}/output" +readonly tmpdir_base="${orig_pwd}/tmp" +readonly profile="${1}" +readonly buildmode="${2}" +readonly install_dir="arch" + +tmpdir="" +tmpdir="$(mktemp --dry-run --directory --tmpdir="${tmpdir_base}")" +gnupg_homedir="" +codesigning_dir="" +codesigning_cert="" +codesigning_key="" +pgp_key_id="" + +print_section_start() { + # gitlab collapsible sections start: https://docs.gitlab.com/ee/ci/jobs/#custom-collapsible-sections + local _section _title + _section="${1}" + _title="${2}" + + printf "\e[0Ksection_start:%(%s)T:%s\r\e[0K%s\n" '-1' "${_section}" "${_title}" +} + +print_section_end() { + # gitlab collapsible sections end: https://docs.gitlab.com/ee/ci/jobs/#custom-collapsible-sections + local _section + _section="${1}" + + printf "\e[0Ksection_end:%(%s)T:%s\r\e[0K\n" '-1' "${_section}" +} + +cleanup() { + # clean up temporary directories + print_section_start "cleanup" "Cleaning up temporary directory" + + if [ -n "${tmpdir_base:-}" ]; then + rm -fr "${tmpdir_base}" + fi + + print_section_end "cleanup" +} + +create_checksums() { + # create checksums for files + # $@: files + local _file_path _file_name _current_pwd + _current_pwd="${PWD}" + + print_section_start "checksums" "Creating checksums" + + for _file_path in "$@"; do + cd "$(dirname "${_file_path}")" + _file_name="$(basename "${_file_path}")" + b2sum "${_file_name}" > "${_file_name}.b2" + md5sum "${_file_name}" > "${_file_name}.md5" + sha1sum "${_file_name}" > "${_file_name}.sha1" + sha256sum "${_file_name}" > "${_file_name}.sha256" + sha512sum "${_file_name}" > "${_file_name}.sha512" + ls -lah "${_file_name}."{b2,md5,sha{1,256,512}} + cat "${_file_name}."{b2,md5,sha{1,256,512}} + done + cd "${_current_pwd}" + + print_section_end "checksums" +} + +create_zsync_delta() { + # create zsync control files for files + # $@: files + local _file + + print_section_start "zsync_delta" "Creating zsync delta" + + for _file in "$@"; do + if [[ "${buildmode}" == "bootstrap" ]]; then + # zsyncmake fails on 'too long between blocks' with default block size on bootstrap image + zsyncmake -v -b 512 -C -u "${_file##*/}" -o "${_file}".zsync "${_file}" + else + zsyncmake -v -C -u "${_file##*/}" -o "${_file}".zsync "${_file}" + fi + done + + print_section_end "zsync_delta" +} + +create_metrics() { + local _metrics="${output}/${profile}/metrics.txt" + # create metrics + print_section_start "metrics" "Creating metrics" + + { + # create metrics based on buildmode + case "${buildmode}" in + iso) + printf 'image_size_mebibytes{image="%s"} %s\n' \ + "${profile}" \ + "$(du -m -- "${output}/${profile}/"*.iso | cut -f1)" + printf 'package_count{image="%s"} %s\n' \ + "${profile}" \ + "$(sort -u -- "${tmpdir}/${profile}/iso/"*/pkglist.*.txt | wc -l)" + if [[ -e "${tmpdir}/${profile}/efiboot.img" ]]; then + printf 'eltorito_efi_image_size_mebibytes{image="%s"} %s\n' \ + "${profile}" \ + "$(du -m -- "${tmpdir}/${profile}/efiboot.img" | cut -f1)" + fi + # shellcheck disable=SC2046 + # shellcheck disable=SC2183 + printf 'initramfs_size_mebibytes{image="%s",initramfs="%s"} %s\n' \ + $(du -m -- "${tmpdir}/${profile}/iso/"*/boot/**/initramfs*.img | \ + awk -v profile="${profile}" \ + 'function basename(file) { + sub(".*/", "", file) + return file + } + { print profile, basename($2), $1 }' + ) + ;; + netboot) + printf 'netboot_size_mebibytes{image="%s"} %s\n' \ + "${profile}" \ + "$(du -m -- "${output}/${profile}/${install_dir}/" | tail -n1 | cut -f1)" + printf 'netboot_package_count{image="%s"} %s\n' \ + "${profile}" \ + "$(sort -u -- "${tmpdir}/${profile}/iso/"*/pkglist.*.txt | wc -l)" + ;; + bootstrap) + printf 'bootstrap_size_mebibytes{image="%s"} %s\n' \ + "${profile}" \ + "$(du -m -- "${output}/${profile}/"*.tar*(.gz|.xz|.zst) | cut -f1)" + printf 'bootstrap_package_count{image="%s"} %s\n' \ + "${profile}" \ + "$(sort -u -- "${tmpdir}/${profile}/"*/bootstrap/root.*/pkglist.*.txt | wc -l)" + ;; + esac + } > "${_metrics}" + ls -lah "${_metrics}" + cat "${_metrics}" + + print_section_end "metrics" +} + +create_ephemeral_pgp_key() { + # create an ephemeral PGP key for signing the rootfs image + print_section_start "ephemeral_pgp_key" "Creating ephemeral PGP key" + + gnupg_homedir="$tmpdir/.gnupg" + mkdir -p "${gnupg_homedir}" + chmod 700 "${gnupg_homedir}" + + cat << __EOF__ > "${gnupg_homedir}"/gpg.conf +quiet +batch +no-tty +no-permission-warning +export-options no-export-attributes,export-clean +list-options no-show-keyring +armor +no-emit-version +__EOF__ + + gpg --homedir "${gnupg_homedir}" --gen-key <> "${codesigning_conf}" + openssl req \ + -newkey rsa:4096 \ + -keyout "${codesigning_key}" \ + -nodes \ + -sha256 \ + -x509 \ + -days 365 \ + -out "${codesigning_cert}" \ + -config "${codesigning_conf}" \ + -subj "${codesigning_subj}" \ + -extensions codesigning + + print_section_end "ephemeral_codesigning_key" +} + +run_mkarchiso() { + # run mkarchiso + create_ephemeral_pgp_key + create_ephemeral_codesigning_key + + print_section_start "mkarchiso" "Running mkarchiso" + mkdir -p "${output}/${profile}" "${tmpdir}/${profile}" + GNUPGHOME="${gnupg_homedir}" ./archiso/mkarchiso \ + -D "${install_dir}" \ + -c "${codesigning_cert} ${codesigning_key}" \ + -g "${pgp_key_id}" \ + -o "${output}/${profile}" \ + -w "${tmpdir}/${profile}" \ + -m "${buildmode}" \ + -v "configs/${profile}" + + print_section_end "mkarchiso" + + if [[ "${buildmode}" =~ "iso" ]]; then + create_zsync_delta "${output}/${profile}/"*.iso + create_checksums "${output}/${profile}/"*.iso + fi + if [[ "${buildmode}" == "bootstrap" ]]; then + create_zsync_delta "${output}/${profile}/"*.tar*(.gz|.xz|.zst) + create_checksums "${output}/${profile}/"*.tar*(.gz|.xz|.zst) + fi + create_metrics + + print_section_start "ownership" "Setting ownership on output" + + if [[ -n "${SUDO_UID:-}" ]] && [[ -n "${SUDO_GID:-}" ]]; then + chown -Rv "${SUDO_UID}:${SUDO_GID}" -- "${output}" + fi + print_section_end "ownership" +} + +trap cleanup EXIT + +run_mkarchiso -- cgit v1.2.3-70-g09d2 From e0514b1d2e63ef50a4ca999dd402a1386e9ad77e Mon Sep 17 00:00:00 2001 From: David Runge Date: Wed, 30 Jun 2021 23:23:21 +0200 Subject: ci: Provide artifacts in top-level output dir .gitlab/ci/build-inside-vm.sh: Change the build script to provide the build artifacts and metrics in the top-level output directory. This goes in line with the soon to be used ci-scripts, allowing code sharing amongst several projects. --- .gitlab/ci/build_archiso.sh | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/.gitlab/ci/build_archiso.sh b/.gitlab/ci/build_archiso.sh index 6424b32..8a6f92f 100755 --- a/.gitlab/ci/build_archiso.sh +++ b/.gitlab/ci/build_archiso.sh @@ -104,7 +104,7 @@ create_zsync_delta() { } create_metrics() { - local _metrics="${output}/${profile}/metrics.txt" + local _metrics="${output}/metrics.txt" # create metrics print_section_start "metrics" "Creating metrics" @@ -114,19 +114,19 @@ create_metrics() { iso) printf 'image_size_mebibytes{image="%s"} %s\n' \ "${profile}" \ - "$(du -m -- "${output}/${profile}/"*.iso | cut -f1)" + "$(du -m -- "${output}/"*.iso | cut -f1)" printf 'package_count{image="%s"} %s\n' \ "${profile}" \ - "$(sort -u -- "${tmpdir}/${profile}/iso/"*/pkglist.*.txt | wc -l)" - if [[ -e "${tmpdir}/${profile}/efiboot.img" ]]; then + "$(sort -u -- "${tmpdir}/iso/"*/pkglist.*.txt | wc -l)" + if [[ -e "${tmpdir}/efiboot.img" ]]; then printf 'eltorito_efi_image_size_mebibytes{image="%s"} %s\n' \ "${profile}" \ - "$(du -m -- "${tmpdir}/${profile}/efiboot.img" | cut -f1)" + "$(du -m -- "${tmpdir}/efiboot.img" | cut -f1)" fi # shellcheck disable=SC2046 # shellcheck disable=SC2183 printf 'initramfs_size_mebibytes{image="%s",initramfs="%s"} %s\n' \ - $(du -m -- "${tmpdir}/${profile}/iso/"*/boot/**/initramfs*.img | \ + $(du -m -- "${tmpdir}/iso/"*/boot/**/initramfs*.img | \ awk -v profile="${profile}" \ 'function basename(file) { sub(".*/", "", file) @@ -138,18 +138,18 @@ create_metrics() { netboot) printf 'netboot_size_mebibytes{image="%s"} %s\n' \ "${profile}" \ - "$(du -m -- "${output}/${profile}/${install_dir}/" | tail -n1 | cut -f1)" + "$(du -m -- "${output}/${install_dir}/" | tail -n1 | cut -f1)" printf 'netboot_package_count{image="%s"} %s\n' \ "${profile}" \ - "$(sort -u -- "${tmpdir}/${profile}/iso/"*/pkglist.*.txt | wc -l)" + "$(sort -u -- "${tmpdir}/iso/"*/pkglist.*.txt | wc -l)" ;; bootstrap) printf 'bootstrap_size_mebibytes{image="%s"} %s\n' \ "${profile}" \ - "$(du -m -- "${output}/${profile}/"*.tar*(.gz|.xz|.zst) | cut -f1)" + "$(du -m -- "${output}/"*.tar*(.gz|.xz|.zst) | cut -f1)" printf 'bootstrap_package_count{image="%s"} %s\n' \ "${profile}" \ - "$(sort -u -- "${tmpdir}/${profile}/"*/bootstrap/root.*/pkglist.*.txt | wc -l)" + "$(sort -u -- "${tmpdir}/"*/bootstrap/root.*/pkglist.*.txt | wc -l)" ;; esac } > "${_metrics}" @@ -235,25 +235,25 @@ run_mkarchiso() { create_ephemeral_codesigning_key print_section_start "mkarchiso" "Running mkarchiso" - mkdir -p "${output}/${profile}" "${tmpdir}/${profile}" + mkdir -p "${output}/" "${tmpdir}/" GNUPGHOME="${gnupg_homedir}" ./archiso/mkarchiso \ -D "${install_dir}" \ -c "${codesigning_cert} ${codesigning_key}" \ -g "${pgp_key_id}" \ - -o "${output}/${profile}" \ - -w "${tmpdir}/${profile}" \ + -o "${output}/" \ + -w "${tmpdir}/" \ -m "${buildmode}" \ -v "configs/${profile}" print_section_end "mkarchiso" if [[ "${buildmode}" =~ "iso" ]]; then - create_zsync_delta "${output}/${profile}/"*.iso - create_checksums "${output}/${profile}/"*.iso + create_zsync_delta "${output}/"*.iso + create_checksums "${output}/"*.iso fi if [[ "${buildmode}" == "bootstrap" ]]; then - create_zsync_delta "${output}/${profile}/"*.tar*(.gz|.xz|.zst) - create_checksums "${output}/${profile}/"*.tar*(.gz|.xz|.zst) + create_zsync_delta "${output}/"*.tar*(.gz|.xz|.zst) + create_checksums "${output}/"*.tar*(.gz|.xz|.zst) fi create_metrics -- cgit v1.2.3-70-g09d2 From d56cbd9537107bf176b20a7aa9fbf472a7035f45 Mon Sep 17 00:00:00 2001 From: David Runge Date: Wed, 30 Jun 2021 23:25:48 +0200 Subject: gitlab-ci: Include from ci-scripts .gitlab-ci.yml: Include a gitlab CI yaml file from the ci-scripts repository, that replaces the build-host.sh script and abstracts the .build target. Move all variables to top-level as they are useful in all build targets. Remove unnecessary before_script definitions because they are abstracted away by ci-scripts. --- .gitlab-ci.yml | 37 ++++++++++++------------------------- 1 file changed, 12 insertions(+), 25 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3f77b7e..ed6aa92 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -2,43 +2,30 @@ # # SPDX-License-Identifier: GPL-3.0-or-later -default: - image: "archlinux:latest" +# NOTE: most functionality (apart from building) is abstracted by this include +include: + - project: 'archlinux/ci-scripts' + ref: master + file: '/prepare_archiso_vm.yml' variables: BUILD_SCRIPT: ./.gitlab/ci/build_archiso.sh + PACKAGE_LIST: dosfstools e2fsprogs erofs-utils jq libisoburn mtools openssl qemu-headless squashfs-tools zsync QEMU_BUILD_TIMEOUT: 2400 + QEMU_COPY_ARTIFACTS_TIMEOUT: 120 QEMU_VM_MEMORY: 3072 ARCHISO_COW_SPACE_SIZE: 2g stages: - - lint + - check - build -shellcheck: - stage: lint +check: before_script: - pacman --noconfirm -Syu --needed make shellcheck script: - make lint - -.build: - variables: - PACKAGE_LIST: dosfstools e2fsprogs erofs-utils jq libisoburn mtools openssl qemu-headless squashfs-tools zsync - stage: build - before_script: - - pacman -Syu --needed --noconfirm qemu-headless - script: - - ./.gitlab/ci/build-host.sh - after_script: - - PROFILE=($BUILD_SCRIPT_ARGS) && cp -- "output/${PROFILE[0]}/metrics.txt" . - artifacts: - name: "output" - paths: - - "output/*/*" - expire_in: 2d - reports: - metrics: metrics.txt + stage: check build_short: extends: .build @@ -49,11 +36,11 @@ build_short: build_long: extends: .build - tags: - - fast-single-thread parallel: matrix: - BUILD_SCRIPT_ARGS: baseline iso - BUILD_SCRIPT_ARGS: baseline netboot - BUILD_SCRIPT_ARGS: releng iso - BUILD_SCRIPT_ARGS: releng netboot + tags: + - fast-single-thread -- cgit v1.2.3-70-g09d2 From f48e9cda40d8a48cfac9ee992a9b86b81f16ea1c Mon Sep 17 00:00:00 2001 From: David Runge Date: Thu, 1 Jul 2021 00:31:38 +0200 Subject: Remove build-host.sh .gitlab/ci/build-host.sh: Remove script, as it has been replaced by ci-scripts. --- .gitlab/ci/build-host.sh | 279 ----------------------------------------------- 1 file changed, 279 deletions(-) delete mode 100755 .gitlab/ci/build-host.sh diff --git a/.gitlab/ci/build-host.sh b/.gitlab/ci/build-host.sh deleted file mode 100755 index 78bb067..0000000 --- a/.gitlab/ci/build-host.sh +++ /dev/null @@ -1,279 +0,0 @@ -#!/usr/bin/env bash -# -# This script runs a build script in a QEMU VM using the latest Arch Linux installation medium. -# The build script is expected to create an './output' directory in the project's directory (when running in the VM) and -# place any build artifacts there. -# After the build script has finished this script will copy all artifacts to a (local) './output' directory and shutdown -# the VM. -# -# Dependencies: -# - coreutils -# - curl -# - libarchive -# - qemu-headless -# - util-linux -# -# Considered environment variables: -# ARCHISO_COW_SPACE_SIZE: The amount of RAM to allocate for the copy-on-write space used by archiso (defaults to 1g - -# see https://man.archlinux.org/man/tmpfs.5 for understood units) -# ARCHITECTURE: A string to set the CPU architecture (defaults to x86_64) -# BUILD_SCRIPT: A script that will be called on the host (defaults to ./build-inside-vm.sh) -# BUILD_SCRIPT_ARGS: The arguments to BUILD_SCRIPT (as a space delimited list) -# PACKAGE_LIST: A space delimited list of packages to install to the virtual machine -# PACMAN_MIRROR: The pacman mirror to use (defaults to "https://mirror.pkgbuild.com") -# QEMU_DISK_SIZE: A string given to fallocate to create a scratch disk to build in (defaults to 8G - see -# https://man.archlinux.org/man/fallocate.1 for understood units) -# QEMU_VM_MEMORY: The amount of RAM (in MiB) allocated for the QEMU virtual machine (defaults to 1024) -# QEMU_LOGIN_TIMEOUT: The maximum time (in seconds) to wait for the initial prompt in the VM to appear (defaults to 60) -# QEMU_PACKAGES_TIMEOUT: The maximum time (in seconds) to wait for output from pacman when installing packages (defaults -# to 120) -# QEMU_BUILD_TIMEOUT: The maximum time (in seconds) to wait for output from the build script (defaults to 1800) -# QEMU_COPY_ARTIFACTS_TIMEOUT: The maximum time (in seconds) to wait for output from the action of copying the build -# artifacts from the VM to a local directory (defaults to 60) - - -set -euo pipefail - -readonly orig_pwd="${PWD}" -readonly output="${PWD}/output" - -# variables with presets/ environmental overrides -arch="${ARCHITECTURE:-x86_64}" -script="${BUILD_SCRIPT:-./build-inside-vm.sh}" -script_args="${BUILD_SCRIPT_ARGS:-}" -mirror="${PACMAN_MIRROR:-https://mirror.pkgbuild.com}" -disk_size="${QEMU_DISK_SIZE:-8G}" -vm_memory="${QEMU_VM_MEMORY:-1024}" -login_timeout="${QEMU_LOGIN_TIMEOUT:-60}" -packages_timeout="${QEMU_PACKAGES_TIMEOUT:-120}" -build_timeout="${QEMU_BUILD_TIMEOUT:-1800}" -copy_artifacts_timeout="${QEMU_COPY_ARTIFACTS_TIMEOUT:-60}" -cow_space_size="${ARCHISO_COW_SPACE_SIZE:-1g}" -packages="${PACKAGE_LIST:-}" - -# variables without presets/ environmental overrides -iso="" -iso_volume_id="" -tmpdir="" -tmpdir="$(mktemp --dry-run --directory --tmpdir="${PWD}/tmp")" - -print_section_start() { - # gitlab collapsible sections start: https://docs.gitlab.com/ee/ci/jobs/#custom-collapsible-sections - local _section _title - _section="${1}" - _title="${2}" - - printf "\e[0Ksection_start:%(%s)T:%s\r\e[0K%s\n" '-1' "${_section}" "${_title}" -} - -print_section_end() { - # gitlab collapsible sections end: https://docs.gitlab.com/ee/ci/jobs/#custom-collapsible-sections - local _section - _section="${1}" - - printf "\e[0Ksection_end:%(%s)T:%s\r\e[0K\n" '-1' "${_section}" -} - -init() { - print_section_start "create_dirs" "Create required directories" - - mkdir -p "${output}" "${tmpdir}" - cd "${tmpdir}" - - print_section_end "create_dirs" -} - -# Do some cleanup when the script exits -cleanup() { - print_section_start "cleanup" "Cleaning up" - - rm -rf -- "${tmpdir}" - jobs -p | xargs --no-run-if-empty kill - - print_section_end "cleanup" -} -trap cleanup EXIT - -# Use local Arch iso or download the latest iso and extract the relevant files -prepare_boot() { - local _latest_iso _iso - local _isos=() - - print_section_start "prepare_boot" "Prepare boot media" - - # retrieve any local images and sort them - for _iso in "${orig_pwd}/"archlinux-*-"${arch}.iso"; do - if [[ -f "${_iso}" ]]; then - _isos+=("${_iso}") - fi - done - if (( ${#_isos[@]} >= 1 )); then - iso="$(printf '%s\n' "${_isos[@]}" | sort -r | head -n1)" - printf "Using local iso: %s\n" "$iso" - fi - - if (( ${#_isos[@]} < 1 )); then - _latest_iso="$( - curl -fs "${mirror}/iso/latest/" | \ - grep -Eo "archlinux-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-${arch}.iso" | \ - head -n 1 - )" - if [[ -z "${_latest_iso}" ]]; then - echo "Error: Could not find latest iso" - exit 1 - fi - curl -fO "${mirror}/iso/latest/${_latest_iso}" - iso="${PWD}/${_latest_iso}" - fi - - # Extract the kernel and initrd so that a custom kernel cmdline can be set: - # console=ttyS0, so that the kernel and systemd send output to the serial. - bsdtar -x -f "${iso}" -C . "arch/boot/${arch}" - iso_volume_id="$(blkid -s LABEL -o value "${iso}")" - - print_section_end "prepare_boot" -} - -start_qemu() { - local _kernel_params=( - "archisobasedir=arch" - "archisolabel=${iso_volume_id}" - "cow_spacesize=${cow_space_size}" - "ip=dhcp" - "net.ifnames=0" - "console=ttyS0" - "mirror=${mirror}" - ) - - print_section_start "start_qemu" "Start VM using QEMU" - - # Used to communicate with qemu - mkfifo guest.out guest.in - # We could use a sparse file but we want to fail early - fallocate -l "${disk_size}" scratch-disk.img - - { qemu-system-x86_64 \ - -machine accel=kvm:tcg \ - -smp "$(nproc)" \ - -m "${vm_memory}" \ - -device virtio-net-pci,romfile=,netdev=net0 \ - -netdev user,id=net0 \ - -kernel "arch/boot/${arch}/vmlinuz-linux" \ - -initrd "arch/boot/${arch}/initramfs-linux.img" \ - -append "${_kernel_params[*]}" \ - -drive file=scratch-disk.img,format=raw,if=virtio \ - -drive "file=${iso},format=raw,if=virtio,media=cdrom,read-only=on" \ - -virtfs "local,path=${orig_pwd},mount_tag=host,security_model=none" \ - -monitor none \ - -serial pipe:guest \ - -nographic || kill "${$}"; } & - - # We want to send the output to both stdout (fd1) and fd10 (used by the expect function) - exec 3>&1 10< <(tee /dev/fd/3 guest.in -} - -main() { - local _pacman_command=( - "pacman -Fy &&" - "pacman -Syu --ignore" - "\$(pacman -Fq --machinereadable /usr/lib/modules/" - "| awk 'BEGIN { FS =\"\\\0\";ORS=\",\" }; { print \$2 }'" - "| sort -ut , | head -c -2)" - "--noconfirm --needed ${packages}\n" - ) - - init - prepare_boot - start_qemu - - print_section_start "init_build_environment" "Initialize build environment" - - # Login - expect "archiso login:" "${login_timeout}" - send "root\n" - expect "# " - - # Switch to bash and shutdown on error - send "bash\n" - expect "# " - send "trap \"shutdown now\" ERR\n" - expect "# " - - # Prepare environment - send "mkdir /mnt/project && mount -t 9p -o trans=virtio host /mnt/project -oversion=9p2000.L\n" - expect "# " - send "mkfs.ext4 /dev/vda && mkdir /mnt/scratch-disk/ && mount /dev/vda /mnt/scratch-disk && cd /mnt/scratch-disk\n" - expect "# " - send "rsync -a --exclude tmp --exclude .git -- /mnt/project/ .\n" - expect "# " - send "mkdir pkg && mount --bind pkg /var/cache/pacman/pkg\n" - expect "# " - - # Wait for pacman-init - send "until systemctl is-active pacman-init; do sleep 1; done\n" - expect "# " - - # Explicitly lookup mirror address as we'd get random failures otherwise during pacman - send "curl -sSo /dev/null ${mirror}\n" - expect "# " - - print_section_end "init_build_environment" - print_section_start "install_packages" "Install packages" - - if [[ -n "${packages}" ]]; then - # Install required packages - send "${_pacman_command[*]}" - expect "# " "${packages_timeout}" - fi - - print_section_end "install_packages" - - ## Start build and copy output to local disk - send "bash -x ${script} ${script_args}\n " - expect "# " "${build_timeout}" - - print_section_start "move_artifacts" "Move artifacts to output directory" - - send "rsync -av -- output /mnt/project/tmp/$(basename "${tmpdir}")/\n" - expect "# " "${copy_artifacts_timeout}" - mv -- output/* "${output}/" - - print_section_end "move_artifacts" - print_section_start "shutdown" "Shutdown the VM" - - # Shutdown the VM - send "systemctl poweroff -i\n" - wait - - print_section_end "shutdown" -} - -main -- cgit v1.2.3-70-g09d2