| index : archiso32 | |
| Archlinux32 iso tools | gitolite user |
| summaryrefslogtreecommitdiff |
| -rwxr-xr-x | .gitlab/ci/build-host.sh | 226 |
diff --git a/.gitlab/ci/build-host.sh b/.gitlab/ci/build-host.sh index 3377ba7..78bb067 100755 --- a/.gitlab/ci/build-host.sh +++ b/.gitlab/ci/build-host.sh @@ -1,88 +1,181 @@ #!/usr/bin/env bash -# build-host.sh runs build-inside-vm.sh in a qemu VM running the latest Arch installer iso # -# nounset: "Treat unset variables and parameters [...] as an error when performing parameter expansion." -# errexit: "Exit immediately if [...] command exits with a non-zero status." -set -o nounset -o errexit -readonly MIRROR="https://mirror.pkgbuild.com" +# This script runs a build script in a QEMU VM using the latest Arch Linux installation medium. +# The build script is expected to create an './output' directory in the project's directory (when running in the VM) and +# place any build artifacts there. +# After the build script has finished this script will copy all artifacts to a (local) './output' directory and shutdown +# the VM. +# +# Dependencies: +# - coreutils +# - curl +# - libarchive +# - qemu-headless +# - util-linux +# +# Considered environment variables: +# ARCHISO_COW_SPACE_SIZE: The amount of RAM to allocate for the copy-on-write space used by archiso (defaults to 1g - +# see https://man.archlinux.org/man/tmpfs.5 for understood units) +# ARCHITECTURE: A string to set the CPU architecture (defaults to x86_64) +# BUILD_SCRIPT: A script that will be called on the host (defaults to ./build-inside-vm.sh) +# BUILD_SCRIPT_ARGS: The arguments to BUILD_SCRIPT (as a space delimited list) +# PACKAGE_LIST: A space delimited list of packages to install to the virtual machine +# PACMAN_MIRROR: The pacman mirror to use (defaults to "https://mirror.pkgbuild.com") +# QEMU_DISK_SIZE: A string given to fallocate to create a scratch disk to build in (defaults to 8G - see +# https://man.archlinux.org/man/fallocate.1 for understood units) +# QEMU_VM_MEMORY: The amount of RAM (in MiB) allocated for the QEMU virtual machine (defaults to 1024) +# QEMU_LOGIN_TIMEOUT: The maximum time (in seconds) to wait for the initial prompt in the VM to appear (defaults to 60) +# QEMU_PACKAGES_TIMEOUT: The maximum time (in seconds) to wait for output from pacman when installing packages (defaults +# to 120) +# QEMU_BUILD_TIMEOUT: The maximum time (in seconds) to wait for output from the build script (defaults to 1800) +# QEMU_COPY_ARTIFACTS_TIMEOUT: The maximum time (in seconds) to wait for output from the action of copying the build +# artifacts from the VM to a local directory (defaults to 60) + + +set -euo pipefail +readonly orig_pwd="${PWD}" +readonly output="${PWD}/output" + +# variables with presets/ environmental overrides +arch="${ARCHITECTURE:-x86_64}" +script="${BUILD_SCRIPT:-./build-inside-vm.sh}" +script_args="${BUILD_SCRIPT_ARGS:-}" +mirror="${PACMAN_MIRROR:-https://mirror.pkgbuild.com}" +disk_size="${QEMU_DISK_SIZE:-8G}" +vm_memory="${QEMU_VM_MEMORY:-1024}" +login_timeout="${QEMU_LOGIN_TIMEOUT:-60}" +packages_timeout="${QEMU_PACKAGES_TIMEOUT:-120}" +build_timeout="${QEMU_BUILD_TIMEOUT:-1800}" +copy_artifacts_timeout="${QEMU_COPY_ARTIFACTS_TIMEOUT:-60}" +cow_space_size="${ARCHISO_COW_SPACE_SIZE:-1g}" +packages="${PACKAGE_LIST:-}" + +# variables without presets/ environmental overrides +iso="" +iso_volume_id="" tmpdir="" +tmpdir="$(mktemp --dry-run --directory --tmpdir="${PWD}/tmp")" -function init() { - readonly ORIG_PWD="${PWD}" - readonly OUTPUT="${PWD}/output" - tmpdir="$(mktemp --dry-run --directory --tmpdir="${PWD}/tmp")" - mkdir -p "${OUTPUT}" "${tmpdir}" +print_section_start() { + # gitlab collapsible sections start: https://docs.gitlab.com/ee/ci/jobs/#custom-collapsible-sections + local _section _title + _section="${1}" + _title="${2}" + + printf "\e[0Ksection_start:%(%s)T:%s\r\e[0K%s\n" '-1' "${_section}" "${_title}" +} +print_section_end() { + # gitlab collapsible sections end: https://docs.gitlab.com/ee/ci/jobs/#custom-collapsible-sections + local _section + _section="${1}" + + printf "\e[0Ksection_end:%(%s)T:%s\r\e[0K\n" '-1' "${_section}" +} + +init() { + print_section_start "create_dirs" "Create required directories" + + mkdir -p "${output}" "${tmpdir}" cd "${tmpdir}" + + print_section_end "create_dirs" } # Do some cleanup when the script exits -function cleanup() { +cleanup() { + print_section_start "cleanup" "Cleaning up" + rm -rf -- "${tmpdir}" jobs -p | xargs --no-run-if-empty kill + + print_section_end "cleanup" } trap cleanup EXIT # Use local Arch iso or download the latest iso and extract the relevant files -function prepare_boot() { - local iso - local isos=() +prepare_boot() { + local _latest_iso _iso + local _isos=() + + print_section_start "prepare_boot" "Prepare boot media" # retrieve any local images and sort them - for iso in "${ORIG_PWD}/"archlinux-*-x86_64.iso; do - if [[ -f "$iso" ]]; then - isos+=("${iso}") + for _iso in "${orig_pwd}/"archlinux-*-"${arch}.iso"; do + if [[ -f "${_iso}" ]]; then + _isos+=("${_iso}") fi done - if (( ${#isos[@]} >= 1 )); then - ISO="$(printf '%s\n' "${isos[@]}" | sort -r | head -n1)" - printf "Using local iso: %s\n" "$ISO" + if (( ${#_isos[@]} >= 1 )); then + iso="$(printf '%s\n' "${_isos[@]}" | sort -r | head -n1)" + printf "Using local iso: %s\n" "$iso" fi - if (( ${#isos[@]} < 1 )); then - LATEST_ISO="$(curl -fs "${MIRROR}/iso/latest/" | grep -Eo 'archlinux-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-x86_64.iso' | head -n 1)" - if [[ -z "${LATEST_ISO}" ]]; then - echo "Error: Couldn't find latest iso'" + if (( ${#_isos[@]} < 1 )); then + _latest_iso="$( + curl -fs "${mirror}/iso/latest/" | \ + grep -Eo "archlinux-[0-9]{4}\.[0-9]{2}\.[0-9]{2}-${arch}.iso" | \ + head -n 1 + )" + if [[ -z "${_latest_iso}" ]]; then + echo "Error: Could not find latest iso" exit 1 fi - curl -fO "${MIRROR}/iso/latest/${LATEST_ISO}" - ISO="${PWD}/${LATEST_ISO}" + curl -fO "${mirror}/iso/latest/${_latest_iso}" + iso="${PWD}/${_latest_iso}" fi - # We need to extract the kernel and initrd so we can set a custom cmdline: - # console=ttyS0, so the kernel and systemd sends output to the serial. - xorriso -osirrox on -indev "${ISO}" -extract arch/boot/x86_64 . - ISO_VOLUME_ID="$(xorriso -indev "${ISO}" |& awk -F : '$1 ~ "Volume id" {print $2}' | tr -d "' ")" + # Extract the kernel and initrd so that a custom kernel cmdline can be set: + # console=ttyS0, so that the kernel and systemd send output to the serial. + bsdtar -x -f "${iso}" -C . "arch/boot/${arch}" + iso_volume_id="$(blkid -s LABEL -o value "${iso}")" + + print_section_end "prepare_boot" } -function start_qemu() { +start_qemu() { + local _kernel_params=( + "archisobasedir=arch" + "archisolabel=${iso_volume_id}" + "cow_spacesize=${cow_space_size}" + "ip=dhcp" + "net.ifnames=0" + "console=ttyS0" + "mirror=${mirror}" + ) + + print_section_start "start_qemu" "Start VM using QEMU" + # Used to communicate with qemu mkfifo guest.out guest.in # We could use a sparse file but we want to fail early - fallocate -l 8G scratch-disk.img + fallocate -l "${disk_size}" scratch-disk.img { qemu-system-x86_64 \ -machine accel=kvm:tcg \ -smp "$(nproc)" \ - -m 4096 \ - -device virtio-net-pci,romfile=,netdev=net0 -netdev user,id=net0 \ - -kernel vmlinuz-linux \ - -initrd initramfs-linux.img \ - -append "archisobasedir=arch archisolabel=${ISO_VOLUME_ID} cow_spacesize=4G ip=dhcp net.ifnames=0 console=ttyS0 mirror=${MIRROR}" \ + -m "${vm_memory}" \ + -device virtio-net-pci,romfile=,netdev=net0 \ + -netdev user,id=net0 \ + -kernel "arch/boot/${arch}/vmlinuz-linux" \ + -initrd "arch/boot/${arch}/initramfs-linux.img" \ + -append "${_kernel_params[*]}" \ -drive file=scratch-disk.img,format=raw,if=virtio \ - -drive file="${ISO}",format=raw,if=virtio,media=cdrom,read-only=on \ - -virtfs "local,path=${ORIG_PWD},mount_tag=host,security_model=none" \ + -drive "file=${iso},format=raw,if=virtio,media=cdrom,read-only=on" \ + -virtfs "local,path=${orig_pwd},mount_tag=host,security_model=none" \ -monitor none \ -serial pipe:guest \ -nographic || kill "${$}"; } & # We want to send the output to both stdout (fd1) and fd10 (used by the expect function) exec 3>&1 10< <(tee /dev/fd/3 <guest.out) + + print_section_end "start_qemu" } # Wait for a specific string from qemu -function expect() { +expect() { local length="${#1}" local i=0 local timeout="${2:-30}" @@ -103,17 +196,28 @@ function expect() { } # Send string to qemu -function send() { +send() { echo -en "${1}" >guest.in } -function main() { +main() { + local _pacman_command=( + "pacman -Fy &&" + "pacman -Syu --ignore" + "\$(pacman -Fq --machinereadable /usr/lib/modules/" + "| awk 'BEGIN { FS =\"\\\0\";ORS=\",\" }; { print \$2 }'" + "| sort -ut , | head -c -2)" + "--noconfirm --needed ${packages}\n" + ) + init prepare_boot start_qemu + print_section_start "init_build_environment" "Initialize build environment" + # Login - expect "archiso login:" 60 + expect "archiso login:" "${login_timeout}" send "root\n" expect "# " @@ -128,7 +232,7 @@ function main() { expect "# " send "mkfs.ext4 /dev/vda && mkdir /mnt/scratch-disk/ && mount /dev/vda /mnt/scratch-disk && cd /mnt/scratch-disk\n" expect "# " - send "cp -a -- /mnt/project/{.gitlab,archiso,configs,scripts} .\n" + send "rsync -a --exclude tmp --exclude .git -- /mnt/project/ .\n" expect "# " send "mkdir pkg && mount --bind pkg /var/cache/pacman/pkg\n" expect "# " @@ -138,22 +242,38 @@ function main() { expect "# " # Explicitly lookup mirror address as we'd get random failures otherwise during pacman - send "curl -sSo /dev/null ${MIRROR}\n" + send "curl -sSo /dev/null ${mirror}\n" expect "# " - # Install required packages - send "pacman -Fy && pacman -Syu --ignore \$(pacman -Fq --machinereadable /usr/lib/modules/ | awk 'BEGIN { FS = \"\\\0\";ORS=\",\" }; { print \$2 } ' | sort -ut , | head -c -2) --noconfirm --needed qemu-headless jq dosfstools erofs-utils e2fsprogs libisoburn mtools squashfs-tools zsync\n" - expect "# " 120 + print_section_end "init_build_environment" + print_section_start "install_packages" "Install packages" + + if [[ -n "${packages}" ]]; then + # Install required packages + send "${_pacman_command[*]}" + expect "# " "${packages_timeout}" + fi + + print_section_end "install_packages" ## Start build and copy output to local disk - send "bash -x ./.gitlab/ci/build-inside-vm.sh ${PROFILE} ${BUILDMODE}\n " - expect "# " 2400 # mksquashfs can take a very long time - send "cp -r --preserve=mode,timestamps -- output /mnt/project/tmp/$(basename "${tmpdir}")/\n" - expect "# " 60 - mv output/* "${OUTPUT}/" + send "bash -x ${script} ${script_args}\n " + expect "# " "${build_timeout}" + + print_section_start "move_artifacts" "Move artifacts to output directory" + + send "rsync -av -- output /mnt/project/tmp/$(basename "${tmpdir}")/\n" + expect "# " "${copy_artifacts_timeout}" + mv -- output/* "${output}/" + + print_section_end "move_artifacts" + print_section_start "shutdown" "Shutdown the VM" # Shutdown the VM send "systemctl poweroff -i\n" wait + + print_section_end "shutdown" } + main |