From 55c2ca1312e649916a9a4469b7e88464f2f20c38 Mon Sep 17 00:00:00 2001 From: Robin Candau Date: Fri, 23 Feb 2024 11:04:03 +0100 Subject: feat(version): let upgrade subcommand also update checksums This commit aims to make 'pkgctl version upgrade' also update checksums in addition of bumping the pkgver and reseting the pkgrel. Component: pkgctl version upgrade --- src/lib/version/upgrade.sh | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) (limited to 'src/lib') diff --git a/src/lib/version/upgrade.sh b/src/lib/version/upgrade.sh index df3b77d..2885182 100644 --- a/src/lib/version/upgrade.sh +++ b/src/lib/version/upgrade.sh @@ -30,14 +30,15 @@ pkgctl_version_upgrade_usage() { Upon execution, it automatically adjusts the PKGBUILD file, ensuring that the pkgver field is set to match the latest version available from the upstream source. In addition to updating the pkgver, this command also resets the pkgrel - to 1. + to 1 and updates checksums. Outputs a summary of upgraded packages, up-to-date packages, and any check failures. OPTIONS - -v, --verbose Display results including up-to-date versions - -h, --help Show this help text + --no-update-checksums Disable computation and update of the checksums + -v, --verbose Display results including up-to-date versions + -h, --help Show this help text EXAMPLES $ ${COMMAND} neovim vim @@ -50,6 +51,7 @@ pkgctl_version_upgrade() { local verbose=0 local exit_code=0 local current_item=0 + local update_checksums=1 while (( $# )); do case $1 in @@ -57,6 +59,10 @@ pkgctl_version_upgrade() { pkgctl_version_upgrade_usage exit 0 ;; + --no-update-checksums) + update_checksums=0 + shift + ;; -v|--verbose) verbose=1 shift @@ -153,6 +159,11 @@ pkgctl_version_upgrade() { # change the PKGBUILD pkgbuild_set_pkgver "${upstream_version}" pkgbuild_set_pkgrel 1 + + # download sources and update the checksums + if (( update_checksums )); then + updpkgsums + fi fi popd >/dev/null -- cgit v1.2.3-70-g09d2 From 9a5181db5bfa78d33d3123145ea4c84375f2e8f2 Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Fri, 19 Apr 2024 21:16:06 +0200 Subject: feat(pkgctl): add internal update checksums to better control output This allows us to have more control over the output and status logs. Using this method we are able to avoid cluttering the version upgrade subcommand while downloading sources for updating the checksums. Having this internally will also allow us in the future to have smart checksums updating by only trying to change the checksums of sources that have actually changed, for example when adjusting a patch file we should avoid trying to overwrite the archive checksums unintentionally. Component: pkgctl version upgrade --- README.md | 1 - doc/man/pkgctl-version-upgrade.1.asciidoc | 1 - src/lib/build/build.sh | 5 ++-- src/lib/util/makepkg.sh | 20 +++++++++++++ src/lib/util/pkgbuild.sh | 50 ++++++++++++++++++++++++++++++- src/lib/version/upgrade.sh | 39 +++++++++++++++++------- 6 files changed, 101 insertions(+), 15 deletions(-) (limited to 'src/lib') diff --git a/README.md b/README.md index 2682aa4..eea9c0a 100644 --- a/README.md +++ b/README.md @@ -91,7 +91,6 @@ Component: pkgctl db remove - bat (pretty printing) - nvchecker (version checking) -- pacman-contrib (--update-checksums and --pkgver=*PKGVER* options for pkgctl build, pkgctl version upgrade) ### Development Dependencies diff --git a/doc/man/pkgctl-version-upgrade.1.asciidoc b/doc/man/pkgctl-version-upgrade.1.asciidoc index 02ceff9..0cec0b6 100644 --- a/doc/man/pkgctl-version-upgrade.1.asciidoc +++ b/doc/man/pkgctl-version-upgrade.1.asciidoc @@ -49,6 +49,5 @@ See Also pkgctl-version(1) nvchecker(1) -updpkgsums(8) include::include/footer.asciidoc[] diff --git a/src/lib/build/build.sh b/src/lib/build/build.sh index c35d70f..c3e05be 100644 --- a/src/lib/build/build.sh +++ b/src/lib/build/build.sh @@ -437,10 +437,11 @@ pkgctl_build() { stat_done fi - # update checksums if any sources are declared if (( UPDATE_CHECKSUMS )) && (( ${#source[@]} >= 1 )); then - updpkgsums + if ! result=$(pkgbuild_update_checksums /dev/stderr); then + die "${result}" + fi fi # re-source the PKGBUILD if it changed diff --git a/src/lib/util/makepkg.sh b/src/lib/util/makepkg.sh index 22df247..d7ec74c 100644 --- a/src/lib/util/makepkg.sh +++ b/src/lib/util/makepkg.sh @@ -22,9 +22,11 @@ makepkg_source_package() { return fi ( + # shellcheck disable=SC2030 disable=SC2031 export LIBMAKEPKG_LINT_PKGBUILD_SH=1 lint_pkgbuild() { :; } + # shellcheck disable=SC2030 disable=SC2031 export LIBMAKEPKG_SRCINFO_SH=1 write_srcinfo() { print_srcinfo; } @@ -35,3 +37,21 @@ makepkg_source_package() { source "$(command -v makepkg)" ) } + +makepkg_generate_integrity() { + if [[ -z ${DEVTOOLS_GENERATE_INTEGRITY} ]]; then + [[ -z ${WORKDIR:-} ]] && setup_workdir + export WORKDIR DEVTOOLS_INCLUDE_COMMON_SH + bash -$- -c "DEVTOOLS_GENERATE_INTEGRITY=1; source '${BASH_SOURCE[0]}' && ${FUNCNAME[0]}" + return + fi + ( + # shellcheck disable=SC2030 disable=SC2031 + export LIBMAKEPKG_LINT_PKGBUILD_SH=1 + lint_pkgbuild() { :; } + + set +e -- --geninteg + # shellcheck source=/usr/bin/makepkg + source "$(command -v makepkg)" + ) +} diff --git a/src/lib/util/pkgbuild.sh b/src/lib/util/pkgbuild.sh index ebf8e5f..245a82f 100644 --- a/src/lib/util/pkgbuild.sh +++ b/src/lib/util/pkgbuild.sh @@ -6,10 +6,13 @@ DEVTOOLS_INCLUDE_UTIL_PKGBUILD_SH=1 _DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@} +# shellcheck source=src/lib/util/makepkg.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/makepkg.sh source /usr/share/makepkg/util/message.sh +source /usr/share/makepkg/util/schema.sh -set -e +set -eo pipefail # set the pkgver variable in a PKGBUILD @@ -41,3 +44,48 @@ pkgbuild_set_pkgrel() { sed --regexp-extended "s|^(pkgrel=)${pkgrel}$|\1${new_pkgrel}|g" --in-place PKGBUILD } +pkgbuild_update_checksums() { + local status_file=$1 + local builddir newbuildfile sumtypes newsums + + [[ -z ${WORKDIR:-} ]] && setup_workdir + + builddir=$(mktemp --tmpdir="${WORKDIR}" --directory update-checksums.XXXXXX) + newbuildfile="${builddir}/PKGBUILD" + + # generate new integrity checksums + if ! newsums=$(BUILDDIR=${builddir} makepkg_generate_integrity 2>"${status_file}"); then + printf 'Failed to generate new checksums' + return 1 + fi + + # early exit if no integrity checksums are needed + if [[ -z ${newsums} ]]; then + return 0 + fi + + # replace the integrity sums and write it to a temporary file + sumtypes=$(IFS='|'; echo "${known_hash_algos[*]}") + if ! awk --assign=sumtypes="${sumtypes}" --assign=newsums="${newsums}" ' + $0 ~"^[[:blank:]]*(" sumtypes ")sums(_[^=]+)?\\+?=", $0 ~ "\\)[[:blank:]]*(#.*)?$" { + if (!w) { + print newsums + w++ + } + next + } + + 1 + END { if (!w) print newsums }' PKGBUILD > "${newbuildfile}"; then + printf 'Failed to replace the generated checksums' + return 1 + fi + + # overwrite the original PKGBUILD while preserving permissions + if ! cat -- "${newbuildfile}" > PKGBUILD; then + printf "Failed to write to the PKGBUILD file" + return 1 + fi + + return 0 +} diff --git a/src/lib/version/upgrade.sh b/src/lib/version/upgrade.sh index 2885182..c57171c 100644 --- a/src/lib/version/upgrade.sh +++ b/src/lib/version/upgrade.sh @@ -115,6 +115,12 @@ pkgctl_version_upgrade() { die "No PKGBUILD found for ${path}" fi + # reset common PKGBUILD variables + unset pkgbase pkgname arch source pkgver pkgrel validpgpkeys + # shellcheck source=contrib/makepkg/PKGBUILD.proto + . ./PKGBUILD + pkgbase=${pkgbase:-$pkgname} + # update the current terminal spinner status (( ++current_item )) pkgctl_version_upgrade_spinner \ @@ -123,13 +129,9 @@ pkgctl_version_upgrade() { "${#out_of_date[@]}" \ "${#failure[@]}" \ "${current_item}" \ - "${#pkgbases[@]}" - - # reset common PKGBUILD variables - unset pkgbase pkgname arch source pkgver pkgrel validpgpkeys - # shellcheck source=contrib/makepkg/PKGBUILD.proto - . ./PKGBUILD - pkgbase=${pkgbase:-$pkgname} + "${#pkgbases[@]}" \ + "${pkgbase}" \ + "query latest version" if ! result=$(get_upstream_version); then result="${BOLD}${pkgbase}${ALL_OFF}: ${result}" @@ -162,7 +164,20 @@ pkgctl_version_upgrade() { # download sources and update the checksums if (( update_checksums )); then - updpkgsums + pkgctl_version_upgrade_spinner \ + "${status_dir}" \ + "${#up_to_date[@]}" \ + "${#out_of_date[@]}" \ + "${#failure[@]}" \ + "${current_item}" \ + "${#pkgbases[@]}" \ + "${pkgbase}" \ + "updating checksums" + + if ! result=$(pkgbuild_update_checksums /dev/null); then + result="${BOLD}${pkgbase}${ALL_OFF}: failed to update checksums for version ${DARK_GREEN}${upstream_version}${ALL_OFF}" + failure+=("${result}") + fi fi fi @@ -242,6 +257,8 @@ pkgctl_version_upgrade_spinner() { local failure_count=$4 local current=$5 local total=$6 + local pkgbase=$7 + local message=$8 local percentage=$(( 100 * current / total )) local tmp_file="${status_dir}/tmp" @@ -254,8 +271,10 @@ pkgctl_version_upgrade_spinner() { "${failure_count}" > "${tmp_file}" # print the progress status - printf "📡 Upgrading: %s/%s [%s] %%spinner%%" \ - "${BOLD}${current}" "${total}" "${percentage}%${ALL_OFF}" \ + printf "📡 %s: %s\n" \ + "${pkgbase}" "${BOLD}${message}${ALL_OFF}" >> "${tmp_file}" + printf "⌛ Upgrading: %s/%s [%s] %%spinner%%" \ + "${BOLD}${current}" "${total}" "${percentage}%${ALL_OFF}" \ >> "${tmp_file}" # swap the status file -- cgit v1.2.3-70-g09d2 From 5780ba0e380d619c7e6632edbf6e357dce73289f Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Fri, 16 Feb 2024 22:34:43 +0100 Subject: fix(release): ensure we release split package uploads as one batch Use a central location in common.sh to define the default rsync options. Switch to use batched uploads by targeting a partial directory which is not taken into account by db update. Instead, once all packages that belong to a split package group are uploaded into the .partial directory, all artifacts are moved in one batch into the staging directory of the repo server. This reduced the window of opportunity for a partial release significantly to a tiny window. Component: pkgctl release --- src/commitpkg.in | 2 +- src/lib/common.sh | 11 +++++++++++ src/offload-build.in | 2 +- 3 files changed, 13 insertions(+), 2 deletions(-) (limited to 'src/lib') diff --git a/src/commitpkg.in b/src/commitpkg.in index e17b270..deb02be 100644 --- a/src/commitpkg.in +++ b/src/commitpkg.in @@ -151,7 +151,7 @@ fi server=${PACKAGING_REPO_RELEASE_HOST} -rsyncopts=(-e ssh -p '--chmod=ug=rw,o=r' -c -h -L --progress --partial -y) +rsyncopts=("${RSYNC_OPTS[@]}" --perms --chmod='ug=rw,o=r') archreleaseopts=() while getopts ':l:a:s:f' flag; do case $flag in diff --git a/src/lib/common.sh b/src/lib/common.sh index 00ece97..7d04c25 100644 --- a/src/lib/common.sh +++ b/src/lib/common.sh @@ -31,6 +31,17 @@ export PACKAGING_REPO_RELEASE_HOST=repos.archlinux.org export PKGBASE_MAINTAINER_URL=https://archlinux.org/packages/pkgbase-maintainer export AUR_URL_SSH=aur@aur.archlinux.org +export RSYNC_OPTS=( + --rsh=ssh + --checksum + --copy-links + --human-readable + --progress + --partial + --partial-dir=.partial + --delay-updates +) + # ensure TERM is set with a fallback to dumb export TERM=${TERM:-dumb} diff --git a/src/offload-build.in b/src/offload-build.in index 7a8c1fd..b272a74 100644 --- a/src/offload-build.in +++ b/src/offload-build.in @@ -23,7 +23,7 @@ fi repo=extra arch=x86_64 server=build.archlinux.org -rsyncopts=(-e ssh -c -h -L --progress --partial -y) +rsyncopts=("${RSYNC_OPTS[@]}") usage() { cat <<- _EOF_ -- cgit v1.2.3-70-g09d2 From dced77d23de2a9b00faed40350d229aaaffaa4b2 Mon Sep 17 00:00:00 2001 From: Christian Heusel Date: Tue, 23 Apr 2024 14:57:55 +0200 Subject: fix(completion): fix erroneous completion variables The architecture definition of the variable was using invalid bash syntax and was previously unused: $ _binary_arch=${DEVTOOLS_VALID_ARCHES[*]:0:-1} bash: -1: substring expression < 0 We therefore fix the definition of the variable and use it for the autocompletion of the offload-build command. Furthermore fix wrongly named architecture variables that have been missed from previous refactoring. Fixes #222 Component: completion Fixes: f961e2e ("completion: implemented structured declarative bash completions") Fixes: 4173e0a ("chore: refactor variable names in valid-{tags,repos}.sh") Signed-off-by: Christian Heusel Co-authored-by: Levente Polyak --- contrib/completion/bash/devtools.in | 10 +++++++--- contrib/completion/zsh/_devtools.in | 9 ++++----- src/lib/valid-tags.sh | 7 ++++++- 3 files changed, 17 insertions(+), 9 deletions(-) (limited to 'src/lib') diff --git a/contrib/completion/bash/devtools.in b/contrib/completion/bash/devtools.in index 6b88203..2d5b7bf 100644 --- a/contrib/completion/bash/devtools.in +++ b/contrib/completion/bash/devtools.in @@ -14,7 +14,6 @@ source "${_DEVTOOLS_LIBRARY_DIR}"/lib/valid-inspect.sh # shellcheck source=src/lib/valid-search.sh source "${_DEVTOOLS_LIBRARY_DIR}"/lib/valid-search.sh -_binary_arch=${DEVTOOLS_VALID_ARCHES[*]:0:-1} _colors=(never always auto) @@ -128,7 +127,7 @@ _offload_build_args=( ) _offload_build_args__repo_opts() { _devtools_completions_build_repo; } _offload_build_args_r_opts() { _offload_build_args__repo_opts; } -_offload_build_args__arch_opts() { _devtools_completions_arch; } +_offload_build_args__arch_opts() { _devtools_completions_binary_arch; } _offload_build_args_a_opts() { _offload_build_args__arch_opts; } _offload_build_args__server_opts() { :; } _offload_build_args_s_opts() { _offload_build_args__server_opts; } @@ -196,7 +195,7 @@ _pkgctl_build_args=( -h --help ) -_pkgctl_build_args__arch_opts() { _devtools_completions_arch; } +_pkgctl_build_args__arch_opts() { _devtools_completions_binary_arch; } _pkgctl_build_args__repo_opts() { _devtools_completions_repo; } _pkgctl_build_args__worker_opts() { :; } _pkgctl_build_args_w_opts() { _pkgctl_build_args__worker_opts; } @@ -244,6 +243,8 @@ _pkgctl_db_remove_args=( -a --arch -h --help ) +_pkgctl_db_remove_args__arch_opts() { _devtools_completions_binary_arch; } +_pkgctl_db_remove_args_a_opts() { _pkgctl_db_remove_args__arch_opts; } _pkgctl_db_remove_opts() { local subcommand args subcommand=(db remove) @@ -434,6 +435,9 @@ _devtools_completions_color() { _devtools_completions_arch() { mapfile -t COMPREPLY < <(compgen -W "${DEVTOOLS_VALID_ARCHES[*]}" -- "$cur") } +_devtools_completions_binary_arch() { + mapfile -t COMPREPLY < <(compgen -W "${DEVTOOLS_VALID_BINARY_ARCHES[*]}" -- "$cur") +} _devtools_completions_repo() { local optional=${1:-} mapfile -t COMPREPLY < <(compgen -W "${optional} ${DEVTOOLS_VALID_REPOS[*]}" -- "$cur") diff --git a/contrib/completion/zsh/_devtools.in b/contrib/completion/zsh/_devtools.in index 7c0fe91..65c7352 100644 --- a/contrib/completion/zsh/_devtools.in +++ b/contrib/completion/zsh/_devtools.in @@ -14,7 +14,6 @@ source "${_DEVTOOLS_LIBRARY_DIR}"/lib/valid-inspect.sh # shellcheck source=src/lib/valid-search.sh source "${_DEVTOOLS_LIBRARY_DIR}"/lib/valid-search.sh -_binary_arch=${DEVTOOLS_VALID_ARCHES[*]:0:-1} _colors=(never always auto) _archbuild_args=( @@ -41,8 +40,8 @@ _pkgctl_auth_status_args=( ) _pkgctl_build_args=( - "--arch=[Specify architectures to build for (disables auto-detection)]:arch:($_arch[*])" - "--repo=[Specify a target repository (disables auto-detection)]:repo:($DEVTOOLS_VALID_REPOS[*])" + "--arch[Specify architectures to build for (disables auto-detection)]:arch:($DEVTOOLS_VALID_BINARY_ARCHES[*])" + "--repo[Specify a target repository (disables auto-detection)]:repo:($DEVTOOLS_VALID_REPOS[*])" '(-s --staging)'{-s,--staging}'[Build against the staging counterpart of the auto-detected repo]' '(-t --testing)'{-t,--testing}'[Build against the testing counterpart of the auto-detected repo]' '(-o --offload)'{-o,--offload}'[Build on a remote server and transfer artifacts afterwards]' @@ -79,7 +78,7 @@ _pkgctl_db_move_args=( ) _pkgctl_db_remove_args=( - '(-a --arch=)'{-a,--arch=}"[Override the architecture (disables auto-detection)]:arch:($_arch[*])" + '(-a --arch)'{-a,--arch}"[Override the architecture (disables auto-detection)]:arch:($DEVTOOLS_VALID_BINARY_ARCHES[*])" '(-h --help)'{-h,--help}'[Display usage]' "1:repo:($DEVTOOLS_VALID_REPOS[*])" '*:pkgbase:_devtools_completions_all_packages' @@ -250,7 +249,7 @@ _sogrep_args=( _offload_build_args=( '(-r --repo)'{-r,--repo}'[Build against a specific repository]:repo:($DEVTOOLS_VALID_BUILDREPOS[*])' - '(-a --arch)'{-a,--arch}'[Build against a specific architecture]:arch:(${_binary_arch[*]})' + '(-a --arch)'{-a,--arch}'[Build against a specific architecture]:arch:(${DEVTOOLS_VALID_BINARY_ARCHES[*]})' '(-s --server)'{-s,--server}'[Offload to a specific Build server]:server:' '(-h --help)'{-h,--help}'[Display usage]' ) diff --git a/src/lib/valid-tags.sh b/src/lib/valid-tags.sh index cef0cc6..62e40ae 100644 --- a/src/lib/valid-tags.sh +++ b/src/lib/valid-tags.sh @@ -4,8 +4,13 @@ : # shellcheck disable=2034 -DEVTOOLS_VALID_ARCHES=( +DEVTOOLS_VALID_BINARY_ARCHES=( x86_64 +) + +# shellcheck disable=2034 +DEVTOOLS_VALID_ARCHES=( + "${DEVTOOLS_VALID_BINARY_ARCHES[@]}" any ) -- cgit v1.2.3-70-g09d2 From 7b553afcb25286d04dcb4cbf12e18745e8b0139a Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Sat, 27 Apr 2024 00:40:57 +0200 Subject: feat(db): add partial split package option to db remove By default passing a pkgbase removes all split packages, debug packages as well as entries from the state repo for all existing architectures. When using the `--partial` option it may most likely lead to undesired effects by leaving debug packages behind as well as dangling entries in the state repository. However, for specific use cases its required to get rid of old split package parts. Fixes #218 Component: pkgctl db remove Signed-off-by: Levente Polyak --- contrib/completion/bash/devtools.in | 1 + contrib/completion/zsh/_devtools.in | 1 + doc/man/pkgctl-db-remove.1.asciidoc | 15 +++++++++++++-- src/lib/db/remove.sh | 32 +++++++++++++++++++++++++++----- 4 files changed, 42 insertions(+), 7 deletions(-) (limited to 'src/lib') diff --git a/contrib/completion/bash/devtools.in b/contrib/completion/bash/devtools.in index 2d5b7bf..ec45b62 100644 --- a/contrib/completion/bash/devtools.in +++ b/contrib/completion/bash/devtools.in @@ -240,6 +240,7 @@ _pkgctl_db_move_opts() { _pkgctl_db_remove_args=( + --partial -a --arch -h --help ) diff --git a/contrib/completion/zsh/_devtools.in b/contrib/completion/zsh/_devtools.in index 65c7352..6dc0340 100644 --- a/contrib/completion/zsh/_devtools.in +++ b/contrib/completion/zsh/_devtools.in @@ -78,6 +78,7 @@ _pkgctl_db_move_args=( ) _pkgctl_db_remove_args=( + '--partial[Remove only partial pkgnames from a split package]' '(-a --arch)'{-a,--arch}"[Override the architecture (disables auto-detection)]:arch:($DEVTOOLS_VALID_BINARY_ARCHES[*])" '(-h --help)'{-h,--help}'[Display usage]' "1:repo:($DEVTOOLS_VALID_REPOS[*])" diff --git a/doc/man/pkgctl-db-remove.1.asciidoc b/doc/man/pkgctl-db-remove.1.asciidoc index a95766d..9fe07c3 100644 --- a/doc/man/pkgctl-db-remove.1.asciidoc +++ b/doc/man/pkgctl-db-remove.1.asciidoc @@ -12,13 +12,24 @@ pkgctl db remove [OPTIONS] [REPO] [PKGBASE]... Description ----------- -Remove packages from pacman repositories. +Remove packages from pacman repositories. By default passing a pkgbase removes +all split packages, debug packages as well as entries from the state repo for +all existing architectures. + +Beware when using the `--partial` option, as it may most likely lead to +undesired effects by leaving debug packages behind as well as dangling entries +in the state repository. Options ------- +*--partial*:: + Remove only partial pkgnames from a split package. This leaves debug + packages behind and pkgbase entries in the state repo. + *-a, --arch* 'ARCH':: - Override the architecture (disables auto-detection) + Remove only one specific architecture (disables auto-detection). + By default all architectures are removed when this option is not used. *-h, --help*:: Show a help text diff --git a/src/lib/db/remove.sh b/src/lib/db/remove.sh index ba21c83..018b793 100644 --- a/src/lib/db/remove.sh +++ b/src/lib/db/remove.sh @@ -17,10 +17,18 @@ pkgctl_db_remove_usage() { cat <<- _EOF_ Usage: ${COMMAND} [OPTIONS] [REPO] [PKGBASE]... - Remove packages from binary repositories. + Remove packages from pacman repositories. By default passing a pkgbase removes + all split packages, debug packages as well as entries from the state repo for + all existing architectures. + + Beware when using the --partial option, as it may most likely lead to + undesired effects by leaving debug packages behind as well as dangling entries + in the state repository. OPTIONS - -a, --arch Override the architecture (disables auto-detection) + -a, --arch Remove only one specific architecture (disables auto-detection) + --partial Remove only partial pkgnames from a split package. This leaves + debug packages behind and pkgbase entries in the state repo. -h, --help Show this help text EXAMPLES @@ -31,8 +39,9 @@ _EOF_ pkgctl_db_remove() { local REPO="" - local ARCH=any local PKGBASES=() + local partial=0 + local dbscripts_options=() # option checking while (( $# )); do @@ -41,9 +50,14 @@ pkgctl_db_remove() { pkgctl_db_remove_usage exit 0 ;; + --partial) + partial=1 + dbscripts_options+=(--partial) + shift + ;; -a|--arch) (( $# <= 1 )) && die "missing argument for %s" "$1" - ARCH=$2 + dbscripts_options+=(--arch "$2") shift 2 ;; -*) @@ -64,6 +78,14 @@ pkgctl_db_remove() { shift PKGBASES+=("$@") + # print explenation about partial removal + if (( partial )); then + echo + msg_warn "${YELLOW}Removing only partial pkgnames from a split package.${ALL_OFF}" + msg_warn "${YELLOW}This leaves debug packages and pkgbase entries in the state repo!${ALL_OFF}" + fi + # shellcheck disable=SC2029 - ssh "${PACKAGING_REPO_RELEASE_HOST}" db-remove "${REPO}" "${ARCH}" "${PKGBASES[@]}" + echo + ssh "${PACKAGING_REPO_RELEASE_HOST}" db-remove "${dbscripts_options[@]}" "${REPO}" "${PKGBASES[@]}" } -- cgit v1.2.3-70-g09d2 From 1d433f600e6eecfe685650a06e58d1a8edae9b5d Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Sat, 27 Apr 2024 01:50:57 +0200 Subject: feat(db): confirm list of all packages that will be removed Sometimes it isn't obvious which set of packages are removed from a split package when the pkgbase matches also a subset of a pkgbase. This can happen for example with bootstrapping packages, when the intention is to just remove a partial part of the bootstrap pkgbase. To make the intention more explicit, list all to be removed packages and await for confirmation. Component: pkgctl db remove Signed-off-by: Levente Polyak --- README.md | 1 + config/pacman/universe.conf | 112 ++++++++++++++++++++++++++++++++++++ config/pacman/unstable.conf | 83 ++++++++++++++++++++++++++ contrib/completion/bash/devtools.in | 1 + contrib/completion/zsh/_devtools.in | 1 + doc/man/pkgctl-db-remove.1.asciidoc | 3 + src/lib/build/build.sh | 2 +- src/lib/db/remove.sh | 57 +++++++++++++++++- src/lib/release.sh | 2 +- src/lib/util/pacman.sh | 36 +++++++++++- src/lib/util/term.sh | 16 ++++++ 11 files changed, 308 insertions(+), 6 deletions(-) create mode 100644 config/pacman/universe.conf create mode 100644 config/pacman/unstable.conf (limited to 'src/lib') diff --git a/README.md b/README.md index eea9c0a..c0bcfdd 100644 --- a/README.md +++ b/README.md @@ -71,6 +71,7 @@ Component: pkgctl db remove - coreutils - curl - diffutils +- expac - fakeroot - findutils - grep diff --git a/config/pacman/universe.conf b/config/pacman/universe.conf new file mode 100644 index 0000000..417114b --- /dev/null +++ b/config/pacman/universe.conf @@ -0,0 +1,112 @@ +# +# /etc/pacman.conf +# +# See the pacman.conf(5) manpage for option and repository directives + +# +# GENERAL OPTIONS +# +[options] +# The following paths are commented out with their default values listed. +# If you wish to use different paths, uncomment and update the paths. +#RootDir = / +#DBPath = /var/lib/pacman/ +#CacheDir = /var/cache/pacman/pkg/ +#LogFile = /var/log/pacman.log +#GPGDir = /etc/pacman.d/gnupg/ +#HookDir = /etc/pacman.d/hooks/ +HoldPkg = pacman glibc +#XferCommand = /usr/bin/curl -L -C - -f -o %o %u +#XferCommand = /usr/bin/wget --passive-ftp -c -O %o %u +#CleanMethod = KeepInstalled +Architecture = auto + +# Pacman won't upgrade packages listed in IgnorePkg and members of IgnoreGroup +#IgnorePkg = +#IgnoreGroup = + +#NoUpgrade = +#NoExtract = + +# Misc options +#UseSyslog +#Color +NoProgressBar +# We cannot check disk space from within a chroot environment +#CheckSpace +VerbosePkgLists +ParallelDownloads = 5 + +# By default, pacman accepts packages signed by keys that its local keyring +# trusts (see pacman-key and its man page), as well as unsigned packages. +SigLevel = Required DatabaseOptional +LocalFileSigLevel = Optional +#RemoteFileSigLevel = Required + +# NOTE: You must run `pacman-key --init` before first using pacman; the local +# keyring can then be populated with the keys of all official Arch Linux +# packagers with `pacman-key --populate archlinux`. + +# +# REPOSITORIES +# - can be defined here or included from another file +# - pacman will search repositories in the order defined here +# - local/custom mirrors can be added here or in separate files +# - repositories listed first will take precedence when packages +# have identical names, regardless of version number +# - URLs will have $repo replaced by the name of the current repo +# - URLs will have $arch replaced by the name of the architecture +# +# Repository entries are of the format: +# [repo-name] +# Server = ServerName +# Include = IncludePath +# +# The header [repo-name] is crucial - it must be present and +# uncommented to enable the repo. +# + +# The testing repositories are disabled by default. To enable, uncomment the +# repo name header and Include lines. You can add preferred servers immediately +# after the header, and they will be used before the default mirrors. + +[gnome-unstable] +Include = /etc/pacman.d/mirrorlist + +[kde-unstable] +Include = /etc/pacman.d/mirrorlist + +[core-staging] +Include = /etc/pacman.d/mirrorlist + +[core-testing] +Include = /etc/pacman.d/mirrorlist + +[core] +Include = /etc/pacman.d/mirrorlist + +[extra-staging] +Include = /etc/pacman.d/mirrorlist + +[extra-testing] +Include = /etc/pacman.d/mirrorlist + +[extra] +Include = /etc/pacman.d/mirrorlist + +# If you want to run 32 bit applications on your x86_64 system, +# enable the multilib repositories as required here. +[multilib-staging] +Include = /etc/pacman.d/mirrorlist + +[multilib-testing] +Include = /etc/pacman.d/mirrorlist + +[multilib] +Include = /etc/pacman.d/mirrorlist + +# An example of a custom package repository. See the pacman manpage for +# tips on creating your own repositories. +#[custom] +#SigLevel = Optional TrustAll +#Server = file:///home/custompkgs diff --git a/config/pacman/unstable.conf b/config/pacman/unstable.conf new file mode 100644 index 0000000..408d0ce --- /dev/null +++ b/config/pacman/unstable.conf @@ -0,0 +1,83 @@ +# +# /etc/pacman.conf +# +# See the pacman.conf(5) manpage for option and repository directives + +# +# GENERAL OPTIONS +# +[options] +# The following paths are commented out with their default values listed. +# If you wish to use different paths, uncomment and update the paths. +#RootDir = / +#DBPath = /var/lib/pacman/ +#CacheDir = /var/cache/pacman/pkg/ +#LogFile = /var/log/pacman.log +#GPGDir = /etc/pacman.d/gnupg/ +#HookDir = /etc/pacman.d/hooks/ +HoldPkg = pacman glibc +#XferCommand = /usr/bin/curl -L -C - -f -o %o %u +#XferCommand = /usr/bin/wget --passive-ftp -c -O %o %u +#CleanMethod = KeepInstalled +Architecture = auto + +# Pacman won't upgrade packages listed in IgnorePkg and members of IgnoreGroup +#IgnorePkg = +#IgnoreGroup = + +#NoUpgrade = +#NoExtract = + +# Misc options +#UseSyslog +#Color +NoProgressBar +# We cannot check disk space from within a chroot environment +#CheckSpace +VerbosePkgLists +ParallelDownloads = 5 + +# By default, pacman accepts packages signed by keys that its local keyring +# trusts (see pacman-key and its man page), as well as unsigned packages. +SigLevel = Required DatabaseOptional +LocalFileSigLevel = Optional +#RemoteFileSigLevel = Required + +# NOTE: You must run `pacman-key --init` before first using pacman; the local +# keyring can then be populated with the keys of all official Arch Linux +# packagers with `pacman-key --populate archlinux`. + +# +# REPOSITORIES +# - can be defined here or included from another file +# - pacman will search repositories in the order defined here +# - local/custom mirrors can be added here or in separate files +# - repositories listed first will take precedence when packages +# have identical names, regardless of version number +# - URLs will have $repo replaced by the name of the current repo +# - URLs will have $arch replaced by the name of the architecture +# +# Repository entries are of the format: +# [repo-name] +# Server = ServerName +# Include = IncludePath +# +# The header [repo-name] is crucial - it must be present and +# uncommented to enable the repo. +# + +# The testing repositories are disabled by default. To enable, uncomment the +# repo name header and Include lines. You can add preferred servers immediately +# after the header, and they will be used before the default mirrors. + +[gnome-unstable] +Include = /etc/pacman.d/mirrorlist + +[kde-unstable] +Include = /etc/pacman.d/mirrorlist + +# An example of a custom package repository. See the pacman manpage for +# tips on creating your own repositories. +#[custom] +#SigLevel = Optional TrustAll +#Server = file:///home/custompkgs diff --git a/contrib/completion/bash/devtools.in b/contrib/completion/bash/devtools.in index ec45b62..5125ceb 100644 --- a/contrib/completion/bash/devtools.in +++ b/contrib/completion/bash/devtools.in @@ -241,6 +241,7 @@ _pkgctl_db_move_opts() { _pkgctl_db_remove_args=( --partial + --noconfirm -a --arch -h --help ) diff --git a/contrib/completion/zsh/_devtools.in b/contrib/completion/zsh/_devtools.in index 6dc0340..48ff373 100644 --- a/contrib/completion/zsh/_devtools.in +++ b/contrib/completion/zsh/_devtools.in @@ -79,6 +79,7 @@ _pkgctl_db_move_args=( _pkgctl_db_remove_args=( '--partial[Remove only partial pkgnames from a split package]' + '--noconfirm[Bypass any confirmation messages, should only be used with caution]' '(-a --arch)'{-a,--arch}"[Override the architecture (disables auto-detection)]:arch:($DEVTOOLS_VALID_BINARY_ARCHES[*])" '(-h --help)'{-h,--help}'[Display usage]' "1:repo:($DEVTOOLS_VALID_REPOS[*])" diff --git a/doc/man/pkgctl-db-remove.1.asciidoc b/doc/man/pkgctl-db-remove.1.asciidoc index 9fe07c3..85f616b 100644 --- a/doc/man/pkgctl-db-remove.1.asciidoc +++ b/doc/man/pkgctl-db-remove.1.asciidoc @@ -31,6 +31,9 @@ Options Remove only one specific architecture (disables auto-detection). By default all architectures are removed when this option is not used. +*--noconfirm*:: + Bypass any confirmation messages, should only be used with caution. + *-h, --help*:: Show a help text diff --git a/src/lib/build/build.sh b/src/lib/build/build.sh index c3e05be..b82011b 100644 --- a/src/lib/build/build.sh +++ b/src/lib/build/build.sh @@ -312,7 +312,7 @@ pkgctl_build() { # Update pacman cache for auto-detection if [[ -z ${REPO} ]]; then - update_pacman_repo_cache + update_pacman_repo_cache multilib # Check valid repos if not resolved dynamically elif ! in_array "${REPO}" "${DEVTOOLS_VALID_REPOS[@]}"; then die "Invalid repository target: %s" "${REPO}" diff --git a/src/lib/db/remove.sh b/src/lib/db/remove.sh index 018b793..6ca091d 100644 --- a/src/lib/db/remove.sh +++ b/src/lib/db/remove.sh @@ -8,6 +8,10 @@ DEVTOOLS_INCLUDE_DB_REMOVE_SH=1 _DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@} # shellcheck source=src/lib/common.sh source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh +# shellcheck source=src/lib/util/pacman.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/pacman.sh +# shellcheck source=src/lib/util/term.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/term.sh set -e @@ -29,6 +33,7 @@ pkgctl_db_remove_usage() { -a, --arch Remove only one specific architecture (disables auto-detection) --partial Remove only partial pkgnames from a split package. This leaves debug packages behind and pkgbase entries in the state repo. + --noconfirm Bypass any confirmation messages, should only be used with caution -h, --help Show this help text EXAMPLES @@ -40,8 +45,12 @@ _EOF_ pkgctl_db_remove() { local REPO="" local PKGBASES=() + local pkgnames=() local partial=0 + local confirm=1 local dbscripts_options=() + local lookup_repo=multilib + local pkgname # option checking while (( $# )); do @@ -60,6 +69,10 @@ pkgctl_db_remove() { dbscripts_options+=(--arch "$2") shift 2 ;; + --noconfirm) + confirm=0 + shift + ;; -*) die "invalid argument: %s" "$1" ;; @@ -77,6 +90,40 @@ pkgctl_db_remove() { REPO=$1 shift PKGBASES+=("$@") + pkgnames=("${PKGBASES[@]}") + + # update pacman cache to query all pkgnames + if (( ! partial )); then + case ${REPO} in + *-unstable) + update_pacman_repo_cache unstable + ;; + *-staging) + update_pacman_repo_cache multilib-staging + ;; + *-testing) + update_pacman_repo_cache multilib-testing + ;; + *) + update_pacman_repo_cache multilib + ;; + esac + + # fetch the pkgnames of all pkgbase as present in the repo + mapfile -t pkgnames < <(get_pkgnames_from_repo_pkgbase "${REPO}" "${PKGBASES[@]}") + echo + + if (( ! ${#pkgnames[@]} )); then + error "Packages not found in %s" "${REPO}" + exit 1 + fi + fi + + # print list of packages + printf "%sRemoving packages from %s:%s\n" "${RED}" "${REPO}" "${ALL_OFF}" + for pkgname in "${pkgnames[@]}"; do + printf "• %s\n" "${pkgname}" + done # print explenation about partial removal if (( partial )); then @@ -85,7 +132,15 @@ pkgctl_db_remove() { msg_warn "${YELLOW}This leaves debug packages and pkgbase entries in the state repo!${ALL_OFF}" fi - # shellcheck disable=SC2029 + # ask for confirmation + if (( confirm )); then + echo + if ! prompt "${GREEN}${BOLD}?${ALL_OFF} Are you sure this is correct?"; then + exit 1 + fi + fi + echo + # shellcheck disable=SC2029 ssh "${PACKAGING_REPO_RELEASE_HOST}" db-remove "${dbscripts_options[@]}" "${REPO}" "${PKGBASES[@]}" } diff --git a/src/lib/release.sh b/src/lib/release.sh index acb3b54..ba21384 100644 --- a/src/lib/release.sh +++ b/src/lib/release.sh @@ -124,7 +124,7 @@ pkgctl_release() { # Update pacman cache for auto-detection if [[ -z ${REPO} ]]; then - update_pacman_repo_cache + update_pacman_repo_cache multilib # Check valid repos if not resolved dynamically elif ! in_array "${REPO}" "${DEVTOOLS_VALID_REPOS[@]}"; then die "Invalid repository target: %s" "${REPO}" diff --git a/src/lib/util/pacman.sh b/src/lib/util/pacman.sh index 620e1a8..4637d28 100644 --- a/src/lib/util/pacman.sh +++ b/src/lib/util/pacman.sh @@ -18,10 +18,12 @@ readonly _DEVTOOLS_MAKEPKG_CONF_DIR=${_DEVTOOLS_LIBRARY_DIR}/makepkg.conf.d update_pacman_repo_cache() { + local repo=${1:-multilib} + mkdir -p "${_DEVTOOLS_PACMAN_CACHE_DIR}" msg "Updating pacman database cache" lock 10 "${_DEVTOOLS_PACMAN_CACHE_DIR}.lock" "Locking pacman database cache" - fakeroot -- pacman --config "${_DEVTOOLS_PACMAN_CONF_DIR}/multilib.conf" \ + fakeroot -- pacman --config "${_DEVTOOLS_PACMAN_CONF_DIR}/${repo}.conf" \ --dbpath "${_DEVTOOLS_PACMAN_CACHE_DIR}" \ -Sy lock_close 10 @@ -29,6 +31,7 @@ update_pacman_repo_cache() { get_pacman_repo_from_pkgbuild() { local path=${1:-PKGBUILD} + local repo=${2:-multilib} # shellcheck source=contrib/makepkg/PKGBUILD.proto mapfile -t pkgnames < <(source "${path}"; printf "%s\n" "${pkgname[@]}") @@ -40,12 +43,12 @@ get_pacman_repo_from_pkgbuild() { # update the pacman repo cache if it doesn't exist yet if [[ ! -d "${_DEVTOOLS_PACMAN_CACHE_DIR}" ]]; then - update_pacman_repo_cache + update_pacman_repo_cache "${repo}" fi slock 10 "${_DEVTOOLS_PACMAN_CACHE_DIR}.lock" "Locking pacman database cache" # query repo of passed pkgname, specify --nodeps twice to skip all dependency checks - mapfile -t repos < <(pacman --config "${_DEVTOOLS_PACMAN_CONF_DIR}/multilib.conf" \ + mapfile -t repos < <(pacman --config "${_DEVTOOLS_PACMAN_CONF_DIR}/${repo}.conf" \ --dbpath "${_DEVTOOLS_PACMAN_CACHE_DIR}" \ --sync \ --nodeps \ @@ -58,3 +61,30 @@ get_pacman_repo_from_pkgbuild() { printf "%s" "${repos[0]}" } + +get_pkgnames_from_repo_pkgbase() { + local repo=$1 + shift + local pkgbases=("$@") + + # update the pacman repo cache if it doesn't exist yet + if [[ ! -d "${_DEVTOOLS_PACMAN_CACHE_DIR}" ]]; then + update_pacman_repo_cache universe + fi + + slock 10 "${_DEVTOOLS_PACMAN_CACHE_DIR}.lock" "Locking pacman database cache" + # query pkgnames of passed pkgbase inside a repo + mapfile -t pkgnames < <(expac --config <(sed "s|#DBPath.*|DBPath = $(realpath "${_DEVTOOLS_PACMAN_CACHE_DIR}")|" < "${_DEVTOOLS_PACMAN_CONF_DIR}/universe.conf") \ + --sync '%r %e %n' 2>/dev/null \ + | sort | awk -v pkgbase="${pkgbases[*]}" \ + 'BEGIN { split(pkgbase, array); for (item in array) filter[array[item]]=1 } $1=="'"${repo}"'" && $2 in filter {print $3}' + ) + lock_close 10 + + if (( ! ${#pkgnames[@]} )); then + return 1 + fi + + printf "%s\n" "${pkgnames[@]}" + return 0 +} diff --git a/src/lib/util/term.sh b/src/lib/util/term.sh index 853dccf..08d044f 100644 --- a/src/lib/util/term.sh +++ b/src/lib/util/term.sh @@ -180,3 +180,19 @@ term_spinner_stop() { # show the cursor after stopping the spinner term_cursor_show } + +prompt() { + local message=$1 + local answer + + read -r -p "${message} (y/N) " answer + + case "${answer}" in + y|Y|yes|Yes|YES) + true + ;; + *) + false + ;; + esac +} -- cgit v1.2.3-70-g09d2 From 35b417d226fc3d11ac0015abdcb3594d2c696c2e Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Sat, 27 Apr 2024 02:18:05 +0200 Subject: fix(db): check valid target repo on db remove Return an error and abort operation when an invalid target repo is passed. Component: pkgctl db remove Signed-off-by: Levente Polyak --- src/lib/db/remove.sh | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'src/lib') diff --git a/src/lib/db/remove.sh b/src/lib/db/remove.sh index 6ca091d..cddcc1d 100644 --- a/src/lib/db/remove.sh +++ b/src/lib/db/remove.sh @@ -12,6 +12,8 @@ source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/pacman.sh # shellcheck source=src/lib/util/term.sh source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/term.sh +# shellcheck source=src/lib/valid-repos.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/valid-repos.sh set -e @@ -92,6 +94,11 @@ pkgctl_db_remove() { PKGBASES+=("$@") pkgnames=("${PKGBASES[@]}") + # check if the target repo is valid + if ! in_array "${REPO}" "${DEVTOOLS_VALID_REPOS[@]}"; then + die "Invalid repository target: %s" "${REPO}" + fi + # update pacman cache to query all pkgnames if (( ! partial )); then case ${REPO} in -- cgit v1.2.3-70-g09d2 From a46b2d4fb7dee11fcc508c6871b86d9bff8d01ae Mon Sep 17 00:00:00 2001 From: Robin Candau Date: Sun, 28 Apr 2024 12:39:20 +0200 Subject: feat(repo): add repo clean command to remove untracked files This introduces the `pkgctl repo clean` command which removes every untracked files from local package repositories (via `git clean`). The usage is as simple as `pkgctl repo clean [OPTION] [PATH]` (where "[PATH]" can be equal to a wildcard "*"). Component: pkgctl repo clean --- contrib/completion/bash/devtools.in | 7 +++ contrib/completion/zsh/_devtools.in | 8 +++ doc/man/pkgctl-repo-clean.1.asciidoc | 40 ++++++++++++ doc/man/pkgctl-repo.1.asciidoc | 4 ++ src/lib/repo.sh | 10 +++ src/lib/repo/clean.sh | 114 +++++++++++++++++++++++++++++++++++ 6 files changed, 183 insertions(+) create mode 100644 doc/man/pkgctl-repo-clean.1.asciidoc create mode 100644 src/lib/repo/clean.sh (limited to 'src/lib') diff --git a/contrib/completion/bash/devtools.in b/contrib/completion/bash/devtools.in index 5125ceb..136beef 100644 --- a/contrib/completion/bash/devtools.in +++ b/contrib/completion/bash/devtools.in @@ -292,6 +292,7 @@ _pkgctl_aur_drop_from_repo_opts() { _filedir -d; } _pkgctl_repo_cmds=( + clean clone configure create @@ -317,6 +318,12 @@ _pkgctl_repo_clone_args__jobs_opts() { :; } _pkgctl_repo_clone_args_j_opts() { _pkgctl_repo_clone_args__jobs_opts; } _pkgctl_repo_clone_opts() { _devtools_completions_all_packages; } +_pkgctl_repo_clean_args=( + -i --interactive + -n --dry-run + -h --help +) +_pkgctl_repo_clean_opts() { _filedir -d; } _pkgctl_repo_configure_args=( --protocol diff --git a/contrib/completion/zsh/_devtools.in b/contrib/completion/zsh/_devtools.in index 48ff373..65d7895 100644 --- a/contrib/completion/zsh/_devtools.in +++ b/contrib/completion/zsh/_devtools.in @@ -114,6 +114,7 @@ _pkgctl_aur_drop_from_repo_args=( _pkgctl_repo_cmds=( "pkgctl repo command" + "clean[Remove untracked files from the working tree]" "clone[Clone a package repository]" "configure[Configure a clone according to distro specs]" "create[Create a new GitLab package repository]" @@ -128,6 +129,13 @@ _pkgctl_repo_switch_args=( '*:git_dir:_files -/' ) +_pkgctl_repo_clean_args=( + '(-i --interactive)'{-i,--interactive}'[Show what would be done and clean files interactively]' + '(-n --dry-run)'{-n,--dry-run}"[Don't remove anything, just show what would be done]" + '(-h --help)'{-h,--help}'[Display usage]' + '*:git_dir:_files -/' +) + _pkgctl_repo_clone_args=( '(-m --maintainer=)'{-m,--maintainer=}'[Clone all packages of the named maintainer]:maintainer:' '--protocol[Clone the repository over https]:proto:(https)' diff --git a/doc/man/pkgctl-repo-clean.1.asciidoc b/doc/man/pkgctl-repo-clean.1.asciidoc new file mode 100644 index 0000000..b39693c --- /dev/null +++ b/doc/man/pkgctl-repo-clean.1.asciidoc @@ -0,0 +1,40 @@ +pkgctl-repo-clean(1) +==================== + +Name +---- + +pkgctl-repo-clean - Remove untracked files from the working tree + +Synopsis +-------- + +pkgctl repo clean [OPTION] [PATH]... + +Description +----------- + +Cleans the working tree by recursively removing files that are not under +version control, starting from the current directory. + +Files unknown to Git as well as ignored files are removed. This can, for +example, be useful to remove all build products. + +Options +------- + +*-i, --interactive*:: + Show what would be done and clean files interactively + +*-n, --dry-run*:: + Don't actually remove anything, just show what would be done + +*-h, --help*:: + Show a help text + +See Also +-------- + +git-clean(1) + +include::include/footer.asciidoc[] diff --git a/doc/man/pkgctl-repo.1.asciidoc b/doc/man/pkgctl-repo.1.asciidoc index 3d0f3f1..c1fd298 100644 --- a/doc/man/pkgctl-repo.1.asciidoc +++ b/doc/man/pkgctl-repo.1.asciidoc @@ -32,6 +32,9 @@ Options Subcommands ----------- +pkgctl repo clean:: + Remove untracked files from the working tree + pkgctl repo clone:: Clone a package repository @@ -50,6 +53,7 @@ pkgctl repo web:: See Also -------- +pkgctl-repo-clean(1) pkgctl-repo-clone(1) pkgctl-repo-configure(1) pkgctl-repo-create(1) diff --git a/src/lib/repo.sh b/src/lib/repo.sh index 9f545e9..8f8dd0a 100644 --- a/src/lib/repo.sh +++ b/src/lib/repo.sh @@ -27,6 +27,7 @@ pkgctl_repo_usage() { without SSH access using read-only HTTPS. COMMANDS + clean Remove untracked files from the working tree clone Clone a package repository configure Configure a clone according to distro specs create Create a new GitLab package repository @@ -37,6 +38,7 @@ pkgctl_repo_usage() { -h, --help Show this help text EXAMPLES + $ ${COMMAND} clean --interactive * $ ${COMMAND} clone libfoo linux libbar $ ${COMMAND} clone --maintainer mynickname $ ${COMMAND} configure * @@ -59,6 +61,14 @@ pkgctl_repo() { pkgctl_repo_usage exit 0 ;; + clean) + _DEVTOOLS_COMMAND+=" $1" + shift + # shellcheck source=src/lib/repo/clean.sh + source "${_DEVTOOLS_LIBRARY_DIR}"/lib/repo/clean.sh + pkgctl_repo_clean "$@" + exit 0 + ;; clone) _DEVTOOLS_COMMAND+=" $1" shift diff --git a/src/lib/repo/clean.sh b/src/lib/repo/clean.sh new file mode 100644 index 0000000..bb8980e --- /dev/null +++ b/src/lib/repo/clean.sh @@ -0,0 +1,114 @@ +#!/bin/bash +# +# SPDX-License-Identifier: GPL-3.0-or-later + +[[ -z ${DEVTOOLS_INCLUDE_REPO_CLEAN_SH:-} ]] || return 0 +DEVTOOLS_INCLUDE_REPO_CLEAN_SH=1 + +_DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@} +# shellcheck source=src/lib/common.sh +source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh + +source /usr/share/makepkg/util/message.sh + +set -eo pipefail + + +pkgctl_repo_clean_usage() { + local -r COMMAND=${_DEVTOOLS_COMMAND:-${BASH_SOURCE[0]##*/}} + cat <<- _EOF_ + Usage: ${COMMAND} [OPTION] [PATH]... + + Cleans the working tree by recursively removing files that are not under + version control, starting from the current directory. + + Files unknown to Git as well as ignored files are removed. This can, for + example, be useful to remove all build products. + + OPTIONS + -i, --interactive Show what would be done and clean files interactively + -n, --dry-run Don't remove anything, just show what would be done + -h, --help Show this help text + + EXAMPLES + $ ${COMMAND} libfoo linux libbar + $ ${COMMAND} --interactive libfoo linux libbar + $ ${COMMAND} --dry-run * +_EOF_ +} + +pkgctl_repo_clean() { + # options + local git_clean_options=() + local paths + + local path pkgbase + + while (( $# )); do + case $1 in + -i|--interactive) + git_clean_options+=("$1") + shift + ;; + -n|--dry-run) + git_clean_options+=("$1") + shift + ;; + -h|--help) + pkgctl_repo_clean_usage + exit 0 + ;; + --) + shift + break + ;; + -*) + die "invalid argument: %s" "$1" + ;; + *) + paths=("$@") + break + ;; + esac + done + + # check if invoked without any path from within a packaging repo + if (( ${#paths[@]} == 0 )); then + paths=(".") + fi + + # print message about the work chunk + printf "🗑️ Removing untracked files from %s working trees\n" "${BOLD}${#paths[@]}${ALL_OFF}" + + for path in "${paths[@]}"; do + # skip paths that are not directories + if [[ ! -d "${path}" ]]; then + continue + fi + + if [[ ! -f "${path}/PKGBUILD" ]]; then + msg_error "Not a package repository: ${path}" + continue + fi + + if [[ ! -d "${path}/.git" ]]; then + msg_error "Not a Git repository: ${path}" + continue + fi + + pkgbase=$(basename "$(realpath "${path}")") + pkgbase=${pkgbase%.git} + + # run dry mode to see if git would clean any files + if [[ ! $(git -C "${path}" clean -x -d --dry-run 2>&1) ]]; then + continue + fi + + # git clean untracked files + msg_success "Cleaning ${BOLD}${pkgbase}${ALL_OFF}" + if ! git -C "${path}" clean -x -d --force "${git_clean_options[@]}"; then + msg_error "Failed to remove untracked files" + fi + echo + done +} -- cgit v1.2.3-70-g09d2 From c484a55cde24457f6a8f6f581f7502da2f873b30 Mon Sep 17 00:00:00 2001 From: Christian Heusel Date: Mon, 29 Apr 2024 14:46:21 +0200 Subject: fix(version): dont die if no PKGBUILD is found So far the commands would stop execution if one of the target directories did not contain a PKGBUILD instead of just reporting failure for that directory. Fix this by replacing the 'die' calls with setting the error for the spinner facility. Component: pkgctl version check Component: pkgctl version upgrade Signed-off-by: Christian Heusel --- src/lib/version/check.sh | 11 +++++++---- src/lib/version/upgrade.sh | 8 ++++++-- 2 files changed, 13 insertions(+), 6 deletions(-) (limited to 'src/lib') diff --git a/src/lib/version/check.sh b/src/lib/version/check.sh index ec90eb4..27509a6 100644 --- a/src/lib/version/check.sh +++ b/src/lib/version/check.sh @@ -114,10 +114,6 @@ pkgctl_version_check() { fi pushd "${path}" >/dev/null - if [[ ! -f "PKGBUILD" ]]; then - die "No PKGBUILD found for ${path}" - fi - # update the current terminal spinner status (( ++current_item )) pkgctl_version_check_spinner \ @@ -128,6 +124,13 @@ pkgctl_version_check() { "${current_item}" \ "${#pkgbases[@]}" + if [[ ! -f "PKGBUILD" ]]; then + result="${BOLD}${path}${ALL_OFF}: no PKGBUILD found" + failure+=("${result}") + popd >/dev/null + continue + fi + # reset common PKGBUILD variables unset pkgbase pkgname arch source pkgver pkgrel validpgpkeys # shellcheck source=contrib/makepkg/PKGBUILD.proto diff --git a/src/lib/version/upgrade.sh b/src/lib/version/upgrade.sh index c57171c..70a4659 100644 --- a/src/lib/version/upgrade.sh +++ b/src/lib/version/upgrade.sh @@ -111,8 +111,13 @@ pkgctl_version_upgrade() { fi pushd "${path}" >/dev/null + (( ++current_item )) + if [[ ! -f "PKGBUILD" ]]; then - die "No PKGBUILD found for ${path}" + result="${BOLD}${path}${ALL_OFF}: no PKGBUILD found" + failure+=("${result}") + popd >/dev/null + continue fi # reset common PKGBUILD variables @@ -122,7 +127,6 @@ pkgctl_version_upgrade() { pkgbase=${pkgbase:-$pkgname} # update the current terminal spinner status - (( ++current_item )) pkgctl_version_upgrade_spinner \ "${status_dir}" \ "${#up_to_date[@]}" \ -- cgit v1.2.3-70-g09d2 From 952f483574db38b4f39960a9dbafc1bbb387ab0b Mon Sep 17 00:00:00 2001 From: Jakub Klinkovský Date: Sun, 21 Jan 2024 20:13:03 +0100 Subject: feat(offload-build): fetch logs after building from the remote server MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since logs from offloaded builds are collected in a temporary directory on the remote server, it is rather difficult/error-prone to get to them, because the path changes in each rebuild. Fetching logs from the server into $LOGDEST makes it easier to investigate them and also brings the behavior of offload-build closer to archbuild. Log files are always downloaded, even for failed builds. Component: offload-build Signed-off-by: Jakub Klinkovský Co-authored-by: Levente Polyak --- src/lib/common.sh | 8 ++++++++ src/offload-build.in | 16 ++++++++++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) (limited to 'src/lib') diff --git a/src/lib/common.sh b/src/lib/common.sh index 7d04c25..641dea5 100644 --- a/src/lib/common.sh +++ b/src/lib/common.sh @@ -354,6 +354,14 @@ is_debug_package() { [[ ${pkgdesc} == "Detached debugging symbols for "* && ${pkgbase}-debug = "${pkgname}" ]] } +# Proxy function to check if a file exists. Using [[ -f ... ]] directly is not +# always wanted because we might want to expand bash globs first. This way we +# can pass unquoted globs to is_globfile() and have them expanded as function +# arguments before being checked. +is_globfile() { + [[ -f $1 ]] +} + join_by() { local IFS="$1" shift diff --git a/src/offload-build.in b/src/offload-build.in index b272a74..f231ed1 100644 --- a/src/offload-build.in +++ b/src/offload-build.in @@ -107,6 +107,7 @@ mapfile -t files < <( cd "$temp" && { bsdtar --strip-components 1 -xvf - && + export LOGDEST="" && script -qefc "'"${archbuild_cmd[@]@Q}"'" /dev/null && printf "%s\n" "" "-> build complete" && printf "\t%s\n" "$temp"/* @@ -120,14 +121,25 @@ mapfile -t files < <( [[ -f "${file}" ]] && printf "%s\n" "${file}" ||: done < <(makepkg --config <(cat "${makepkg_user_config}" "${makepkg_config}" 2>/dev/null) --packagelist) && printf "%s\n" "${temp}/PKGBUILD" + + find "${temp}" -name "*.log" ') if (( ${#files[@]} )); then - printf '%s\n' '' '-> copying files...' + msg 'Downloading files...' rsync "${rsyncopts[@]}" "${files[@]/#/$server:}" "${TEMPDIR}/" || die - mv "${TEMPDIR}"/*.pkg.tar* "${PKGDEST:-${PWD}}/" + + if is_globfile "${TEMPDIR}"/*.log; then + mv "${TEMPDIR}"/*.log "${LOGDEST:-${PWD}}/" + fi + # missing PKGBUILD download means the build failed + if [[ ! -f "${TEMPDIR}/PKGBUILD" ]]; then + error "Build failed, check logs in ${LOGDEST:-${PWD}}" + exit 1 + fi mv "${TEMPDIR}/PKGBUILD" "${PWD}/" + mv "${TEMPDIR}"/*.pkg.tar* "${PKGDEST:-${PWD}}/" else exit 1 fi -- cgit v1.2.3-70-g09d2 From d1790c295a054982734aa9b1b3eb4f7d4de234f6 Mon Sep 17 00:00:00 2001 From: Levente Polyak Date: Fri, 10 May 2024 21:22:23 +0200 Subject: fix(version): escape pkgbase in nvchecker toml This fixes issues with packages containing plus signs, that need to be escaped in toml as well as the extended grep regex. Component: pkgctl version check --- src/lib/version/check.sh | 4 ++-- src/lib/version/setup.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'src/lib') diff --git a/src/lib/version/check.sh b/src/lib/version/check.sh index 27509a6..0449c60 100644 --- a/src/lib/version/check.sh +++ b/src/lib/version/check.sh @@ -267,13 +267,13 @@ nvchecker_check_config() { done # check if the config contains a pkgbase section - if [[ -n ${pkgbase} ]] && ! grep --max-count=1 --extended-regexp --quiet "^\\[\"?${pkgbase}\"?\\]" < "${config}"; then + if [[ -n ${pkgbase} ]] && ! grep --max-count=1 --extended-regexp --quiet "^\\[\"?${pkgbase//+/\\+}\"?\\]" < "${config}"; then printf "missing pkgbase section in %s: %s" "${config}" "${pkgbase}" return 1 fi # check if the config contains any section other than pkgbase - if [[ -n ${pkgbase} ]] && property=$(grep --max-count=1 --perl-regexp "^\\[(?!\"?${pkgbase}\"?\\]).+\\]" < "${config}"); then + if [[ -n ${pkgbase} ]] && property=$(grep --max-count=1 --perl-regexp "^\\[(?!\"?${pkgbase//+/\\+}\"?\\]).+\\]" < "${config}"); then printf "non-pkgbase section not supported in %s: %s" "${config}" "${property}" return 1 fi diff --git a/src/lib/version/setup.sh b/src/lib/version/setup.sh index 123862c..cdfbeac 100644 --- a/src/lib/version/setup.sh +++ b/src/lib/version/setup.sh @@ -252,7 +252,7 @@ nvchecker_setup() { # escape the section if it contains toml subsection chars section="${pkgbase}" - if [[ ${section} == *.* ]]; then + if [[ ${section} == *.* ]] || [[ ${section} == *+* ]]; then section="\"${section}\"" fi -- cgit v1.2.3-70-g09d2 From 0e2b16b0ac63e33e32a6cb889bc4b047e0d451e9 Mon Sep 17 00:00:00 2001 From: Christian Heusel Date: Sat, 18 May 2024 02:00:19 +0200 Subject: fix: disable systemd-nspawn terminal coloring Systemd 256 introduces functionality which colors the terminal background on systemd-nspawn invocations which makes the pkgctl output look weird. Disable this bevaviour for pkgctl, so it stays active for arch-nspawn (for now). Component: pkgctl Signed-off-by: Christian Heusel --- src/lib/common.sh | 3 +++ 1 file changed, 3 insertions(+) (limited to 'src/lib') diff --git a/src/lib/common.sh b/src/lib/common.sh index 641dea5..b087bb1 100644 --- a/src/lib/common.sh +++ b/src/lib/common.sh @@ -15,6 +15,9 @@ $DEVTOOLS_INCLUDE_COMMON_SH # Avoid any encoding problems export LANG=C.UTF-8 +# Avoid systemd trying to color the terminal on systemd-nspawn +export SYSTEMD_TINT_BACKGROUND=no + # Set buildtool properties export BUILDTOOL=devtools export BUILDTOOLVER=@buildtoolver@ -- cgit v1.2.3-70-g09d2 From 144f9a871ee554b44e1d9733a56d6b4dc99803fe Mon Sep 17 00:00:00 2001 From: David Runge Date: Thu, 6 Jun 2024 16:26:35 +0200 Subject: fix(version): Ignore warnings when nvchecker ignores invalid versions Since version 2.15.1 nvchecker emits a warning for version strings that it consideres invalid (e.g. in the case of PyPI). These warning messages get in the way (the first version emitted via a warning is used as version to compare against) of retrieving the latest version of an upstream and therefore we ignore them. Component: pkgctl version check Signed-off-by: David Runge --- src/lib/version/check.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/lib') diff --git a/src/lib/version/check.sh b/src/lib/version/check.sh index 0449c60..4a2b5fa 100644 --- a/src/lib/version/check.sh +++ b/src/lib/version/check.sh @@ -227,7 +227,7 @@ get_upstream_version() { fi if ! output=$(GIT_TERMINAL_PROMPT=0 nvchecker --file "${config}" --logger json "${opts[@]}" 2>&1 | \ - jq --raw-output 'select(.level != "debug")'); then + jq --raw-output 'select((.level != "debug") and (.event != "ignoring invalid version"))'); then printf "failed to run nvchecker: %s" "${output}" return 1 fi -- cgit v1.2.3-70-g09d2 From 1df0979da6bbe5de8549693d12cb3e547bc6d94a Mon Sep 17 00:00:00 2001 From: Christian Heusel Date: Sat, 15 Jun 2024 14:23:21 +0200 Subject: fix(common): guard the WORKDIR environment var This avoids the unwanted removal of the folder if someone has already pre-defined the variable. Fixes #219 Suggested-by: Levente Polyak Signed-off-by: Christian Heusel --- src/lib/common.sh | 2 ++ 1 file changed, 2 insertions(+) (limited to 'src/lib') diff --git a/src/lib/common.sh b/src/lib/common.sh index b087bb1..5416eaf 100644 --- a/src/lib/common.sh +++ b/src/lib/common.sh @@ -120,6 +120,8 @@ print_workdir_error() { } _setup_workdir=false +# Ensure that there is no outside value for WORKDIR leaking in +unset WORKDIR setup_workdir() { [[ -z ${WORKDIR:-} ]] && WORKDIR=$(mktemp -d --tmpdir "${0##*/}.XXXXXXXXXX") _setup_workdir=true -- cgit v1.2.3-70-g09d2 From 9ff63503b94e7f523cba1a5f9e5dfdc9c8855471 Mon Sep 17 00:00:00 2001 From: Chih-Hsuan Yen <645432-yan12125@users.noreply.gitlab.com> Date: Wed, 19 Jun 2024 17:37:18 +0800 Subject: fix(pkgctl): make sure git signing uses PGP Component: pkgctl repo configure --- src/lib/repo/configure.sh | 1 + 1 file changed, 1 insertion(+) (limited to 'src/lib') diff --git a/src/lib/repo/configure.sh b/src/lib/repo/configure.sh index 0980fd1..e9ec5a4 100644 --- a/src/lib/repo/configure.sh +++ b/src/lib/repo/configure.sh @@ -271,6 +271,7 @@ pkgctl_repo_configure() { if [[ -n $GPGKEY ]]; then git config commit.gpgsign true git config user.signingKey "${GPGKEY}" + git config gpg.format openpgp fi # set default git exclude -- cgit v1.2.3-70-g09d2 From 27eebe383d0b571c08cba991e4824768d7623602 Mon Sep 17 00:00:00 2001 From: Jaroslav Lichtblau Date: Wed, 26 Jun 2024 16:12:56 +0000 Subject: doc: fix the example command in the help text the '--pkgver' argument is not space-separated but instead specified with an equals sign. Component: pkgctl build --- src/lib/build/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/lib') diff --git a/src/lib/build/build.sh b/src/lib/build/build.sh index b82011b..9f724dd 100644 --- a/src/lib/build/build.sh +++ b/src/lib/build/build.sh @@ -80,7 +80,7 @@ pkgctl_build_usage() { EXAMPLES $ ${COMMAND} $ ${COMMAND} --rebuild --staging --message 'libyay 0.42 rebuild' libfoo libbar - $ ${COMMAND} --pkgver 1.42 --release --db-update + $ ${COMMAND} --pkgver=1.42 --release --db-update _EOF_ } -- cgit v1.2.3-70-g09d2