From f2cafa3cb0941be8235025434620adbf5849a432 Mon Sep 17 00:00:00 2001
From: Levente Polyak <anthraxx@archlinux.org>
Date: Sun, 3 Dec 2023 19:46:09 +0100
Subject: feat(clone): speedup maintainer and universe clone query

There is a single endpoint now to list all pkgbases and their current
maintainers. Use this endpoint for speeding up the clone of all packages
of a maintainer by only issuing a single API call.

Component: pkgctl repo clone
Signed-off-by: Levente Polyak <anthraxx@archlinux.org>
---
 src/lib/repo/clone.sh | 30 +++++++++---------------------
 1 file changed, 9 insertions(+), 21 deletions(-)

(limited to 'src/lib/repo')

diff --git a/src/lib/repo/clone.sh b/src/lib/repo/clone.sh
index fb927d2..ef6a0e2 100644
--- a/src/lib/repo/clone.sh
+++ b/src/lib/repo/clone.sh
@@ -8,6 +8,8 @@ DEVTOOLS_INCLUDE_REPO_CLONE_SH=1
 _DEVTOOLS_LIBRARY_DIR=${_DEVTOOLS_LIBRARY_DIR:-@pkgdatadir@}
 # shellcheck source=src/lib/common.sh
 source "${_DEVTOOLS_LIBRARY_DIR}"/lib/common.sh
+# shellcheck source=src/lib/api/archweb.sh
+source "${_DEVTOOLS_LIBRARY_DIR}"/lib/api/archweb.sh
 # shellcheck source=src/lib/api/gitlab.sh
 source "${_DEVTOOLS_LIBRARY_DIR}"/lib/api/gitlab.sh
 # shellcheck source=src/lib/repo/configure.sh
@@ -18,6 +20,7 @@ source "${_DEVTOOLS_LIBRARY_DIR}"/lib/util/git.sh
 source /usr/share/makepkg/util/message.sh
 
 set -e
+set -o pipefail
 
 
 pkgctl_repo_clone_usage() {
@@ -137,33 +140,18 @@ pkgctl_repo_clone() {
 
 	# Query packages of a maintainer
 	if [[ -n ${MAINTAINER} ]]; then
-		stat_busy "Query packages"
-		max_pages=$(curl --silent --location --fail --retry 3 --retry-delay 3 "https://archlinux.org/packages/search/json/?sort=name&maintainer=${MAINTAINER}" | jq -r '.num_pages')
-		if [[ ! ${max_pages} =~ ([[:digit:]]) ]]; then
-			stat_done
-			warning "found no packages for maintainer ${MAINTAINER}"
-			exit 0
+		mapfile -t pkgbases < <(archweb_query_maintainer_packages "${MAINTAINER}")
+		if ! wait $!; then
+			die "Failed to query maintainer packages"
 		fi
-		mapfile -t pkgbases < <(for page in $(seq "${max_pages}"); do
-			curl --silent --location --fail --retry 3 --retry-delay 3 "https://archlinux.org/packages/search/json/?sort=name&maintainer=${MAINTAINER}&page=${page}" | jq -r '.results[].pkgbase'
-			stat_progress
-		done | sort --unique)
-		stat_done
 	fi
 
 	# Query all released packages
 	if (( CLONE_ALL )); then
-		stat_busy "Query all released packages"
-		max_pages=$(curl --silent --location --fail --retry 3 --retry-delay 3 "https://archlinux.org/packages/search/json/?sort=name" | jq -r '.num_pages')
-		if [[ ! ${max_pages} =~ ([[:digit:]]) ]]; then
-			stat_done
-			die "failed to query packages"
+		mapfile -t pkgbases < <(archweb_query_all_packages)
+		if ! wait $!; then
+			die "Failed to query all packages"
 		fi
-		mapfile -t pkgbases < <(for page in $(seq "${max_pages}"); do
-			curl --silent --location --fail --retry 3 --retry-delay 3 "https://archlinux.org/packages/search/json/?sort=name&page=${page}" | jq -r '.results[].pkgbase'
-			stat_progress
-		done | sort --unique)
-		stat_done
 	fi
 
 	# parallelization
-- 
cgit v1.2.3-70-g09d2