lib/common-functions: failsafe_rsync() new

6 files changed, 37 insertions, 14 deletions

diff --git a/bin/copy-to-build-support b/bin/copy-to-build-support
index 70e26e7..7456235 100755
--- a/bin/copy-to-build-support
+++ b/bin/copy-to-build-support
@@ -167,7 +167,7 @@ sort -u "${tmp_dir}/repo-archs" | \
 
 while read -r repo_arch; do
   mkdir "${tmp_dir}/${repo_arch}"
-  ${master_mirror_rsync_command} \
+  failsafe_rsync \
     "${master_mirror_rsync_directory}/${repo_arch}/build-support/build-support.db."* \
     "${master_mirror_rsync_directory}/${repo_arch}/build-support/build-support.files."* \
     "${tmp_dir}/${repo_arch}/"
@@ -179,7 +179,7 @@ if [ -s "${tmp_dir}/to-copy" ]; then
   while read -r repo_arch; do
     mkdir "${tmp_dir}/transit/${repo_arch}/"
     # shellcheck disable=SC2046
-    ${master_mirror_rsync_command} \
+    failsafe_rsync \
       $(sed -n 's,^'"$(str_to_regex "${repo_arch}")"' ,,;T;p' "${tmp_dir}/to-copy") \
       "${tmp_dir}/transit/${repo_arch}/"
     repo-add "${tmp_dir}/${repo_arch}/build-support.db.tar.gz" \
@@ -198,7 +198,7 @@ while read -r repo_arch; do
     "${tmp_dir}" \
     "${tmp_dir}/${repo_arch}/build-support."*".tar.gz" \
     "${tmp_dir}/${repo_arch}/build-support."*".tar.gz.old"
-  ${master_mirror_rsync_command} \
+  failsafe_rsync \
     "${tmp_dir}/${repo_arch}/build-support.db."* \
     "${tmp_dir}/${repo_arch}/build-support.files."* \
     "${master_mirror_rsync_directory}/${repo_arch}/build-support/"
diff --git a/bin/db-update b/bin/db-update
index 367fb36..9e5a59b 100755
--- a/bin/db-update
+++ b/bin/db-update
@@ -356,7 +356,7 @@ for source_stability in \
   while read -r arch repo; do
     mkdir -p "${tmp_dir}/dbs/${arch}/${repo}"
     # shellcheck disable=SC2086
-    ${master_mirror_rsync_command} \
+    failsafe_rsync \
       "${master_mirror_rsync_directory}/${arch}/${repo}/${repo}.db."* \
       "${master_mirror_rsync_directory}/${arch}/${repo}/${repo}.files."* \
       "${tmp_dir}/dbs/${arch}/${repo}/"
@@ -456,7 +456,7 @@ for source_stability in \
       "${tmp_dir}/dbs/${arch}/${repo}/${repo}."*".tar.gz" \
       "${tmp_dir}/dbs/${arch}/${repo}/${repo}."*".tar.gz.old"
     # shellcheck disable=SC2086
-    ${master_mirror_rsync_command} \
+    failsafe_rsync \
       "${tmp_dir}/dbs/${arch}/${repo}/${repo}.db."* \
       "${tmp_dir}/dbs/${arch}/${repo}/${repo}.files."* \
       "${master_mirror_rsync_directory}/${arch}/${repo}/"
diff --git a/bin/delete-packages b/bin/delete-packages
index 50fc853..0c1689f 100755
--- a/bin/delete-packages
+++ b/bin/delete-packages
@@ -326,7 +326,7 @@ mkdir "${tmp_dir}/repos"
 
 while read -r arch repo; do
   mkdir -p "${tmp_dir}/repos/${arch}"
-  ${master_mirror_rsync_command} \
+  failsafe_rsync \
     "${master_mirror_rsync_directory}/${arch}/${repo}/${repo}.db.tar.gz" \
     "${master_mirror_rsync_directory}/${arch}/${repo}/${repo}.files.tar.gz" \
     "${tmp_dir}/repos/${arch}"
@@ -342,7 +342,7 @@ while read -r arch repo; do
       "${tmp_dir}" \
       "${tmp_dir}/repos/${arch}/${repo}."*".tar.gz" \
       "${tmp_dir}/repos/${arch}/${repo}."*".tar.gz.old"
-    ${master_mirror_rsync_command} \
+    failsafe_rsync \
       "${tmp_dir}/repos/${arch}/${repo}.db."* \
       "${tmp_dir}/repos/${arch}/${repo}.files."* \
       "${master_mirror_rsync_directory}/${arch}/${repo}/"
diff --git a/bin/return-assignment b/bin/return-assignment
index d8c6028..b29764c 100755
--- a/bin/return-assignment
+++ b/bin/return-assignment
@@ -586,7 +586,7 @@ cut -d' ' -f4,5 "${tmp_dir}/repository-ids" | \
   while read -r arch repo; do
 
     mkdir -p "${arch}/${repo}"
-    ${master_mirror_rsync_command} \
+    failsafe_rsync \
       "${master_mirror_rsync_directory}/${arch}/${repo}/${repo}.db."* \
       "${master_mirror_rsync_directory}/${arch}/${repo}/${repo}.files."* \
       "${arch}/${repo}/"
@@ -599,7 +599,7 @@ cut -d' ' -f4,5 "${tmp_dir}/repository-ids" | \
   done
 
 # upload the packages into /pool
-${master_mirror_rsync_command} -c --copy-dest=/.transfer \
+failsafe_rsync -c --copy-dest=/.transfer \
   ./*".pkg.tar.xz" \
   ./*".pkg.tar.xz.sig" \
   "${master_mirror_rsync_directory}/pool/"
@@ -621,7 +621,7 @@ cut -d' ' -f4,5 "${tmp_dir}/repository-ids" | \
       "${arch}/${repo}/${repo}."*".tar.gz" \
       "${arch}/${repo}/${repo}."*".tar.gz.old"
     # shellcheck disable=SC2046
-    ${master_mirror_rsync_command} \
+    failsafe_rsync \
       "${arch}/${repo}/${repo}.db."* \
       "${arch}/${repo}/${repo}.files."* \
       $(
diff --git a/bin/sanity-check b/bin/sanity-check
index 8baa7cf..5e74edb 100755
--- a/bin/sanity-check
+++ b/bin/sanity-check
@@ -306,7 +306,7 @@ while [ $# -gt 0 ]; do
             i_am_insane
           fi
 
-          ${master_mirror_rsync_command} \
+          failsafe_rsync \
             "${master_mirror_rsync_directory}/${arch}/${repo}/${repo}.db.tar.gz" \
             "${master_mirror_rsync_directory}/${arch}/${repo}/${repo}.files.tar.gz" \
             "${tmp_dir}/"
diff --git a/lib/common-functions b/lib/common-functions
index 6e0892d..7776725 100755
--- a/lib/common-functions
+++ b/lib/common-functions
@@ -225,12 +225,12 @@ remove_old_package_versions() {
     # repo-remove packages
     while read -r arch repo pkgname; do
       mkdir "${tmp_dir}/transit"
-      ${master_mirror_rsync_command} \
+      failsafe_rsync \
         "${master_mirror_rsync_directory}/${arch}/${repo}/${repo}.db."* \
         "${master_mirror_rsync_directory}/${arch}/${repo}/${repo}.files."* \
         "${tmp_dir}/transit/"
       repo-remove "${tmp_dir}/transit/${repo}.db.tar.gz" "${pkgname}"
-      ${master_mirror_rsync_command} \
+      failsafe_rsync \
         "${tmp_dir}/transit/${repo}.db."* \
         "${tmp_dir}/transit/${repo}.files."* \
         "${master_mirror_rsync_directory}/${arch}/${repo}/"
@@ -709,7 +709,7 @@ trigger_mirror_refreshs() {
   tmp_file=$(mktemp "tmp.common-functions.trigger_mirror_refreshs.XXXXXXXXXX" --tmpdir)
   date '+%s' > \
     "${tmp_file}"
-  ${master_mirror_rsync_command} \
+  failsafe_rsync \
     "${tmp_file}" \
     "${master_mirror_rsync_directory}/lastupdate"
   rm "${tmp_file}"
@@ -882,3 +882,26 @@ failsafe_sftp() {
   ) || \
     return $?
 }
+
+# failsafe_rsync
+# execute rsync with the given parameters on the master mirror, retrying
+# if unsuccessful
+# caveats:
+#  - output might be garbled
+#  - error code will be projected into {0,1}
+failsafe_rsync() {
+  local trial_counter
+
+  trial_counter=20
+  if ${master_mirror_rsync_command} "$@"; then
+    return 0
+  fi
+  while [ ${trial_counter} -gt 0 ]; do
+    wait_some_time 30
+    if ${master_mirror_rsync_command} "$@"; then
+      return 0
+    fi
+    trial_counter=$((trial_counter-1))
+  done
+  return 1
+}