download: add retries argument

Allow multiple attempts to download an asset

fluster/fluster.py

@@ -514,7 +514,7 @@ def _global_stats(
             print(output)
 
     def download_test_suites(
-        self, test_suites: List[str], jobs: int, keep_file: bool
+        self, test_suites: List[str], jobs: int, keep_file: bool, retries: int
     ) -> None:
         """Download a group of test suites"""
         self._load_test_suites()
@@ -527,5 +527,9 @@ def download_test_suites(
 
         for test_suite in download_test_suites:
             test_suite.download(
-                jobs, self.resources_dir, verify=True, keep_file=keep_file
+                jobs,
+                self.resources_dir,
+                verify=True,
+                keep_file=keep_file,
+                retries=retries,
             )

fluster/main.py

@@ -330,6 +330,13 @@ def _add_download_cmd(self, subparsers: Any) -> None:
             "files such as .zip, .tar.gz, etc",
             action="store_true",
         )
+        subparser.add_argument(
+            "-r",
+            "--retries",
+            help="number of retries, before failing",
+            type=int,
+            default=1,
+        )
         subparser.add_argument(
             "testsuites", help="list of testsuites to download", nargs="*"
         )
@@ -385,5 +392,8 @@ def _reference_cmd(self, args: Any, fluster: Fluster) -> None:
     def _download_cmd(self, args: Any, fluster: Fluster) -> None:
         args.jobs = args.jobs if args.jobs > 0 else multiprocessing.cpu_count()
         fluster.download_test_suites(
-            test_suites=args.testsuites, jobs=args.jobs, keep_file=args.keep
+            test_suites=args.testsuites,
+            jobs=args.jobs,
+            keep_file=args.keep,
+            retries=args.retries,
         )

fluster/test_suite.py

@@ -25,6 +25,7 @@
 from time import perf_counter
 from shutil import rmtree
 from typing import cast, List, Dict, Optional, Type, Any
+import urllib.error
 
 
 from fluster.test_vector import TestVector
@@ -45,13 +46,15 @@ def __init__(
         keep_file: bool,
         test_suite_name: str,
         test_vector: TestVector,
+        retries: int,
     ):
         self.out_dir = out_dir
         self.verify = verify
         self.extract_all = extract_all
         self.keep_file = keep_file
         self.test_suite_name = test_suite_name
         self.test_vector = test_vector
+        self.retries = retries
 
 
 class Context:
@@ -185,10 +188,23 @@ def _download_worker(self, ctx: DownloadWork) -> None:
         # This avoids:
         # Error sending result: '<multiprocessing.pool.ExceptionWithTraceback object at 0x7fd7811ecee0>'.
         # Reason: 'TypeError("cannot pickle '_io.BufferedReader' object")'
-        try:
-            utils.download(test_vector.source, dest_dir)
-        except Exception as ex:
-            raise Exception(str(ex)) from ex
+        for i in range(ctx.retries):
+            try:
+                exception_str = ""
+                utils.download(test_vector.source, dest_dir)
+            except urllib.error.URLError as ex:
+                exception_str = str(ex)
+                print(
+                    f"\tUnable to download {test_vector.source} to {dest_dir}, {exception_str}, retry count={i+1}"
+                )
+                continue
+            except Exception as ex:
+                raise Exception(str(ex)) from ex
+            break
+
+        if exception_str:
+            raise Exception(exception_str)
+
         if test_vector.source_checksum != "__skip__":
             checksum = utils.file_checksum(dest_path)
             if test_vector.source_checksum != checksum:
@@ -214,6 +230,7 @@ def download(
         verify: bool,
         extract_all: bool = False,
         keep_file: bool = False,
+        retries: int = 1,
     ) -> None:
         """Download the test suite"""
         if not os.path.exists(out_dir):
@@ -235,6 +252,7 @@ def _callback_error(err: Any) -> None:
                     keep_file,
                     self.name,
                     test_vector,
+                    retries,
                 )
                 downloads.append(
                     pool.apply_async(