Add urls to returned results (#146)

parfive/downloader.py

@@ -319,7 +319,8 @@ def _format_results(self, retvals, main_pb):
             elif isinstance(res, Exception):
                 raise res
             else:
-                results.append(res)
+                requested_url, filepath = res
+                results.append(path=filepath, url=requested_url)
 
         return results
 
@@ -551,7 +552,8 @@ async def _get_http(
                             "File %s already exists and overwrite is False; skipping download.",
                             filepath,
                         )
-                        return str(filepath)
+                        return url, str(filepath)
+
                     if callable(file_pb):
                         file_pb = file_pb(
                             position=token.n,
@@ -618,9 +620,11 @@ async def _get_http(
             await asyncio.gather(*tasks)
             # join() waits till all the items in the queue have been processed
             await downloaded_chunk_queue.join()
+
             for callback in self.config.done_callbacks:
                 callback(filepath, url, None)
-            return str(filepath)
+
+            return url, str(filepath)
 
         except (Exception, asyncio.CancelledError) as e:
             for task in tasks:
@@ -810,7 +814,7 @@ async def _get_ftp(
                         "File %s already exists and overwrite is False; skipping download.",
                         filepath,
                     )
-                    return str(filepath)
+                    return url, str(filepath)
 
                 if callable(file_pb):
                     total_size = await get_ftp_size(client, parse.path)
@@ -845,7 +849,7 @@ async def _get_ftp(
                     for callback in self.config.done_callbacks:
                         callback(filepath, url, None)
 
-                    return str(filepath)
+                    return url, str(filepath)
 
         except (Exception, asyncio.CancelledError) as e:
             if writer is not None:

parfive/results.py

@@ -22,14 +22,16 @@ class Results(UserList):
     """
     The results of a download from `parfive.Downloader.download`.
 
-    This object contains the filenames of successful downloads as well
-    as a list of any errors encountered in the `~parfive.Results.errors`
+    This object contains the filenames of successful downloads as well,
+    a list of all urls requested in the `~parfive.Results.urls` property
+    and a list of any errors encountered in the `~parfive.Results.errors`
     property.
     """
 
-    def __init__(self, *args, errors=None):
+    def __init__(self, *args, errors=None, urls=None):
         super().__init__(*args)
         self._errors = errors or list()
+        self._urls = urls or list()
 
     def _get_nice_resp_repr(self, response):
         # This is a modified version of aiohttp.ClientResponse.__repr__
@@ -63,6 +65,10 @@ def __repr__(self):
         out += str(self)
         return out
 
+    def append(self, *, path, url):
+        super().append(path)
+        self._urls.append(url)
+
     def add_error(self, filename, url, exception):
         """
         Add an error to the results.
@@ -82,3 +88,11 @@ def errors(self):
         ``exception`` is the error raised during download.
         """
         return self._errors
+
+    @property
+    def urls(self):
+        """
+        A list of requested urls.
+
+        """
+        return self._urls

parfive/tests/test_downloader.py

@@ -47,6 +47,7 @@ def test_download(httpserver, tmpdir):
     assert dl.queued_downloads == 1
 
     f = dl.download()
+    f.urls == [httpserver.url]
     validate_test_file(f)
 
 
@@ -302,7 +303,10 @@ def test_failed_download():
 def test_results():
     res = Results()
 
-    res.append("hello")
+    res.append(path="hello", url="aurl")
+
+    assert res[0] == "hello"
+    assert res.urls[0] == "aurl"
 
     res.add_error("wibble", "notaurl", "out of cheese")