wip

twitchdl/commands/download.py

@@ -17,7 +17,16 @@
 from twitchdl.exceptions import ConsoleError
 from twitchdl.http import download_all, download_file
 from twitchdl.naming import clip_filename, video_filename, video_placeholders
-from twitchdl.output import blue, bold, green, print_error, print_log, underlined, yellow
+from twitchdl.output import (
+    blue,
+    bold,
+    green,
+    print_error,
+    print_exception,
+    print_log,
+    underlined,
+    yellow,
+)
 from twitchdl.playlists import (
     Playlist,
     enumerate_vods,
@@ -297,10 +306,10 @@ def _download_video(video: Video, args: DownloadOptions) -> None:
     )
 
     if not result.ok:
-        from pprint import pp
-
-        pp(result.results)
-        raise ConsoleError(f"Download failed: {result.exception}")
+        for ex in result.exceptions:
+            print()
+            print_exception(ex)
+        raise ConsoleError("Download failed")
 
     join_playlist = make_join_playlist(vods_m3u8, vods, targets)
     join_playlist_path = cache_dir / "playlist_downloaded.m3u8"

twitchdl/http.py

@@ -24,7 +24,7 @@
 RETRY_COUNT = 5
 """Number of times to retry failed downloads before aborting."""
 
-TIMEOUT = 30
+TIMEOUT = 3
 """
 Number of seconds to wait before aborting when there is no network activity.
 https://www.python-httpx.org/advanced/#timeout-configuration
@@ -148,7 +148,7 @@ async def download_with_retries(
 ) -> TaskResult:
     if skip_existing and target.exists():
         size = os.path.getsize(target)
-        progress.already_downloaded(task_id, size)
+        progress.already_downloaded(task_id, source, target, size)
         return TaskSuccess(task_id, source, target, size, existing=True)
 
     progress.start(task_id, source, target)
@@ -162,10 +162,10 @@ async def download_with_retries(
                 progress.failed(task_id, ex)
                 return TaskError(task_id, source, target, ex)
             else:
-                # TODO: abort task before starting
                 progress.abort(task_id, ex)
+        # Don't retry on other exceptions
         except Exception as ex:
-            # Don't retry on other exceptions
+            progress.failed(task_id, ex)
             return TaskError(task_id, source, target, ex)
 
     raise Exception("Should not happen")
@@ -174,7 +174,10 @@ async def download_with_retries(
 class DownloadAllResult(NamedTuple):
     ok: bool
     results: List[TaskResult]
-    exceptions: Optional[List[Exception]] = None
+
+    @property
+    def exceptions(self):
+        return [r.exception for r in self.results if isinstance(r, TaskError)]
 
 
 async def download_all(
@@ -250,10 +253,11 @@ async def worker(client: httpx.AsyncClient, worker_id: int):
         success = producer_task.done()
 
         # Cancel tasks and wait until they are cancelled
-        for task in worker_tasks:
+        for task in worker_tasks + [producer_task]:
             task.cancel()
 
-        await asyncio.gather(*worker_tasks, return_exceptions=True)
+        await asyncio.gather(producer_task, *worker_tasks, return_exceptions=True)
+
         results = [
             results_map.get(t.task_id, TaskCanceled(t.task_id, t.url, t.target)) for t in tasks
         ]

twitchdl/output.py

@@ -1,5 +1,6 @@
 import json
 import sys
+import traceback
 from itertools import islice
 from typing import Any, Callable, Generator, List, Literal, Mapping, Optional, TypeVar
 
@@ -40,6 +41,11 @@ def print_error(message: Any):
     click.secho(message, err=True, fg="red")
 
 
+def print_exception(ex: BaseException):
+    for line in traceback.format_exception_only(ex):  # type: ignore
+        print_error(line)
+
+
 _prev_transient = False
 
 

twitchdl/progress.py

@@ -32,6 +32,10 @@ class Sample(NamedTuple):
 
 
 class Progress:
+    def already_downloaded(self, task_id: TaskID, source: str, target: Path, size: int):
+        """Skipping download since it's already been downloaded"""
+        pass
+
     def start(self, task_id: TaskID, source: str, target: Path):
         """A new download task is started"""
         pass
@@ -44,10 +48,6 @@ def advance(self, task_id: TaskID, size: int):
         """Advancing a download task by {size} bytes"""
         pass
 
-    def already_downloaded(self, task_id: TaskID, size: int):
-        """Skipping download task since it's already been downloaded"""
-        pass
-
     def abort(self, task_id: TaskID, ex: Exception):
         """Download restarting from beginning due to an error."""
         pass
@@ -62,29 +62,29 @@ def end(self, task_id: TaskID):
 
 
 class PrintingProgress(Progress):
+    @override
+    def already_downloaded(self, task_id: TaskID, source: str, target: Path, size: int):
+        print("already_downloaded", task_id, size)
+
     @override
     def start(self, task_id: TaskID, source: str, target: Path):
-        print("start", task_id, source)
+        print("start", task_id)
 
     @override
     def content_length(self, task_id: TaskID, size: int):
         print("start", task_id, size)
 
     @override
     def advance(self, task_id: TaskID, size: int):
-        print("advance", task_id, size)
-
-    @override
-    def already_downloaded(self, task_id: TaskID, size: int):
-        print("already_downloaded", task_id, size)
+        pass
 
     @override
     def abort(self, task_id: TaskID, ex: Exception):
-        print("abort", task_id, ex)
+        print("abort", task_id, repr(ex))
 
     @override
     def failed(self, task_id: TaskID, ex: Exception):
-        print("failed", task_id, ex)
+        print("failed", task_id, repr(ex))
 
     @override
     def end(self, task_id: TaskID):
@@ -133,7 +133,7 @@ def advance(self, task_id: TaskID, size: int):
         self.print()
 
     @override
-    def already_downloaded(self, task_id: TaskID, size: int):
+    def already_downloaded(self, task_id: TaskID, source: str, target: Path, size: int):
         if task_id in self.tasks:
             raise ValueError(f"Task {task_id}: cannot mark as downloaded, already started")
 
@@ -166,12 +166,10 @@ def end(self, task_id: TaskID):
         self.print()
 
     def _recalculate(self):
-        if self.tasks and self.file_count:
-            self.estimated_total = int(
-                mean(t.size for t in self.tasks.values() if t.size) * self.file_count
-            )
-        else:
-            self.estimated_total = None
+        if self.file_count:
+            sizes = [t.size for t in self.tasks.values() if t.size]
+            if sizes:
+                self.estimated_total = int(mean(sizes) * self.file_count)
 
         self.speed = self._calculate_speed()
         self.progress_perc = (