From 8e06e52ed74a1a91a3ff63592d39df9c5b99bbd0 Mon Sep 17 00:00:00 2001 From: Octavio Simone <70800577+unbekanntes-pferd@users.noreply.github.com> Date: Sat, 25 Jun 2022 11:00:33 +0200 Subject: [PATCH 1/5] bumped dependencies, updated to tqdm and callbacks --- dccmd/__init__.py | 24 ++++---- dccmd/main/auth/__init__.py | 2 +- dccmd/main/auth/util.py | 34 ++++++----- dccmd/main/crypto/util.py | 4 +- dccmd/main/models/__init__.py | 51 ++++++++++++++++ dccmd/main/models/errors.py | 15 +++-- dccmd/main/upload/__init__.py | 67 +++++++++++---------- dccmd/main/users/manage.py | 10 ++-- poetry.lock | 109 +++++++++++----------------------- pyproject.toml | 6 +- 10 files changed, 179 insertions(+), 143 deletions(-) diff --git a/dccmd/__init__.py b/dccmd/__init__.py index 55301ed..9c891b5 100644 --- a/dccmd/__init__.py +++ b/dccmd/__init__.py @@ -4,7 +4,7 @@ """ -__version__ = "0.2.0" +__version__ = "0.3.0" # std imports import sys @@ -16,9 +16,9 @@ from dracoon import DRACOON, OAuth2ConnectionType from dracoon.nodes.models import NodeType from dracoon.errors import ( + DRACOONHttpError, HTTPConflictError, HTTPForbiddenError, - HTTPStatusError, InvalidPathError, InvalidFileError, FileConflictError, @@ -58,6 +58,7 @@ from dccmd.main.crypto.keys import distribute_missing_keys from dccmd.main.crypto.util import get_keypair, init_keypair from dccmd.main.upload import create_folder_struct, bulk_upload, is_directory, is_file +from dccmd.main.models import DCTransfer, DCTransferList from dccmd.main.models.errors import (DCPathParseError, DCClientParseError, ConnectError) @@ -203,7 +204,7 @@ async def _upload(): format_error_message(msg=f"Target path not found. ({target_path})") ) sys.exit(2) - except HTTPStatusError: + except DRACOONHttpError: await dracoon.logout() typer.echo( format_error_message(msg="An error ocurred uploading the file.") @@ -296,7 +297,7 @@ async def _create_folder(): ) ) sys.exit(1) - except HTTPStatusError: + except DRACOONHttpError: await dracoon.logout() typer.echo( format_error_message( @@ -379,7 +380,7 @@ async def _create_room(): ) ) sys.exit(1) - except HTTPStatusError: + except DRACOONHttpError: await dracoon.logout() typer.echo( format_error_message( @@ -481,7 +482,7 @@ async def _delete_node(): format_error_message(msg="Insufficient permissions (delete required).") ) sys.exit(1) - except HTTPStatusError: + except DRACOONHttpError: await dracoon.logout() typer.echo( format_error_message( @@ -583,7 +584,7 @@ async def _list_nodes(): format_error_message(msg="Insufficient permissions (delete required).") ) sys.exit(1) - except HTTPStatusError: + except DRACOONHttpError: await dracoon.logout() typer.echo(format_error_message(msg="Error listing nodes.")) sys.exit(1) @@ -627,7 +628,7 @@ async def _list_nodes(): ) ) sys.exit(1) - except HTTPStatusError: + except DRACOONHttpError: await dracoon.logout() typer.echo(format_error_message(msg="Error listing nodes.")) sys.exit(1) @@ -722,12 +723,15 @@ async def _download(): dracoon=dracoon, base_url=base_url, crypto_secret=crypto_secret ) + transfer = DCTransferList(total=node_info.size, file_count=1) + download_job = DCTransfer(transfer=transfer) + try: await dracoon.download( file_path=parsed_path, target_path=target_dir_path, - display_progress=True, raise_on_err=True, + callback_fn=download_job.update ) # to do: replace with handling via PermissionError except UnboundLocalError: @@ -751,7 +755,7 @@ async def _download(): ) ) sys.exit(1) - except HTTPStatusError: + except DRACOONHttpError: await dracoon.logout() typer.echo(format_error_message(msg="Error downloading file.")) sys.exit(1) diff --git a/dccmd/main/auth/__init__.py b/dccmd/main/auth/__init__.py index 90ba1ca..b50c16b 100644 --- a/dccmd/main/auth/__init__.py +++ b/dccmd/main/auth/__init__.py @@ -14,7 +14,7 @@ import httpx from dracoon import DRACOON, OAuth2ConnectionType from dracoon.client import DRACOONConnection -from dracoon.errors import HTTPUnauthorizedError, HTTPStatusError, HTTPNotFoundError +from dracoon.errors import HTTPUnauthorizedError, DRACOONHttpError, HTTPNotFoundError # internal imports from .credentials import ( diff --git a/dccmd/main/auth/util.py b/dccmd/main/auth/util.py index 42bd9c9..76d62b0 100644 --- a/dccmd/main/auth/util.py +++ b/dccmd/main/auth/util.py @@ -9,8 +9,7 @@ import typer import httpx from dracoon import DRACOON, OAuth2ConnectionType -from dracoon.client import DRACOONConnection -from dracoon.errors import HTTPUnauthorizedError, HTTPStatusError, HTTPBadRequestError +from dracoon.errors import HTTPUnauthorizedError, DRACOONHttpError, HTTPBadRequestError from dccmd import __version__ as dccmd_version @@ -64,7 +63,8 @@ async def login( log_stream=debug, raise_on_err=True ) - + + # set custom user agent dracoon_user_agent = dracoon.client.http.headers["User-Agent"] dracoon.client.http.headers["User-Agent"] = f"{dccmd_name}|{dccmd_version}|{dracoon_user_agent}" @@ -75,6 +75,9 @@ async def login( except HTTPUnauthorizedError: typer.echo(format_error_message(msg='Wrong username/password.')) sys.exit(1) + except DRACOONHttpError: + typer.echo(format_error_message(msg='An error ocurred during login')) + sys.exit(1) # refresh token elif refresh_token: try: @@ -82,6 +85,9 @@ async def login( # invalid refresh token except HTTPBadRequestError: dracoon = await _login_prompt(base_url, dracoon) + except DRACOONHttpError: + typer.echo(format_error_message(msg='An error ocurred during login')) + sys.exit(1) # auth code flow else: @@ -90,6 +96,9 @@ async def login( except HTTPBadRequestError: typer.echo(format_error_message(msg='Invalid authorization code.')) sys.exit(1) + except DRACOONHttpError: + typer.echo(format_error_message(msg='An error ocurred during login')) + sys.exit(1) return dracoon @@ -152,13 +161,11 @@ async def _login_password_flow( sys.exit(1) else: typer.echo(format_error_message(msg="Wrong username or password.")) - except HTTPStatusError: - await graceful_exit(dracoon=dracoon) - typer.echo(format_error_message(msg="Login failed.")) - except httpx.ConnectError: + except DRACOONHttpError: await graceful_exit(dracoon=dracoon) typer.echo(format_error_message(msg="Login failed.")) + save_creds = typer.confirm("Save credentials?", abort=False, default=True) if save_creds: @@ -179,7 +186,7 @@ async def _login_prompt(base_url: str, dracoon: DRACOON) -> DRACOON: except HTTPUnauthorizedError: graceful_exit(dracoon=dracoon) typer.echo(format_error_message(msg="Wrong authorization code.")) - except httpx.ConnectError: + except DRACOONHttpError: graceful_exit(dracoon=dracoon) typer.echo(format_error_message(msg="Login failed (check authorization code).")) @@ -196,9 +203,8 @@ async def _login_prompt(base_url: str, dracoon: DRACOON) -> DRACOON: async def _login_refresh_token( base_url: str, dracoon: DRACOON, refresh_token: str ) -> DRACOON: - dracoon.client.connection = DRACOONConnection(None, None, None, refresh_token, None) try: - await dracoon.connect(connection_type=OAuth2ConnectionType.refresh_token) + await dracoon.connect(connection_type=OAuth2ConnectionType.refresh_token, refresh_token=refresh_token) except HTTPUnauthorizedError as err: await graceful_exit(dracoon=dracoon) @@ -215,13 +221,11 @@ async def _login_refresh_token( else: delete_credentials(base_url=base_url) typer.echo(format_error_message(msg="Refresh token expired.")) - except HTTPStatusError: - graceful_exit(dracoon=dracoon) - typer.echo(format_error_message(msg="Login failed.")) - except httpx.ConnectError: + except DRACOONHttpError: graceful_exit(dracoon=dracoon) typer.echo(format_error_message(msg="Login failed.")) + # store new refresh token store_credentials( base_url=base_url, refresh_token=dracoon.client.connection.refresh_token @@ -245,7 +249,7 @@ async def is_dracoon_url(base_url: str) -> bool: res.raise_for_status() except httpx.ConnectError: return False - except httpx.HTTPStatusError: + except DRACOONHttpError: return False return True diff --git a/dccmd/main/crypto/util.py b/dccmd/main/crypto/util.py index 0558b1f..250a10d 100644 --- a/dccmd/main/crypto/util.py +++ b/dccmd/main/crypto/util.py @@ -5,7 +5,7 @@ import typer from dracoon import DRACOON -from dracoon.errors import HTTPNotFoundError, HTTPStatusError +from dracoon.errors import HTTPNotFoundError, DRACOONHttpError from dccmd.main.util import graceful_exit, format_error_message from dccmd.main.auth.credentials import store_crypto_credentials @@ -27,7 +27,7 @@ async def get_keypair(dracoon: DRACOON, crypto_secret: str): ) ) sys.exit(2) - except HTTPStatusError: + except DRACOONHttpError: await graceful_exit(dracoon=dracoon) typer.echo(format_error_message(msg="An error ocurred getting the keypair.")) sys.exit(2) diff --git a/dccmd/main/models/__init__.py b/dccmd/main/models/__init__.py index e69de29..df74ee3 100644 --- a/dccmd/main/models/__init__.py +++ b/dccmd/main/models/__init__.py @@ -0,0 +1,51 @@ +from dracoon.nodes.models import TransferJob +from tqdm import tqdm +from .errors import DCInvalidArgumentError + + +class DCTransferList: + """ object to manage one or multiple transfers with progress bar """ + def __init__(self, total: int, file_count: int): + + if file_count <= 0 or total <= 0: + raise DCInvalidArgumentError(msg="Total and file count must be a positive number.") + + self.total = total + self.file_count = file_count + self.file_progress = tqdm(unit='file', total=self.file_count, unit_scale=True) + self.progress = tqdm(unit='iMB',unit_divisor=1024, total=self.total, unit_scale=True) + + def update_byte_progress(self, val: int): + """ updates transferred bytes """ + self.progress.update(val) + + def update_file_count(self): + """ updates file count """ + self.file_progress.update(1) + + def __del__(self): + self.progress.close() + self.file_progress.close() + + +class DCTransfer(TransferJob): + """ a single transfer managed by DCTransferList """ + + def __init__(self, transfer: DCTransferList): + super().__init__() + + self.transfer = transfer + + def update(self, val: int, total: int = None): + """ callback function to track progress """ + if total is not None and val is 0: + self.total += total + + self.update_progress(val) + self.transfer.update_byte_progress(val) + + if self.progress == 1: + self.transfer.update_file_count() + + + \ No newline at end of file diff --git a/dccmd/main/models/errors.py b/dccmd/main/models/errors.py index fff6bec..86a486b 100644 --- a/dccmd/main/models/errors.py +++ b/dccmd/main/models/errors.py @@ -4,29 +4,36 @@ # re-import from httpx import ConnectError +class DCBaseError(Exception): + """ base client error """ + def __init__(self, message: str): + super().__init__() + self.message = message + + # error parsing a DRACOON base url -class DCPathParseError(Exception): +class DCPathParseError(DCBaseError): """ error raised if DRACOON url format invalid """ #pylint: disable=W0235 def __init__(self, msg: str = "Invalid DRACOON url"): super().__init__(msg) # error parsing DRACOON client credentials -class DCClientParseError(Exception): +class DCClientParseError(DCBaseError): """ error raised if client creds cannot be parsed """ #pylint: disable=W0235 def __init__(self, msg: str): super().__init__(msg) # error parsing DRACOON client credentials -class DCClientNotFoundError(Exception): +class DCClientNotFoundError(DCBaseError): """ error raised if a client config is not found """ #pylint: disable=W0235 def __init__(self, msg: str): super().__init__(msg) # invalid CLI argument combination -class DCInvalidArgumentError(Exception): +class DCInvalidArgumentError(DCBaseError): """ error raised for invalid arguments """ #pylint: disable=W0235 def __init__(self, msg: str): diff --git a/dccmd/main/upload/__init__.py b/dccmd/main/upload/__init__.py index c866193..1b5bc52 100644 --- a/dccmd/main/upload/__init__.py +++ b/dccmd/main/upload/__init__.py @@ -17,9 +17,9 @@ InvalidPathError, HTTPConflictError, HTTPForbiddenError, - HTTPStatusError, + DRACOONHttpError, ) - +from ..models import DCTransfer, DCTransferList from ..util import format_error_message, to_readable_size from ..auth.credentials import get_crypto_credentials, store_crypto_credentials @@ -79,11 +79,13 @@ def __init__(self, dir_path: str, base_path: str): self.dir_path = dir_path.replace("\\", "/") else: self.dir_path = dir_path + self.x_path = Path(dir_path) self.abs_path = self.dir_path.replace(base_path, "") self.name = self.abs_path.split("/")[-1] self.parent_path = ("/").join(self.abs_path.split("/")[:-1]) self.level = len(self.abs_path.split("/")) - 1 self.size = os.path.getsize(self.dir_path) + self.x_size = self.x_path.stat().st_size class FileItemList: @@ -111,7 +113,16 @@ def sort_by_size(self): """sort files by file size""" self.file_list.sort(key=lambda item: item.size, reverse=True) + @property + def file_count(self): + """ returns file count """ + return len(self.file_list) + @property + def total_size(self): + """ return total size in bytes """ + return sum([item.size for item in self.file_list]) + def convert_to_dir_items(dir_list: list[str], base_dir: str) -> list[DirectoryItem]: """convert a list of paths to a list of directory items (helper class)""" @@ -191,7 +202,7 @@ async def process_batch(batch): ) ) sys.exit(2) - except HTTPStatusError: + except DRACOONHttpError: await dracoon.logout() typer.echo( format_error_message(msg="An error ocurred creating the folder.") @@ -231,6 +242,7 @@ async def bulk_upload( file_list = FileItemList(source_path=source) file_list.sort_by_size() + transfer_list = DCTransferList(total=file_list.total_size, file_count=file_list.file_count) if velocity > 10: velocity = 10 @@ -239,48 +251,43 @@ async def bulk_upload( concurrent_reqs = velocity * 5 - typer.echo(f"{len(file_list.file_list)} files to upload.") + upload_reqs = [] - upload_reqs = [ - dracoon.upload( + for item in file_list.file_list: + upload_job = DCTransfer(transfer=transfer_list) + req = dracoon.upload( file_path=item.dir_path, target_path=(target + item.parent_path), resolution_strategy=resolution_strategy, display_progress=False, + callback_fn=upload_job.update ) - for item in file_list.file_list - ] - total_files = len(upload_reqs) - total_size = sum([item.size for item in file_list.file_list]) - - typer.echo(f"{to_readable_size(size=total_size)} total.") + upload_reqs.append(req) - with typer.progressbar(length=total_files, label="Uploading files...") as progress: - for batch in dracoon.batch_process( + for batch in dracoon.batch_process( coro_list=upload_reqs, batch_size=concurrent_reqs ): - progress.update(len(batch)) - try: - await asyncio.gather(*batch) - except HTTPConflictError: - # ignore file already exists error - pass - except HTTPForbiddenError: - await dracoon.logout() - typer.echo( + try: + await asyncio.gather(*batch) + except HTTPConflictError: + # ignore file already exists error + pass + except HTTPForbiddenError: + await dracoon.logout() + typer.echo( format_error_message( msg="Insufficient permissions (create / esdit required)." ) ) - sys.exit(2) - except HTTPStatusError: - await dracoon.logout() - typer.echo( + sys.exit(2) + except DRACOONHttpError: + await dracoon.logout() + typer.echo( format_error_message(msg="An error ocurred uploading files.") ) - sys.exit(2) - except WriteTimeout: - continue + sys.exit(2) + except WriteTimeout: + continue def create_folder(name: str, parent_id: int, dracoon: DRACOON): diff --git a/dccmd/main/users/manage.py b/dccmd/main/users/manage.py index 8ed242e..060ff3e 100644 --- a/dccmd/main/users/manage.py +++ b/dccmd/main/users/manage.py @@ -15,7 +15,7 @@ HTTPConflictError, HTTPBadRequestError, HTTPForbiddenError, - HTTPStatusError, + DRACOONHttpError, ) from pydantic import BaseModel import typer @@ -182,7 +182,7 @@ async def create_users( continue except HTTPBadRequestError: continue - except HTTPStatusError: + except DRACOONHttpError: continue @@ -233,7 +233,7 @@ async def delete_user(dracoon: DRACOON, user_id: int): ) ) sys.exit(1) - except HTTPStatusError: + except DRACOONHttpError: await dracoon.logout() typer.echo( format_error_message( @@ -262,7 +262,7 @@ async def get_users(dracoon: DRACOON, search_string: str = ''): ) ) sys.exit(1) - except HTTPStatusError: + except DRACOONHttpError: await dracoon.logout() typer.echo( format_error_message( @@ -297,7 +297,7 @@ async def find_user_by_username(dracoon: DRACOON, user_name: str): ) ) sys.exit(1) - except HTTPStatusError: + except DRACOONHttpError: await dracoon.logout() typer.echo( format_error_message( diff --git a/poetry.lock b/poetry.lock index 07cd133..c70cf98 100644 --- a/poetry.lock +++ b/poetry.lock @@ -47,7 +47,7 @@ tests_no_zope = ["coverage[toml] (>=5.0.2)", "hypothesis", "pympler", "pytest (> [[package]] name = "certifi" -version = "2022.5.18.1" +version = "2022.6.15" description = "Python package for providing Mozilla's CA Bundle." category = "main" optional = false @@ -77,7 +77,7 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} [[package]] name = "colorama" -version = "0.4.4" +version = "0.4.5" description = "Cross-platform colored terminal text." category = "main" optional = false @@ -85,7 +85,7 @@ python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" [[package]] name = "cryptography" -version = "37.0.2" +version = "37.0.3" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." category = "main" optional = false @@ -104,7 +104,7 @@ test = ["pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-subtests", [[package]] name = "dracoon" -version = "1.4.1" +version = "1.5.1" description = "DRACOON API wrapper in Python" category = "main" optional = false @@ -171,22 +171,6 @@ category = "main" optional = false python-versions = ">=3.5" -[[package]] -name = "importlib-metadata" -version = "4.11.4" -description = "Read metadata from Python packages" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.dependencies] -zipp = ">=0.5" - -[package.extras] -docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"] -perf = ["ipython"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "packaging", "pyfakefs", "flufl.flake8", "pytest-perf (>=0.9.2)", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)", "importlib-resources (>=1.3)"] - [[package]] name = "jeepney" version = "0.8.0" @@ -201,14 +185,13 @@ trio = ["trio", "async-generator"] [[package]] name = "keyring" -version = "23.5.1" +version = "23.6.0" description = "Store and access your passwords safely." category = "main" optional = false python-versions = ">=3.7" [package.dependencies] -importlib-metadata = ">=3.6" jeepney = {version = ">=0.4.2", markers = "sys_platform == \"linux\""} pywin32-ctypes = {version = "<0.1.0 || >0.1.0,<0.1.1 || >0.1.1", markers = "sys_platform == \"win32\""} SecretStorage = {version = ">=3.2", markers = "sys_platform == \"linux\""} @@ -403,22 +386,10 @@ category = "dev" optional = false python-versions = "*" -[[package]] -name = "zipp" -version = "3.8.0" -description = "Backport of pathlib-compatible object wrapper for zip files" -category = "main" -optional = false -python-versions = ">=3.7" - -[package.extras] -docs = ["sphinx", "jaraco.packaging (>=9)", "rst.linker (>=1.9)"] -testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytest-cov", "pytest-enabler (>=1.0.1)", "jaraco.itertools", "func-timeout", "pytest-black (>=0.3.7)", "pytest-mypy (>=0.9.1)"] - [metadata] lock-version = "1.1" python-versions = "^3.10" -content-hash = "1bca7574003268605676d9246322a80352c170bdca85c4d7873461ec3e7d339e" +content-hash = "44e64f25bb58220d605c282dfef5c825fa32c9e337476bea22f853bbdee920da" [metadata.files] anyio = [ @@ -440,8 +411,8 @@ attrs = [ {file = "attrs-21.4.0.tar.gz", hash = "sha256:626ba8234211db98e869df76230a137c4c40a12d72445c45d5f5b716f076e2fd"}, ] certifi = [ - {file = "certifi-2022.5.18.1-py3-none-any.whl", hash = "sha256:f1d53542ee8cbedbe2118b5686372fb33c297fcd6379b050cca0ef13a597382a"}, - {file = "certifi-2022.5.18.1.tar.gz", hash = "sha256:9c5705e395cd70084351dd8ad5c41e65655e08ce46f2ec9cf6c2c08390f71eb7"}, + {file = "certifi-2022.6.15-py3-none-any.whl", hash = "sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412"}, + {file = "certifi-2022.6.15.tar.gz", hash = "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d"}, ] cffi = [ {file = "cffi-1.15.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:c2502a1a03b6312837279c8c1bd3ebedf6c12c4228ddbad40912d671ccc8a962"}, @@ -500,36 +471,36 @@ click = [ {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, ] colorama = [ - {file = "colorama-0.4.4-py2.py3-none-any.whl", hash = "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"}, - {file = "colorama-0.4.4.tar.gz", hash = "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b"}, + {file = "colorama-0.4.5-py2.py3-none-any.whl", hash = "sha256:854bf444933e37f5824ae7bfc1e98d5bce2ebe4160d46b5edf346a89358e99da"}, + {file = "colorama-0.4.5.tar.gz", hash = "sha256:e6c6b4334fc50988a639d9b98aa429a0b57da6e17b9a44f0451f930b6967b7a4"}, ] cryptography = [ - {file = "cryptography-37.0.2-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:ef15c2df7656763b4ff20a9bc4381d8352e6640cfeb95c2972c38ef508e75181"}, - {file = "cryptography-37.0.2-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:3c81599befb4d4f3d7648ed3217e00d21a9341a9a688ecdd615ff72ffbed7336"}, - {file = "cryptography-37.0.2-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:2bd1096476aaac820426239ab534b636c77d71af66c547b9ddcd76eb9c79e004"}, - {file = "cryptography-37.0.2-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:31fe38d14d2e5f787e0aecef831457da6cec68e0bb09a35835b0b44ae8b988fe"}, - {file = "cryptography-37.0.2-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:093cb351031656d3ee2f4fa1be579a8c69c754cf874206be1d4cf3b542042804"}, - {file = "cryptography-37.0.2-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:59b281eab51e1b6b6afa525af2bd93c16d49358404f814fe2c2410058623928c"}, - {file = "cryptography-37.0.2-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:0cc20f655157d4cfc7bada909dc5cc228211b075ba8407c46467f63597c78178"}, - {file = "cryptography-37.0.2-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:f8ec91983e638a9bcd75b39f1396e5c0dc2330cbd9ce4accefe68717e6779e0a"}, - {file = "cryptography-37.0.2-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:46f4c544f6557a2fefa7ac8ac7d1b17bf9b647bd20b16decc8fbcab7117fbc15"}, - {file = "cryptography-37.0.2-cp36-abi3-win32.whl", hash = "sha256:731c8abd27693323b348518ed0e0705713a36d79fdbd969ad968fbef0979a7e0"}, - {file = "cryptography-37.0.2-cp36-abi3-win_amd64.whl", hash = "sha256:471e0d70201c069f74c837983189949aa0d24bb2d751b57e26e3761f2f782b8d"}, - {file = "cryptography-37.0.2-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a68254dd88021f24a68b613d8c51d5c5e74d735878b9e32cc0adf19d1f10aaf9"}, - {file = "cryptography-37.0.2-pp37-pypy37_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:a7d5137e556cc0ea418dca6186deabe9129cee318618eb1ffecbd35bee55ddc1"}, - {file = "cryptography-37.0.2-pp38-pypy38_pp73-macosx_10_10_x86_64.whl", hash = "sha256:aeaba7b5e756ea52c8861c133c596afe93dd716cbcacae23b80bc238202dc023"}, - {file = "cryptography-37.0.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95e590dd70642eb2079d280420a888190aa040ad20f19ec8c6e097e38aa29e06"}, - {file = "cryptography-37.0.2-pp38-pypy38_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:1b9362d34363f2c71b7853f6251219298124aa4cc2075ae2932e64c91a3e2717"}, - {file = "cryptography-37.0.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:e53258e69874a306fcecb88b7534d61820db8a98655662a3dd2ec7f1afd9132f"}, - {file = "cryptography-37.0.2-pp39-pypy39_pp73-macosx_10_10_x86_64.whl", hash = "sha256:1f3bfbd611db5cb58ca82f3deb35e83af34bb8cf06043fa61500157d50a70982"}, - {file = "cryptography-37.0.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:419c57d7b63f5ec38b1199a9521d77d7d1754eb97827bbb773162073ccd8c8d4"}, - {file = "cryptography-37.0.2-pp39-pypy39_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:dc26bb134452081859aa21d4990474ddb7e863aa39e60d1592800a8865a702de"}, - {file = "cryptography-37.0.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:3b8398b3d0efc420e777c40c16764d6870bcef2eb383df9c6dbb9ffe12c64452"}, - {file = "cryptography-37.0.2.tar.gz", hash = "sha256:f224ad253cc9cea7568f49077007d2263efa57396a2f2f78114066fd54b5c68e"}, + {file = "cryptography-37.0.3-cp36-abi3-macosx_10_10_universal2.whl", hash = "sha256:d10413d493e98075060d3e62e5826de372912ea653ccc948f3c41b21ddca087f"}, + {file = "cryptography-37.0.3-cp36-abi3-macosx_10_10_x86_64.whl", hash = "sha256:cd64147ff16506632893ceb2569624b48c84daa3ba4d89695f7c7bc24188eee9"}, + {file = "cryptography-37.0.3-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:17c74f7d9e9e9bb7e84521243695c1b4bdc3a0e44ca764e6bcf8f05f3de3d0df"}, + {file = "cryptography-37.0.3-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_24_aarch64.whl", hash = "sha256:0713bee6c8077786c56bdec9c5d3f099d40d2c862ff3200416f6862e9dd63156"}, + {file = "cryptography-37.0.3-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b9c2008417741cdfbe945ef2d16b7b7ba0790886a0b49e1de533acf93eb66ed6"}, + {file = "cryptography-37.0.3-cp36-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:646905ff7a712e415bf0d0f214e0eb669dd2257c4d7a27db1e8baec5d2a1d55f"}, + {file = "cryptography-37.0.3-cp36-abi3-manylinux_2_24_x86_64.whl", hash = "sha256:dcafadb5a06cb7a6bb49fb4c1de7414ee2f8c8e12b047606d97c3175d690f582"}, + {file = "cryptography-37.0.3-cp36-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:0b4bfc5ccfe4e5c7de535670680398fed4a0bbc5dfd52b3a295baad42230abdf"}, + {file = "cryptography-37.0.3-cp36-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:a03dbc0d8ce8c1146c177cd0e3a66ea106f36733fb1b997ea4d051f8a68539ff"}, + {file = "cryptography-37.0.3-cp36-abi3-win32.whl", hash = "sha256:190a24c14e91c1fa3101069aac7e77d11c5a73911c3904128367f52946bbb6fd"}, + {file = "cryptography-37.0.3-cp36-abi3-win_amd64.whl", hash = "sha256:b05c5478524deb7a019e240f2a970040c4b0f01f58f0425e6262c96b126c6a3e"}, + {file = "cryptography-37.0.3-pp37-pypy37_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:891ed8312840fd43e0696468a6520a582a033c0109f7b14b96067bfe1123226b"}, + {file = "cryptography-37.0.3-pp37-pypy37_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:30d6aabf623a01affc7c0824936c3dde6590076b61f5dd299df3cc2c75fc5915"}, + {file = "cryptography-37.0.3-pp38-pypy38_pp73-macosx_10_10_x86_64.whl", hash = "sha256:31a7c1f1c2551f013d4294d06e22848e2ccd77825f0987cba3239df6ebf7b020"}, + {file = "cryptography-37.0.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a94fd1ff80001cb97add71d07f596d8b865b716f25ef501183e0e199390e50d3"}, + {file = "cryptography-37.0.3-pp38-pypy38_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:8a85dbcc770256918b40c2f40bd3ffd3b2ae45b0cf19068b561db8f8d61bf492"}, + {file = "cryptography-37.0.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:773d5b5f2e2bd2c7cbb1bd24902ad41283c88b9dd463a0f82adc9a2870d9d066"}, + {file = "cryptography-37.0.3-pp39-pypy39_pp73-macosx_10_10_x86_64.whl", hash = "sha256:0f9193428a55a4347af2d4fd8141a2002dedbcc26487e67fd2ae19f977ee8afc"}, + {file = "cryptography-37.0.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7bf652c73e8f7c32a3f92f7184bf7f9106dacdf5ef59c3c3683d7dae2c4972fb"}, + {file = "cryptography-37.0.3-pp39-pypy39_pp73-manylinux_2_24_x86_64.whl", hash = "sha256:c3c8b1ad2c266fdf7adc041cc4156d6a3d14db93de2f81b26a5af97ef3f209e5"}, + {file = "cryptography-37.0.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:2383d6c3088e863304c37c65cd2ea404b7fbb4886823eab1d74137cc27f3d2ee"}, + {file = "cryptography-37.0.3.tar.gz", hash = "sha256:ae430d51c67ac638dfbb42edf56c669ca9c74744f4d225ad11c6f3d355858187"}, ] dracoon = [ - {file = "dracoon-1.4.1-py3-none-any.whl", hash = "sha256:2f36ae6ffe5ed1919de9736173e25436c9a90e2028d641fdbf63259aa5866305"}, - {file = "dracoon-1.4.1.tar.gz", hash = "sha256:423b6cd83ee5f3d9ebc94525c59db9e6a5e18b68d3d8a95678cb8478200e8b97"}, + {file = "dracoon-1.5.1-py3-none-any.whl", hash = "sha256:ebfa7c0d2d16962011e4ead6d5f34cbbf4a681e37ede4701d6f014b0f920b705"}, + {file = "dracoon-1.5.1.tar.gz", hash = "sha256:bc22707ac833c0623b01e6ecd36bad72214b780e8c9274a730eea37723866c08"}, ] h11 = [ {file = "h11-0.12.0-py3-none-any.whl", hash = "sha256:36a3cb8c0a032f56e2da7084577878a035d3b61d104230d4bd49c0c6b555a9c6"}, @@ -547,17 +518,13 @@ idna = [ {file = "idna-3.3-py3-none-any.whl", hash = "sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff"}, {file = "idna-3.3.tar.gz", hash = "sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"}, ] -importlib-metadata = [ - {file = "importlib_metadata-4.11.4-py3-none-any.whl", hash = "sha256:c58c8eb8a762858f49e18436ff552e83914778e50e9d2f1660535ffb364552ec"}, - {file = "importlib_metadata-4.11.4.tar.gz", hash = "sha256:5d26852efe48c0a32b0509ffbc583fda1a2266545a78d104a6f4aff3db17d700"}, -] jeepney = [ {file = "jeepney-0.8.0-py3-none-any.whl", hash = "sha256:c0a454ad016ca575060802ee4d590dd912e35c122fa04e70306de3d076cce755"}, {file = "jeepney-0.8.0.tar.gz", hash = "sha256:5efe48d255973902f6badc3ce55e2aa6c5c3b3bc642059ef3a91247bcfcc5806"}, ] keyring = [ - {file = "keyring-23.5.1-py3-none-any.whl", hash = "sha256:9ef58314bcc823f426b49ec787539a2d73571b37de4cd498f839803b01acff1e"}, - {file = "keyring-23.5.1.tar.gz", hash = "sha256:dee502cdf18a98211bef428eea11456a33c00718b2f08524fd5727c7f424bffd"}, + {file = "keyring-23.6.0-py3-none-any.whl", hash = "sha256:372ff2fc43ab779e3f87911c26e6c7acc8bb440cbd82683e383ca37594cb0617"}, + {file = "keyring-23.6.0.tar.gz", hash = "sha256:3ac00c26e4c93739e19103091a9986a9f79665a78cf15a4df1dba7ea9ac8da2f"}, ] more-itertools = [ {file = "more-itertools-8.13.0.tar.gz", hash = "sha256:a42901a0a5b169d925f6f217cd5a190e32ef54360905b9c39ee7db5313bfec0f"}, @@ -656,7 +623,3 @@ wcwidth = [ {file = "wcwidth-0.2.5-py2.py3-none-any.whl", hash = "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784"}, {file = "wcwidth-0.2.5.tar.gz", hash = "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83"}, ] -zipp = [ - {file = "zipp-3.8.0-py3-none-any.whl", hash = "sha256:c4f6e5bbf48e74f7a38e7cc5b0480ff42b0ae5178957d564d18932525d5cf099"}, - {file = "zipp-3.8.0.tar.gz", hash = "sha256:56bf8aadb83c24db6c4b577e13de374ccfb67da2078beba1d037c17980bf43ad"}, -] diff --git a/pyproject.toml b/pyproject.toml index f6b305d..7fcb3e5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dccmd" -version = "0.2.0" +version = "0.3.0" description = "DRACOON Commander – CLI client for DRACOON Cloud (dracoon.com)" authors = ["Octavio Simone <70800577+unbekanntes-pferd@users.noreply.github.com>"] license = "Apache-2.0" @@ -9,8 +9,8 @@ readme = "README.md" [tool.poetry.dependencies] python = "^3.10" typer = "^0.4.0" -dracoon = "^1.4.1" -keyring = "^23.5.0" +dracoon = "^1.5.1" +keyring = "^23.6.0" SecretStorage = "^3.3.1" [tool.poetry.dev-dependencies] From 7f017245de94ea37374d283034a5af210d54fef5 Mon Sep 17 00:00:00 2001 From: Octavio Simone <70800577+unbekanntes-pferd@users.noreply.github.com> Date: Sat, 2 Jul 2022 08:51:06 +0200 Subject: [PATCH 2/5] added bulk download, fixed timeout issues --- README.md | 57 ++++++++- dccmd/__init__.py | 200 +++++++++++++++++++++----------- dccmd/main/auth/util.py | 8 ++ dccmd/main/download/__init__.py | 199 +++++++++++++++++++++++++++++++ dccmd/main/models/__init__.py | 2 +- poetry.lock | 130 ++++++++++++--------- pyproject.toml | 2 +- 7 files changed, 466 insertions(+), 132 deletions(-) create mode 100644 dccmd/main/download/__init__.py diff --git a/README.md b/README.md index d20f2be..eaae934 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ The tool is built on top of dracoon, an async API wrapper for DRACOON based on h The project is managed with poetry (dependencies, release build and publishing). ## Getting Started -In order to get started, download the latest tarball from Github: +In order to get started, download the latest tarball from Github or install dccmd from pip: [Releases]() ### Prerequisites @@ -265,7 +265,60 @@ To download a file, use the `download` command: ```bash dccmd download your-dracoon.domain.com/your/cool-file.mp4 /target/directory ``` -*Note: Currently, the download of a container (room, folder) is not supported.* + +To download a room or a folder, use the `download` command with `--recursive` (`-r`) flag: + +```bash +dccmd download -r your-dracoon.domain.com/your/cool-folder /target/directory +``` + +#### Advanced usage +If you download folders recursively, you might encounter performance issues, specifically when downloading many small files. +You can therefore adjust concurrent file uploads via the `--velocity` (`-v`) flag: + +```bash +dccmd upload -r /path/to/folder your-dracoon.domain.com/ -v 3 +``` +The default value is 2 - it does not coincide with real request value.
+Maximum (although not recommended) value is 10. Entering higher numbers will result in max value use.
+Minimum value is 1 - this will not download a folder per file but is the minimum concurrent request value. Entering lower numbers will result in min value use. + +### User operations + +You can list, edit and import users with relevant `dccmd users` command: + +* csv-import Add a list of users to DRACOON from a CSV file +* ls Get a list of users in DRACOON +* rm Delete a user + +#### Importing users + +You can import users by using the `csv-import` command and providing a path to the csv file: + +```bash +dccmd users csv-import /path/to/users.csv your-dracoon.domain.com/ +``` + +#### Listing users + +You can list all users using the `ls` command: + +```bash +dccmd users ls your-dracoon.domain.com/ +``` + +You can get all users also as csv format by using the `--csv` flag: + +```bash +dccmd users ls your-dracoon.domain.com/ --csv > users.csv +``` + +#### Deleting users + +You can delete a user by providing the username: +```bash +dccmd users rm your-dracoon.domain.com/ user123 +``` ## Configuration / administration diff --git a/dccmd/__init__.py b/dccmd/__init__.py index 9c891b5..baa40c0 100644 --- a/dccmd/__init__.py +++ b/dccmd/__init__.py @@ -58,16 +58,17 @@ from dccmd.main.crypto.keys import distribute_missing_keys from dccmd.main.crypto.util import get_keypair, init_keypair from dccmd.main.upload import create_folder_struct, bulk_upload, is_directory, is_file +from dccmd.main.download import create_download_list, bulk_download from dccmd.main.models import DCTransfer, DCTransferList from dccmd.main.models.errors import (DCPathParseError, DCClientParseError, ConnectError) # initialize CLI app app = typer.Typer() -app.add_typer(typer_instance=client_app, name="client") -app.add_typer(typer_instance=auth_app, name="auth") -app.add_typer(typer_instance=crypto_app, name="crypto") -app.add_typer(typer_instance=users_app, name="users") +app.add_typer(typer_instance=client_app, name="client", help="Manage client info") +app.add_typer(typer_instance=auth_app, name="auth", help="Manage authentication credentials") +app.add_typer(typer_instance=crypto_app, name="crypto", help="Manage crypto credentials") +app.add_typer(typer_instance=users_app, name="users", help="Manage users") @app.command() @@ -532,6 +533,9 @@ def ls( debug: bool = typer.Option( False, help="When active, sets log level to DEBUG and streams log" ), + all_items: bool = typer.Option( + False, help="When active, returns all items without prompt" + ), username: str = typer.Argument( None, help="Username to log in to DRACOON - only works with active cli mode" ), @@ -606,9 +610,12 @@ async def _list_nodes(): # handle more than 500 items if nodes.range.total > 500: - show_all = typer.confirm( + if not all_items: + show_all = typer.confirm( f"More than 500 nodes in {parsed_path} - display all?" ) + else: + show_all = all_items if not show_all: typer.echo(f"{nodes.range.total} nodes – only 500 displayed.") @@ -647,9 +654,9 @@ async def _list_nodes(): ) sys.exit(1) - if long_list and parent_id is not 0 and human_readable: + if long_list and parent_id != 0 and human_readable: typer.echo(f"total {to_readable_size(parent_node.size)}") - elif long_list and parent_id is not 0: + elif long_list and parent_id != 0: typer.echo(f"total {parent_node.size}") for node in nodes.items: @@ -679,6 +686,15 @@ def download( debug: bool = typer.Option( False, help="When active, sets log level to DEBUG and streams log" ), + recursive: bool = typer.Option( + False, "--recursive", "-r", help="Download a folder / room content recursively" + ), + velocity: int = typer.Option( + 2, + "--velocity", + "-v", + help="Concurrent requests factor (1: slow, 10: max)", + ), username: str = typer.Argument( None, help="Username to log in to DRACOON - only works with active cli mode" ), @@ -712,9 +728,7 @@ async def _download(): if not node_info: typer.echo(format_error_message(msg=f"Node not found ({parsed_path}).")) sys.exit(1) - if node_info.type != NodeType.file: - typer.echo(format_error_message(msg=f"Node not a file ({parsed_path})")) - sys.exit(1) + if node_info and node_info.isEncrypted is True: @@ -722,76 +736,122 @@ async def _download(): await init_keypair( dracoon=dracoon, base_url=base_url, crypto_secret=crypto_secret ) - - transfer = DCTransferList(total=node_info.size, file_count=1) - download_job = DCTransfer(transfer=transfer) - - try: - await dracoon.download( - file_path=parsed_path, - target_path=target_dir_path, - raise_on_err=True, - callback_fn=download_job.update - ) - # to do: replace with handling via PermissionError - except UnboundLocalError: - typer.echo( - format_error_message(msg=f"Insufficient permissions on target path ({target_dir_path})") - ) - sys.exit(1) - except InvalidPathError: - typer.echo( - format_error_message(msg=f"Path must be a folder ({target_dir_path})") - ) - sys.exit(1) - except InvalidFileError: - await dracoon.logout() - typer.echo(format_error_message(msg=f"File does not exist ({parsed_path})")) - sys.exit(1) - except FileConflictError: + + is_container = node_info == NodeType.folder or node_info.type == NodeType.room + is_file_path = node_info.type == NodeType.file + + if is_container and not recursive: typer.echo( format_error_message( - msg=f"File already exists on target path ({target_dir_path})" + msg="Folder or room can only be downloaded via recursive (-r) flag." ) ) - sys.exit(1) - except DRACOONHttpError: - await dracoon.logout() - typer.echo(format_error_message(msg="Error downloading file.")) - sys.exit(1) - except PermissionError: - await dracoon.logout() - typer.echo( - format_error_message( - msg=f"Cannot write on target path ({target_dir_path})" + elif is_container and recursive: + try: + download_list = await create_download_list(dracoon=dracoon, parent_id=node_info.id, + target_path=target_dir_path) + except InvalidPathError: + typer.echo( + format_error_message( + msg=f"Target path does not exist ({target_dir_path})" + ) ) - ) - sys.exit(1) - except TimeoutError: - typer.echo( - format_error_message( - msg="Connection timeout - could not download file." + sys.exit(1) + finally: + await dracoon.logout() + try: + await bulk_download(dracoon=dracoon, download_list=download_list, velocity=velocity) + except FileConflictError: + typer.echo( + format_error_message( + msg=f"File already exists on target path ({target_dir_path})" + ) ) - ) - sys.exit(1) - except ConnectError: - typer.echo( - format_error_message( - msg="Connection error - could not download file." + sys.exit(1) + except DRACOONHttpError: + await dracoon.logout() + typer.echo(format_error_message(msg="Error downloading file.")) + sys.exit(1) + except PermissionError: + await dracoon.logout() + typer.echo( + format_error_message( + msg=f"Cannot write on target path ({target_dir_path})" + ) + ) + finally: + await dracoon.logout() + elif is_file_path: + transfer = DCTransferList(total=node_info.size, file_count=1) + download_job = DCTransfer(transfer=transfer) + + try: + await dracoon.download( + file_path=parsed_path, + target_path=target_dir_path, + raise_on_err=True, + callback_fn=download_job.update + ) + # to do: replace with handling via PermissionError + except UnboundLocalError: + typer.echo( + format_error_message(msg=f"Insufficient permissions on target path ({target_dir_path})") + ) + sys.exit(1) + except InvalidPathError: + typer.echo( + format_error_message(msg=f"Path must be a folder ({target_dir_path})") + ) + sys.exit(1) + except InvalidFileError: + await dracoon.logout() + typer.echo(format_error_message(msg=f"File does not exist ({parsed_path})")) + sys.exit(1) + except FileConflictError: + typer.echo( + format_error_message( + msg=f"File already exists on target path ({target_dir_path})" + ) + ) + sys.exit(1) + except DRACOONHttpError: + await dracoon.logout() + typer.echo(format_error_message(msg="Error downloading file.")) + sys.exit(1) + except PermissionError: + await dracoon.logout() + typer.echo( + format_error_message( + msg=f"Cannot write on target path ({target_dir_path})" + ) + ) + sys.exit(1) + except TimeoutError: + typer.echo( + format_error_message( + msg="Connection timeout - could not download file." + ) + ) + sys.exit(1) + except ConnectError: + typer.echo( + format_error_message( + msg="Connection error - could not download file." + ) ) + sys.exit(1) + except KeyboardInterrupt: + await dracoon.logout() + typer.echo( + f'{format_success_message(f"Download canceled ({file_name}).")}' ) - sys.exit(1) - except KeyboardInterrupt: - await dracoon.logout() - typer.echo( - f'{format_success_message(f"Download canceled ({file_name}).")}' - ) + finally: + await dracoon.logout() - typer.echo( - f'{format_success_message(f"File {file_name} downloaded to {target_dir_path}.")}' - ) - await dracoon.logout() + typer.echo( + f'{format_success_message(f"File {file_name} downloaded to {target_dir_path}.")}' + ) asyncio.run(_download()) diff --git a/dccmd/main/auth/util.py b/dccmd/main/auth/util.py index 76d62b0..cd58063 100644 --- a/dccmd/main/auth/util.py +++ b/dccmd/main/auth/util.py @@ -26,6 +26,8 @@ get_credentials, ) +DEFAULT_TIMEOUT_CONFIG = httpx.Timeout(None, connect=None, read=None) + async def login( base_url: str, refresh_token: str = None, @@ -61,6 +63,7 @@ async def login( client_secret=client_secret, log_level=log_level, log_stream=debug, + log_file_out=True, raise_on_err=True ) @@ -68,6 +71,11 @@ async def login( dracoon_user_agent = dracoon.client.http.headers["User-Agent"] dracoon.client.http.headers["User-Agent"] = f"{dccmd_name}|{dccmd_version}|{dracoon_user_agent}" + # set custom client with no timeout, up- and downloader with same client !!! + dracoon.client.http = httpx.AsyncClient(headers=dracoon.client.headers, timeout=DEFAULT_TIMEOUT_CONFIG) + dracoon.client.uploader = httpx.AsyncClient(timeout=DEFAULT_TIMEOUT_CONFIG) + dracoon.client.downloader = dracoon.client.uploader + # password flow if cli_mode: try: diff --git a/dccmd/main/download/__init__.py b/dccmd/main/download/__init__.py new file mode 100644 index 0000000..6714a1f --- /dev/null +++ b/dccmd/main/download/__init__.py @@ -0,0 +1,199 @@ +import os +import asyncio +from pathlib import Path +from enum import Enum +from dataclasses import dataclass + + +import typer +from tqdm import tqdm +from dracoon import DRACOON +from dracoon.errors import InvalidPathError +from dracoon.nodes.responses import NodeList, Node + +from ..models import DCTransfer, DCTransferList + +class NodeType(Enum): + """ represents DRACOON node types """ + FILE = "file" + FOLDER = "folder" + ROOM = "room" + + +class DownloadDirectoryItem: + """object representing a directory with all required path elements""" + def __init__(self, dir_path: str, base_path: str): + base_path = Path(base_path) + self.abs_path = base_path.joinpath(dir_path[1:]) + self.path = dir_path + self.name = self.abs_path.name + self.parent_path = self.abs_path.parent + self.level = len(self.path.split("/")) - 1 + +class DownloadFileItem: + """ object representing a single file with all required path infos """ + def __init__(self, dir_path: str, base_path: str, node_info: Node): + base_path = Path(base_path) + self.abs_path = base_path.joinpath(dir_path[1:]) + self.path = dir_path + self.name = self.abs_path.name + self.parent_path = self.abs_path.parent + self.level = len(self.path.split("/")) - 1 + self.node = node_info + + +class DownloadList: + """ list of files and folders within a room (subrooms excluded) """ + def __init__(self, file_list: NodeList, folder_list: NodeList, node: Node, target_path: str): + self.file_list = file_list + self.folder_list = folder_list + self.node = node + self.target_path = target_path + + def get_level(self, level: int) -> list[DownloadDirectoryItem]: + """get all directories in a depth level""" + level_list = [dir for dir in self.folder_items if dir.level == level] + level_list.sort(key=lambda dir: dir.path) + return level_list + + def get_by_parent(self, parent: str): + """get all directories by parent""" + return [dir for dir in self.folder_items if dir.parent_path == parent] + + def get_batches(self, level: int) -> list[list[DownloadDirectoryItem]]: + """create batches based on depth levels""" + parent_list = set([dir.parent_path for dir in self.get_level(level=level)]) + + return [self.get_by_parent(parent) for parent in parent_list] + + @property + def total_size(self) -> int: + """ returns total download size """ + return sum([node.size for node in self.file_list.items]) + + @property + def file_paths(self) -> list[str]: + """ returns all file paths in alphabetical order """ + return sorted([f"{node.parentPath}{node.name}" for node in self.file_list.items]) + + @property + def folder_paths(self) -> list[str]: + """ returns all folder paths in alphabetical order """ + return sorted([f"{node.parentPath}{node.name}" for node in self.folder_list.items]) + + @property + def folder_items(self) -> list[DownloadDirectoryItem]: + """ returns folder item including level """ + return [DownloadDirectoryItem(dir_path=dir_path, base_path=self.target_path) for dir_path in self.folder_paths] + + @property + def file_items(self) -> list[DownloadFileItem]: + """ returns folder item including level """ + return [DownloadFileItem(dir_path=f"{node.parentPath}{node.name}", base_path=self.target_path, node_info=node) for node in self.file_list.items] + + @property + def levels(self): + """ returns levels """ + return set([dir.level for dir in self.folder_items]) + + +def create_folder(name: str, target_dir_path: str): + """ creates a folder in a target directory """ + target_path = Path(target_dir_path) + + if not target_path.is_dir(): + raise InvalidPathError() + + target_path = target_path.joinpath(name) + target_path.mkdir() + + +async def get_nodes(dracoon: DRACOON, parent_id: int, node_type: NodeType, depth_level: int = -1) -> NodeList: + """ get all files for a given parent id """ + node_filter = 'type:eq:' + + + if node_type == NodeType.FILE: + node_filter += NodeType.FILE.value + if node_type == NodeType.FOLDER: + node_filter += NodeType.FOLDER.value + if node_type == NodeType.ROOM: + node_filter += NodeType.ROOM.value + + node_list = await dracoon.nodes.search_nodes(search="*", parent_id=parent_id, depth_level=depth_level, filter=node_filter) + if node_list.range.total > 500: + node_reqs = [ + dracoon.nodes.search_nodes(search="*", parent_id=parent_id, depth_level=depth_level, offset=offset, filter=node_filter) + for offset in range(500, node_list.range.total, 500) + ] + for reqs in dracoon.batch_process(coro_list=node_reqs, batch_size=20): + node_lists = await asyncio.gather(*reqs) + for item in node_lists: + node_list.items.extend(item.items) + + return node_list + + +async def create_download_list(dracoon: DRACOON, parent_id: int, target_path: str) -> DownloadList: + """ returns a list of files and folders for bulk download operations """ + + target_path_check = Path(target_path) + if not target_path_check.is_dir(): + raise InvalidPathError() + + # get all files and folders within path + all_files = await get_nodes(dracoon=dracoon, parent_id=parent_id, node_type=NodeType.FILE) + all_folders = await get_nodes(dracoon=dracoon, parent_id=parent_id, node_type=NodeType.FOLDER) + node_info = await dracoon.nodes.get_node(node_id=parent_id) + + # only consider those with authParentId of the parent room + all_files.items = [item for item in all_files.items if item.authParentId == parent_id] + all_folders.items = [item for item in all_folders.items if item.authParentId == parent_id] + + return DownloadList(file_list=all_files, folder_list=all_folders, node=node_info, target_path=target_path) + +async def bulk_download(dracoon: DRACOON, download_list: DownloadList, velocity: int = 2): + """ download all files within a room (excluded: sub rooms) """ + + if velocity > 10: + velocity = 10 + elif velocity < 1: + velocity = 1 + + concurrent_reqs = velocity * 5 + + # create main folder + try: + create_folder(name=download_list.node.name, target_dir_path=download_list.target_path) + except FileExistsError: + pass + + progress = tqdm(unit='folder level', total=len(download_list.levels), unit_scale=True) + for level in download_list.levels: + + for batch in download_list.get_batches(level): + for folder in batch: + try: + create_folder(name=folder.name,target_dir_path=folder.parent_path) + except FileExistsError: + continue + progress.update() + + progress.close() + + transfer_list = DCTransferList(total=download_list.total_size, file_count=len(download_list.file_items)) + + + download_reqs = [] + + for file_item in download_list.file_items: + download_job = DCTransfer(transfer=transfer_list) + download_req = dracoon.download(target_path=file_item.parent_path, callback_fn=download_job.update, + source_node_id=file_item.node.id, chunksize=1048576) + download_reqs.append(download_req) + + for downloads in dracoon.batch_process(coro_list=download_reqs, batch_size=concurrent_reqs): + await asyncio.gather(*downloads) + + + diff --git a/dccmd/main/models/__init__.py b/dccmd/main/models/__init__.py index df74ee3..f329093 100644 --- a/dccmd/main/models/__init__.py +++ b/dccmd/main/models/__init__.py @@ -38,7 +38,7 @@ def __init__(self, transfer: DCTransferList): def update(self, val: int, total: int = None): """ callback function to track progress """ - if total is not None and val is 0: + if total is not None and val == 0: self.total += total self.update_progress(val) diff --git a/poetry.lock b/poetry.lock index c70cf98..65e392b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -55,7 +55,7 @@ python-versions = ">=3.6" [[package]] name = "cffi" -version = "1.15.0" +version = "1.15.1" description = "Foreign Function Interface for Python calling C code." category = "main" optional = false @@ -104,7 +104,7 @@ test = ["pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-subtests", [[package]] name = "dracoon" -version = "1.5.1" +version = "1.5.3" description = "DRACOON API wrapper in Python" category = "main" optional = false @@ -372,7 +372,7 @@ test = ["shellingham (>=1.3.0,<2.0.0)", "pytest (>=4.4.0,<5.4.0)", "pytest-cov ( [[package]] name = "typing-extensions" -version = "4.2.0" +version = "4.3.0" description = "Backported and Experimental Type Hints for Python 3.7+" category = "main" optional = false @@ -389,7 +389,7 @@ python-versions = "*" [metadata] lock-version = "1.1" python-versions = "^3.10" -content-hash = "44e64f25bb58220d605c282dfef5c825fa32c9e337476bea22f853bbdee920da" +content-hash = "aa462ba08684d8d8e97f893194954b2c4de88a40609b9ffd9002e6e04483698a" [metadata.files] anyio = [ @@ -415,56 +415,70 @@ certifi = [ {file = "certifi-2022.6.15.tar.gz", hash = "sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d"}, ] cffi = [ - {file = "cffi-1.15.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:c2502a1a03b6312837279c8c1bd3ebedf6c12c4228ddbad40912d671ccc8a962"}, - {file = "cffi-1.15.0-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:23cfe892bd5dd8941608f93348c0737e369e51c100d03718f108bf1add7bd6d0"}, - {file = "cffi-1.15.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:41d45de54cd277a7878919867c0f08b0cf817605e4eb94093e7516505d3c8d14"}, - {file = "cffi-1.15.0-cp27-cp27m-win32.whl", hash = "sha256:4a306fa632e8f0928956a41fa8e1d6243c71e7eb59ffbd165fc0b41e316b2474"}, - {file = "cffi-1.15.0-cp27-cp27m-win_amd64.whl", hash = "sha256:e7022a66d9b55e93e1a845d8c9eba2a1bebd4966cd8bfc25d9cd07d515b33fa6"}, - {file = "cffi-1.15.0-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:14cd121ea63ecdae71efa69c15c5543a4b5fbcd0bbe2aad864baca0063cecf27"}, - {file = "cffi-1.15.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:d4d692a89c5cf08a8557fdeb329b82e7bf609aadfaed6c0d79f5a449a3c7c023"}, - {file = "cffi-1.15.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0104fb5ae2391d46a4cb082abdd5c69ea4eab79d8d44eaaf79f1b1fd806ee4c2"}, - {file = "cffi-1.15.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:91ec59c33514b7c7559a6acda53bbfe1b283949c34fe7440bcf917f96ac0723e"}, - {file = "cffi-1.15.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:f5c7150ad32ba43a07c4479f40241756145a1f03b43480e058cfd862bf5041c7"}, - {file = "cffi-1.15.0-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:00c878c90cb53ccfaae6b8bc18ad05d2036553e6d9d1d9dbcf323bbe83854ca3"}, - {file = "cffi-1.15.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:abb9a20a72ac4e0fdb50dae135ba5e77880518e742077ced47eb1499e29a443c"}, - {file = "cffi-1.15.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a5263e363c27b653a90078143adb3d076c1a748ec9ecc78ea2fb916f9b861962"}, - {file = "cffi-1.15.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f54a64f8b0c8ff0b64d18aa76675262e1700f3995182267998c31ae974fbc382"}, - {file = "cffi-1.15.0-cp310-cp310-win32.whl", hash = "sha256:c21c9e3896c23007803a875460fb786118f0cdd4434359577ea25eb556e34c55"}, - {file = "cffi-1.15.0-cp310-cp310-win_amd64.whl", hash = "sha256:5e069f72d497312b24fcc02073d70cb989045d1c91cbd53979366077959933e0"}, - {file = "cffi-1.15.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:64d4ec9f448dfe041705426000cc13e34e6e5bb13736e9fd62e34a0b0c41566e"}, - {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2756c88cbb94231c7a147402476be2c4df2f6078099a6f4a480d239a8817ae39"}, - {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b96a311ac60a3f6be21d2572e46ce67f09abcf4d09344c49274eb9e0bf345fc"}, - {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:75e4024375654472cc27e91cbe9eaa08567f7fbdf822638be2814ce059f58032"}, - {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:59888172256cac5629e60e72e86598027aca6bf01fa2465bdb676d37636573e8"}, - {file = "cffi-1.15.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:27c219baf94952ae9d50ec19651a687b826792055353d07648a5695413e0c605"}, - {file = "cffi-1.15.0-cp36-cp36m-win32.whl", hash = "sha256:4958391dbd6249d7ad855b9ca88fae690783a6be9e86df65865058ed81fc860e"}, - {file = "cffi-1.15.0-cp36-cp36m-win_amd64.whl", hash = "sha256:f6f824dc3bce0edab5f427efcfb1d63ee75b6fcb7282900ccaf925be84efb0fc"}, - {file = "cffi-1.15.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:06c48159c1abed75c2e721b1715c379fa3200c7784271b3c46df01383b593636"}, - {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:c2051981a968d7de9dd2d7b87bcb9c939c74a34626a6e2f8181455dd49ed69e4"}, - {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:fd8a250edc26254fe5b33be00402e6d287f562b6a5b2152dec302fa15bb3e997"}, - {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:91d77d2a782be4274da750752bb1650a97bfd8f291022b379bb8e01c66b4e96b"}, - {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:45db3a33139e9c8f7c09234b5784a5e33d31fd6907800b316decad50af323ff2"}, - {file = "cffi-1.15.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:263cc3d821c4ab2213cbe8cd8b355a7f72a8324577dc865ef98487c1aeee2bc7"}, - {file = "cffi-1.15.0-cp37-cp37m-win32.whl", hash = "sha256:17771976e82e9f94976180f76468546834d22a7cc404b17c22df2a2c81db0c66"}, - {file = "cffi-1.15.0-cp37-cp37m-win_amd64.whl", hash = "sha256:3415c89f9204ee60cd09b235810be700e993e343a408693e80ce7f6a40108029"}, - {file = "cffi-1.15.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:4238e6dab5d6a8ba812de994bbb0a79bddbdf80994e4ce802b6f6f3142fcc880"}, - {file = "cffi-1.15.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:0808014eb713677ec1292301ea4c81ad277b6cdf2fdd90fd540af98c0b101d20"}, - {file = "cffi-1.15.0-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:57e9ac9ccc3101fac9d6014fba037473e4358ef4e89f8e181f8951a2c0162024"}, - {file = "cffi-1.15.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8b6c2ea03845c9f501ed1313e78de148cd3f6cad741a75d43a29b43da27f2e1e"}, - {file = "cffi-1.15.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:10dffb601ccfb65262a27233ac273d552ddc4d8ae1bf93b21c94b8511bffe728"}, - {file = "cffi-1.15.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:786902fb9ba7433aae840e0ed609f45c7bcd4e225ebb9c753aa39725bb3e6ad6"}, - {file = "cffi-1.15.0-cp38-cp38-win32.whl", hash = "sha256:da5db4e883f1ce37f55c667e5c0de439df76ac4cb55964655906306918e7363c"}, - {file = "cffi-1.15.0-cp38-cp38-win_amd64.whl", hash = "sha256:181dee03b1170ff1969489acf1c26533710231c58f95534e3edac87fff06c443"}, - {file = "cffi-1.15.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:45e8636704eacc432a206ac7345a5d3d2c62d95a507ec70d62f23cd91770482a"}, - {file = "cffi-1.15.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:31fb708d9d7c3f49a60f04cf5b119aeefe5644daba1cd2a0fe389b674fd1de37"}, - {file = "cffi-1.15.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:6dc2737a3674b3e344847c8686cf29e500584ccad76204efea14f451d4cc669a"}, - {file = "cffi-1.15.0-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:74fdfdbfdc48d3f47148976f49fab3251e550a8720bebc99bf1483f5bfb5db3e"}, - {file = "cffi-1.15.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffaa5c925128e29efbde7301d8ecaf35c8c60ffbcd6a1ffd3a552177c8e5e796"}, - {file = "cffi-1.15.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3f7d084648d77af029acb79a0ff49a0ad7e9d09057a9bf46596dac9514dc07df"}, - {file = "cffi-1.15.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ef1f279350da2c586a69d32fc8733092fd32cc8ac95139a00377841f59a3f8d8"}, - {file = "cffi-1.15.0-cp39-cp39-win32.whl", hash = "sha256:2a23af14f408d53d5e6cd4e3d9a24ff9e05906ad574822a10563efcef137979a"}, - {file = "cffi-1.15.0-cp39-cp39-win_amd64.whl", hash = "sha256:3773c4d81e6e818df2efbc7dd77325ca0dcb688116050fb2b3011218eda36139"}, - {file = "cffi-1.15.0.tar.gz", hash = "sha256:920f0d66a896c2d99f0adbb391f990a84091179542c205fa53ce5787aff87954"}, + {file = "cffi-1.15.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2"}, + {file = "cffi-1.15.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2"}, + {file = "cffi-1.15.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914"}, + {file = "cffi-1.15.1-cp27-cp27m-win32.whl", hash = "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3"}, + {file = "cffi-1.15.1-cp27-cp27m-win_amd64.whl", hash = "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e"}, + {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162"}, + {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b"}, + {file = "cffi-1.15.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21"}, + {file = "cffi-1.15.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4"}, + {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01"}, + {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e"}, + {file = "cffi-1.15.1-cp310-cp310-win32.whl", hash = "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2"}, + {file = "cffi-1.15.1-cp310-cp310-win_amd64.whl", hash = "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d"}, + {file = "cffi-1.15.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac"}, + {file = "cffi-1.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c"}, + {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef"}, + {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8"}, + {file = "cffi-1.15.1-cp311-cp311-win32.whl", hash = "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d"}, + {file = "cffi-1.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104"}, + {file = "cffi-1.15.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e"}, + {file = "cffi-1.15.1-cp36-cp36m-win32.whl", hash = "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf"}, + {file = "cffi-1.15.1-cp36-cp36m-win_amd64.whl", hash = "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497"}, + {file = "cffi-1.15.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426"}, + {file = "cffi-1.15.1-cp37-cp37m-win32.whl", hash = "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9"}, + {file = "cffi-1.15.1-cp37-cp37m-win_amd64.whl", hash = "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045"}, + {file = "cffi-1.15.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192"}, + {file = "cffi-1.15.1-cp38-cp38-win32.whl", hash = "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314"}, + {file = "cffi-1.15.1-cp38-cp38-win_amd64.whl", hash = "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5"}, + {file = "cffi-1.15.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585"}, + {file = "cffi-1.15.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27"}, + {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76"}, + {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3"}, + {file = "cffi-1.15.1-cp39-cp39-win32.whl", hash = "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee"}, + {file = "cffi-1.15.1-cp39-cp39-win_amd64.whl", hash = "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c"}, + {file = "cffi-1.15.1.tar.gz", hash = "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9"}, ] click = [ {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, @@ -499,8 +513,8 @@ cryptography = [ {file = "cryptography-37.0.3.tar.gz", hash = "sha256:ae430d51c67ac638dfbb42edf56c669ca9c74744f4d225ad11c6f3d355858187"}, ] dracoon = [ - {file = "dracoon-1.5.1-py3-none-any.whl", hash = "sha256:ebfa7c0d2d16962011e4ead6d5f34cbbf4a681e37ede4701d6f014b0f920b705"}, - {file = "dracoon-1.5.1.tar.gz", hash = "sha256:bc22707ac833c0623b01e6ecd36bad72214b780e8c9274a730eea37723866c08"}, + {file = "dracoon-1.5.3-py3-none-any.whl", hash = "sha256:f4aaf984edbf2abc100cd86d9eb676d908576058f727ee7777ced954049cf325"}, + {file = "dracoon-1.5.3.tar.gz", hash = "sha256:e5b6ef8bc57caf4f7ef5024872aee4569e6926a55e1368e363753a8d90d9efe3"}, ] h11 = [ {file = "h11-0.12.0-py3-none-any.whl", hash = "sha256:36a3cb8c0a032f56e2da7084577878a035d3b61d104230d4bd49c0c6b555a9c6"}, @@ -616,8 +630,8 @@ typer = [ {file = "typer-0.4.1.tar.gz", hash = "sha256:5646aef0d936b2c761a10393f0384ee6b5c7fe0bb3e5cd710b17134ca1d99cff"}, ] typing-extensions = [ - {file = "typing_extensions-4.2.0-py3-none-any.whl", hash = "sha256:6657594ee297170d19f67d55c05852a874e7eb634f4f753dbd667855e07c1708"}, - {file = "typing_extensions-4.2.0.tar.gz", hash = "sha256:f1c24655a0da0d1b67f07e17a5e6b2a105894e6824b92096378bb3668ef02376"}, + {file = "typing_extensions-4.3.0-py3-none-any.whl", hash = "sha256:25642c956049920a5aa49edcdd6ab1e06d7e5d467fc00e0506c44ac86fbfca02"}, + {file = "typing_extensions-4.3.0.tar.gz", hash = "sha256:e6d2677a32f47fc7eb2795db1dd15c1f34eff616bcaf2cfb5e997f854fa1c4a6"}, ] wcwidth = [ {file = "wcwidth-0.2.5-py2.py3-none-any.whl", hash = "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784"}, diff --git a/pyproject.toml b/pyproject.toml index 7fcb3e5..ce29652 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ readme = "README.md" [tool.poetry.dependencies] python = "^3.10" typer = "^0.4.0" -dracoon = "^1.5.1" +dracoon = "^1.5.3" keyring = "^23.6.0" SecretStorage = "^3.3.1" From 52af7900a31d0d00c734cb508458f8a445d07019 Mon Sep 17 00:00:00 2001 From: Octavio Simone <70800577+unbekanntes-pferd@users.noreply.github.com> Date: Sun, 3 Jul 2022 22:02:43 +0200 Subject: [PATCH 3/5] download refactor, oidc user import, readme --- README.md | 21 +++++++++++++++++++++ dccmd/__init__.py | 10 +++++----- dccmd/main/download/__init__.py | 16 ++++++++-------- dccmd/main/users/__init__.py | 3 ++- dccmd/main/users/manage.py | 2 +- 5 files changed, 37 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index eaae934..f2a5469 100644 --- a/README.md +++ b/README.md @@ -299,6 +299,20 @@ You can import users by using the `csv-import` command and providing a path to t dccmd users csv-import /path/to/users.csv your-dracoon.domain.com/ ``` +The csv file must contain a header and should include the following attributes: + +* first name +* last name +* email +* login (optional) + +By default, local users are created - if you want to import oidc users, you need pass the oidc config id: + +```bash +#example with OIDC config 5 +dccmd users csv-import /path/to/users.csv your-dracoon.domain.com/ 5 +``` + #### Listing users You can list all users using the `ls` command: @@ -313,6 +327,13 @@ You can get all users also as csv format by using the `--csv` flag: dccmd users ls your-dracoon.domain.com/ --csv > users.csv ``` +To find a user, you can pass a search string to search for either first name, last name or user name (search string applies to all): + +```bash +# will return all users with either first name, last name or user name containing 'yourname' +dccmd users ls your-dracoon.domain.com/ yourname +``` + #### Deleting users You can delete a user by providing the username: diff --git a/dccmd/__init__.py b/dccmd/__init__.py index baa40c0..5f7e5e4 100644 --- a/dccmd/__init__.py +++ b/dccmd/__init__.py @@ -737,9 +737,9 @@ async def _download(): dracoon=dracoon, base_url=base_url, crypto_secret=crypto_secret ) - is_container = node_info == NodeType.folder or node_info.type == NodeType.room + is_container = node_info.type == NodeType.folder or node_info.type == NodeType.room is_file_path = node_info.type == NodeType.file - + if is_container and not recursive: typer.echo( format_error_message( @@ -748,7 +748,7 @@ async def _download(): ) elif is_container and recursive: try: - download_list = await create_download_list(dracoon=dracoon, parent_id=node_info.id, + download_list = await create_download_list(dracoon=dracoon, node_info=node_info, target_path=target_dir_path) except InvalidPathError: typer.echo( @@ -756,9 +756,9 @@ async def _download(): msg=f"Target path does not exist ({target_dir_path})" ) ) - sys.exit(1) - finally: await dracoon.logout() + sys.exit(1) + try: await bulk_download(dracoon=dracoon, download_list=download_list, velocity=velocity) except FileConflictError: diff --git a/dccmd/main/download/__init__.py b/dccmd/main/download/__init__.py index 6714a1f..d0798f0 100644 --- a/dccmd/main/download/__init__.py +++ b/dccmd/main/download/__init__.py @@ -134,7 +134,7 @@ async def get_nodes(dracoon: DRACOON, parent_id: int, node_type: NodeType, depth return node_list -async def create_download_list(dracoon: DRACOON, parent_id: int, target_path: str) -> DownloadList: +async def create_download_list(dracoon: DRACOON, node_info: Node, target_path: str) -> DownloadList: """ returns a list of files and folders for bulk download operations """ target_path_check = Path(target_path) @@ -142,13 +142,13 @@ async def create_download_list(dracoon: DRACOON, parent_id: int, target_path: st raise InvalidPathError() # get all files and folders within path - all_files = await get_nodes(dracoon=dracoon, parent_id=parent_id, node_type=NodeType.FILE) - all_folders = await get_nodes(dracoon=dracoon, parent_id=parent_id, node_type=NodeType.FOLDER) - node_info = await dracoon.nodes.get_node(node_id=parent_id) - - # only consider those with authParentId of the parent room - all_files.items = [item for item in all_files.items if item.authParentId == parent_id] - all_folders.items = [item for item in all_folders.items if item.authParentId == parent_id] + all_files = await get_nodes(dracoon=dracoon, parent_id=node_info.id, node_type=NodeType.FILE) + all_folders = await get_nodes(dracoon=dracoon, parent_id=node_info.id, node_type=NodeType.FOLDER) + + # only consider those with authParentId of the parent room if the source is a room (exclude sub rooms) + if node_info.type == NodeType.ROOM: + all_files.items = [item for item in all_files.items if item.authParentId == node_info.id] + all_folders.items = [item for item in all_folders.items if item.authParentId == node_info.id] return DownloadList(file_list=all_files, folder_list=all_folders, node=node_info, target_path=target_path) diff --git a/dccmd/main/users/__init__.py b/dccmd/main/users/__init__.py index 288c88e..2ca9202 100644 --- a/dccmd/main/users/__init__.py +++ b/dccmd/main/users/__init__.py @@ -31,6 +31,7 @@ def csv_import( help="Full path to a CSV file containing user data to bulk import", ), target_path: str = typer.Argument(..., help="DRACOON url to import users to"), + oidc_id: int = typer.Argument(None, help="Numeric id of OIDC config"), cli_mode: bool = typer.Option( False, help="When active, accepts username and password" ), @@ -59,7 +60,7 @@ async def _import_users(): user_list = parse_csv(source_path=source_path) - await create_users(dracoon=dracoon, user_list=user_list) + await create_users(dracoon=dracoon, user_list=user_list, oidc_id=oidc_id) await dracoon.logout() asyncio.run(_import_users()) diff --git a/dccmd/main/users/manage.py b/dccmd/main/users/manage.py index 060ff3e..00c59e9 100644 --- a/dccmd/main/users/manage.py +++ b/dccmd/main/users/manage.py @@ -250,7 +250,7 @@ async def get_users(dracoon: DRACOON, search_string: str = ''): search_filter = None else: search_filter = f'userName:cn:{search_string}|firstName:cn:{search_string}|lastName:cn:{search_string}' - search_filter = parse.quote(search_filter) + #search_filter = parse.quote(search_filter) try: user_list = await dracoon.users.get_users(filter=search_filter, offset=0) From 97b9b5a15338905965a9139e1d79b611f796c213 Mon Sep 17 00:00:00 2001 From: Octavio Simone <70800577+unbekanntes-pferd@users.noreply.github.com> Date: Sun, 3 Jul 2022 23:46:22 +0200 Subject: [PATCH 4/5] fixed recursive download: nested level support --- dccmd/main/download/__init__.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/dccmd/main/download/__init__.py b/dccmd/main/download/__init__.py index d0798f0..1ff34e4 100644 --- a/dccmd/main/download/__init__.py +++ b/dccmd/main/download/__init__.py @@ -49,6 +49,7 @@ def __init__(self, file_list: NodeList, folder_list: NodeList, node: Node, targe self.folder_list = folder_list self.node = node self.target_path = target_path + print(self.base_level) def get_level(self, level: int) -> list[DownloadDirectoryItem]: """get all directories in a depth level""" @@ -79,7 +80,7 @@ def file_paths(self) -> list[str]: @property def folder_paths(self) -> list[str]: """ returns all folder paths in alphabetical order """ - return sorted([f"{node.parentPath}{node.name}" for node in self.folder_list.items]) + return sorted([f"{normalize_parent_path(parent_path=node.parentPath, level=self.base_level)}{node.name}" for node in self.folder_list.items]) @property def folder_items(self) -> list[DownloadDirectoryItem]: @@ -89,13 +90,30 @@ def folder_items(self) -> list[DownloadDirectoryItem]: @property def file_items(self) -> list[DownloadFileItem]: """ returns folder item including level """ - return [DownloadFileItem(dir_path=f"{node.parentPath}{node.name}", base_path=self.target_path, node_info=node) for node in self.file_list.items] + return [DownloadFileItem(dir_path=f"{normalize_parent_path(parent_path=node.parentPath, level=self.base_level)}{node.name}", + base_path=self.target_path, node_info=node) for node in self.file_list.items] @property def levels(self): """ returns levels """ return set([dir.level for dir in self.folder_items]) + @property + def base_level(self): + """ return base level of the source container """ + path_comp = self.node.parentPath.split("/") + + if len(path_comp) == 2: + return 0 + elif len(path_comp) >= 3: + return len(path_comp) - 2 + +def normalize_parent_path(parent_path: str, level: int): + """ remove parent path components if root on specific level """ + path_comp = parent_path.split('/') + normalized_comp = path_comp[level+1:] + return "/" + "/".join(normalized_comp) + def create_folder(name: str, target_dir_path: str): """ creates a folder in a target directory """ From 9c44e4f875ee8fb4062f4ae127cb40064f2a7b74 Mon Sep 17 00:00:00 2001 From: Octavio Simone <70800577+unbekanntes-pferd@users.noreply.github.com> Date: Mon, 4 Jul 2022 00:00:44 +0200 Subject: [PATCH 5/5] catch no file content in bulk download --- dccmd/__init__.py | 2 ++ dccmd/main/download/__init__.py | 11 +++++++++++ 2 files changed, 13 insertions(+) diff --git a/dccmd/__init__.py b/dccmd/__init__.py index 5f7e5e4..17db7ed 100644 --- a/dccmd/__init__.py +++ b/dccmd/__init__.py @@ -759,6 +759,8 @@ async def _download(): await dracoon.logout() sys.exit(1) + + try: await bulk_download(dracoon=dracoon, download_list=download_list, velocity=velocity) except FileConflictError: diff --git a/dccmd/main/download/__init__.py b/dccmd/main/download/__init__.py index 1ff34e4..a2fbb0f 100644 --- a/dccmd/main/download/__init__.py +++ b/dccmd/main/download/__init__.py @@ -1,4 +1,10 @@ +""" +Module implementing bulk download from DRACOON + +""" + import os +import sys import asyncio from pathlib import Path from enum import Enum @@ -12,6 +18,7 @@ from dracoon.nodes.responses import NodeList, Node from ..models import DCTransfer, DCTransferList +from ..util import format_error_message class NodeType(Enum): """ represents DRACOON node types """ @@ -173,6 +180,10 @@ async def create_download_list(dracoon: DRACOON, node_info: Node, target_path: s async def bulk_download(dracoon: DRACOON, download_list: DownloadList, velocity: int = 2): """ download all files within a room (excluded: sub rooms) """ + if len(download_list.file_list.items) <= 0: + typer.echo(format_error_message(f"No files to download in {download_list.node.parentPath}{download_list.node.name}")) + sys.exit(1) + if velocity > 10: velocity = 10 elif velocity < 1: