From 2eab4d2fbef39d73321ee5eb8a9525486eef03fb Mon Sep 17 00:00:00 2001 From: Octavio Simone Date: Thu, 25 May 2023 18:03:55 +0200 Subject: [PATCH 1/2] refactoring, fetch new token on batches --- dccmd/__init__.py | 42 +++++++++++++++++++++-------------- dccmd/main/upload/__init__.py | 9 ++++++-- pyproject.toml | 2 +- 3 files changed, 33 insertions(+), 20 deletions(-) diff --git a/dccmd/__init__.py b/dccmd/__init__.py index 9affc30..93da846 100644 --- a/dccmd/__init__.py +++ b/dccmd/__init__.py @@ -236,8 +236,10 @@ async def _upload(): distrib_node_id = node_info.authParentId if node_info.type == NodeType.room: distrib_node_id = node_info.id + else: + distrib_node_id = None - if node_info.isEncrypted is True: + if node_info.isEncrypted is True and distrib_node_id is not None: await distribute_missing_keys(dracoon=dracoon, room_id=distrib_node_id) await dracoon.logout() @@ -368,12 +370,12 @@ async def _create_room(): # remove base url from path parsed_path = parse_new_path(full_path=dir_path) - if parsed_path != "/": - parent_node = await dracoon.nodes.get_node_from_path(path=parsed_path) - parent_id = parent_node.id - elif parsed_path == "/": + if parsed_path == "/": parent_node = None parent_id = 0 + else: + parent_node = await dracoon.nodes.get_node_from_path(path=parsed_path) + parent_id = parent_node.id room_name = parse_file_name(full_path=dir_path) @@ -397,10 +399,10 @@ async def _create_room(): if admin_user and parent_id != 0: user_info = await find_user_by_username(dracoon=dracoon, user_name=admin_user, as_user_manager=False, room_id=parent_id) - payload = dracoon.nodes.make_room(name=room_name, parent_id=parent_id, inherit_perms=False, admin_ids=[user_info.userInfo.id]) + payload = dracoon.nodes.make_room(name=room_name, parent_id=parent_id, inherit_perms=False, admin_ids=[user_info.userInfo.id]) # type: ignore if admin_user and parent_id == 0: user_info = await find_user_by_username(dracoon=dracoon, user_name=admin_user) - payload = dracoon.nodes.make_room(name=room_name, inherit_perms=False, admin_ids=[user_info.id], parent_id=None) + payload = dracoon.nodes.make_room(name=room_name, inherit_perms=False, admin_ids=[user_info.id], parent_id=None) # type: ignore else: payload = dracoon.nodes.make_room( name=room_name, parent_id=parent_id, inherit_perms=True @@ -612,14 +614,12 @@ async def _list_nodes(): # remove base url from path parsed_path = parse_path(full_path=source_path) - - if parsed_path != "/": - parent_node = await dracoon.nodes.get_node_from_path(path=parsed_path) - elif parsed_path == "/": + + if parsed_path == "/": parent_node = None parent_id = 0 - - if parent_node: + else: + parent_node = await dracoon.nodes.get_node_from_path(path=parsed_path) parent_id = parent_node.id if parent_node is None and parsed_path != "/": @@ -721,9 +721,13 @@ async def _list_nodes(): ) sys.exit(1) - if long_list and parent_id != 0 and human_readable: - typer.echo(f"total {to_readable_size(parent_node.size)}") - elif long_list and parent_id != 0: + if long_list and parent_node is not None and human_readable: + if parent_node.size: + size = parent_node.size + else: + size = 0 + typer.echo(f"total {to_readable_size(size)}") + elif long_list and parent_node is not None: typer.echo(f"total {parent_node.size}") for node in nodes.items: @@ -861,7 +865,11 @@ async def _download(): finally: await dracoon.logout() elif is_file_path: - transfer = DCTransferList(total=node_info.size, file_count=1) + if node_info.size: + size = node_info.size + else: + size = 0 + transfer = DCTransferList(total=size, file_count=1) download_job = DCTransfer(transfer=transfer) try: diff --git a/dccmd/main/upload/__init__.py b/dccmd/main/upload/__init__.py index f47c2a2..b881a0e 100644 --- a/dccmd/main/upload/__init__.py +++ b/dccmd/main/upload/__init__.py @@ -12,7 +12,7 @@ import typer from httpx import WriteTimeout -from dracoon import DRACOON +from dracoon import DRACOON, OAuth2ConnectionType from dracoon.errors import ( InvalidPathError, HTTPConflictError, @@ -201,6 +201,8 @@ async def process_batch(batch): # process 10 folders per batch for reqs in dracoon.batch_process(coro_list=folder_reqs, batch_size=velocity): try: + # get new token to avoid rate limiting + await dracoon.connect(connection_type=OAuth2ConnectionType.refresh_token) await asyncio.gather(*reqs) except HTTPConflictError: pass @@ -242,6 +244,7 @@ async def process_batch(batch): # process 3 batches in parallel per level for batch in dracoon.batch_process(coro_list=folder_reqs, batch_size=3): try: + await dracoon.connect(connection_type=OAuth2ConnectionType.refresh_token) await asyncio.gather(*batch) except DRACOONHttpError: for req in folder_reqs: @@ -297,6 +300,8 @@ async def bulk_upload( coro_list=upload_reqs, batch_size=concurrent_reqs ): try: + # get fresh token per batch (avoid rate limiting) + await dracoon.connect(connection_type=OAuth2ConnectionType.refresh_token) await asyncio.gather(*batch) except HTTPConflictError: # ignore file already exists error @@ -329,7 +334,7 @@ def create_folder(name: str, parent_id: int, dracoon: DRACOON): folder = dracoon.nodes.make_folder(name=name, parent_id=parent_id) return dracoon.nodes.create_folder(folder=folder, raise_on_err=True) -def validate_file_name(name: str) -> str: +def validate_file_name(name: str) -> bool: """ validate file name """ # return false if name length is more than 150 chars if len(name) > 150: diff --git a/pyproject.toml b/pyproject.toml index 1e7292d..86ce71f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dccmd" -version = "0.4.1" +version = "0.4.2" description = "DRACOON Commander – CLI client for DRACOON Cloud (dracoon.com)" authors = ["Octavio Simone <70800577+unbekanntes-pferd@users.noreply.github.com>"] license = "Apache-2.0" From e951ca2a3301ada8c2fcd3e5f9550ea15b534204 Mon Sep 17 00:00:00 2001 From: Octavio Simone <70800577+unbekanntes-pferd@users.noreply.github.com> Date: Fri, 26 May 2023 21:54:19 +0200 Subject: [PATCH 2/2] fix reconnect on folder create --- dccmd/__init__.py | 2 +- dccmd/main/upload/__init__.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/dccmd/__init__.py b/dccmd/__init__.py index 93da846..8d25060 100644 --- a/dccmd/__init__.py +++ b/dccmd/__init__.py @@ -4,7 +4,7 @@ """ -__version__ = "0.4.2-SNAPSHOT" +__version__ = "0.4.2" # std imports import sys diff --git a/dccmd/main/upload/__init__.py b/dccmd/main/upload/__init__.py index b881a0e..fc1781b 100644 --- a/dccmd/main/upload/__init__.py +++ b/dccmd/main/upload/__init__.py @@ -244,7 +244,6 @@ async def process_batch(batch): # process 3 batches in parallel per level for batch in dracoon.batch_process(coro_list=folder_reqs, batch_size=3): try: - await dracoon.connect(connection_type=OAuth2ConnectionType.refresh_token) await asyncio.gather(*batch) except DRACOONHttpError: for req in folder_reqs: