From 2a770c3278314e0f770013ae1f512b4000e039d0 Mon Sep 17 00:00:00 2001 From: KBolashev Date: Thu, 7 Nov 2024 12:55:35 +0200 Subject: [PATCH 1/2] Expose rclone_init in the storage docs --- dagshub/storage/__init__.py | 4 +-- dagshub/storage/rclone.py | 55 ++++++++++--------------------------- 2 files changed, 17 insertions(+), 42 deletions(-) diff --git a/dagshub/storage/__init__.py b/dagshub/storage/__init__.py index f3f4a3a7..960ae701 100644 --- a/dagshub/storage/__init__.py +++ b/dagshub/storage/__init__.py @@ -1,3 +1,3 @@ -from .rclone import mount, unmount, sync +from .rclone import mount, unmount, sync, rclone_init -__all__ = [mount.__name__, unmount.__name__, sync.__name__] +__all__ = [mount.__name__, unmount.__name__, sync.__name__, rclone_init.__name__] diff --git a/dagshub/storage/rclone.py b/dagshub/storage/rclone.py index 2bed44b8..64346ea8 100644 --- a/dagshub/storage/rclone.py +++ b/dagshub/storage/rclone.py @@ -47,9 +47,7 @@ def check_and_provide_install_script(quiet=False): # Format a string listing all missing packages missing_packages_list = ", ".join([pkg for pkg, _ in missing_packages]) if not quiet: - log_message( - f"The following packages are not installed: {missing_packages_list}." - ) + log_message(f"The following packages are not installed: {missing_packages_list}.") if platform.system().lower() == "linux" and shutil.which("apt") is not None: response = prompt_user("Do you want to install all missing packages?") @@ -62,45 +60,34 @@ def check_and_provide_install_script(quiet=False): except subprocess.CalledProcessError as e: logger.error(f"Failed to install {package}: {e}") # Non-Linux OS or 'apt' not available + log_message("Manual installation required. Please install Rclone and fuse3 for your system.") + log_message("Rclone installation instructions: https://rclone.org/install/") log_message( - "Manual installation required. Please install Rclone and fuse3 for your system." - ) - log_message( - "Rclone installation instructions: https://rclone.org/install/" - ) - log_message( - "For fuse3, please consult your operating system's package manager or " - "documentation." + "For fuse3, please consult your operating system's package manager or " "documentation." ) else: log_message("Skipping installation of missing packages.") else: # Non-Linux OS or 'apt' not available - log_message( - "Manual installation required. Please install Rclone and fuse3 for your system." - ) + log_message("Manual installation required. Please install Rclone and fuse3 for your system.") log_message("Rclone installation instructions: https://rclone.org/install/") - log_message( - "For fuse3, please consult your operating system's package manager or documentation." - ) + log_message("For fuse3, please consult your operating system's package manager or documentation.") return else: if not quiet: log_message("All packages are installed.") -def rclone_init( - repo_owner: str, conf_path: Optional[Path] = None, update=False, quiet=False -) -> Tuple[str, Path]: +def rclone_init(repo_owner: str, conf_path: Optional[Path] = None, update=False, quiet=False) -> Tuple[str, Path]: """ Initializes or updates the Rclone configuration for a DAGsHub repository. :param repo_owner: The owner of the repository. This is used to create a unique section in the Rclone configuration. :param conf_path: Optional. The path to the Rclone configuration file. If not provided, the default path is used. :param update: Optional. A boolean flag indicating whether to update the configuration if it already exists. - Defaults to False. + Defaults to False. :param quiet: Optional. A boolean flag that controls the output of the function. If False, the function will - print messages about its operation. + print messages about its operation. :return: Name of the remote for rclone + The absolute path to the Rclone configuration file. """ # Make sure RClone and fuse3 are properly installed @@ -139,9 +126,7 @@ def rclone_init( if not quiet: # Inform the user about the remote name - log_message( - f"Configuration complete. The remote '{remote_name}' has been created/updated in '{conf_path}'." - ) + log_message(f"Configuration complete. The remote '{remote_name}' has been created/updated in '{conf_path}'.") log_message( f"Example usage with rclone: `rclone ls {remote_name}:` to list the contents of " f"'your-bucket-name'." @@ -150,9 +135,7 @@ def rclone_init( return remote_name, conf_path.absolute() -def sync( - repo: str, local_path: Union[str, os.PathLike], remote_path: Union[str, os.PathLike] -): +def sync(repo: str, local_path: Union[str, os.PathLike], remote_path: Union[str, os.PathLike]): """ Synchronizes the contents of a local directory with a specified remote directory in a DAGsHub repository using Rclone. @@ -193,13 +176,9 @@ def sync( for line in p.stdout: log_message(line) - log_message( - f"Successfully synchronized {local_path_str} to DagsHub Storage {remote_path}." - ) + log_message(f"Successfully synchronized {local_path_str} to DagsHub Storage {remote_path}.") except subprocess.CalledProcessError as e: - log_message( - f"Failed to synchronize {local_path_str} to DagsHub Storage {remote_path}: {e}" - ) + log_message(f"Failed to synchronize {local_path_str} to DagsHub Storage {remote_path}: {e}") def mount(repo: str, cache: bool = False, path: Path = None) -> os.PathLike: @@ -265,12 +244,8 @@ def mount(repo: str, cache: bool = False, path: Path = None) -> os.PathLike: try: # Execute the mount command subprocess.run(mount_command, check=True) - log_message( - f"Successfully mounted DagsHub Storage in '{repo_name}' to '{mount_dir}'." - ) - log_message( - f'To unmount, run `dagshub.storage.unmount(repo="{repo}", path="{mount_dir}")`.' - ) + log_message(f"Successfully mounted DagsHub Storage in '{repo_name}' to '{mount_dir}'.") + log_message(f'To unmount, run `dagshub.storage.unmount(repo="{repo}", path="{mount_dir}")`.') return mount_dir except subprocess.CalledProcessError as e: logger.error(f"Failed to mount DagsHub Storage in '{repo_name}': {e}") From f40fbcb591aa8c962e2f928007be3fa999543aac Mon Sep 17 00:00:00 2001 From: KBolashev Date: Thu, 7 Nov 2024 14:36:44 +0200 Subject: [PATCH 2/2] Add note about using RClone for mount/sync --- dagshub/storage/rclone.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dagshub/storage/rclone.py b/dagshub/storage/rclone.py index 64346ea8..8f77d96e 100644 --- a/dagshub/storage/rclone.py +++ b/dagshub/storage/rclone.py @@ -197,6 +197,10 @@ def mount(repo: str, cache: bool = False, path: Path = None) -> os.PathLike: :param path: Optional. A Path object specifying the custom mount path. If not provided, the mount directory is determined based on the current working directory and the repository name. + + .. note:: + This function, as well :func:`sync` are using `RClone `_ to work. + If it's not installed, you'll get instructions how to install it. """ # Parse the repo string to get the repo owner and name repo_owner, repo_name = repo.split("/")