diff --git a/.github/workflows/pypi-release.yml b/.github/workflows/pypi-release.yml index 67120ef..f5b382a 100644 --- a/.github/workflows/pypi-release.yml +++ b/.github/workflows/pypi-release.yml @@ -7,7 +7,7 @@ defaults: shell: bash jobs: - build-sdist: + build: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -19,70 +19,38 @@ jobs: if [[ "v$version" != "$tag" ]]; then exit 1 fi - - uses: actions/setup-python@v5 - with: - python-version: 3.9 - - run: | - python -m pip install -U pip setuptools wheel build - - run: | - python -m build - - run: | - pip install dist/*.tar.gz - - uses: actions/upload-artifact@v4 - with: - name: sdist - path: dist/*.tar.gz - build-wheels: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - run: | - python -m pip install -U pip setuptools wheel build - - run: | - python -m build - - run: | - python -m pip install dist/*.whl + - name: Install uv + uses: astral-sh/setup-uv@v3 + with: + version: "0.4.27" + enable-cache: true + cache-dependency-glob: "**/pyproject.toml" + - name: Set up Python + run: uv python install 3.12 + - name: Build sdist and wheel + run: uv build + - name: Test installation of sdist and wheel + run: | + uv venv --no-project + uv pip install dist/*.tar.gz + uv pip install dist/*.whl - uses: actions/upload-artifact@v4 with: - name: wheel-${{ matrix.python-version }} - path: dist/*.whl + name: build + path: dist/* publish-artifacts: runs-on: ubuntu-latest - needs: [build-sdist, build-wheels] + needs: [build] environment: name: release url: https://pypi.org/p/coqui-tts-trainer permissions: id-token: write steps: - - run: | - mkdir dist - - uses: actions/download-artifact@v4 - with: - name: "sdist" - path: "dist/" - - uses: actions/download-artifact@v4 - with: - name: "wheel-3.9" - path: "dist/" - - uses: actions/download-artifact@v4 - with: - name: "wheel-3.10" - path: "dist/" - - uses: actions/download-artifact@v4 - with: - name: "wheel-3.11" - path: "dist/" - uses: actions/download-artifact@v4 with: - name: "wheel-3.12" path: "dist/" + name: build - run: | ls -lh dist/ - name: Publish package distributions to PyPI diff --git a/.github/workflows/style_check.yml b/.github/workflows/style_check.yml index 6755f44..a146213 100644 --- a/.github/workflows/style_check.yml +++ b/.github/workflows/style_check.yml @@ -15,21 +15,13 @@ jobs: python-version: [3.9] steps: - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + - name: Install uv + uses: astral-sh/setup-uv@v3 with: - python-version: ${{ matrix.python-version }} - architecture: x64 - cache: 'pip' - cache-dependency-path: 'requirements*' - - name: check OS - run: cat /etc/os-release - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y git make gcc - - name: Install/upgrade dev dependencies - run: python3 -m pip install -r requirements.dev.txt + version: "0.4.27" + enable-cache: true + cache-dependency-glob: "**/pyproject.toml" + - name: Set up Python ${{ matrix.python-version }} + run: uv python install ${{ matrix.python-version }} - name: Lint check - run: | - make lint + run: make lint diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 441941b..5ab4a4e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -16,26 +16,16 @@ jobs: uv-resolution: ["lowest-direct", "highest"] steps: - uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + - name: Install uv + uses: astral-sh/setup-uv@v3 with: - python-version: ${{ matrix.python-version }} - architecture: x64 - cache: 'pip' - cache-dependency-path: 'requirements*' - - name: check OS - run: cat /etc/os-release - - name: Install dependencies - run: | - sudo apt-get update - sudo apt-get install -y --no-install-recommends git make gcc - - name: Install/upgrade Python setup deps - run: python3 -m pip install --upgrade pip setuptools uv - - name: Install Trainer - run: | - python3 -m uv pip install --resolution=${{ matrix.uv-resolution }} --system "coqui-tts-trainer[dev,test] @ ." + version: "0.4.27" + enable-cache: true + cache-dependency-glob: "**/pyproject.toml" + - name: Set up Python ${{ matrix.python-version }} + run: uv python install ${{ matrix.python-version }} - name: Unit tests - run: make test_all + run: uv run --resolution=${{ matrix.uv-resolution }} --all-extras coverage run -m pytest trainer tests - name: Upload coverage data uses: actions/upload-artifact@v4 with: @@ -49,18 +39,17 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - name: Install uv + uses: astral-sh/setup-uv@v3 with: - python-version: "3.12" + version: "0.4.27" - uses: actions/download-artifact@v4 with: pattern: coverage-data-* merge-multiple: true - name: Combine coverage run: | - python -Im pip install --upgrade coverage[toml] - - python -Im coverage combine - python -Im coverage html --skip-covered --skip-empty - - python -Im coverage report --format=markdown >> $GITHUB_STEP_SUMMARY + uv python install + uvx coverage combine + uvx coverage html --skip-covered --skip-empty + uvx coverage report --format=markdown >> $GITHUB_STEP_SUMMARY diff --git a/.gitignore b/.gitignore index e17302b..be4e374 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +uv.lock + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 615dc54..be693ec 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,20 +1,13 @@ repos: - repo: "https://github.com/pre-commit/pre-commit-hooks" - rev: v4.6.0 + rev: v5.0.0 hooks: - id: check-yaml - id: end-of-file-fixer - id: trailing-whitespace - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.10 + rev: v0.6.9 hooks: - id: ruff args: [--fix, --exit-non-zero-on-fix] - id: ruff-format - - repo: local - hooks: - - id: generate_requirements.py - name: generate_requirements.py - language: system - entry: python bin/generate_requirements.py - files: "pyproject.toml|requirements.*\\.txt|tools/generate_requirements.py" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 18390e9..dccce67 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -29,28 +29,36 @@ If you have a new feature or a bug to squash, go ahead and send a ✨**PR**✨. Please use the following steps for a ✨**PR**✨. Let us know if you encounter a problem along the way. -The following steps are tested on an Ubuntu system. +The following steps are tested on an Ubuntu system and require +[uv](https://docs.astral.sh/uv/) for virtual environment management. Choose your +preferred [installation +method](https://docs.astral.sh/uv/getting-started/installation/), e.g. the +standalone installer: + +```bash +curl -LsSf https://astral.sh/uv/install.sh | sh +``` 1. Fork 👟[https://github.com/idiap/coqui-ai-Trainer] by clicking the fork button at the top right corner of the project page. 2. Clone 👟 and add the main repo as a new remote named ```upsteam```. ```bash - $ git clone git@github.com:/coqui-ai-Trainer.git - $ cd coqui-ai-Trainer - $ git remote add upstream https://github.com/idiap/coqui-ai-Trainer.git + git clone git@github.com:/coqui-ai-Trainer.git + cd coqui-ai-Trainer + git remote add upstream https://github.com/idiap/coqui-ai-Trainer.git ``` 3. Install 👟 for development. ```bash - $ make install + make install ``` 4. Create a new branch with an informative name for your goal. ```bash - $ git checkout -b an_informative_name_for_my_branch + git checkout -b an_informative_name_for_my_branch ``` 5. Implement your changes on your new branch. @@ -62,34 +70,37 @@ The following steps are tested on an Ubuntu system. 8. Run the tests to see how your updates work with the rest of the project. You can repeat this step multiple times as you implement your changes to make sure you are on the right direction. ```bash - $ make test # stop at the first error - $ make test_all # run all the tests, report all the errors + make test # stop at the first error + make test_all # run all the tests, report all the errors ``` 9. Format your code. We use ```ruff``` for code formatting. ```bash - $ make style + make style ``` -10. Run the linter and correct the issues raised. We use ```ruff``` for linting. It helps to enforce a coding standard, offers simple refactoring suggestions. +10. Run the linter and correct the issues raised. We use ```ruff``` for linting. + It helps to enforce a coding standard, offers simple refactoring + suggestions. The formatter and linter are also run automatically via + pre-commit. ```bash - $ make lint + make lint ``` 11. When things are good, add new files and commit your changes. ```bash - $ git add my_file1.py my_file2.py ... - $ git commit + git add my_file1.py my_file2.py ... + git commit ``` It's a good practice to regularly sync your local copy of the project with the upstream code to keep up with the recent updates. ```bash - $ git fetch upstream - $ git rebase upstream/main + git fetch upstream + git rebase upstream/main ``` 12. Send a PR to ```main``` branch. @@ -97,7 +108,7 @@ The following steps are tested on an Ubuntu system. Push your branch to your fork. ```bash - $ git push -u origin an_informative_name_for_my_branch + git push -u origin an_informative_name_for_my_branch ``` Then go to your fork's Github page and click on 'Pull request' to send your ✨**PR**✨. diff --git a/MANIFEST.in b/MANIFEST.in index 97413e4..56c9b76 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,5 @@ include README.md include LICENSE.txt -include requirements.dev.txt recursive-include trainer *.json recursive-include trainer *.html recursive-include trainer *.png diff --git a/Makefile b/Makefile index eacf888..1869cad 100644 --- a/Makefile +++ b/Makefile @@ -7,23 +7,21 @@ help: target_dirs := bin examples tests trainer test_all: ## run tests and don't stop on an error. - coverage run -m pytest trainer tests + uv run coverage run -m pytest trainer tests test: ## run tests. - coverage run -m pytest -x trainer tests + uv run coverage run -m pytest -x trainer tests test_failed: ## only run tests failed the last time. - coverage run -m pytest --ff trainer tests + uv run coverage run -m pytest --ff trainer tests style: ## update code style. - ruff format ${target_dirs} + uv run --only-dev ruff format ${target_dirs} lint: ## run linter. - ruff check ${target_dirs} - -dev-deps: ## install development deps - pip install -r requirements.dev.txt + uv run --only-dev ruff check ${target_dirs} + uv run --only-dev ruff format --check ${target_dirs} install: ## install 🐸 Trainer for development. - pip install -e .[dev,test] - pre-commit install + uv sync --all-extras + uv run pre-commit install diff --git a/README.md b/README.md index 26817ca..bdb57af 100644 --- a/README.md +++ b/README.md @@ -13,18 +13,18 @@ Fork of the [original, unmaintained repository](https://github.com/coqui-ai/Trai ## Installation -From Github: +From PyPI: ```console -git clone https://github.com/idiap/coqui-ai-Trainer -cd coqui-ai-Trainer -make install +pip install coqui-tts-trainer ``` -From PyPI: +From Github: ```console -pip install coqui-tts-trainer +git clone https://github.com/idiap/coqui-ai-Trainer +cd coqui-ai-Trainer +pip install -e . ``` ## Implementing a model diff --git a/bin/generate_requirements.py b/bin/generate_requirements.py deleted file mode 100644 index bbd32ba..0000000 --- a/bin/generate_requirements.py +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env python -"""Generate requirements/*.txt files from pyproject.toml. - -Adapted from: -https://github.com/numpy/numpydoc/blob/e7c6baf00f5f73a4a8f8318d0cb4e04949c9a5d1/tools/generate_requirements.py -""" - -import sys -from pathlib import Path - -try: # standard module since Python 3.11 - import tomllib as toml -except ImportError: - try: # available for older Python via pip - import tomli as toml - except ImportError: - sys.exit("Please install `tomli` first: `pip install tomli`") - -script_pth = Path(__file__) -repo_dir = script_pth.parent.parent -script_relpth = script_pth.relative_to(repo_dir) -header = [ - f"# Generated via {script_relpth.as_posix()} and pre-commit hook.", - "# Do not edit this file; modify pyproject.toml instead.", -] - - -def generate_requirement_file(name: str, req_list: list[str]) -> None: - req_fname = repo_dir / f"requirements.{name}.txt" - req_fname.write_text("\n".join(header + req_list) + "\n") - - -def main() -> None: - pyproject = toml.loads((repo_dir / "pyproject.toml").read_text()) - generate_requirement_file("dev", pyproject["project"]["optional-dependencies"]["dev"]) - - -if __name__ == "__main__": - main() diff --git a/pyproject.toml b/pyproject.toml index 46fc7c2..7376384 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ include = ["trainer*"] [project] name = "coqui-tts-trainer" -version = "0.1.5" +version = "0.1.6" description = "General purpose model trainer for PyTorch that is more flexible than it should be, by 🐸Coqui." readme = "README.md" requires-python = ">=3.9, <3.13" @@ -43,20 +43,20 @@ dependencies = [ "fsspec>=2023.6.0", "numpy>=1.24.3; python_version < '3.12'", "numpy>=1.26.0; python_version >= '3.12'", + "packaging>=21.0", "psutil>=5", "soundfile>=0.12.0", "tensorboard>=2.17.0", - "torch>=2.0", + "torch>=2.1; python_version < '3.12'", + "torch>=2.3; python_version >= '3.12'", ] -[project.optional-dependencies] -# Development dependencies +[dependency-groups] dev = [ "coverage>=7", "pre-commit>=3", "pytest>=8", - "ruff==0.4.10", - "tomli>=2; python_version < '3.11'", + "ruff==0.6.9", ] # Dependencies for running the tests test = [ @@ -64,6 +64,9 @@ test = [ "torchvision>=0.15.1", ] +[tool.uv] +default-groups = ["dev", "test"] + [project.urls] Homepage = "https://github.com/idiap/coqui-ai-Trainer" Repository = "https://github.com/idiap/coqui-ai-Trainer" diff --git a/requirements.dev.txt b/requirements.dev.txt deleted file mode 100644 index ae2d8bf..0000000 --- a/requirements.dev.txt +++ /dev/null @@ -1,7 +0,0 @@ -# Generated via bin/generate_requirements.py and pre-commit hook. -# Do not edit this file; modify pyproject.toml instead. -coverage -pre-commit -pytest -ruff==0.4.10 -tomli; python_version < '3.11' diff --git a/trainer/generic_utils.py b/trainer/generic_utils.py index b02bf97..d8b24bc 100644 --- a/trainer/generic_utils.py +++ b/trainer/generic_utils.py @@ -5,10 +5,16 @@ import fsspec import torch +from packaging.version import Version from trainer.logger import logger +def is_pytorch_at_least_2_4() -> bool: + """Check if the installed Pytorch version is 2.4 or higher.""" + return Version(torch.__version__) >= Version("2.4") + + def isimplemented(obj, method_name) -> bool: """Check if a method is implemented in a class.""" if method_name in dir(obj) and callable(getattr(obj, method_name)): diff --git a/trainer/io.py b/trainer/io.py index 615ad0a..532c721 100644 --- a/trainer/io.py +++ b/trainer/io.py @@ -11,8 +11,13 @@ import torch from coqpit import Coqpit +from trainer.generic_utils import is_pytorch_at_least_2_4 from trainer.logger import logger +# `torch.serialization.add_safe_globals` is needed for weights_only=True to work +# with all Coqui models and only available from Pytorch >=2.4 +_WEIGHTS_ONLY = is_pytorch_at_least_2_4() + def get_user_data_dir(appname: str) -> Path: TTS_HOME = os.environ.get("TTS_HOME") @@ -77,10 +82,10 @@ def load_fsspec( filecache={"cache_storage": str(get_user_data_dir("tts_cache"))}, mode="rb", ) as f: - return torch.load(f, map_location=map_location, weights_only=True, **kwargs) + return torch.load(f, map_location=map_location, weights_only=_WEIGHTS_ONLY, **kwargs) else: with fsspec.open(str(path), "rb") as f: - return torch.load(f, map_location=map_location, weights_only=True, **kwargs) + return torch.load(f, map_location=map_location, weights_only=_WEIGHTS_ONLY, **kwargs) def load_checkpoint( diff --git a/trainer/trainer.py b/trainer/trainer.py index 2300baa..2e9d639 100644 --- a/trainer/trainer.py +++ b/trainer/trainer.py @@ -25,6 +25,7 @@ count_parameters, get_experiment_folder_path, get_git_branch, + is_pytorch_at_least_2_4, isimplemented, remove_experiment_folder, set_partial_state_dict, @@ -883,7 +884,10 @@ def _model_train_step( def _get_autocast_args(self, mixed_precision: bool, precision: str): device = "cpu" - dtype = torch.get_autocast_cpu_dtype() + if is_pytorch_at_least_2_4(): + dtype = torch.get_autocast_dtype("cpu") + else: + dtype = torch.get_autocast_cpu_dtype() if self.use_cuda: device = "cuda" dtype = torch.float32