From 6c5650a9054d266d5d77c57efa2cd6e98f1dfeec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Morales?= Date: Fri, 22 Nov 2024 11:14:04 -0600 Subject: [PATCH] ci: use spawn for multiprocessing in nbdev_test --- .github/workflows/ci.yaml | 2 +- .pre-commit-config.yaml | 3 ++- action_files/nbdev_test | 52 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 2 deletions(-) create mode 100755 action_files/nbdev_test diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index fa981ba80..301b037e9 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -68,7 +68,7 @@ jobs: CIBW_BUILD: cp${{ matrix.python-version }}-${{ matrix.os-platform[1] }} CIBW_TEST_COMMAND: > uv pip install -r {project}/setup.py --extra dev --extra plotly && - nbdev_test --path {project}/nbs/ --skip_file_re "(distributed|prophet).*.ipynb" --pause 1.0 --do_print --timing --n_workers 0 + ./{project}/action_files/nbdev_test --path {project}/nbs/ --skip_file_re "(distributed|prophet).*.ipynb" --pause 1.0 --do_print --timing - uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 if: github.event_name == 'push' diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ecf6e5de1..156436fdb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -20,9 +20,10 @@ repos: rev: v0.2.1 hooks: - id: ruff + files: python/statsforecast - repo: https://github.com/pre-commit/mirrors-mypy rev: v1.8.0 hooks: - id: mypy args: [--ignore-missing-imports] - exclude: (experiments|setup.py) + files: python/statsforecast diff --git a/action_files/nbdev_test b/action_files/nbdev_test new file mode 100755 index 000000000..9cdda4ba7 --- /dev/null +++ b/action_files/nbdev_test @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +import time,os,sys,traceback,contextlib, inspect +from fastcore.basics import * +from fastcore.imports import * +from fastcore.foundation import * +from fastcore.parallel import * +from fastcore.script import * +from fastcore.meta import delegates + +from nbdev.config import * +from nbdev.doclinks import * +from nbdev.process import NBProcessor, nb_lang +from nbdev.frontmatter import FrontmatterProc +from nbdev.test import _keep_file, test_nb + +from execnb.nbio import * +from execnb.shell import * + + +@call_parse +@delegates(nbglob_cli) +def nbdev_test( + path:str=None, # A notebook name or glob to test + flags:str='', # Space separated list of test flags to run that are normally ignored + n_workers:int=None, # Number of workers + timing:bool=False, # Time each notebook to see which are slow + do_print:bool=False, # Print start and end of each notebook + pause:float=0.01, # Pause time (in seconds) between notebooks to avoid race conditions + ignore_fname:str='.notest', # Filename that will result in siblings being ignored + **kwargs): + "Test in parallel notebooks matching `path`, passing along `flags`" + skip_flags = get_config().tst_flags.split() + force_flags = flags.split() + files = nbglob(path, as_path=True, **kwargs) + files = [f.absolute() for f in sorted(files) if _keep_file(f, ignore_fname)] + if len(files)==0: return print('No files were eligible for testing') + + if n_workers is None: n_workers = 0 if len(files)==1 else min(num_cpus(), 8) + kw = {'method': 'spawn'} + wd_pth = get_config().nbs_path + with working_directory(wd_pth if (wd_pth and wd_pth.exists()) else os.getcwd()): + results = parallel(test_nb, files, skip_flags=skip_flags, force_flags=force_flags, n_workers=n_workers, + basepath=get_config().config_path, pause=pause, do_print=do_print, **kw) + passed,times = zip(*results) + if all(passed): print("Success.") + else: + _fence = '='*50 + failed = '\n\t'.join(f.name for p,f in zip(passed,files) if not p) + sys.stderr.write(f"\nnbdev Tests Failed On The Following Notebooks:\n{_fence}\n\t{failed}\n") + sys.exit(1) + if timing: + for i,t in sorted(enumerate(times), key=lambda o:o[1], reverse=True): print(f"{files[i].name}: {int(t)} secs")