From 4ffcceac5ced2a1fae2c1c52a211edc9995cbd6e Mon Sep 17 00:00:00 2001 From: pgoslatara Date: Mon, 22 Jul 2024 08:43:01 +0200 Subject: [PATCH] Deprecate dbt-artifacts-dir arg --- .github/workflows/ci_pipeline.yml | 6 ++--- README.md | 12 +++++---- action.yml | 6 +---- dbt-bouncer-example.yml | 2 ++ dbt_bouncer/main.py | 12 +++------ tests/conftest.py | 8 +++--- tests/test_logger.py | 4 +-- tests/test_main.py | 44 +++++++++++++------------------ 8 files changed, 40 insertions(+), 54 deletions(-) diff --git a/.github/workflows/ci_pipeline.yml b/.github/workflows/ci_pipeline.yml index 7af70d46..c07c7a44 100644 --- a/.github/workflows/ci_pipeline.yml +++ b/.github/workflows/ci_pipeline.yml @@ -189,7 +189,7 @@ jobs: docker run --rm \ --volume "$PWD/dbt_project/target":/dbt_project/target \ ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:ci \ - /dbt-bouncer.pex --config-file dbt-bouncer-example.yml --dbt-artifacts-dir dbt_project/target + /dbt-bouncer.pex --config-file dbt-bouncer-example.yml github-action-tests: needs: [pre-commit] @@ -213,7 +213,7 @@ jobs: - name: Run action uses: ./ with: - dbt-artifacts-dir: dbt_project/target + config-file: dbt-bouncer-example.yml pex-tests: @@ -258,4 +258,4 @@ jobs: run: make build-pex - name: Test pex file - run: ./dist/dbt-bouncer.pex --config-file dbt-bouncer-example.yml --dbt-artifacts-dir dbt_project/target + run: ./dist/dbt-bouncer.pex --config-file dbt-bouncer-example.yml diff --git a/README.md b/README.md index c05d78c3..a9d9b093 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,9 @@ Configure and enforce conventions for your dbt project. # How to use -Generate a `manifest.json` by running `dbt parse`. Once completed, run `dbt-bouncer` to validate that your conventions are being maintained. +1. Generate a `manifest.json` by running `dbt parse`. +1. Create a `dbt-bouncer.yml` config file. +1. Run `dbt-bouncer` to validate that your conventions are being maintained. You can use GitHub Actions, Docker or a `.pex` file to run `dbt-bouncer`. ## GitHub Actions @@ -21,7 +23,7 @@ steps: - uses: godatadriven/dbt-bouncer@v0 with: - dbt-artifacts-dir: .//target + config-file: ./ ... ``` @@ -36,17 +38,17 @@ docker pull ghcr.io/godatadriven/dbt-bouncer:v0 docker run --rm \ --volume "$PWD//target"://target \ ghcr.io/godatadriven/dbt-bouncer:v0 \ - /dbt-bouncer.pex --dbt-artifacts-dir /target + /dbt-bouncer.pex --config-file ``` ## Pex -You can also run the `.pex` artifact directly once you have a python executable installed: +You can also run the `.pex` ([Python EXecutable](https://docs.pex-tool.org/whatispex.html#whatispex)) artifact directly once you have a python executable installed: ```bash wget https://github.com/godatadriven/dbt-bouncer/releases/download/vX.X.X/dbt-bouncer.pex -O dbt-bouncer.pex -dbt-bouncer.pex --dbt-artifacts-dir /target +dbt-bouncer.pex --config-file ``` # Development diff --git a/action.yml b/action.yml index 47f42c4d..e6cc0dad 100644 --- a/action.yml +++ b/action.yml @@ -5,10 +5,6 @@ inputs: default: 'dbt-bouncer.yml' description: 'Location of the YML config file.' required: false - dbt-artifacts-dir: - default: '.' - description: 'Directory where the dbt artifacts exists, generally the `target` directory inside a dbt project.' - required: false runs: using: 'composite' @@ -23,4 +19,4 @@ runs: shell: bash run: | wget https://github.com/godatadriven/dbt-bouncer/releases/download/vX.X.X/dbt-bouncer.pex -O ${{ github.action_path }}/dist/dbt-bouncer.pex - ${{ github.action_path }}/dist/dbt-bouncer.pex --config-file ${{ inputs.config-file }} --dbt-artifacts-dir ${{ inputs.dbt-artifacts-dir }} + ${{ github.action_path }}/dist/dbt-bouncer.pex --config-file ${{ inputs.config-file }} diff --git a/dbt-bouncer-example.yml b/dbt-bouncer-example.yml index 25dedda5..09dbb73a 100644 --- a/dbt-bouncer-example.yml +++ b/dbt-bouncer-example.yml @@ -1,3 +1,5 @@ +dbt-artifacts-dir: dbt_project/target # [Optional] Directory where the dbt artifacts exists, generally the `target` directory inside a dbt project. Defaults to `./target`. + checks: - name: check_model_names include: ^intermediate diff --git a/dbt_bouncer/main.py b/dbt_bouncer/main.py index 86f9fc4c..cd7f0036 100644 --- a/dbt_bouncer/main.py +++ b/dbt_bouncer/main.py @@ -19,14 +19,8 @@ required=False, type=click.Path(exists=True), ) -@click.option( - "--dbt-artifacts-dir", - help="Directory where the dbt artifacts exists, generally the `target` directory inside a dbt project.", - required=True, - type=click.Path(exists=True), -) @click.version_option() -def cli(config_file, dbt_artifacts_dir): +def cli(config_file): logger.info(f"Running dbt-bouncer ({version()})...") # Load config @@ -53,7 +47,9 @@ def cli(config_file, dbt_artifacts_dir): logger.debug(f"{config=}") # Load manifest - manifest_json_path = Path(dbt_artifacts_dir) / "manifest.json" + manifest_json_path = ( + Path(bouncer_config.get("dbt-artifacts-dir", "./target")) / "manifest.json" + ) logger.info(f"Loading manifest.json from {manifest_json_path}...") if not manifest_json_path.exists(): raise FileNotFoundError(f"No manifest.json found at {manifest_json_path}.") diff --git a/tests/conftest.py b/tests/conftest.py index dc919b97..03526846 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,7 +1,3 @@ -def pytest_configure(config): - config.addinivalue_line("markers", "iterate_over_models: Tests that should run once per model") - - import json from pathlib import Path @@ -9,6 +5,10 @@ def pytest_configure(config): from dbt_artifacts_parser.parser import parse_manifest +def pytest_configure(config): + config.addinivalue_line("markers", "iterate_over_models: Tests that should run once per model") + + @pytest.fixture(scope="session") def manifest_obj(): manifest_json_path = Path("dbt_project") / "target/manifest.json" diff --git a/tests/test_logger.py b/tests/test_logger.py index 42df7fe2..5244d6e4 100644 --- a/tests/test_logger.py +++ b/tests/test_logger.py @@ -15,8 +15,6 @@ def test_logger_debug() -> None: [ "--config-file", "dbt-bouncer-example.yml", - "--dbt-artifacts-dir", - "dbt_project/target", ], ) @@ -36,6 +34,6 @@ def test_logger_info(caplog) -> None: runner = CliRunner() runner.invoke( cli, - ["--config-file", "dbt-bouncer-example.yml", "--dbt-artifacts-dir", "dbt_project/target"], + ["--config-file", "dbt-bouncer-example.yml"], ) assert "Running dbt-bouncer (0.0.0)..." in caplog.text diff --git a/tests/test_main.py b/tests/test_main.py index 3e001ecd..3819297e 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -1,4 +1,7 @@ +from pathlib import Path + import pytest +import yaml from click.testing import CliRunner from dbt_bouncer.main import cli @@ -7,7 +10,7 @@ @pytest.mark.parametrize( "cli_args", [ - ("--config-file dbt-bouncer-example.yml --dbt-artifacts-dir dbt_project/target"), + ("--config-file dbt-bouncer-example.yml"), ], ) def test_cli_happy_path(caplog, cli_args): @@ -29,10 +32,7 @@ def test_cli_happy_path(caplog, cli_args): "cli_args", [ (""), - ("--config-file dbt-bouncer-example.yml"), - ("--dbt-artifacts-dir dbt_project/target"), - ("--config-file non-existing.yml --dbt-artifacts-dir dbt_project/target"), - ("--config-file dbt-bouncer-example.yml --dbt-artifacts-dir dbt_project"), + ("--config-file non-existing.yml"), ], ) def test_cli_unhappy_path(caplog, cli_args): @@ -48,21 +48,6 @@ def test_cli_unhappy_path(caplog, cli_args): assert result.exit_code != 0 -def test_cli_dbt_dir_doesnt_exist(): - runner = CliRunner() - result = runner.invoke( - cli, - [ - "--config-file", - "dbt-bouncer-example.yml", - "--dbt-artifacts-dir", - "non-existent-directory/target", - ], - ) - assert type(result.exception) in [SystemExit] - assert result.exit_code != 0 - - def test_cli_config_doesnt_exist(tmp_path): runner = CliRunner() result = runner.invoke( @@ -70,8 +55,6 @@ def test_cli_config_doesnt_exist(tmp_path): [ "--config-file", "non-existent-file.yml", - "--dbt-artifacts-dir", - "dbt_project/target", ], ) assert type(result.exception) in [SystemExit] @@ -79,16 +62,25 @@ def test_cli_config_doesnt_exist(tmp_path): def test_cli_manifest_doesnt_exist(tmp_path): + with Path.open(Path("dbt-bouncer-example.yml"), "r") as f: + bouncer_config = yaml.safe_load(f) + + bouncer_config["dbt-artifacts-dir"] = "non-existent-dir/target" + + with Path(tmp_path / "dbt-bouncer-example.yml").open("w") as f: + yaml.dump(bouncer_config, f) + runner = CliRunner() result = runner.invoke( cli, [ "--config-file", - "dbt-bouncer-example.yml", - "--dbt-artifacts-dir", - tmp_path, + Path(tmp_path / "dbt-bouncer-example.yml").__str__(), ], ) assert type(result.exception) in [FileNotFoundError] - assert result.exception.args[0] == f"No manifest.json found at {tmp_path / 'manifest.json'}." + assert ( + result.exception.args[0] + == "No manifest.json found at non-existent-dir/target/manifest.json." + ) assert result.exit_code != 0