meltano.yml

# Meltano Configuration File
# 
# One-time workstation setup:
# > meltano install  # Install the plugins declared by the project
#
# Sample usage for end-to-end development:
# > meltano run el    # Run the job titled 'el' to extract and load data
# > meltano run t     # Run the job titled 't' to transform data
# > meltano run bi    # Build and serve the Evidence BI reports
#
# Repeat the same actions as above on "prod":
# > meltano --environment=prod run elt evidence:build

version: 1

jobs:
  # Sample usage: `meltano run el`, `meltano run t`, `meltano run el t`, `meltano run elt`
- name: el         # Extract and load the raw data
  tasks:
  - tap-jaffle-shop target-duckdb
- name: t          # Transform the raw data
  tasks:
  - dbt-duckdb:run
  - dbt-duckdb:test
- name: elt        # Extract, Load, and Transform
  tasks:
  - tap-jaffle-shop target-duckdb
  - dbt-duckdb:run
  - dbt-duckdb:test
- name: bi         # Launch the Evidence BI dev environment
  tasks:
  - evidence:dev
- name: bi-compile # Build BI reports and test for breakages
  tasks:
  - evidence:build-strict
- name: full-build # End-to-end build and test
  tasks:
  - tap-jaffle-shop target-duckdb
  - dbt-duckdb:run
  - dbt-duckdb:test
  - evidence:build-strict

default_environment: dev
environments:
- name: dev
  env:
    JAFFLE_DB_PATH: ${MELTANO_PROJECT_ROOT}/reports/jaffle_shop.${MELTANO_ENVIRONMENT}-duckdb
    JAFFLE_DB_NAME: jaffle_shop
    JAFFLE_RAW_SCHEMA: tap_jaffle_shop
    TAP_JAFFLE_SHOP_YEARS: '1'
- name: staging
  env:
    JAFFLE_DB_PATH: ${MELTANO_PROJECT_ROOT}/reports/jaffle_shop.${MELTANO_ENVIRONMENT}-duckdb
    JAFFLE_DB_NAME: jaffle_shop
    JAFFLE_RAW_SCHEMA: tap_jaffle_shop
    TAP_JAFFLE_SHOP_YEARS: '3'
- name: prod
  env:
    JAFFLE_DB_PATH: ${MELTANO_PROJECT_ROOT}/reports/jaffle_shop.${MELTANO_ENVIRONMENT}-duckdb
    JAFFLE_DB_NAME: jaffle_shop
    JAFFLE_RAW_SCHEMA: tap_jaffle_shop
    TAP_JAFFLE_SHOP_YEARS: '5'

plugins:
  extractors:
  - name: tap-jaffle-shop
    namespace: tap_jaffle_shop
    variant: meltanolabs
    pip_url: git+https://github.com/MeltanoLabs/tap-jaffle-shop.git@v0.2.1
    capabilities:
    - catalog
    - discover
    config:
      years: 1
      stream_name_prefix: ${JAFFLE_RAW_SCHEMA}-raw_
  loaders:
  - name: target-duckdb
    variant: jwills
    pip_url: target-duckdb~=0.4
    config:
      filepath: ${JAFFLE_DB_PATH}
  - name: target-parquet
    variant: estrategiahq
    pip_url: git+https://github.com/estrategiahq/target-parquet.git
  utilities:
  - name: dbt-duckdb
    variant: jwills
    pip_url: dbt-core~=1.4.5 dbt-duckdb~=1.4.0 git+https://github.com/meltano/dbt-ext.git@v0.1.0
    config:
      project_dir: ${MELTANO_PROJECT_ROOT}
      profiles_dir: ${MELTANO_PROJECT_ROOT}
      path: ${JAFFLE_DB_PATH}
  - name: evidence
    variant: meltanolabs
    pip_url: evidence-ext>=0.5
    commands:
      dev: dev
    config:
      home_dir: ${MELTANO_PROJECT_ROOT}/reports
      settings:
        duckdb:
          # filename: ${MELTANO_PROJECT_ROOT}/reports/${JAFFLE_DB_NAME}.${MELTANO_ENVIRONMENT}.duckdb
          filename: ${JAFFLE_DB_NAME}.${MELTANO_ENVIRONMENT}-duckdb
project_id: ff061732-bd27-4021-916f-e8f8b55fcf9d