feat: allow to include DBs instead of just excluding them

@Tecnativa TT32631

Dockerfile

@@ -14,7 +14,6 @@ ENV CRONTAB_15MIN='*/15 * * * *' \
     CRONTAB_DAILY='0 2 * * MON-SAT' \
     CRONTAB_WEEKLY='0 1 * * SUN' \
     CRONTAB_MONTHLY='0 5 1 * *' \
-    DBS_TO_EXCLUDE='$^' \
     DST='' \
     EMAIL_FROM='' \
     EMAIL_SUBJECT='Backup report: {hostname} - {periodicity} - {result}' \
@@ -100,15 +99,17 @@ ENV JOB_500_WHAT='dup full $SRC $DST' \
 
 FROM base AS postgres
 
-RUN apk add --no-cache --repository http://dl-cdn.alpinelinux.org/alpine/v3.13/main postgresql-client \
+RUN apk add --no-cache postgresql-client \
 	&& psql --version \
     && pg_dump --version
 
 # Install full version of grep to support more options
 RUN apk add --no-cache grep
 
-ENV JOB_200_WHAT set -euo pipefail; psql -0Atd postgres -c \"SELECT datname FROM pg_database WHERE NOT datistemplate AND datname != \'postgres\'\" | grep --null-data --invert-match -E \"\$DBS_TO_EXCLUDE\" | xargs -0tI DB pg_dump --dbname DB --no-owner --no-privileges --file \"\$SRC/DB.sql\"
+ENV JOB_200_WHAT set -euo pipefail; psql -0Atd postgres -c \"SELECT datname FROM pg_database WHERE NOT datistemplate AND datname != \'postgres\'\" | grep --null-data -E \"\$DBS_TO_INCLUDE\" | grep --null-data --invert-match -E \"\$DBS_TO_EXCLUDE\" | xargs -0tI DB pg_dump --dbname DB --no-owner --no-privileges --file \"\$SRC/DB.sql\"
 ENV JOB_200_WHEN='daily weekly' \
+    DBS_TO_INCLUDE='.*' \
+    DBS_TO_EXCLUDE='$^' \
     PGHOST=db
 
 

README.md

@@ -16,7 +16,7 @@
 - [Tags](#tags)
 - [Environment variables available](#environment-variables-available)
   - [`CRONTAB_{15MIN,HOURLY,DAILY,WEEKLY,MONTHLY}`](#crontab_15minhourlydailyweeklymonthly)
-  - [`DBS_TO_EXCLUDE`](#dbs_to_exclude)
+  - [`DBS_TO_{INCLUDE,EXCLUDE}`](#dbs_to_includeexclude)
   - [`DST`](#dst)
   - [`EMAIL_FROM`](#email_from)
   - [`EMAIL_SUBJECT`](#email_subject)
@@ -121,7 +121,9 @@ CRONTAB_WEEKLY=0 1 * * SUN
 CRONTAB_MONTHLY=0 5 1 * *
 ```
 
-### `DBS_TO_EXCLUDE`
+### `DBS_TO_{INCLUDE,EXCLUDE}`
+
+Only available in the [PostgreSQL flavors](#postgresql-docker-duplicity-postgres).
 
 Define a Regular Expression to filter databases that shouldn't be included in the DB
 dump.
@@ -136,6 +138,12 @@ use:
 DBS_TO_EXCLUDE="^(DB1|DB2)$"
 ```
 
+Or, if you only want to include those databases that start with `prod`:
+
+```sh
+DBS_TO_INCLUDE="^prod"
+```
+
 ### `DST`
 
 Where to store the backup.

tests/conftest.py

@@ -1,11 +1,15 @@
+import json
 import logging
 from contextlib import contextmanager
 from pathlib import Path
+from time import sleep
 
 import pytest
-from plumbum import local
+from plumbum import ProcessExecutionError, local
 from plumbum.cmd import docker
 
+MIN_PG = 13.0
+
 _logger = logging.getLogger(__name__)
 
 
@@ -86,3 +90,29 @@ def _container(tag_name):
                 )
 
     return _container
+
+
+@pytest.fixture
+def postgres_container():
+    """Give a running postgres container ID."""
+    container_id = docker(
+        "container",
+        "run",
+        "--detach",
+        "--env=POSTGRES_USER=postgres",
+        "--env=POSTGRES_PASSWORD=password",
+        f"postgres:{MIN_PG}-alpine",
+    ).strip()
+    container_info = json.loads(docker("container", "inspect", container_id))[0]
+    for attempt in range(10):
+        _logger.debug("Attempt %d of waiting for postgres to start.", attempt)
+        try:
+            docker("container", "exec", "--user=postgres", container_id, "psql", "-l")
+            break
+        except ProcessExecutionError:
+            sleep(3)
+            if attempt == 9:
+                raise
+    yield container_info
+    _logger.info(f"Removing {container_id}...")
+    docker("container", "rm", "-f", container_id)

tests/test_service.py

@@ -1,6 +1,9 @@
+import re
+
+import pytest
 from plumbum.cmd import docker
 
-MIN_PG = 13.0
+from .conftest import MIN_PG
 
 
 def test_containers_start(container_factory):
@@ -38,3 +41,53 @@ def test_postgres_bin(container_factory):
                 assert binary_name == app
                 assert product == "(PostgreSQL)"
                 assert float(version) >= MIN_PG
+
+
+@pytest.mark.parametrize("tag", ("postgres-s3", "postgres"))
+@pytest.mark.parametrize(
+    "dbs_to_include, dbs_to_exclude, dbs_matched",
+    (
+        (None, "^demo", ["prod1", "prod2"]),
+        ("^prod", None, ["prod1", "prod2"]),
+        ("prod", "2", ["prod1"]),
+    ),
+)
+def test_postgres_db_filters(
+    container_factory,
+    tag: str,
+    postgres_container,
+    dbs_to_include: str,
+    dbs_to_exclude: str,
+    dbs_matched: list,
+):
+    # Create some DBs to test
+    existing_dbs = ["prod1", "prod2", "demo1", "demo2"]
+    with container_factory(tag) as duplicity_container:
+        # Build docker exec command
+        exc = docker[
+            "exec",
+            "--env=PGUSER=postgres",
+            "--env=PGPASSWORD=password",
+            "--env=PASSPHRASE=good",
+            f"--env=PGHOST={postgres_container['NetworkSettings']['IPAddress']}",
+            "--env=DST=file:///mnt/backup/dst",
+        ]
+        if dbs_to_include is not None:
+            exc = exc[f"--env=DBS_TO_INCLUDE={dbs_to_include}"]
+        if dbs_to_exclude is not None:
+            exc = exc[f"--env=DBS_TO_EXCLUDE={dbs_to_exclude}"]
+        exc = exc[duplicity_container]
+        # Create all those test dbs
+        list(map(exc["createdb"], existing_dbs))
+        # Back up
+        exc("/etc/periodic/daily/jobrunner", retcode=None)
+        # Check backup files. Output looks like this:
+        #   Local and Remote metadata are synchronized, no sync needed.
+        #   Last full backup date: Fri Oct 29 12:29:35 2021
+        #   Fri Oct 29 12:29:34 2021 .
+        #   Fri Oct 29 12:29:34 2021 demo1.sql
+        #   Fri Oct 29 12:29:34 2021 demo2.sql
+        output = exc("dup", "list-current-files", "file:///mnt/backup/dst")
+        # Assert we backed up the correct DBs
+        backed = re.findall(r" (\w+)\.sql", output)
+        assert backed == dbs_matched