diff --git a/Dockerfile b/Dockerfile index 80ca224c..2e4f4461 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,6 @@ ENV CRONTAB_15MIN='*/15 * * * *' \ CRONTAB_DAILY='0 2 * * MON-SAT' \ CRONTAB_WEEKLY='0 1 * * SUN' \ CRONTAB_MONTHLY='0 5 1 * *' \ - DBS_TO_EXCLUDE='$^' \ DST='' \ EMAIL_FROM='' \ EMAIL_SUBJECT='Backup report: {hostname} - {periodicity} - {result}' \ @@ -100,15 +99,17 @@ ENV JOB_500_WHAT='dup full $SRC $DST' \ FROM base AS postgres -RUN apk add --no-cache --repository http://dl-cdn.alpinelinux.org/alpine/v3.13/main postgresql-client \ +RUN apk add --no-cache postgresql-client \ && psql --version \ && pg_dump --version # Install full version of grep to support more options RUN apk add --no-cache grep -ENV JOB_200_WHAT set -euo pipefail; psql -0Atd postgres -c \"SELECT datname FROM pg_database WHERE NOT datistemplate AND datname != \'postgres\'\" | grep --null-data --invert-match -E \"\$DBS_TO_EXCLUDE\" | xargs -0tI DB pg_dump --dbname DB --no-owner --no-privileges --file \"\$SRC/DB.sql\" +ENV JOB_200_WHAT set -euo pipefail; psql -0Atd postgres -c \"SELECT datname FROM pg_database WHERE NOT datistemplate AND datname != \'postgres\'\" | grep --null-data -E \"\$DBS_TO_INCLUDE\" | grep --null-data --invert-match -E \"\$DBS_TO_EXCLUDE\" | xargs -0tI DB pg_dump --dbname DB --no-owner --no-privileges --file \"\$SRC/DB.sql\" ENV JOB_200_WHEN='daily weekly' \ + DBS_TO_INCLUDE='.*' \ + DBS_TO_EXCLUDE='$^' \ PGHOST=db diff --git a/README.md b/README.md index 9cab725b..204f9854 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ - [Tags](#tags) - [Environment variables available](#environment-variables-available) - [`CRONTAB_{15MIN,HOURLY,DAILY,WEEKLY,MONTHLY}`](#crontab_15minhourlydailyweeklymonthly) - - [`DBS_TO_EXCLUDE`](#dbs_to_exclude) + - [`DBS_TO_{INCLUDE,EXCLUDE}`](#dbs_to_includeexclude) - [`DST`](#dst) - [`EMAIL_FROM`](#email_from) - [`EMAIL_SUBJECT`](#email_subject) @@ -121,7 +121,9 @@ CRONTAB_WEEKLY=0 1 * * SUN CRONTAB_MONTHLY=0 5 1 * * ``` -### `DBS_TO_EXCLUDE` +### `DBS_TO_{INCLUDE,EXCLUDE}` + +Only available in the [PostgreSQL flavors](#postgresql-docker-duplicity-postgres). Define a Regular Expression to filter databases that shouldn't be included in the DB dump. @@ -136,6 +138,12 @@ use: DBS_TO_EXCLUDE="^(DB1|DB2)$" ``` +Or, if you only want to include those databases that start with `prod`: + +```sh +DBS_TO_INCLUDE="^prod" +``` + ### `DST` Where to store the backup. diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/conftest.py b/tests/conftest.py index 27c708c0..df714a41 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,11 +1,15 @@ +import json import logging from contextlib import contextmanager from pathlib import Path +from time import sleep import pytest -from plumbum import local +from plumbum import ProcessExecutionError, local from plumbum.cmd import docker +MIN_PG = 13.0 + _logger = logging.getLogger(__name__) @@ -86,3 +90,29 @@ def _container(tag_name): ) return _container + + +@pytest.fixture +def postgres_container(): + """Give a running postgres container ID.""" + container_id = docker( + "container", + "run", + "--detach", + "--env=POSTGRES_USER=postgres", + "--env=POSTGRES_PASSWORD=password", + f"postgres:{MIN_PG}-alpine", + ).strip() + container_info = json.loads(docker("container", "inspect", container_id))[0] + for attempt in range(10): + _logger.debug("Attempt %d of waiting for postgres to start.", attempt) + try: + docker("container", "exec", "--user=postgres", container_id, "psql", "-l") + break + except ProcessExecutionError: + sleep(3) + if attempt == 9: + raise + yield container_info + _logger.info(f"Removing {container_id}...") + docker("container", "rm", "-f", container_id) diff --git a/tests/test_service.py b/tests/test_service.py index 3a1c651d..6badd3d0 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -1,6 +1,9 @@ +import re + +import pytest from plumbum.cmd import docker -MIN_PG = 13.0 +from .conftest import MIN_PG def test_containers_start(container_factory): @@ -38,3 +41,53 @@ def test_postgres_bin(container_factory): assert binary_name == app assert product == "(PostgreSQL)" assert float(version) >= MIN_PG + + +@pytest.mark.parametrize("tag", ("postgres-s3", "postgres")) +@pytest.mark.parametrize( + "dbs_to_include, dbs_to_exclude, dbs_matched", + ( + (None, "^demo", ["prod1", "prod2"]), + ("^prod", None, ["prod1", "prod2"]), + ("prod", "2", ["prod1"]), + ), +) +def test_postgres_db_filters( + container_factory, + tag: str, + postgres_container, + dbs_to_include: str, + dbs_to_exclude: str, + dbs_matched: list, +): + # Create some DBs to test + existing_dbs = ["prod1", "prod2", "demo1", "demo2"] + with container_factory(tag) as duplicity_container: + # Build docker exec command + exc = docker[ + "exec", + "--env=PGUSER=postgres", + "--env=PGPASSWORD=password", + "--env=PASSPHRASE=good", + f"--env=PGHOST={postgres_container['NetworkSettings']['IPAddress']}", + "--env=DST=file:///mnt/backup/dst", + ] + if dbs_to_include is not None: + exc = exc[f"--env=DBS_TO_INCLUDE={dbs_to_include}"] + if dbs_to_exclude is not None: + exc = exc[f"--env=DBS_TO_EXCLUDE={dbs_to_exclude}"] + exc = exc[duplicity_container] + # Create all those test dbs + list(map(exc["createdb"], existing_dbs)) + # Back up + exc("/etc/periodic/daily/jobrunner", retcode=None) + # Check backup files. Output looks like this: + # Local and Remote metadata are synchronized, no sync needed. + # Last full backup date: Fri Oct 29 12:29:35 2021 + # Fri Oct 29 12:29:34 2021 . + # Fri Oct 29 12:29:34 2021 demo1.sql + # Fri Oct 29 12:29:34 2021 demo2.sql + output = exc("dup", "list-current-files", "file:///mnt/backup/dst") + # Assert we backed up the correct DBs + backed = re.findall(r" (\w+)\.sql", output) + assert backed == dbs_matched