From 2e304edabc66d208f41f7a464dadf1498a4b5264 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik <37010174+vitabaks@users.noreply.github.com> Date: Fri, 24 May 2024 02:01:51 +0300 Subject: [PATCH 01/18] Molecule: Avoid requests 2.32.0 during testing (#663) --- .config/python/dev/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/.config/python/dev/requirements.txt b/.config/python/dev/requirements.txt index bb70d770a..66994e4a6 100644 --- a/.config/python/dev/requirements.txt +++ b/.config/python/dev/requirements.txt @@ -1,3 +1,4 @@ +requests==2.31.0 docker==6.1.3 molecule==6.0.3 molecule-plugins==23.5.3 From 8c7089999c054207ea76466d35e5105a75ca0dbd Mon Sep 17 00:00:00 2001 From: IT <113095009+garry-t@users.noreply.github.com> Date: Sun, 26 May 2024 16:04:04 +0300 Subject: [PATCH 02/18] Syntax problem with cron wal-g command (#658) --- molecule/tests/variables/asserts/wal_g_cron_jobs.yml | 12 ++++++------ vars/main.yml | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/molecule/tests/variables/asserts/wal_g_cron_jobs.yml b/molecule/tests/variables/asserts/wal_g_cron_jobs.yml index 959ee5fe2..92f920f02 100644 --- a/molecule/tests/variables/asserts/wal_g_cron_jobs.yml +++ b/molecule/tests/variables/asserts/wal_g_cron_jobs.yml @@ -24,9 +24,9 @@ run_once: true ansible.builtin.assert: that: - - wal_g_backup_command[0] == "[ $(curl -s -o /dev/null -w '%{http_code}' http://{{ inventory_hostname }}:{{ patroni_restapi_port }}) = '200' ]" + - wal_g_backup_command[0] == "curl -I -s http://{{ inventory_hostname }}:{{ patroni_restapi_port }} | grep 200" - wal_g_backup_command[1] == " && wal-g backup-push {{ postgresql_data_dir }} > {{ postgresql_log_dir }}/walg_backup.log 2>&1" - - wal_g_delete_command[0] == "[ $(curl -s -o /dev/null -w '%{http_code}' http://{{ inventory_hostname }}:{{ patroni_restapi_port }}) = '200' ]" + - wal_g_delete_command[0] == "curl -I -s http://{{ inventory_hostname }}:{{ patroni_restapi_port }} | grep 200" - wal_g_delete_command[1] == " && wal-g delete retain FULL 4 --confirm > {{ postgresql_log_dir }}/walg_delete.log 2>&1" fail_msg: "Test failed: wal_g_backup_command or wal_g_delete_command do not have the expected content." success_msg: "Test passed: wal_g_backup_command and wal_g_delete_command have the expected content." @@ -45,7 +45,7 @@ - name: Molecule.tests.variables.asserts.wal_g_cron_jobs | Debian | Define Expected First wal_g_cron Job run_once: true ansible.builtin.set_fact: # yamllint disable rule:line-length - origin_wal_g_cron_jobs_create_job: "[ $(curl -s -o /dev/null -w '%{http_code}' http://{{ inventory_hostname }}:{{ patroni_restapi_port }}) = '200' ] && wal-g backup-push {{ postgresql_data_dir }} > {{ postgresql_log_dir }}/walg_backup.log 2>&1" + origin_wal_g_cron_jobs_create_job: "curl -I -s http://{{ inventory_hostname }}:{{ patroni_restapi_port }} | grep 200 && wal-g backup-push {{ postgresql_data_dir }} > {{ postgresql_log_dir }}/walg_backup.log 2>&1" # 🖨️ Display the first wal_g_cron job for Debian for debugging purposes - name: Molecule.tests.variables.asserts.wal_g_cron_jobs | Debian | Debug First wal_g_cron Job @@ -66,7 +66,7 @@ - name: Molecule.tests.variables.asserts.wal_g_cron_jobs | Debian | Define Expected Second wal_g_cron Job run_once: true ansible.builtin.set_fact: # yamllint disable rule:line-length - origin_wal_g_cron_jobs_delete_job: "[ $(curl -s -o /dev/null -w '%{http_code}' http://{{ inventory_hostname }}:{{ patroni_restapi_port }}) = '200' ] && wal-g delete retain FULL 4 --confirm > {{ postgresql_log_dir }}/walg_delete.log 2>&1" + origin_wal_g_cron_jobs_delete_job: "curl -I -s http://{{ inventory_hostname }}:{{ patroni_restapi_port }} | grep 200 && wal-g delete retain FULL 4 --confirm > {{ postgresql_log_dir }}/walg_delete.log 2>&1" # 🖨️ Display the second wal_g_cron job for Debian for debugging purposes - name: Molecule.tests.variables.asserts.wal_g_cron_jobs | Debian | Debug Second wal_g_cron Job @@ -97,7 +97,7 @@ - name: Molecule.tests.variables.asserts.wal_g_cron_jobs | RedHat | Define Expected First wal_g_cron Job run_once: true ansible.builtin.set_fact: # yamllint disable rule:line-length - origin_wal_g_cron_jobs_create_job: "[ $(curl -s -o /dev/null -w '%{http_code}' http://{{ inventory_hostname }}:{{ patroni_restapi_port }}) = '200' ] && wal-g backup-push {{ postgresql_data_dir }} > {{ postgresql_log_dir }}/walg_backup.log 2>&1" + origin_wal_g_cron_jobs_create_job: "curl -I -s http://{{ inventory_hostname }}:{{ patroni_restapi_port }} | grep 200 && wal-g backup-push {{ postgresql_data_dir }} > {{ postgresql_log_dir }}/walg_backup.log 2>&1" # 🖨️ Display the first wal_g_cron job for RedHat for debugging purposes - name: Molecule.tests.variables.asserts.wal_g_cron_jobs | RedHat | Debug First wal_g_cron Job @@ -118,7 +118,7 @@ - name: Molecule.tests.variables.asserts.wal_g_cron_jobs | RedHat | Define Expected Second wal_g_cron Job run_once: true ansible.builtin.set_fact: # yamllint disable rule:line-length - origin_wal_g_cron_jobs_delete_job: "[ $(curl -s -o /dev/null -w '%{http_code}' http://{{ inventory_hostname }}:{{ patroni_restapi_port }}) = '200' ] && wal-g delete retain FULL 4 --confirm > {{ postgresql_log_dir }}/walg_delete.log 2>&1" + origin_wal_g_cron_jobs_delete_job: "curl -I -s http://{{ inventory_hostname }}:{{ patroni_restapi_port }} | grep 200 && wal-g delete retain FULL 4 --confirm > {{ postgresql_log_dir }}/walg_delete.log 2>&1" # 🖨️ Display the second wal_g_cron job for RedHat for debugging purposes - name: Molecule.tests.variables.asserts.wal_g_cron_jobs | RedHat | Debug Second wal_g_cron Job diff --git a/vars/main.yml b/vars/main.yml index 40ae769e2..6fdb22d39 100644 --- a/vars/main.yml +++ b/vars/main.yml @@ -481,10 +481,10 @@ wal_g_patroni_cluster_bootstrap_command: "wal-g backup-fetch {{ postgresql_data_ # Define job_parts outside of wal_g_cron_jobs # ⚠️ Ensure there is a space at the beginning of each part to prevent commands from concatenating. wal_g_backup_command: - - "[ $(curl -s -o /dev/null -w '%{http_code}' http://{{ inventory_hostname }}:{{ patroni_restapi_port }}) = '200' ]" + - "curl -I -s http://{{ inventory_hostname }}:{{ patroni_restapi_port }} | grep 200" - " && wal-g backup-push {{ postgresql_data_dir }} > {{ postgresql_log_dir }}/walg_backup.log 2>&1" wal_g_delete_command: - - "[ $(curl -s -o /dev/null -w '%{http_code}' http://{{ inventory_hostname }}:{{ patroni_restapi_port }}) = '200' ]" + - "curl -I -s http://{{ inventory_hostname }}:{{ patroni_restapi_port }} | grep 200" - " && wal-g delete retain FULL 4 --confirm > {{ postgresql_log_dir }}/walg_delete.log 2>&1" wal_g_cron_jobs: From 6d2878bf583147a8cc5801f9e91366eff40c5e71 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik <37010174+vitabaks@users.noreply.github.com> Date: Mon, 10 Jun 2024 13:21:10 +0300 Subject: [PATCH 03/18] Remove CentOS Stream 8 support (EOL) (#674) --- .github/workflows/molecule.yml | 3 -- .github/workflows/molecule_pg_upgrade.yml | 3 -- .../workflows/schedule_pg_centosstream8.yml | 33 ------------------- README.md | 6 ++-- 4 files changed, 2 insertions(+), 43 deletions(-) delete mode 100644 .github/workflows/schedule_pg_centosstream8.yml diff --git a/.github/workflows/molecule.yml b/.github/workflows/molecule.yml index 609200d53..aa449cc78 100644 --- a/.github/workflows/molecule.yml +++ b/.github/workflows/molecule.yml @@ -52,9 +52,6 @@ jobs: - distro: centosstream9 tag: latest namespace: glillico - - distro: centosstream8 - tag: latest - namespace: glillico steps: - name: Set TERM environment variable diff --git a/.github/workflows/molecule_pg_upgrade.yml b/.github/workflows/molecule_pg_upgrade.yml index a7a9ecd68..d68632623 100644 --- a/.github/workflows/molecule_pg_upgrade.yml +++ b/.github/workflows/molecule_pg_upgrade.yml @@ -48,9 +48,6 @@ jobs: - distro: centosstream9 tag: latest namespace: glillico - - distro: centosstream8 - tag: latest - namespace: glillico steps: - name: Set TERM environment variable diff --git a/.github/workflows/schedule_pg_centosstream8.yml b/.github/workflows/schedule_pg_centosstream8.yml deleted file mode 100644 index 14253a858..000000000 --- a/.github/workflows/schedule_pg_centosstream8.yml +++ /dev/null @@ -1,33 +0,0 @@ ---- -name: scheduled PostgreSQL (CentOS Stream 8) - -on: - schedule: - - cron: "0 0 * * *" - -jobs: - test: - runs-on: ubuntu-latest - - steps: - - name: Set TERM environment variable - run: echo "TERM=xterm" >> $GITHUB_ENV - - - name: Checkout - uses: actions/checkout@v3 - - - name: Set up Python 3.10 - uses: actions/setup-python@v4 - with: - python-version: "3.10" - - - name: Install dependencies - run: make bootstrap-dev - - - name: Run Molecule tests - run: make molecule-test - env: - PY_COLORS: "1" - ANSIBLE_FORCE_COLOR: "1" - IMAGE_DISTRO: centosstream8 - IMAGE_NAMESPACE: glillico diff --git a/README.md b/README.md index d3498e88e..8a4ee7a56 100644 --- a/README.md +++ b/README.md @@ -90,9 +90,8 @@ RedHat and Debian based distros (x86_64) ###### Supported Linux Distributions: - **Debian**: 10, 11, 12 - **Ubuntu**: 20.04, 22.04 -- **CentOS**: 7, 8 -- **CentOS Stream**: 8, 9 -- **Oracle Linux**: 7, 8, 9 +- **CentOS Stream**: 9 +- **Oracle Linux**: 8, 9 - **Rocky Linux**: 8, 9 - **AlmaLinux**: 8, 9 @@ -109,7 +108,6 @@ _Table of results of daily automated testing of cluster deployment:_ | Debian 12 | [![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/vitabaks/postgresql_cluster/schedule_pg_debian11.yml?branch=master)](https://github.com/vitabaks/postgresql_cluster/actions/workflows/schedule_pg_debian12.yml) | | Ubuntu 20.04 | [![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/vitabaks/postgresql_cluster/schedule_pg_ubuntu2004.yml?branch=master)](https://github.com/vitabaks/postgresql_cluster/actions/workflows/schedule_pg_ubuntu2004.yml) | | Ubuntu 22.04 | [![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/vitabaks/postgresql_cluster/schedule_pg_ubuntu2204.yml?branch=master)](https://github.com/vitabaks/postgresql_cluster/actions/workflows/schedule_pg_ubuntu2204.yml) | -| CentOS Stream 8 | [![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/vitabaks/postgresql_cluster/schedule_pg_centosstream8.yml?branch=master)](https://github.com/vitabaks/postgresql_cluster/actions/workflows/schedule_pg_centosstream8.yml) | | CentOS Stream 9 | [![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/vitabaks/postgresql_cluster/schedule_pg_centosstream9.yml?branch=master)](https://github.com/vitabaks/postgresql_cluster/actions/workflows/schedule_pg_centosstream9.yml) | | Oracle Linux 8 | [![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/vitabaks/postgresql_cluster/schedule_pg_oracle_linux8.yml?branch=master)](https://github.com/vitabaks/postgresql_cluster/actions/workflows/schedule_pg_oracle_linux8.yml) | | Oracle Linux 9 | [![GitHub Workflow Status](https://img.shields.io/github/actions/workflow/status/vitabaks/postgresql_cluster/schedule_pg_oracle_linux9.yml?branch=master)](https://github.com/vitabaks/postgresql_cluster/actions/workflows/schedule_pg_oracle_linux9.yml) | From 9ac0428bb8c7c70845959e006c628f41df5a8f5b Mon Sep 17 00:00:00 2001 From: dissolution <50195845+SDV109@users.noreply.github.com> Date: Mon, 10 Jun 2024 17:25:08 +0700 Subject: [PATCH 04/18] Replacing restart pgbouncer with reload pgbouncer (#673) --- roles/pgbouncer/config/tasks/main.yml | 2 +- roles/pgbouncer/tasks/main.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/pgbouncer/config/tasks/main.yml b/roles/pgbouncer/config/tasks/main.yml index 46d8def58..84fd3821d 100644 --- a/roles/pgbouncer/config/tasks/main.yml +++ b/roles/pgbouncer/config/tasks/main.yml @@ -19,7 +19,7 @@ loop_control: index_var: idx label: "{{ 'pgbouncer' if idx == 0 else 'pgbouncer-%d' % (idx + 1) }}" - notify: "restart pgbouncer" + notify: "reload pgbouncer" when: existing_pgcluster is not defined or not existing_pgcluster|bool tags: pgbouncer, pgbouncer_conf diff --git a/roles/pgbouncer/tasks/main.yml b/roles/pgbouncer/tasks/main.yml index a91386f3e..d9d24f288 100644 --- a/roles/pgbouncer/tasks/main.yml +++ b/roles/pgbouncer/tasks/main.yml @@ -133,7 +133,7 @@ loop_control: index_var: idx label: "{{ 'pgbouncer' if idx == 0 else 'pgbouncer-%d' % (idx + 1) }}" - notify: "restart pgbouncer" + notify: "reload pgbouncer" when: existing_pgcluster is not defined or not existing_pgcluster|bool tags: pgbouncer_conf, pgbouncer @@ -225,7 +225,7 @@ loop_control: index_var: idx label: "{{ 'pgbouncer' if idx == 0 else 'pgbouncer-%d' % (idx + 1) }}" - notify: "restart pgbouncer" + notify: "reload pgbouncer" when: pgbouncer_listen_addr != "0.0.0.0" when: existing_pgcluster is defined and existing_pgcluster|bool tags: pgbouncer_conf, pgbouncer From 254c0685fd5daf41ad47953be2dacc41d98af823 Mon Sep 17 00:00:00 2001 From: Ruslan Nigmatullin <60605660+rrrru@users.noreply.github.com> Date: Sun, 23 Jun 2024 22:20:02 +0300 Subject: [PATCH 05/18] Add wal_g_path variable; fix "wal-g: not found" (issue 658) (#679) --- .../tests/variables/asserts/wal_g_cron_jobs.yml | 12 ++++++------ roles/wal-g/tasks/main.yml | 16 ++++++++-------- vars/main.yml | 13 +++++++------ 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/molecule/tests/variables/asserts/wal_g_cron_jobs.yml b/molecule/tests/variables/asserts/wal_g_cron_jobs.yml index 92f920f02..aff193f4f 100644 --- a/molecule/tests/variables/asserts/wal_g_cron_jobs.yml +++ b/molecule/tests/variables/asserts/wal_g_cron_jobs.yml @@ -25,9 +25,9 @@ ansible.builtin.assert: that: - wal_g_backup_command[0] == "curl -I -s http://{{ inventory_hostname }}:{{ patroni_restapi_port }} | grep 200" - - wal_g_backup_command[1] == " && wal-g backup-push {{ postgresql_data_dir }} > {{ postgresql_log_dir }}/walg_backup.log 2>&1" + - wal_g_backup_command[1] == " && {{ wal_g_path }} backup-push {{ postgresql_data_dir }} > {{ postgresql_log_dir }}/walg_backup.log 2>&1" - wal_g_delete_command[0] == "curl -I -s http://{{ inventory_hostname }}:{{ patroni_restapi_port }} | grep 200" - - wal_g_delete_command[1] == " && wal-g delete retain FULL 4 --confirm > {{ postgresql_log_dir }}/walg_delete.log 2>&1" + - wal_g_delete_command[1] == " && {{ wal_g_path }} delete retain FULL 4 --confirm > {{ postgresql_log_dir }}/walg_delete.log 2>&1" fail_msg: "Test failed: wal_g_backup_command or wal_g_delete_command do not have the expected content." success_msg: "Test passed: wal_g_backup_command and wal_g_delete_command have the expected content." @@ -45,7 +45,7 @@ - name: Molecule.tests.variables.asserts.wal_g_cron_jobs | Debian | Define Expected First wal_g_cron Job run_once: true ansible.builtin.set_fact: # yamllint disable rule:line-length - origin_wal_g_cron_jobs_create_job: "curl -I -s http://{{ inventory_hostname }}:{{ patroni_restapi_port }} | grep 200 && wal-g backup-push {{ postgresql_data_dir }} > {{ postgresql_log_dir }}/walg_backup.log 2>&1" + origin_wal_g_cron_jobs_create_job: "curl -I -s http://{{ inventory_hostname }}:{{ patroni_restapi_port }} | grep 200 && {{ wal_g_path }} backup-push {{ postgresql_data_dir }} > {{ postgresql_log_dir }}/walg_backup.log 2>&1" # 🖨️ Display the first wal_g_cron job for Debian for debugging purposes - name: Molecule.tests.variables.asserts.wal_g_cron_jobs | Debian | Debug First wal_g_cron Job @@ -66,7 +66,7 @@ - name: Molecule.tests.variables.asserts.wal_g_cron_jobs | Debian | Define Expected Second wal_g_cron Job run_once: true ansible.builtin.set_fact: # yamllint disable rule:line-length - origin_wal_g_cron_jobs_delete_job: "curl -I -s http://{{ inventory_hostname }}:{{ patroni_restapi_port }} | grep 200 && wal-g delete retain FULL 4 --confirm > {{ postgresql_log_dir }}/walg_delete.log 2>&1" + origin_wal_g_cron_jobs_delete_job: "curl -I -s http://{{ inventory_hostname }}:{{ patroni_restapi_port }} | grep 200 && {{ wal_g_path }} delete retain FULL 4 --confirm > {{ postgresql_log_dir }}/walg_delete.log 2>&1" # 🖨️ Display the second wal_g_cron job for Debian for debugging purposes - name: Molecule.tests.variables.asserts.wal_g_cron_jobs | Debian | Debug Second wal_g_cron Job @@ -97,7 +97,7 @@ - name: Molecule.tests.variables.asserts.wal_g_cron_jobs | RedHat | Define Expected First wal_g_cron Job run_once: true ansible.builtin.set_fact: # yamllint disable rule:line-length - origin_wal_g_cron_jobs_create_job: "curl -I -s http://{{ inventory_hostname }}:{{ patroni_restapi_port }} | grep 200 && wal-g backup-push {{ postgresql_data_dir }} > {{ postgresql_log_dir }}/walg_backup.log 2>&1" + origin_wal_g_cron_jobs_create_job: "curl -I -s http://{{ inventory_hostname }}:{{ patroni_restapi_port }} | grep 200 && {{ wal_g_path }} backup-push {{ postgresql_data_dir }} > {{ postgresql_log_dir }}/walg_backup.log 2>&1" # 🖨️ Display the first wal_g_cron job for RedHat for debugging purposes - name: Molecule.tests.variables.asserts.wal_g_cron_jobs | RedHat | Debug First wal_g_cron Job @@ -118,7 +118,7 @@ - name: Molecule.tests.variables.asserts.wal_g_cron_jobs | RedHat | Define Expected Second wal_g_cron Job run_once: true ansible.builtin.set_fact: # yamllint disable rule:line-length - origin_wal_g_cron_jobs_delete_job: "curl -I -s http://{{ inventory_hostname }}:{{ patroni_restapi_port }} | grep 200 && wal-g delete retain FULL 4 --confirm > {{ postgresql_log_dir }}/walg_delete.log 2>&1" + origin_wal_g_cron_jobs_delete_job: "curl -I -s http://{{ inventory_hostname }}:{{ patroni_restapi_port }} | grep 200 && {{ wal_g_path }} delete retain FULL 4 --confirm > {{ postgresql_log_dir }}/walg_delete.log 2>&1" # 🖨️ Display the second wal_g_cron job for RedHat for debugging purposes - name: Molecule.tests.variables.asserts.wal_g_cron_jobs | RedHat | Debug Second wal_g_cron Job diff --git a/roles/wal-g/tasks/main.yml b/roles/wal-g/tasks/main.yml index f43b8d52c..ee3d9dd14 100644 --- a/roles/wal-g/tasks/main.yml +++ b/roles/wal-g/tasks/main.yml @@ -3,7 +3,7 @@ - name: Check if WAL-G is already installed ansible.builtin.shell: | set -o pipefail; - wal-g --version | awk {'print $3'} | tr -d 'v' + "{{ wal_g_path }}" --version | awk {'print $3'} | tr -d 'v' args: executable: /bin/bash changed_when: false @@ -42,7 +42,7 @@ - name: Copy WAL-G binary file to /usr/local/bin/ ansible.builtin.copy: src: "/tmp/wal-g-pg-ubuntu-{{ ansible_distribution_version }}-amd64" - dest: /usr/local/bin/wal-g + dest: "{{ wal_g_path }}" mode: u+x,g+x,o+x remote_src: true when: @@ -184,10 +184,10 @@ - --no-same-owner remote_src: true - - name: Copy WAL-G binary file to /usr/local/bin/ + - name: Copy WAL-G binary file to "{{ wal_g_path }}" ansible.builtin.copy: src: "/tmp/wal-g" - dest: /usr/local/bin/ + dest: "{{ wal_g_path }}" mode: u+x,g+x,o+x remote_src: true when: @@ -208,10 +208,10 @@ extra_opts: - --no-same-owner - - name: Copy WAL-G binary file to /usr/local/bin/ + - name: Copy WAL-G binary file to "{{ wal_g_path }}" ansible.builtin.copy: src: "/tmp/{{ wal_g_package_file.split('.tar.gz')[0] | basename }}" - dest: /usr/local/bin/wal-g + dest: "{{ wal_g_path }}" mode: u+x,g+x,o+x remote_src: true when: @@ -231,10 +231,10 @@ extra_opts: - --no-same-owner - - name: Copy WAL-G binary file to /usr/local/bin/ + - name: Copy WAL-G binary file to "{{ wal_g_path }}" ansible.builtin.copy: src: "/tmp/wal-g" - dest: /usr/local/bin/ + dest: "{{ wal_g_path }}" mode: u+x,g+x,o+x remote_src: true when: diff --git a/vars/main.yml b/vars/main.yml index 6fdb22d39..54c57701f 100644 --- a/vars/main.yml +++ b/vars/main.yml @@ -422,7 +422,7 @@ pgbackrest: - { option: "keep_data", value: "True" } - { option: "no_params", value: "True" } wal_g: - - { option: "command", value: "wal-g backup-fetch {{ postgresql_data_dir }} LATEST" } + - { option: "command", value: "{{ wal_g_path }} backup-fetch {{ postgresql_data_dir }} LATEST" } - { option: "no_params", value: "True" } basebackup: - { option: "max-rate", value: "100M" } @@ -434,7 +434,7 @@ pg_probackup: # "restore_command" written to recovery.conf when configuring follower (create replica) postgresql_restore_command: "" -# postgresql_restore_command: "wal-g wal-fetch %f %p" # restore WAL-s using WAL-G +# postgresql_restore_command: "{{ wal_g_path }} wal-fetch %f %p" # restore WAL-s using WAL-G # postgresql_restore_command: "pgbackrest --stanza={{ pgbackrest_stanza }} archive-get %f %p" # restore WAL-s using pgbackrest # postgresql_restore_command: "pg_probackup-{{ pg_probackup_version }} archive-get -B @@ -462,6 +462,7 @@ pg_probackup_patroni_cluster_bootstrap_command: "{{ pg_probackup_command_parts | # WAL-G wal_g_install: false # or 'true' wal_g_version: "3.0.0" +wal_g_path: "/usr/local/bin/wal-g" wal_g_json: # config https://github.com/wal-g/wal-g#configuration - { option: "AWS_ACCESS_KEY_ID", value: "{{ AWS_ACCESS_KEY_ID | default('') }}" } # define values or pass via --extra-vars - { option: "AWS_SECRET_ACCESS_KEY", value: "{{ AWS_SECRET_ACCESS_KEY | default('') }}" } # define values or pass via --extra-vars @@ -475,17 +476,17 @@ wal_g_json: # config https://github.com/wal-g/wal-g#configuration # - { option: "AWS_S3_FORCE_PATH_STYLE", value: "true" } # to use Minio.io S3-compatible storage # - { option: "AWS_ENDPOINT", value: "http://minio:9000" } # to use Minio.io S3-compatible storage # - { option: "", value: "" } -wal_g_archive_command: "wal-g wal-push %p" -wal_g_patroni_cluster_bootstrap_command: "wal-g backup-fetch {{ postgresql_data_dir }} LATEST" +wal_g_archive_command: "{{ wal_g_path }} wal-push %p" +wal_g_patroni_cluster_bootstrap_command: "{{ wal_g_path }} backup-fetch {{ postgresql_data_dir }} LATEST" # Define job_parts outside of wal_g_cron_jobs # ⚠️ Ensure there is a space at the beginning of each part to prevent commands from concatenating. wal_g_backup_command: - "curl -I -s http://{{ inventory_hostname }}:{{ patroni_restapi_port }} | grep 200" - - " && wal-g backup-push {{ postgresql_data_dir }} > {{ postgresql_log_dir }}/walg_backup.log 2>&1" + - " && {{ wal_g_path }} backup-push {{ postgresql_data_dir }} > {{ postgresql_log_dir }}/walg_backup.log 2>&1" wal_g_delete_command: - "curl -I -s http://{{ inventory_hostname }}:{{ patroni_restapi_port }} | grep 200" - - " && wal-g delete retain FULL 4 --confirm > {{ postgresql_log_dir }}/walg_delete.log 2>&1" + - " && {{ wal_g_path }} delete retain FULL 4 --confirm > {{ postgresql_log_dir }}/walg_delete.log 2>&1" wal_g_cron_jobs: - name: "WAL-G: Create daily backup" From e4b4812415cb77a21813bec2e7d48fb84a1d84bd Mon Sep 17 00:00:00 2001 From: abyss-ms <13432337+abyss-ms@users.noreply.github.com> Date: Mon, 1 Jul 2024 14:59:35 +0200 Subject: [PATCH 06/18] Make keepalived role more configurable (issue #683) (#684) --- roles/keepalived/defaults/main.yml | 15 ++++++ roles/keepalived/handlers/main.yml | 2 +- roles/keepalived/tasks/main.yml | 8 +++- roles/keepalived/templates/keepalived.conf.j2 | 48 +++++++++++-------- 4 files changed, 50 insertions(+), 23 deletions(-) create mode 100644 roles/keepalived/defaults/main.yml diff --git a/roles/keepalived/defaults/main.yml b/roles/keepalived/defaults/main.yml new file mode 100644 index 000000000..df4b992ad --- /dev/null +++ b/roles/keepalived/defaults/main.yml @@ -0,0 +1,15 @@ +--- + +keepalived_instances: + - name: VI_1 + state: BACKUP + interface: "{{ vip_interface }}" + virtual_router_id: "{{ keepalived_virtual_router_id | default(123) }}" + priority: 100 + advert_int: 2 + check_status_command: /usr/libexec/keepalived/haproxy_check.sh + authentication: + auth_type: PASS + auth_pass: "1ce24b6e" + virtual_ipaddresses: + - "{{ cluster_vip }}" diff --git a/roles/keepalived/handlers/main.yml b/roles/keepalived/handlers/main.yml index 32d6c081e..a716b586b 100644 --- a/roles/keepalived/handlers/main.yml +++ b/roles/keepalived/handlers/main.yml @@ -27,7 +27,7 @@ ansible_ssh_port | default(22) ) }} - ignore_errors: true # show the error and continue the playbook execution + ignore_errors: true # noqa ignore-errors # show the error and continue the playbook execution listen: "restart keepalived" ... diff --git a/roles/keepalived/tasks/main.yml b/roles/keepalived/tasks/main.yml index ecb7d411e..ffd946295 100644 --- a/roles/keepalived/tasks/main.yml +++ b/roles/keepalived/tasks/main.yml @@ -28,6 +28,7 @@ state: directory owner: root group: root + mode: "0750" tags: keepalived_conf, keepalived - name: Create vrrp_script "/usr/libexec/keepalived/haproxy_check.sh" @@ -46,6 +47,9 @@ ansible.builtin.template: src: templates/keepalived.conf.j2 dest: /etc/keepalived/keepalived.conf + owner: root + group: root + mode: "0644" notify: "restart keepalived" when: add_balancer is not defined or not add_balancer|bool tags: keepalived_conf, keepalived @@ -88,13 +92,13 @@ when: add_balancer is defined and add_balancer|bool tags: keepalived_conf, keepalived -- name: selinux | change the keepalived_t domain to permissive +- name: Selinux | Change the keepalived_t domain to permissive community.general.selinux_permissive: name: keepalived_t permissive: true when: ansible_selinux.status is defined and ansible_selinux.status == 'enabled' - ignore_errors: true + ignore_errors: true # noqa ignore-errors tags: keepalived, keepalived_selinux ... diff --git a/roles/keepalived/templates/keepalived.conf.j2 b/roles/keepalived/templates/keepalived.conf.j2 index c17a9c3e6..80d524e9e 100644 --- a/roles/keepalived/templates/keepalived.conf.j2 +++ b/roles/keepalived/templates/keepalived.conf.j2 @@ -3,27 +3,35 @@ global_defs { enable_script_security script_user root } - -vrrp_script haproxy_check { - script "/usr/libexec/keepalived/haproxy_check.sh" + +{% for instance in keepalived_instances %} +{% if instance.check_status_command is defined %} +vrrp_script chk_command_{{ instance.virtual_router_id }} { + script "{{ instance.check_status_command }}" interval 2 weight 2 } - -vrrp_instance VI_1 { - interface {{ vip_interface }} - virtual_router_id {{ keepalived_virtual_router_id | default(123) }} - priority 100 - advert_int 2 - state BACKUP - virtual_ipaddress { - {{ cluster_vip }} - } - track_script { - haproxy_check - } - authentication { - auth_type PASS - auth_pass 1ce24b6e - } +{% endif %} + +vrrp_instance {{ instance.name }} { + interface {{ instance.interface }} + virtual_router_id {{ instance.virtual_router_id }} + priority {{ instance.priority }} + advert_int {{ instance.advert_int }} + state {{ instance.state }} + virtual_ipaddress { + {% for ip in instance.virtual_ipaddresses %} + {{ ip }} + {% endfor %} + } + {% if instance.check_status_command is defined %} + track_script { + chk_command_{{ instance.virtual_router_id }} + } + {% endif %} + authentication { + auth_type {{ instance.authentication.auth_type }} + auth_pass {{ instance.authentication.auth_pass }} + } } +{% endfor %} From b10d907dd1efcbcfdd8ea4dd006810a0f0e467f3 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik <37010174+vitabaks@users.noreply.github.com> Date: Thu, 4 Jul 2024 02:24:23 +0300 Subject: [PATCH 07/18] Update vip-manager to v2.5.0 (#688) --- vars/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/main.yml b/vars/main.yml index 54c57701f..6b02b59f5 100644 --- a/vars/main.yml +++ b/vars/main.yml @@ -51,7 +51,7 @@ keepalived_virtual_router_id: "{{ cluster_vip.split('.')[3] | int }}" # The last # virtual_router_id - must be unique in the network (available values are 0..255). # vip-manager (if 'cluster_vip' is specified and 'with_haproxy_load_balancing' is 'false') -vip_manager_version: "2.4.0" # version to install +vip_manager_version: "2.5.0" # version to install vip_manager_conf: "/etc/patroni/vip-manager.yml" vip_manager_interval: "1000" # time (in milliseconds) after which vip-manager wakes up and checks if it needs to register or release ip addresses. vip_manager_iface: "{{ vip_interface }}" # interface to which the virtual ip will be added From c9a634f99647cfc8fbffcd1e73cf2a90338a435d Mon Sep 17 00:00:00 2001 From: Boris <91973490+jimnydev@users.noreply.github.com> Date: Mon, 8 Jul 2024 19:52:29 +0200 Subject: [PATCH 08/18] haproxy: Add optional log-format (#687) --- roles/confd/templates/haproxy.tmpl.j2 | 12 ++++------- roles/haproxy/templates/haproxy.cfg.j2 | 12 ++++------- vars/main.yml | 29 ++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 16 deletions(-) diff --git a/roles/confd/templates/haproxy.tmpl.j2 b/roles/confd/templates/haproxy.tmpl.j2 index f7996b8ec..a0acb597f 100644 --- a/roles/confd/templates/haproxy.tmpl.j2 +++ b/roles/confd/templates/haproxy.tmpl.j2 @@ -12,6 +12,10 @@ global defaults mode tcp log global + option tcplog +{% if haproxy_log_format is defined %} + log-format '{{ haproxy_log_format }}' +{% endif %} retries 2 timeout queue 5s timeout connect 5s @@ -32,7 +36,6 @@ listen master bind {{ inventory_hostname }}:{{ haproxy_listen_port.master }} {% endif %} maxconn {{ haproxy_maxconn.master }} - option tcplog option httpchk OPTIONS /primary http-check expect status 200 default-server inter 3s fastinter 1s fall 3 rise 4 on-marked-down shutdown-sessions @@ -53,7 +56,6 @@ listen master_direct bind {{ inventory_hostname }}:{{ haproxy_listen_port.master_direct }} {% endif %} maxconn {{ haproxy_maxconn.master }} - option tcplog option httpchk OPTIONS /primary http-check expect status 200 default-server inter 3s fastinter 1s fall 3 rise 4 on-marked-down shutdown-sessions @@ -68,7 +70,6 @@ listen replicas bind {{ inventory_hostname }}:{{ haproxy_listen_port.replicas }} {% endif %} maxconn {{ haproxy_maxconn.replica }} - option tcplog {% if balancer_tags | default('') | length > 0 %} option httpchk OPTIONS /replica?lag={{ patroni_maximum_lag_on_replica }}{{ '&' + balancer_tags.split(',') | map('trim') | map('regex_replace', '([^=]+)=(.*)', 'tag_\\1=\\2') | join('&') }} {% else %} @@ -94,7 +95,6 @@ listen replicas_direct bind {{ inventory_hostname }}:{{ haproxy_listen_port.replicas_direct }} {% endif %} maxconn {{ haproxy_maxconn.replica }} - option tcplog {% if balancer_tags | default('') | length > 0 %} option httpchk OPTIONS /replica?lag={{ patroni_maximum_lag_on_replica }}{{ '&' + balancer_tags.split(',') | map('trim') | map('regex_replace', '([^=]+)=(.*)', 'tag_\\1=\\2') | join('&') }} {% else %} @@ -114,7 +114,6 @@ listen replicas_sync bind {{ inventory_hostname }}:{{ haproxy_listen_port.replicas_sync }} {% endif %} maxconn {{ haproxy_maxconn.replica }} - option tcplog {% if balancer_tags | default('') | length > 0 %} option httpchk OPTIONS /sync{{ '?' + balancer_tags.split(',') | map('trim') | map('regex_replace', '([^=]+)=(.*)', 'tag_\\1=\\2') | join('&') }} {% else %} @@ -140,7 +139,6 @@ listen replicas_sync_direct bind {{ inventory_hostname }}:{{ haproxy_listen_port.replicas_sync_direct }} {% endif %} maxconn {{ haproxy_maxconn.replica }} - option tcplog {% if balancer_tags | default('') | length > 0 %} option httpchk OPTIONS /sync{{ '?' + balancer_tags.split(',') | map('trim') | map('regex_replace', '([^=]+)=(.*)', 'tag_\\1=\\2') | join('&') }} {% else %} @@ -160,7 +158,6 @@ listen replicas_async bind {{ inventory_hostname }}:{{ haproxy_listen_port.replicas_async }} {% endif %} maxconn {{ haproxy_maxconn.replica }} - option tcplog {% if balancer_tags | default('') | length > 0 %} option httpchk OPTIONS /async?lag={{ patroni_maximum_lag_on_replica }}{{ '&' + balancer_tags.split(',') | map('trim') | map('regex_replace', '([^=]+)=(.*)', 'tag_\\1=\\2') | join('&') }} {% else %} @@ -186,7 +183,6 @@ listen replicas_async_direct bind {{ inventory_hostname }}:{{ haproxy_listen_port.replicas_async_direct }} {% endif %} maxconn {{ haproxy_maxconn.replica }} - option tcplog {% if balancer_tags | default('') | length > 0 %} option httpchk OPTIONS /async?lag={{ patroni_maximum_lag_on_replica }}{{ '&' + balancer_tags.split(',') | map('trim') | map('regex_replace', '([^=]+)=(.*)', 'tag_\\1=\\2') | join('&') }} {% else %} diff --git a/roles/haproxy/templates/haproxy.cfg.j2 b/roles/haproxy/templates/haproxy.cfg.j2 index 80849cdf5..d7299e347 100644 --- a/roles/haproxy/templates/haproxy.cfg.j2 +++ b/roles/haproxy/templates/haproxy.cfg.j2 @@ -12,6 +12,10 @@ global defaults mode tcp log global + option tcplog +{% if haproxy_log_format is defined %} + log-format '{{ haproxy_log_format }}' +{% endif %} retries 2 timeout queue 5s timeout connect 5s @@ -32,7 +36,6 @@ listen master bind {{ inventory_hostname }}:{{ haproxy_listen_port.master }} {% endif %} maxconn {{ haproxy_maxconn.master }} - option tcplog option httpchk OPTIONS /primary http-check expect status 200 default-server inter 3s fastinter 1s fall 3 rise 4 on-marked-down shutdown-sessions @@ -55,7 +58,6 @@ listen master_direct bind {{ inventory_hostname }}:{{ haproxy_listen_port.master_direct }} {% endif %} maxconn {{ haproxy_maxconn.master }} - option tcplog option httpchk OPTIONS /primary http-check expect status 200 default-server inter 3s fastinter 1s fall 3 rise 4 on-marked-down shutdown-sessions @@ -71,7 +73,6 @@ listen replicas bind {{ inventory_hostname }}:{{ haproxy_listen_port.replicas }} {% endif %} maxconn {{ haproxy_maxconn.replica }} - option tcplog {% if balancer_tags | default('') | length > 0 %} option httpchk OPTIONS /replica?lag={{ patroni_maximum_lag_on_replica }}{{ '&' + balancer_tags.split(',') | map('trim') | map('regex_replace', '([^=]+)=(.*)', 'tag_\\1=\\2') | join('&') }} {% else %} @@ -99,7 +100,6 @@ listen replicas_direct bind {{ inventory_hostname }}:{{ haproxy_listen_port.replicas_direct }} {% endif %} maxconn {{ haproxy_maxconn.replica }} - option tcplog {% if balancer_tags | default('') | length > 0 %} option httpchk OPTIONS /replica?lag={{ patroni_maximum_lag_on_replica }}{{ '&' + balancer_tags.split(',') | map('trim') | map('regex_replace', '([^=]+)=(.*)', 'tag_\\1=\\2') | join('&') }} {% else %} @@ -120,7 +120,6 @@ listen replicas_sync bind {{ inventory_hostname }}:{{ haproxy_listen_port.replicas_sync }} {% endif %} maxconn {{ haproxy_maxconn.replica }} - option tcplog {% if balancer_tags | default('') | length > 0 %} option httpchk OPTIONS /sync{{ '?' + balancer_tags.split(',') | map('trim') | map('regex_replace', '([^=]+)=(.*)', 'tag_\\1=\\2') | join('&') }} {% else %} @@ -148,7 +147,6 @@ listen replicas_sync_direct bind {{ inventory_hostname }}:{{ haproxy_listen_port.replicas_sync_direct }} {% endif %} maxconn {{ haproxy_maxconn.replica }} - option tcplog {% if balancer_tags | default('') | length > 0 %} option httpchk OPTIONS /sync{{ '?' + balancer_tags.split(',') | map('trim') | map('regex_replace', '([^=]+)=(.*)', 'tag_\\1=\\2') | join('&') }} {% else %} @@ -169,7 +167,6 @@ listen replicas_async bind {{ inventory_hostname }}:{{ haproxy_listen_port.replicas_async }} {% endif %} maxconn {{ haproxy_maxconn.replica }} - option tcplog {% if balancer_tags | default('') | length > 0 %} option httpchk OPTIONS /async?lag={{ patroni_maximum_lag_on_replica }}{{ '&' + balancer_tags.split(',') | map('trim') | map('regex_replace', '([^=]+)=(.*)', 'tag_\\1=\\2') | join('&') }} {% else %} @@ -197,7 +194,6 @@ listen replicas_async_direct bind {{ inventory_hostname }}:{{ haproxy_listen_port.replicas_async_direct }} {% endif %} maxconn {{ haproxy_maxconn.replica }} - option tcplog {% if balancer_tags | default('') | length > 0 %} option httpchk OPTIONS /async?lag={{ patroni_maximum_lag_on_replica }}{{ '&' + balancer_tags.split(',') | map('trim') | map('regex_replace', '([^=]+)=(.*)', 'tag_\\1=\\2') | join('&') }} {% else %} diff --git a/vars/main.yml b/vars/main.yml index 6b02b59f5..731541579 100644 --- a/vars/main.yml +++ b/vars/main.yml @@ -45,6 +45,35 @@ haproxy_maxconn: haproxy_timeout: client: "60m" server: "60m" +# Optionally declare log format for haproxy. +# Uncomment following lines (and remove extra space in front of variable definition) for JSON structured log format. +# haproxy_log_format: "{ +# \"pid\":%pid,\ +# \"haproxy_frontend_type\":\"tcp\",\ +# \"haproxy_process_concurrent_connections\":%ac,\ +# \"haproxy_frontend_concurrent_connections\":%fc,\ +# \"haproxy_backend_concurrent_connections\":%bc,\ +# \"haproxy_server_concurrent_connections\":%sc,\ +# \"haproxy_backend_queue\":%bq,\ +# \"haproxy_server_queue\":%sq,\ +# \"haproxy_queue_wait_time\":%Tw,\ +# \"haproxy_server_wait_time\":%Tc,\ +# \"response_time\":%Td,\ +# \"session_duration\":%Tt,\ +# \"request_termination_state\":\"%tsc\",\ +# \"haproxy_server_connection_retries\":%rc,\ +# \"remote_addr\":\"%ci\",\ +# \"remote_port\":%cp,\ +# \"frontend_addr\":\"%fi\",\ +# \"frontend_port\":%fp,\ +# \"frontend_ssl_version\":\"%sslv\",\ +# \"frontend_ssl_ciphers\":\"%sslc\",\ +# \"haproxy_frontend_name\":\"%f\",\ +# \"haproxy_backend_name\":\"%b\",\ +# \"haproxy_server_name\":\"%s\",\ +# \"response_size\":%B,\ +# \"request_size\":%U\ +# }" # keepalived (if 'cluster_vip' is specified and 'with_haproxy_load_balancing' is 'true') keepalived_virtual_router_id: "{{ cluster_vip.split('.')[3] | int }}" # The last octet of 'cluster_vip' IP address is used by default. From 6cefdd61474b8243a7cf156939acbf61defa25a0 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik <37010174+vitabaks@users.noreply.github.com> Date: Fri, 12 Jul 2024 00:33:05 +0300 Subject: [PATCH 09/18] Update WAL-G to v3.0.2 (#693) --- roles/wal-g/tasks/main.yml | 7 +++++++ vars/main.yml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/roles/wal-g/tasks/main.yml b/roles/wal-g/tasks/main.yml index ee3d9dd14..34e5c2627 100644 --- a/roles/wal-g/tasks/main.yml +++ b/roles/wal-g/tasks/main.yml @@ -132,6 +132,13 @@ version: v{{ wal_g_version }} dest: /tmp/wal-g + - name: Run go mod tidy to ensure dependencies are correct + ansible.builtin.command: go mod tidy + args: + chdir: /tmp/wal-g + environment: + PATH: "{{ ansible_env.PATH }}:/usr/local/go/bin" + - name: Build WAL-G deps become: true become_user: root diff --git a/vars/main.yml b/vars/main.yml index 731541579..0bdbc6c45 100644 --- a/vars/main.yml +++ b/vars/main.yml @@ -490,7 +490,7 @@ pg_probackup_patroni_cluster_bootstrap_command: "{{ pg_probackup_command_parts | # WAL-G wal_g_install: false # or 'true' -wal_g_version: "3.0.0" +wal_g_version: "3.0.2" wal_g_path: "/usr/local/bin/wal-g" wal_g_json: # config https://github.com/wal-g/wal-g#configuration - { option: "AWS_ACCESS_KEY_ID", value: "{{ AWS_ACCESS_KEY_ID | default('') }}" } # define values or pass via --extra-vars From a4cc0d32ce964767748e9aec2918cb99265d2708 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik <37010174+vitabaks@users.noreply.github.com> Date: Fri, 12 Jul 2024 22:45:20 +0300 Subject: [PATCH 10/18] Update Patroni to v3.3.2 (#694) --- vars/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/main.yml b/vars/main.yml index 0bdbc6c45..8ae8c8748 100644 --- a/vars/main.yml +++ b/vars/main.yml @@ -12,7 +12,7 @@ vip_interface: "{{ ansible_default_ipv4.interface }}" # interface name (e.g., " # Note: VIP-based solutions such as keepalived or vip-manager may not function correctly in cloud environments like AWS. patroni_cluster_name: "postgres-cluster" # the cluster name (must be unique for each cluster) -patroni_install_version: "3.3.0" # or 'latest' +patroni_install_version: "3.3.2" # or 'latest' patroni_superuser_username: "postgres" patroni_superuser_password: "postgres-pass" # please change password From 3532496f1c77b32b2a394184cef94f7c903b6a73 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik <37010174+vitabaks@users.noreply.github.com> Date: Sat, 13 Jul 2024 00:29:56 +0300 Subject: [PATCH 11/18] pgBackRest: Add '--no-online' option to stanza-create (#695) --- roles/pgbackrest/stanza-create/tasks/main.yml | 2 +- vars/main.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/pgbackrest/stanza-create/tasks/main.yml b/roles/pgbackrest/stanza-create/tasks/main.yml index 8395f97e6..9b508888c 100644 --- a/roles/pgbackrest/stanza-create/tasks/main.yml +++ b/roles/pgbackrest/stanza-create/tasks/main.yml @@ -55,7 +55,7 @@ become_user: "{{ pgbackrest_repo_user }}" delegate_to: "{{ groups['pgbackrest'][0] }}" run_once: true - ansible.builtin.command: "pgbackrest --stanza={{ pgbackrest_stanza }} stanza-create" + ansible.builtin.command: "pgbackrest --stanza={{ pgbackrest_stanza }} --no-online stanza-create" register: stanza_create_result changed_when: - stanza_create_result.rc == 0 diff --git a/vars/main.yml b/vars/main.yml index 8ae8c8748..f46c0f8b0 100644 --- a/vars/main.yml +++ b/vars/main.yml @@ -612,7 +612,7 @@ pgbackrest_cron_jobs: day: "*" month: "*" weekday: "0" - job: "pgbackrest --type=full --stanza={{ pgbackrest_stanza }} backup" + job: "pgbackrest --stanza={{ pgbackrest_stanza }} --type=full backup" # job: "if [ $(psql -tAXc 'select pg_is_in_recovery()') = 'f' ]; then pgbackrest --type=full --stanza={{ pgbackrest_stanza }} backup; fi" - name: "pgBackRest: Diff Backup" file: "/etc/cron.d/pgbackrest-{{ patroni_cluster_name }}" @@ -622,7 +622,7 @@ pgbackrest_cron_jobs: day: "*" month: "*" weekday: "1-6" - job: "pgbackrest --type=diff --stanza={{ pgbackrest_stanza }} backup" + job: "pgbackrest --stanza={{ pgbackrest_stanza }} --type=diff backup" # job: "if [ $(psql -tAXc 'select pg_is_in_recovery()') = 'f' ]; then pgbackrest --type=diff --stanza={{ pgbackrest_stanza }} backup; fi" From 3b24f6ee18481adedf663c25bac520f5e8105600 Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik <37010174+vitabaks@users.noreply.github.com> Date: Tue, 16 Jul 2024 12:06:46 +0300 Subject: [PATCH 12/18] Fix (Upgrade): Delegate maintenance mode tasks to 'balancers' group (#699) --- roles/haproxy/tasks/main.yml | 2 +- roles/upgrade/tasks/maintenance_disable.yml | 10 ++++++++++ roles/upgrade/tasks/maintenance_enable.yml | 17 +++++++++++++++++ roles/upgrade/tasks/update_config.yml | 6 ++++++ 4 files changed, 34 insertions(+), 1 deletion(-) diff --git a/roles/haproxy/tasks/main.yml b/roles/haproxy/tasks/main.yml index d41f016d5..93f408e14 100644 --- a/roles/haproxy/tasks/main.yml +++ b/roles/haproxy/tasks/main.yml @@ -1,6 +1,6 @@ --- -- name: Gather facts for hosts postres_cluster +- name: Gather facts from postgres_cluster hosts ansible.builtin.setup: delegate_to: "{{ item }}" delegate_facts: true diff --git a/roles/upgrade/tasks/maintenance_disable.yml b/roles/upgrade/tasks/maintenance_disable.yml index 38a80bbd4..65936259b 100644 --- a/roles/upgrade/tasks/maintenance_disable.yml +++ b/roles/upgrade/tasks/maintenance_disable.yml @@ -8,20 +8,30 @@ dest: /etc/haproxy/haproxy.cfg owner: haproxy group: haproxy + delegate_to: "{{ item }}" + loop: "{{ groups.balancers | default([]) | list }}" + run_once: true - name: Reload haproxy service ansible.builtin.systemd: name: haproxy state: reloaded + delegate_to: "{{ item }}" + loop: "{{ groups.balancers | default([]) | list }}" + run_once: true - name: Start confd service ansible.builtin.service: name: confd state: started + delegate_to: "{{ item }}" + loop: "{{ groups.balancers | default([]) | list }}" + run_once: true become: true become_user: root ignore_errors: true # show the error and continue the playbook execution when: + - groups.balancers | default([]) | length > 0 - with_haproxy_load_balancing | bool - pgbouncer_install | bool - pgbouncer_pool_pause | bool diff --git a/roles/upgrade/tasks/maintenance_enable.yml b/roles/upgrade/tasks/maintenance_enable.yml index 1a560b892..6317acc64 100644 --- a/roles/upgrade/tasks/maintenance_enable.yml +++ b/roles/upgrade/tasks/maintenance_enable.yml @@ -16,10 +16,20 @@ # Temporarily disable http-checks in order to keep database connections after stopping the Patroni service # and then pause the pgbouncer pools. - block: + - name: Gather facts from balancers + ansible.builtin.setup: + delegate_to: "{{ item }}" + delegate_facts: true + loop: "{{ groups.balancers | default([]) | list }}" + run_once: true + - name: Stop confd service ansible.builtin.service: name: confd state: stopped + delegate_to: "{{ item }}" + loop: "{{ groups.balancers | default([]) | list }}" + run_once: true when: dcs_type == "etcd" - name: Update haproxy conf file (disable http-checks) @@ -28,14 +38,21 @@ dest: /etc/haproxy/haproxy.cfg owner: haproxy group: haproxy + delegate_to: "{{ item }}" + loop: "{{ groups.balancers | default([]) | list }}" + run_once: true - name: Reload haproxy service ansible.builtin.systemd: name: haproxy state: reloaded + delegate_to: "{{ item }}" + loop: "{{ groups.balancers | default([]) | list }}" + run_once: true become: true become_user: root when: + - groups.balancers | default([]) | length > 0 - with_haproxy_load_balancing | bool - pgbouncer_install | bool - pgbouncer_pool_pause | bool diff --git a/roles/upgrade/tasks/update_config.yml b/roles/upgrade/tasks/update_config.yml index a680f5354..0e96ff120 100644 --- a/roles/upgrade/tasks/update_config.yml +++ b/roles/upgrade/tasks/update_config.yml @@ -7,6 +7,12 @@ dest: "{{ patroni_config_file }}.bkp" remote_src: true +# https://github.com/vitabaks/postgresql_cluster/issues/666 +- name: Remove patroni.dynamic.json file + ansible.builtin.file: + path: "{{ pg_old_datadir }}/patroni.dynamic.json" + state: absent + # Update the directory path to a new version of PostgresSQL - name: "Edit patroni.yml | update parameters: data_dir, bin_dir, config_dir" ansible.builtin.replace: From 5d5a1857ab42ff730672552a09793e3baa1471fb Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik <37010174+vitabaks@users.noreply.github.com> Date: Wed, 17 Jul 2024 14:54:44 +0300 Subject: [PATCH 13/18] Fix (Upgrade): Update pg-path in local pgbackrest.conf (#700) --- roles/upgrade/tasks/post_upgrade.yml | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/roles/upgrade/tasks/post_upgrade.yml b/roles/upgrade/tasks/post_upgrade.yml index 2ecd9c4d9..d524d979c 100644 --- a/roles/upgrade/tasks/post_upgrade.yml +++ b/roles/upgrade/tasks/post_upgrade.yml @@ -96,58 +96,57 @@ # pgBackRest (local) - block: - name: pgbackrest | Check pg-path option - ansible.builtin.command: "grep -c '^pg.*-path=' {{ pgbackrest_conf_file }}" + ansible.builtin.command: "grep -c '^pg[0-9]*-path=' {{ pgbackrest_conf_file }}" register: pg_path_count changed_when: false - name: pgbackrest | Update pg-path in pgbackrest.conf - ansible.builtin.lineinfile: + ansible.builtin.replace: path: "{{ pgbackrest_conf_file }}" - regexp: '^pg{{ idx + 1 }}-path=' - line: 'pg{{ idx + 1 }}-path={{ pg_new_datadir }}' + regexp: '^pg{{ idx + 1 }}-path=.*$' + replace: 'pg{{ idx + 1 }}-path={{ pg_new_datadir }}' loop: "{{ range(0, pg_path_count.stdout | int) | list }}" loop_control: index_var: idx label: "pg{{ idx + 1 }}-path={{ pg_new_datadir }}" - when: pg_path_count.stdout | length > 0 + when: pg_path_count.stdout | int > 0 - name: pgbackrest | Upgrade stanza "{{ pgbackrest_stanza }}" ansible.builtin.command: "pgbackrest --stanza={{ pgbackrest_stanza }} --no-online stanza-upgrade" - when: pg_path_count.stdout | length > 0 and pgbackrest_stanza_upgrade | bool + when: pg_path_count.stdout | int > 0 and pgbackrest_stanza_upgrade | bool and pgbackrest_repo_host | length < 1 become: true become_user: postgres ignore_errors: true # show the error and continue the playbook execution when: - pgbackrest_install | bool - - pgbackrest_repo_host | length < 1 # pgBackRest (dedicated) - block: - name: pgbackrest | Check pg-path option delegate_to: "{{ groups['pgbackrest'][0] }}" run_once: true - ansible.builtin.command: "grep -c '^pg.*-path=' {{ pgbackrest_conf_file | dirname }}/conf.d/{{ pgbackrest_stanza }}.conf" + ansible.builtin.command: "grep -c '^pg[0-9]*-path=' {{ pgbackrest_conf_file | dirname }}/conf.d/{{ pgbackrest_stanza }}.conf" register: pg_path_count changed_when: false - name: pgbackrest | Update pg-path in pgbackrest.conf delegate_to: "{{ groups['pgbackrest'][0] }}" run_once: true - ansible.builtin.lineinfile: + ansible.builtin.replace: path: "{{ pgbackrest_conf_file | dirname }}/conf.d/{{ pgbackrest_stanza }}.conf" - regexp: '^pg{{ idx + 1 }}-path=' - line: 'pg{{ idx + 1 }}-path={{ pg_new_datadir }}' + regexp: '^pg{{ idx + 1 }}-path=.*$' + replace: 'pg{{ idx + 1 }}-path={{ pg_new_datadir }}' loop: "{{ range(0, pg_path_count.stdout | int) | list }}" loop_control: index_var: idx label: "pg{{ idx + 1 }}-path={{ pg_new_datadir }}" - when: pg_path_count.stdout | length > 0 + when: pg_path_count.stdout | int > 0 - name: pgbackrest | Upgrade stanza "{{ pgbackrest_stanza }}" delegate_to: "{{ groups['pgbackrest'][0] }}" run_once: true ansible.builtin.command: "pgbackrest --stanza={{ pgbackrest_stanza }} --no-online stanza-upgrade" - when: pg_path_count.stdout | length > 0 and pgbackrest_stanza_upgrade | bool + when: pg_path_count.stdout | int > 0 and pgbackrest_stanza_upgrade | bool become: true become_user: "{{ pgbackrest_repo_user }}" ignore_errors: true # show the error and continue the playbook execution From a7a116c0f46b910b6e4259d4d4bb958352daa98c Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik <37010174+vitabaks@users.noreply.github.com> Date: Thu, 18 Jul 2024 12:37:43 +0300 Subject: [PATCH 14/18] WAL-G: Specify the '--config' option (#703) --- vars/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/main.yml b/vars/main.yml index f46c0f8b0..920b23806 100644 --- a/vars/main.yml +++ b/vars/main.yml @@ -491,7 +491,7 @@ pg_probackup_patroni_cluster_bootstrap_command: "{{ pg_probackup_command_parts | # WAL-G wal_g_install: false # or 'true' wal_g_version: "3.0.2" -wal_g_path: "/usr/local/bin/wal-g" +wal_g_path: "/usr/local/bin/wal-g --config {{ postgresql_home_dir }}/.walg.json" wal_g_json: # config https://github.com/wal-g/wal-g#configuration - { option: "AWS_ACCESS_KEY_ID", value: "{{ AWS_ACCESS_KEY_ID | default('') }}" } # define values or pass via --extra-vars - { option: "AWS_SECRET_ACCESS_KEY", value: "{{ AWS_SECRET_ACCESS_KEY | default('') }}" } # define values or pass via --extra-vars From 8c7d2b020ac7b00a86393c74f80aaf40c0953d9a Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik <37010174+vitabaks@users.noreply.github.com> Date: Thu, 18 Jul 2024 14:40:11 +0300 Subject: [PATCH 15/18] pgBackRest: Ensure directories exist with correct permissions (#704) --- roles/pgbackrest/stanza-create/tasks/main.yml | 4 +- roles/pgbackrest/tasks/main.yml | 59 +++++++++++++++++-- 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/roles/pgbackrest/stanza-create/tasks/main.yml b/roles/pgbackrest/stanza-create/tasks/main.yml index 9b508888c..334393eda 100644 --- a/roles/pgbackrest/stanza-create/tasks/main.yml +++ b/roles/pgbackrest/stanza-create/tasks/main.yml @@ -12,7 +12,7 @@ state: directory owner: postgres group: postgres - mode: "0755" + mode: "0750" when: repo1_path | length > 0 - name: Create stanza "{{ pgbackrest_stanza }}" @@ -47,7 +47,7 @@ state: directory owner: "{{ pgbackrest_repo_user }}" group: "{{ pgbackrest_repo_user }}" - mode: "0755" + mode: "0750" when: repo1_path | length > 0 - name: Create stanza "{{ pgbackrest_stanza }}" diff --git a/roles/pgbackrest/tasks/main.yml b/roles/pgbackrest/tasks/main.yml index 90a014e6b..e6f2e01e2 100644 --- a/roles/pgbackrest/tasks/main.yml +++ b/roles/pgbackrest/tasks/main.yml @@ -106,7 +106,31 @@ tags: pgbackrest, pgbackrest_install - block: - - name: Ensure spool directory exist + - name: Ensure log directory exists + ansible.builtin.file: + path: "{{ item.value }}" + state: directory + owner: postgres + group: postgres + mode: "0755" + loop: "{{ pgbackrest_conf.global }}" + when: item.option == 'log-path' + loop_control: + label: "{{ item.value }}" + + - name: Ensure repo directory exists + ansible.builtin.file: + path: "{{ item.value }}" + state: directory + owner: postgres + group: postgres + mode: "0750" + loop: "{{ pgbackrest_conf.global }}" + when: item.option == 'repo1-path' and pgbackrest_repo_host | length < 1 + loop_control: + label: "{{ item.value }}" + + - name: Ensure spool directory exists ansible.builtin.file: path: "{{ item.value }}" state: directory @@ -118,12 +142,13 @@ loop_control: label: "{{ item.value }}" - - name: Ensure config directory exist + - name: Ensure config directory exists ansible.builtin.file: path: "{{ pgbackrest_conf_file | dirname }}" state: directory owner: postgres group: postgres + mode: "0750" - name: "Generate conf file {{ pgbackrest_conf_file }}" ansible.builtin.template: @@ -137,19 +162,45 @@ # Dedicated pgbackrest server (if "repo_host" is set) - block: - - name: Ensure config directory exist + - name: Ensure log directory exists + ansible.builtin.file: + path: "{{ item.value }}" + state: directory + owner: "{{ pgbackrest_repo_user }}" + group: "{{ pgbackrest_repo_user }}" + mode: "0755" + loop: "{{ pgbackrest_server_conf.global }}" + when: item.option == 'log-path' + loop_control: + label: "{{ item.value }}" + + - name: Ensure repo directory exists + ansible.builtin.file: + path: "{{ item.value }}" + state: directory + owner: "{{ pgbackrest_repo_user }}" + group: "{{ pgbackrest_repo_user }}" + mode: "0750" + loop: "{{ pgbackrest_server_conf.global }}" + when: item.option == 'repo1-path' + loop_control: + label: "{{ item.value }}" + + - name: Ensure config directory exists ansible.builtin.file: path: "{{ pgbackrest_conf_file | dirname }}" state: directory owner: "{{ pgbackrest_repo_user }}" group: "{{ pgbackrest_repo_user }}" + mode: "0750" - - name: Ensure stanza config directory exist + - name: Ensure stanza config directory exists ansible.builtin.file: path: "{{ pgbackrest_conf_file | dirname }}/conf.d" state: directory owner: "{{ pgbackrest_repo_user }}" group: "{{ pgbackrest_repo_user }}" + mode: "0750" - name: "Generate global conf file {{ pgbackrest_conf_file }}" ansible.builtin.template: From b97312747bfc7bcbb39f57f89e83e3a83129687b Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik <37010174+vitabaks@users.noreply.github.com> Date: Thu, 25 Jul 2024 23:24:07 +0300 Subject: [PATCH 16/18] ssh-keys: Fix the typo in the condition (#706) --- roles/ssh-keys/tasks/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/ssh-keys/tasks/main.yml b/roles/ssh-keys/tasks/main.yml index 83a393a98..996cd6511 100644 --- a/roles/ssh-keys/tasks/main.yml +++ b/roles/ssh-keys/tasks/main.yml @@ -47,7 +47,7 @@ loop: "{{ ssh_known_host_results.results }}" ignore_errors: true when: - - anable_ssh_key_based_authentication is defined + - enable_ssh_key_based_authentication is defined - enable_ssh_key_based_authentication | bool - not ansible_check_mode tags: ssh_keys From 62302d2b9e0bb5de6e19d6d7ca63e65931bcc7bc Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik <37010174+vitabaks@users.noreply.github.com> Date: Tue, 30 Jul 2024 13:53:43 +0300 Subject: [PATCH 17/18] Update vip-manager to v2.6.0 (#708) --- vars/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/main.yml b/vars/main.yml index 920b23806..cf3f4203e 100644 --- a/vars/main.yml +++ b/vars/main.yml @@ -80,7 +80,7 @@ keepalived_virtual_router_id: "{{ cluster_vip.split('.')[3] | int }}" # The last # virtual_router_id - must be unique in the network (available values are 0..255). # vip-manager (if 'cluster_vip' is specified and 'with_haproxy_load_balancing' is 'false') -vip_manager_version: "2.5.0" # version to install +vip_manager_version: "2.6.0" # version to install vip_manager_conf: "/etc/patroni/vip-manager.yml" vip_manager_interval: "1000" # time (in milliseconds) after which vip-manager wakes up and checks if it needs to register or release ip addresses. vip_manager_iface: "{{ vip_interface }}" # interface to which the virtual ip will be added From 69a2c44850d47b990e648747167cb04fc0a503cc Mon Sep 17 00:00:00 2001 From: Vitaliy Kukharik <37010174+vitabaks@users.noreply.github.com> Date: Tue, 30 Jul 2024 16:09:34 +0300 Subject: [PATCH 18/18] Update etcd to v3.5.15 (#709) --- vars/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vars/main.yml b/vars/main.yml index cf3f4203e..19decc77f 100644 --- a/vars/main.yml +++ b/vars/main.yml @@ -93,7 +93,7 @@ dcs_exists: false # or 'true' if you don't want to deploy a new etcd cluster dcs_type: "etcd" # or 'consul' # if dcs_type: "etcd" and dcs_exists: false -etcd_version: "3.5.11" # version for deploy etcd cluster +etcd_version: "3.5.15" # version for deploy etcd cluster etcd_data_dir: "/var/lib/etcd" etcd_cluster_name: "etcd-{{ patroni_cluster_name }}" # ETCD_INITIAL_CLUSTER_TOKEN