From bb7cb8274958e08661f2c520fed7bbcdb84e233d Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Wed, 10 Apr 2024 17:56:34 +0200 Subject: [PATCH 1/4] restore-module. Do not overwrite node vars IP address and port number are allocated by the cluster: do not overwrite them during the restore from backup. Traefik must be restarted to expand restored credentials. --- imageroot/actions/restore-module/06copyenv | 2 -- imageroot/actions/restore-module/90restart | 8 ++++++++ 2 files changed, 8 insertions(+), 2 deletions(-) create mode 100755 imageroot/actions/restore-module/90restart diff --git a/imageroot/actions/restore-module/06copyenv b/imageroot/actions/restore-module/06copyenv index 1f19667..f861062 100755 --- a/imageroot/actions/restore-module/06copyenv +++ b/imageroot/actions/restore-module/06copyenv @@ -29,11 +29,9 @@ request = json.load(sys.stdin) original_environment = request['environment'] for evar in [ - "LOKI_ADDR", "LOKI_API_AUTH_USERNAME", "LOKI_API_AUTH_PASSWORD", "LOKI_LOGS_INGRESS_TOKEN", - "LOKI_HTTP_PORT", "LOKI_RETENTION_PERIOD", ]: agent.set_env(evar, original_environment[evar]) diff --git a/imageroot/actions/restore-module/90restart b/imageroot/actions/restore-module/90restart new file mode 100755 index 0000000..20c3035 --- /dev/null +++ b/imageroot/actions/restore-module/90restart @@ -0,0 +1,8 @@ +#!/usr/bin/env sh + +# +# Copyright (C) 2024 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +systemctl --user restart loki.service From abc7c42768e083e8c2a85ed18b807354dbc2813f Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Wed, 10 Apr 2024 17:57:42 +0200 Subject: [PATCH 2/4] Fixed internal port number for Traefik As the pod network namespace is private, we can use a fixed port number to avoid confusion. The allocated dynamic port number is necessary only for the public side of the pod. --- imageroot/systemd/user/loki.service | 2 +- imageroot/systemd/user/traefik.service | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/imageroot/systemd/user/loki.service b/imageroot/systemd/user/loki.service index f8bf480..5d5f3df 100644 --- a/imageroot/systemd/user/loki.service +++ b/imageroot/systemd/user/loki.service @@ -13,7 +13,7 @@ ExecStartPre=/usr/bin/podman pod create \ --infra-conmon-pidfile %t/loki.pid \ --pod-id-file %t/loki.pod-id \ --name loki \ - --publish ${LOKI_HTTP_PORT}:${LOKI_HTTP_PORT} \ + --publish ${LOKI_HTTP_PORT}:8080 \ --replace \ --network=slirp4netns:allow_host_loopback=true ExecStart=/usr/bin/podman pod start --pod-id-file %t/loki.pod-id diff --git a/imageroot/systemd/user/traefik.service b/imageroot/systemd/user/traefik.service index cef0e2f..e6e7f5a 100644 --- a/imageroot/systemd/user/traefik.service +++ b/imageroot/systemd/user/traefik.service @@ -19,7 +19,7 @@ ExecStart=/usr/bin/podman run \ --name=%N \ --volume=./traefik.yaml:/etc/traefik.yaml:Z \ ${TRAEFIK_IMAGE} \ - --entryPoints.loki.address=:${LOKI_HTTP_PORT} \ + --entryPoints.loki.address=:8080 \ --providers.file.filename=/etc/traefik.yaml ExecStop=/usr/bin/podman stop --ignore --cidfile %t/traefik.ctr-id -t 10 ExecStopPost=/usr/bin/podman rm --ignore -f --cidfile %t/traefik.ctr-id From f8e0d506a41e4bd431d5dedbcba01114bd952041 Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Thu, 11 Apr 2024 15:08:46 +0200 Subject: [PATCH 3/4] Use the default Podman private network implementation --- imageroot/systemd/user/loki.service | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/imageroot/systemd/user/loki.service b/imageroot/systemd/user/loki.service index 5d5f3df..0181d7c 100644 --- a/imageroot/systemd/user/loki.service +++ b/imageroot/systemd/user/loki.service @@ -14,8 +14,7 @@ ExecStartPre=/usr/bin/podman pod create \ --pod-id-file %t/loki.pod-id \ --name loki \ --publish ${LOKI_HTTP_PORT}:8080 \ - --replace \ - --network=slirp4netns:allow_host_loopback=true + --replace ExecStart=/usr/bin/podman pod start --pod-id-file %t/loki.pod-id ExecStop=/usr/bin/podman pod stop --ignore --pod-id-file %t/loki.pod-id -t 10 ExecStopPost=/usr/bin/podman pod rm --ignore -f --pod-id-file %t/loki.pod-id From aafad6d104203f9b87af732cc65435d57ee43346 Mon Sep 17 00:00:00 2001 From: Davide Principi Date: Wed, 10 Apr 2024 18:34:19 +0200 Subject: [PATCH 4/4] Fix backup/restore procedure - Clean up volume before Restic restores it - Restore the "active from" timestamp - Restore the "active to" timestamp, by reading it from volume - Do not restore credentials and tokens: use new ones --- imageroot/actions/restore-module/06copyenv | 25 +++--------- .../actions/restore-module/07reinit_data | 14 +++++++ .../restore-module/85restore_active_to | 38 +++++++++++++++++++ .../{90restart => 90start_server} | 4 +- 4 files changed, 60 insertions(+), 21 deletions(-) create mode 100755 imageroot/actions/restore-module/07reinit_data create mode 100755 imageroot/actions/restore-module/85restore_active_to rename imageroot/actions/restore-module/{90restart => 90start_server} (69%) diff --git a/imageroot/actions/restore-module/06copyenv b/imageroot/actions/restore-module/06copyenv index f861062..2e11311 100755 --- a/imageroot/actions/restore-module/06copyenv +++ b/imageroot/actions/restore-module/06copyenv @@ -1,37 +1,22 @@ #!/usr/bin/env python3 # -# Copyright (C) 2022 Nethesis S.r.l. -# http://www.nethesis.it - nethserver@nethesis.it -# -# This script is part of NethServer. -# -# NethServer is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, -# or any later version. -# -# NethServer is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with NethServer. If not, see COPYING. +# Copyright (C) 2024 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later # import sys import json import agent +import os request = json.load(sys.stdin) original_environment = request['environment'] for evar in [ - "LOKI_API_AUTH_USERNAME", - "LOKI_API_AUTH_PASSWORD", - "LOKI_LOGS_INGRESS_TOKEN", "LOKI_RETENTION_PERIOD", + "LOKI_ACTIVE_FROM", + # NOTE: LOKI_ACTIVE_TO is restored by a later step ]: agent.set_env(evar, original_environment[evar]) diff --git a/imageroot/actions/restore-module/07reinit_data b/imageroot/actions/restore-module/07reinit_data new file mode 100755 index 0000000..bde4302 --- /dev/null +++ b/imageroot/actions/restore-module/07reinit_data @@ -0,0 +1,14 @@ +#!/usr/bin/env sh + +# +# Copyright (C) 2024 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +exec 1>&2 + +# Stop the pod +systemctl --user stop loki.service + +# At index 20, volume contents are restored by Restic. +podman volume rm loki-server-data diff --git a/imageroot/actions/restore-module/85restore_active_to b/imageroot/actions/restore-module/85restore_active_to new file mode 100755 index 0000000..b8b2437 --- /dev/null +++ b/imageroot/actions/restore-module/85restore_active_to @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 + +# +# Copyright (C) 2024 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-3.0-or-later +# + +import sys +import agent +import subprocess +import os +import sys + +try: + # Find the biggest modification timestamp among Loki's subdirs, + # and print it in ISO8601 format. Note that the timezone requires + # the ":" separator, otherwise the parsing fails. + find_latest_change_script = """find /loki -type d | + xargs -- stat -c %Y | + awk 'BEGIN { ts = 0 } ; { ts = $1 > ts ? $1 : ts ; } ; END { print ts }' | + xargs -IDATE -- date -u -d @DATE +%Y-%m-%dT%H:%M:%S.0+00:00 + """ + proc_active_to = subprocess.run(["podman", "run", "-i", + "--name=loki-server-restore", "--replace", "--rm", + "--network=none", "--volume=loki-server-data:/loki:z", + '--entrypoint=["ash","-s"]', + os.environ["LOKI_IMAGE"], + ], + stdout=subprocess.PIPE, + stderr=sys.stderr, + input=find_latest_change_script, + text=True, + check=True, + ) +except Exception as ex: + print(agent.SD_WARNING + "Cannot restore LOKI_ACTIVE_TO:", str(ex), file=sys.stderr) +else: + agent.set_env("LOKI_ACTIVE_TO", proc_active_to.stdout.strip()) diff --git a/imageroot/actions/restore-module/90restart b/imageroot/actions/restore-module/90start_server similarity index 69% rename from imageroot/actions/restore-module/90restart rename to imageroot/actions/restore-module/90start_server index 20c3035..c7d6b76 100755 --- a/imageroot/actions/restore-module/90restart +++ b/imageroot/actions/restore-module/90start_server @@ -5,4 +5,6 @@ # SPDX-License-Identifier: GPL-3.0-or-later # -systemctl --user restart loki.service +exec 1>&2 + +systemctl --user start loki.service