wip #20

build-alpine-netboot-zfs.sh

@@ -4,61 +4,63 @@
 # Build netboot image with zfs kernel module included
 
 # USAGE:
-# podman run  -it --rm -v $(pwd):/root/workdir alpine sh /root/workdir/build-alpine-netboot-zfs.sh
+
+# podman run  -it --rm -v $(pwd)/scratch:/root/workdir alpine sh /root/workdir/build-alpine-netboot-zfs.sh
 
 
 set -x
 
-apk add alpine-sdk build-base apk-tools alpine-conf busybox fakeroot syslinux xorriso squashfs-tools sudo git grub grub-efi
+apk add alpine-sdk build-base apk-tools busybox fakeroot syslinux xorriso squashfs-tools sudo git grub grub-efi
+
+# Note we build alpine-conf from source due to issue https://github.com/KarmaComputing/server-bootstrap/issues/20
+# Clone and build latest alpine-conf
+git clone https://gitlab.alpinelinux.org/alpine/alpine-conf.git
+cd alpine-conf
+make
+make install
+cd -
 
 
+# Start build
 adduser build --disabled-password -G abuild
 # Set password non interactively
 echo -e "password\npassword" | passwd build
 echo "build ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers.d/abuild
-cp -R /root/workdir /home/build/
-chown -R build /home/build/workdir
+
 su - build << 'EOF'
 set -x
 SUDO=sudo abuild-keygen -n -i -a
-cd workdir
+# aports contains build utilities such as mkimage.sh
 git clone --depth 1 https://gitlab.alpinelinux.org/alpine/aports
 cd aports
-mkdir -p ~/iso
-# Enable zfs kernel module
+
+# Create & build alpine netboot profile with zfs kernel module enabled
 cat > ./scripts/mkimg.zfsnetboot.sh << 'EOFINNER'
+
 profile_zfsnetboot() {
         profile_standard
-        kernel_cmdline="unionfs_size=512M console=tty0 console=ttyS0,115200"
+        kernel_cmdline="overlay_size=0 console=tty0 console=ttyS0,115200"
         syslinux_serial="0 115200"
         kernel_addons="zfs"
-        apks="$apks zfs-scripts zfs zfs-utils-py python3
-                mkinitfs
-                syslinux util-linux"
+        apks="$apks zfs-scripts zfs zfs-utils-py python3 mkinitfs syslinux util-linux linux-firmware"
         initfs_features="base network squashfs usb virtio"
-        local _k _a
-        for _k in $kernel_flavors; do
-                apks="$apks linux-$_k"
-                for _a in $kernel_addons; do
-                        apks="$apks $_a-$_k"
-                done
-        done
-        apks="$apks linux-firmware"
         output_format="netboot"
         image_ext="tar.gz"
 }
 EOFINNER
 cat ./scripts/mkimg.zfsnetboot.sh
 echo Running mkimage.sh
+mkdir -p ~/iso
 ./scripts/mkimage.sh --outdir ~/iso --arch x86_64 --repository http://dl-cdn.alpinelinux.org/alpine/edge/main --profile zfsnetboot
 EOF
 
 
+ls -l /home/build/iso
 mkdir -p /root/workdir/iso
 cp /home/build/iso/alpine-zfsnetboot-*.tar.gz /root/workdir/iso
 exit
-# back on the host machine
+# We're back outside the container at this point
 ls -ltr | tail -n 1 # latest build
 # Upload (scp) and extract latest build (e.g.
-# alpine-netboot-230813-x86_64.tar.gz to boot server)
+# alpine-zfsnetboot-*-x86_64.tar.gz to boot server)
 

src/playbooks/servers.yaml

@@ -16,6 +16,33 @@
       repo: deb http://deb.debian.org/debian bullseye-backports main contrib
       state: present
 
+  # - name: Template ~/.vimrc
+  #   ansible.builtin.template:
+  #     src: ./.vimrc
+  #     dest: /root/.vimrc
+  #     owner: root
+  #     group: root
+  #     mode: '0644'
+  #   tags:
+  #   - dotfiles
+
+  # template minimal (loopback*) /etc/network/interfaces 
+  # during bootstrap (interfaces are already configured with global IPs at this point,
+  # this is to satisfy `setup-ntp busybox` which can't operate on an
+  # empty /etc/network/interfaces file.
+  # *only loopback is needed because iPXE has configured interfaces already,
+  # there's no need to persist that to disk since we're booted into a minumal
+  # alpine image at this point (netboot) which will be blown away after
+  # Fedora/persistant operating system is installed
+  - name: Template /etc/network/interfaces
+    ansible.builtin.template:
+      src: etc/network/interfaces
+      dest: /etc/network/interfaces
+      owner: root
+      group: root
+      mode: '0644'
+    tags:
+    - network
 
   - name: Install openssh-server
     ansible.builtin.apt:
@@ -30,10 +57,37 @@
       - dpkg-dev
       - linux-headers-amd64
 
-  - name: Install zfsutils-linux
-    ansible.builtin.apt:
-      name: zfsutils-linux
-      state: latest
+  - apk:
+      name: eudev,lsblk,sgdisk,jq,wipefs
+      update_cache: yes
+    tags:
+    - packages
+
+  - name: Run udev
+    command: setup-devd udev
+    tags:
+    - udev
+
+
+  - name: Copy file wipe-all-disks.sh
+    ansible.builtin.copy:
+      src: ./scripts/wipe-all-disks.sh
+      dest: /root/wipe-all-disks.sh
+      owner: root
+      group: root
+      mode: '0755'
+    tags:
+    - scripts
+
+  - name: Copy file install-fedora-root-on-zfs.sh
+    ansible.builtin.copy:
+      src: ./scripts/install-fedora-root-on-zfs.sh
+      dest: /root/install-fedora-root-on-zfs.sh
+      owner: root
+      group: root
+      mode: '0755'
+    tags:
+    - scripts
 
   - name: Disable swap during play
     command: swapoff --all

src/requirements.txt

@@ -5,3 +5,6 @@ gunicorn
 PyMySQL
 coloredlogs
 python-dotenv
+ansible
+strictyaml
+apiflask

src/web-ui/app.py

@@ -24,12 +24,146 @@
     "IDRAC_SCRIPTS_BASE_PATH", "./iDRAC-Redfish-Scripting/Redfish Python/"
 )
 
-HOST_HEALTHCHECK_POLL_IP = os.getenv("HOST_HEALTHCHECK_POLL_IP")
+HOST_HEALTHCHECK_POLL_IP = settings.get("HOST_HEALTHCHECK_POLL_IP")
 
 session_requests = requests.Session()
 session_requests.verify = False
 
 
+def countdown(seconds):
+    log.info(f"Sleeping for {seconds} seconds")
+    for remaining in range(seconds, 0, -1):
+        sys.stdout.write(f"\rTime left: {remaining} seconds")
+        sys.stdout.flush()
+        time.sleep(1)
+    # Clear the line after countdown ends
+    sys.stdout.write("\rCountdown finished!\n")
+
+
+def ConnectToVPN():
+    """
+    Attempt to connect to VPN
+    Assumptions:
+    - Any existing VPN connection will be torn down
+    - Credentials for VPN will be fetched using secret(s)
+      required to fetch them
+    - VPN tunnel (wireguard) will be started
+    """
+    log.info(
+        "Tear down any existing VPN connection "
+        "(assumes wg-quick is used for WireGuard"
+    )
+    subprocess.run(["wg-quick", "down", "wg0"], check=False)
+
+    log.info("Download the psonoci tool")
+    subprocess.run(
+        [
+            "curl",
+            "https://get.psono.com/psono/psono-ci/x86_64-linux/psonoci",
+            "--output",
+            "./psonoci",
+        ],
+        check=True,
+    )
+
+    log.info("Mark psonoci as executable")
+    subprocess.run(["chmod", "+x", "./psonoci"], check=True)
+
+    # Fetch credentials using the psonoci tool
+    PSONO_CI_VPN_SECRET_NOTE_ID = settings.get(
+        "PSONO_CI_VPN_SECRET_NOTE_ID"
+    ).value  # noqa: E501
+
+    try:
+        os.environ["PSONO_CI_API_KEY_ID"] = settings.get(
+            "PSONO_CI_API_KEY_ID"
+        ).value  # noqa: E501
+        os.environ["PSONO_CI_API_SECRET_KEY_HEX"] = settings.get(
+            "PSONO_CI_API_SECRET_KEY_HEX"
+        ).value
+        os.environ["PSONO_CI_SERVER_URL"] = settings.get(
+            "PSONO_CI_SERVER_URL"
+        ).value  # noqa: E501
+        result = subprocess.run(
+            [
+                "./psonoci",
+                "secret",
+                "get",
+                PSONO_CI_VPN_SECRET_NOTE_ID,
+                "notes",
+            ],  # noqa: E501
+            check=True,
+            capture_output=True,
+            text=True,
+            env=os.environ,
+        )
+        log.info(result)
+    except Exception as e:
+        log.error(e)
+
+    vpn_config = result.stdout.strip()
+
+    log.debug("Write the VPN configuration to /etc/wireguard/wg0.conf")
+    with open("/etc/wireguard/wg0.conf", "w") as vpn_file:
+        vpn_file.write(vpn_config)
+
+    try:
+        log.debug("Start the VPN tunnel")
+        sleep(3)
+        subprocess.run(["wg-quick", "up", "wg0"], check=False)
+        print("VPN connected successfully.")
+
+    except subprocess.CalledProcessError as e:
+        print(f"An error occurred while executing a command: {e}")
+    except Exception as e:
+        print(f"An unexpected error occurred: {e}")
+
+
+def recover_from_error_vpn_not_active(retry_state):
+    """Attempt to recover from error VPN
+    not active.
+    """
+    log.debug(retry_state)
+    ConnectToVPN()
+
+
+@retry(
+    wait=wait_exponential(multiplier=1, min=5, max=10),
+    before=before_log(log, logging.DEBUG),
+    stop=stop_after_attempt(4),
+    retry_error_callback=recover_from_error_vpn_not_active,
+)
+def vpn_must_be_up(f):
+    """
+    Checks for a route to the IDRAC_HOST
+    The/a valid VPN connection
+    must be up for the majority of the server
+    bootstrap process to work.
+
+    If the VPN is *up*, then the IDRAC_HOST
+    will be reachable (there will be a route to
+    that host/IP).
+    If the VPN is *down* then the IDRAC_HOST will
+    likely be 'no route to host'.
+    """
+
+    @wraps(f)
+    def wrapper(*args, **kwds):
+        log.info("Calling wrapper vpn_must_be_up")
+        try:
+            print(settings.get('IDRAC_HOST'))
+            url = f"https://{settings.get('IDRAC_HOST')}/start.html"
+            log.info(f"Contacting: {url}")
+            #requests.get(url, verify=False, timeout=DEFAULT_HTTP_REQ_TIMEOUT)
+        except Exception as e:
+            log.error(f"Verify VPN connection is up & functioning. {e}")
+            log.debug("Attempting reconnect of VPN")
+            #ConnectToVPN()
+        return f(*args, **kwds)
+
+    return wrapper
+
+
 def api_response(req):
     return (
         jsonify({"resp": req.text, "status_code": req.status_code}),
@@ -39,9 +173,12 @@ def api_response(req):
 
 def api_call(path=None, method=None, payload=None, raw_payload=False):
     assert method is not None
-    url = f"https://{os.getenv('IDRAC_HOST')}/redfish/v1/{path}"
+    if "redfish" not in path and "http" not in path:
+        url = f"https://{settings.get('IDRAC_HOST')}/redfish/v1/{path}"
+    if "redfish" in path:
+        url = f"https://{settings.get('IDRAC_HOST')}/{path}"
     authHeaders = HTTPBasicAuth(
-        os.getenv("IDRAC_USERNAME"), os.getenv("IDRAC_PASSWORD")
+        settings.get("IDRAC_USERNAME"), settings.get("IDRAC_PASSWORD")
     )  # noqa: E501
 
     # Making the request
@@ -97,7 +234,7 @@ def load_idrac_settings():
 
 @app.context_processor
 def inject_settings():
-    return dict(IDRAC_HOST=os.getenv("IDRAC_HOST"))
+    return dict(IDRAC_HOST=settings.get("IDRAC_HOST"))
 
 
 @app.route("/")