diff --git a/build-alpine-netboot-zfs.sh b/build-alpine-netboot-zfs.sh index 71fda95..c7ed6ab 100644 --- a/build-alpine-netboot-zfs.sh +++ b/build-alpine-netboot-zfs.sh @@ -4,61 +4,63 @@ # Build netboot image with zfs kernel module included # USAGE: -# podman run -it --rm -v $(pwd):/root/workdir alpine sh /root/workdir/build-alpine-netboot-zfs.sh + +# podman run -it --rm -v $(pwd)/scratch:/root/workdir alpine sh /root/workdir/build-alpine-netboot-zfs.sh set -x -apk add alpine-sdk build-base apk-tools alpine-conf busybox fakeroot syslinux xorriso squashfs-tools sudo git grub grub-efi +apk add alpine-sdk build-base apk-tools busybox fakeroot syslinux xorriso squashfs-tools sudo git grub grub-efi + +# Note we build alpine-conf from source due to issue https://github.com/KarmaComputing/server-bootstrap/issues/20 +# Clone and build latest alpine-conf +git clone https://gitlab.alpinelinux.org/alpine/alpine-conf.git +cd alpine-conf +make +make install +cd - +# Start build adduser build --disabled-password -G abuild # Set password non interactively echo -e "password\npassword" | passwd build echo "build ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers.d/abuild -cp -R /root/workdir /home/build/ -chown -R build /home/build/workdir + su - build << 'EOF' set -x SUDO=sudo abuild-keygen -n -i -a -cd workdir +# aports contains build utilities such as mkimage.sh git clone --depth 1 https://gitlab.alpinelinux.org/alpine/aports cd aports -mkdir -p ~/iso -# Enable zfs kernel module + +# Create & build alpine netboot profile with zfs kernel module enabled cat > ./scripts/mkimg.zfsnetboot.sh << 'EOFINNER' + profile_zfsnetboot() { profile_standard - kernel_cmdline="unionfs_size=512M console=tty0 console=ttyS0,115200" + kernel_cmdline="overlay_size=0 console=tty0 console=ttyS0,115200" syslinux_serial="0 115200" kernel_addons="zfs" - apks="$apks zfs-scripts zfs zfs-utils-py python3 - mkinitfs - syslinux util-linux" + apks="$apks zfs-scripts zfs zfs-utils-py python3 mkinitfs syslinux util-linux linux-firmware" initfs_features="base network squashfs usb virtio" - local _k _a - for _k in $kernel_flavors; do - apks="$apks linux-$_k" - for _a in $kernel_addons; do - apks="$apks $_a-$_k" - done - done - apks="$apks linux-firmware" output_format="netboot" image_ext="tar.gz" } EOFINNER cat ./scripts/mkimg.zfsnetboot.sh echo Running mkimage.sh +mkdir -p ~/iso ./scripts/mkimage.sh --outdir ~/iso --arch x86_64 --repository http://dl-cdn.alpinelinux.org/alpine/edge/main --profile zfsnetboot EOF +ls -l /home/build/iso mkdir -p /root/workdir/iso cp /home/build/iso/alpine-zfsnetboot-*.tar.gz /root/workdir/iso exit -# back on the host machine +# We're back outside the container at this point ls -ltr | tail -n 1 # latest build # Upload (scp) and extract latest build (e.g. -# alpine-netboot-230813-x86_64.tar.gz to boot server) +# alpine-zfsnetboot-*-x86_64.tar.gz to boot server) diff --git a/src/playbooks/servers.yaml b/src/playbooks/servers.yaml index a4d8c6e..a938803 100644 --- a/src/playbooks/servers.yaml +++ b/src/playbooks/servers.yaml @@ -16,6 +16,33 @@ repo: deb http://deb.debian.org/debian bullseye-backports main contrib state: present + # - name: Template ~/.vimrc + # ansible.builtin.template: + # src: ./.vimrc + # dest: /root/.vimrc + # owner: root + # group: root + # mode: '0644' + # tags: + # - dotfiles + + # template minimal (loopback*) /etc/network/interfaces + # during bootstrap (interfaces are already configured with global IPs at this point, + # this is to satisfy `setup-ntp busybox` which can't operate on an + # empty /etc/network/interfaces file. + # *only loopback is needed because iPXE has configured interfaces already, + # there's no need to persist that to disk since we're booted into a minumal + # alpine image at this point (netboot) which will be blown away after + # Fedora/persistant operating system is installed + - name: Template /etc/network/interfaces + ansible.builtin.template: + src: etc/network/interfaces + dest: /etc/network/interfaces + owner: root + group: root + mode: '0644' + tags: + - network - name: Install openssh-server ansible.builtin.apt: @@ -30,10 +57,37 @@ - dpkg-dev - linux-headers-amd64 - - name: Install zfsutils-linux - ansible.builtin.apt: - name: zfsutils-linux - state: latest + - apk: + name: eudev,lsblk,sgdisk,jq,wipefs + update_cache: yes + tags: + - packages + + - name: Run udev + command: setup-devd udev + tags: + - udev + + + - name: Copy file wipe-all-disks.sh + ansible.builtin.copy: + src: ./scripts/wipe-all-disks.sh + dest: /root/wipe-all-disks.sh + owner: root + group: root + mode: '0755' + tags: + - scripts + + - name: Copy file install-fedora-root-on-zfs.sh + ansible.builtin.copy: + src: ./scripts/install-fedora-root-on-zfs.sh + dest: /root/install-fedora-root-on-zfs.sh + owner: root + group: root + mode: '0755' + tags: + - scripts - name: Disable swap during play command: swapoff --all diff --git a/src/requirements.txt b/src/requirements.txt index 38841a2..674f85c 100644 --- a/src/requirements.txt +++ b/src/requirements.txt @@ -5,3 +5,6 @@ gunicorn PyMySQL coloredlogs python-dotenv +ansible +strictyaml +apiflask diff --git a/src/web-ui/app.py b/src/web-ui/app.py index 0b6a56f..1e860b8 100644 --- a/src/web-ui/app.py +++ b/src/web-ui/app.py @@ -24,12 +24,146 @@ "IDRAC_SCRIPTS_BASE_PATH", "./iDRAC-Redfish-Scripting/Redfish Python/" ) -HOST_HEALTHCHECK_POLL_IP = os.getenv("HOST_HEALTHCHECK_POLL_IP") +HOST_HEALTHCHECK_POLL_IP = settings.get("HOST_HEALTHCHECK_POLL_IP") session_requests = requests.Session() session_requests.verify = False +def countdown(seconds): + log.info(f"Sleeping for {seconds} seconds") + for remaining in range(seconds, 0, -1): + sys.stdout.write(f"\rTime left: {remaining} seconds") + sys.stdout.flush() + time.sleep(1) + # Clear the line after countdown ends + sys.stdout.write("\rCountdown finished!\n") + + +def ConnectToVPN(): + """ + Attempt to connect to VPN + Assumptions: + - Any existing VPN connection will be torn down + - Credentials for VPN will be fetched using secret(s) + required to fetch them + - VPN tunnel (wireguard) will be started + """ + log.info( + "Tear down any existing VPN connection " + "(assumes wg-quick is used for WireGuard" + ) + subprocess.run(["wg-quick", "down", "wg0"], check=False) + + log.info("Download the psonoci tool") + subprocess.run( + [ + "curl", + "https://get.psono.com/psono/psono-ci/x86_64-linux/psonoci", + "--output", + "./psonoci", + ], + check=True, + ) + + log.info("Mark psonoci as executable") + subprocess.run(["chmod", "+x", "./psonoci"], check=True) + + # Fetch credentials using the psonoci tool + PSONO_CI_VPN_SECRET_NOTE_ID = settings.get( + "PSONO_CI_VPN_SECRET_NOTE_ID" + ).value # noqa: E501 + + try: + os.environ["PSONO_CI_API_KEY_ID"] = settings.get( + "PSONO_CI_API_KEY_ID" + ).value # noqa: E501 + os.environ["PSONO_CI_API_SECRET_KEY_HEX"] = settings.get( + "PSONO_CI_API_SECRET_KEY_HEX" + ).value + os.environ["PSONO_CI_SERVER_URL"] = settings.get( + "PSONO_CI_SERVER_URL" + ).value # noqa: E501 + result = subprocess.run( + [ + "./psonoci", + "secret", + "get", + PSONO_CI_VPN_SECRET_NOTE_ID, + "notes", + ], # noqa: E501 + check=True, + capture_output=True, + text=True, + env=os.environ, + ) + log.info(result) + except Exception as e: + log.error(e) + + vpn_config = result.stdout.strip() + + log.debug("Write the VPN configuration to /etc/wireguard/wg0.conf") + with open("/etc/wireguard/wg0.conf", "w") as vpn_file: + vpn_file.write(vpn_config) + + try: + log.debug("Start the VPN tunnel") + sleep(3) + subprocess.run(["wg-quick", "up", "wg0"], check=False) + print("VPN connected successfully.") + + except subprocess.CalledProcessError as e: + print(f"An error occurred while executing a command: {e}") + except Exception as e: + print(f"An unexpected error occurred: {e}") + + +def recover_from_error_vpn_not_active(retry_state): + """Attempt to recover from error VPN + not active. + """ + log.debug(retry_state) + ConnectToVPN() + + +@retry( + wait=wait_exponential(multiplier=1, min=5, max=10), + before=before_log(log, logging.DEBUG), + stop=stop_after_attempt(4), + retry_error_callback=recover_from_error_vpn_not_active, +) +def vpn_must_be_up(f): + """ + Checks for a route to the IDRAC_HOST + The/a valid VPN connection + must be up for the majority of the server + bootstrap process to work. + + If the VPN is *up*, then the IDRAC_HOST + will be reachable (there will be a route to + that host/IP). + If the VPN is *down* then the IDRAC_HOST will + likely be 'no route to host'. + """ + + @wraps(f) + def wrapper(*args, **kwds): + log.info("Calling wrapper vpn_must_be_up") + try: + print(settings.get('IDRAC_HOST')) + url = f"https://{settings.get('IDRAC_HOST')}/start.html" + log.info(f"Contacting: {url}") + #requests.get(url, verify=False, timeout=DEFAULT_HTTP_REQ_TIMEOUT) + except Exception as e: + log.error(f"Verify VPN connection is up & functioning. {e}") + log.debug("Attempting reconnect of VPN") + #ConnectToVPN() + return f(*args, **kwds) + + return wrapper + + def api_response(req): return ( jsonify({"resp": req.text, "status_code": req.status_code}), @@ -39,9 +173,12 @@ def api_response(req): def api_call(path=None, method=None, payload=None, raw_payload=False): assert method is not None - url = f"https://{os.getenv('IDRAC_HOST')}/redfish/v1/{path}" + if "redfish" not in path and "http" not in path: + url = f"https://{settings.get('IDRAC_HOST')}/redfish/v1/{path}" + if "redfish" in path: + url = f"https://{settings.get('IDRAC_HOST')}/{path}" authHeaders = HTTPBasicAuth( - os.getenv("IDRAC_USERNAME"), os.getenv("IDRAC_PASSWORD") + settings.get("IDRAC_USERNAME"), settings.get("IDRAC_PASSWORD") ) # noqa: E501 # Making the request @@ -97,7 +234,7 @@ def load_idrac_settings(): @app.context_processor def inject_settings(): - return dict(IDRAC_HOST=os.getenv("IDRAC_HOST")) + return dict(IDRAC_HOST=settings.get("IDRAC_HOST")) @app.route("/")