NIC Configuration Updates and MGMT VM Automation (#10)

crucible.spec

@@ -28,6 +28,8 @@
 # https://github.com/openSUSE/python-rpm-macros#terminology
 %define pythons %(echo $PYTHON_BIN)
 
+%define conf_dir /etc/%{name}
+
 # python*-devel is not listed because we do not ship the ability to rebuild our PIP package.
 AutoReqProv: no
 BuildRequires: python-rpm-generators
@@ -73,8 +75,12 @@ sed -i 's:^#!.*$:#!%{install_dir}/bin/python:' %{buildroot}%{install_dir}/bin/%{
 # Add the PoC mock files
 cp -pr poc-mocks %{buildroot}%{install_dir}
 
-mkdir -p %{buildroot}/usr/bin/
-ln -snf %{install_dir}/bin/%{name} %{buildroot}/usr/bin/%{name}
+install -D -m 755 -d %{buildroot}%{_bindir}
+ln -snf %{install_dir}/scripts/lsnics.sh %{buildroot}%{_bindir}/lsnics
+ln -snf %{install_dir}/bin/%{name} %{buildroot}%{_bindir}/%{name}
+
+install -D -m 755 -d %{buildroot}%{conf_dir}
+install -m 644 %{name}/network/ifname.yml %{buildroot}%{conf_dir}/ifname.yml
 
 find %{buildroot}%{install_dir} | sed 's:'${RPM_BUILD_ROOT}'::' | tee -a INSTALLED_FILES
 cat INSTALLED_FILES | xargs -i sh -c 'test -f $RPM_BUILD_ROOT{} && echo {} || test -L $RPM_BUILD_ROOT{} && echo {} || echo %dir {}' | sort -u > FILES
@@ -83,6 +89,8 @@ cat INSTALLED_FILES | xargs -i sh -c 'test -f $RPM_BUILD_ROOT{} && echo {} || te
 
 %files -f FILES
 /usr/bin/%{name}
+/usr/bin/lsnics
+%config(noreplace) %{conf_dir}/ifname.yml
 %doc README.adoc
 %defattr(755,root,root)
 %license LICENSE

crucible/cli.py

@@ -30,6 +30,7 @@
 import click
 from click_option_group import optgroup
 from click_option_group import MutuallyExclusiveOptionGroup
+from crucible import vms
 
 from crucible.install import install_to_disk
 from crucible.network import config
@@ -321,3 +322,53 @@ def install(**kwargs) -> None:
     """
     LOG.info('Calling install with: %s', kwargs)
     install_to_disk(**kwargs)
+
+
+@crucible.group()
+def vm() -> None:
+    # pylint: disable=invalid-name
+    """
+    Functions for configuring VMs.
+
+    \f
+    """
+    LOG.info('Invoked vm group.')
+
+
+@vm.command()
+@click.option(
+    '--capacity',
+    default=100,
+    type=int,
+    is_flag=False,
+    metavar='<int>',
+    help="Capacity of the management VM disk in Gigabytes (default 100).",
+)
+@click.option(
+    '--interface',
+    default='lan0',
+    type=str,
+    is_flag=False,
+    help="Interface to use for the external network.",
+)
+@click.option(
+    '--ssh-key-path',
+    default='/root/.ssh/id_rsa.pub',
+    type=str,
+    is_flag=False,
+    help="Path to an SSH public key to add to the VM's root user.",
+)
+def start(**kwargs) -> None:
+    """
+    Starts the management VM.
+    :param kwargs:
+    """
+    vms.start(**kwargs)
+
+
+@vm.command()
+def reset() -> None:
+    """
+    Resets the VM, purges all assets.
+    """
+    vms.reset()

crucible/network/ifname.py

@@ -363,7 +363,14 @@ def _ifname_meta() -> dict:
     """
     Opens the ifname.yml datafile.
     """
-    ifname_yml_path = os.path.join(os.path.dirname(__file__), 'ifname.yml')
+    default_yaml_path = '/etc/crucible/ifname'
+    if os.path.exists(f'{default_yaml_path}.yml'):
+        ifname_yml_path = os.path.join(f'{default_yaml_path}.yml')
+    elif os.path.exists(f'{default_yaml_path}.yaml'):
+        ifname_yml_path = os.path.join(f'{default_yaml_path}.yaml')
+    else:
+        ifname_yml_path = os.path.join(os.path.dirname(__file__), 'ifname.yml')
+    LOG.info('Using NIC database file: %s',ifname_yml_path)
     with open(ifname_yml_path, 'r', encoding='utf-8') as ifname_yml:
         return safe_load(ifname_yml.read())
 

crucible/os.py

@@ -38,25 +38,27 @@
 
 from crucible.logger import Logger
 
-LOG = Logger(__name__)
+LOG = Logger(__file__)
 
 
 class _CLI:
     """
     An object to abstract the return result from ``run_command``.
-
     """
-    _stdout = ''
-    _stderr = ''
+
+    _stdout = b''
+    _stderr = b''
     _return_code = None
     _duration = None
 
     def __init__(self, args: [str, list], shell: bool = False) -> None:
         """
+        Create a ``Popen`` object.
+
         If shell==True then the arguments will be converted to a string if
         a list was passed.
         The conversion is recommended by Popen's documentation:
-            https://docs.python.org/3/library/subprocess.html
+            https://docs.python.org/3/library/subprocess.html.
 
         :param args: The arguments (as a list or string) to run with Popen.
         :param shell: Whether to run Popen in a shell (default: False)
@@ -70,26 +72,34 @@ def __init__(self, args: [str, list], shell: bool = False) -> None:
 
     def _run(self) -> None:
         """
+        Invoke the loaded command with ``Popen``.
+
         Run the arguments and set the object's class variables with the
         results.
         """
         start_time = time()
         try:
             with Popen(
-                self.args,
-                stdout=PIPE,
-                stderr=PIPE,
-                shell=self.shell,
+                    self.args,
+                    stdout=PIPE,
+                    stderr=PIPE,
+                    shell=self.shell,
             ) as command:
                 stdout, stderr = command.communicate()
         except IOError as error:
             self._stderr = error.strerror
             self._return_code = error.errno
             LOG.error('Could not find command for given args: %s', self.args)
         else:
-            self._stdout = stdout.decode('utf8')
-            self._stderr = stderr.decode('utf8')
-            self._return_code = command.returncode
+            try:
+                self._stdout = stdout
+                self._stderr = stderr
+                self._return_code = command.returncode
+            except UnicodeDecodeError as error:
+                self._stderr = error
+                self._return_code = 1
+                LOG.error('Could not decode stdout or stderr recieved from given args: %s. \
+stdout: %s, stderr %s', self.args, stdout, stderr)
         self._duration = time() - start_time
         if self._return_code and self._duration:
             LOG.info(
@@ -100,14 +110,14 @@ def _run(self) -> None:
             )
 
     @property
-    def stdout(self) -> str:
+    def stdout(self) -> [str, bytes]:
         """
         ``stdout`` from the command.
         """
         return self._stdout
 
     @property
-    def stderr(self) -> str:
+    def stderr(self) -> [str, bytes]:
         """
         ``stderr`` from the command.
         """
@@ -116,7 +126,7 @@ def stderr(self) -> str:
     @property
     def return_code(self) -> int:
         """
-        return code from the command.
+        Return code from the command.
         """
         return self._return_code
 
@@ -127,11 +137,35 @@ def duration(self) -> float:
         """
         return self._duration
 
+    def decode(self, charset: str) -> None:
+        """
+        Decode ``self.stdout`` and ``self.stderr``.
+
+        Decodes ``self._stdout`` and ``self._stderr`` with the given ``charset``.
+        :param charset: The character set to decode with.
+        """
+        if not isinstance(self._stdout, str):
+            try:
+                self._stdout = self._stdout.decode(charset)
+            except ValueError as error:
+                LOG.error("Command output was requested to be decoded as"
+                          " %s but failed: %s", charset, error)
+                raise error
+        if not isinstance(self._stderr, str):
+            try:
+                self._stderr = self._stderr.decode(charset)
+            except ValueError as error:
+                LOG.error("Command output was requested to be decoded as"
+                          " %s but failed: %s", charset, error)
+                raise error
+
 
 @contextmanager
 def chdir(directory: str, create: bool = False) -> None:
     """
-    Changes into a given directory and returns to the original directory on
+    Change directories and run a command.
+
+    Change into a given directory and returns to the original directory on
     exit.
 
     .. note::
@@ -140,7 +174,7 @@ def chdir(directory: str, create: bool = False) -> None:
 
     .. code-block:: python
 
-        from crucible.os import chdir
+        from libcsm.os import chdir
 
         print('doing things in /dir/foo')
         with chdir('/some/other/dir'):
@@ -164,22 +198,27 @@ def chdir(directory: str, create: bool = False) -> None:
 
 
 def run_command(
-    args: [list, str],
-    in_shell: bool = False,
-    silence: bool = False, ) -> _CLI:
+        args: [list, str],
+        in_shell: bool = False,
+        silence: bool = False,
+        charset: str = None,
+) -> _CLI:
     """
-    Runs a given command or list of commands by instantiating a ``CLI`` object.
+    Run a given command or list of commands by instantiating a ``CLI`` object.
 
     .. code-block:: python
 
-        from crucible.os import run_command
+        from libcsm.os import run_command
 
         result = run_command(['my', 'args'])
         print(vars(result))
 
     :param args: List of arguments to run, can also be a string. If a string,
     :param in_shell: Whether to use a shell when invoking the command.
     :param silence: Tells this not to output the command to console.
+    :param charset: Returns the command ``stdout`` and ``stderr`` as a
+                    string instead of bytes, and decoded with the given
+                    ``charset``.
     """
     args_string = [str(x) for x in args]
     if not silence:
@@ -188,7 +227,10 @@ def run_command(
             ' '.join(args_string),
             in_shell
         )
-    return _CLI(args_string, shell=in_shell)
+    result = _CLI(args_string, shell=in_shell)
+    if charset:
+        result.decode(charset)
+    return result
 
 
 def supported_platforms() -> (bool, str):

crucible/scripts/install.sh

@@ -284,6 +284,7 @@ function partition_os {
 
     # metadata=0.9 for boot files.
     mdadm_raid_devices="--raid-devices=$metal_disks"
+    [ "$metal_disks" -eq 1 ] && mdadm_raid_devices="$mdadm_raid_devices --force"
     mdadm --create /dev/md/BOOT --run --verbose --assume-clean --metadata=0.9 --level="$metal_md_level" "$mdadm_raid_devices" "${boot_raid_parts[@]}" || metal_die -b "Failed to make filesystem on /dev/md/BOOT"
     mdadm --create /dev/md/SQFS --run --verbose --assume-clean --metadata=1.2 --level="$metal_md_level" "$mdadm_raid_devices" "${sqfs_raid_parts[@]}" || metal_die -b "Failed to make filesystem on /dev/md/SQFS"
     mdadm --create /dev/md/ROOT --assume-clean --run --verbose --metadata=1.2 --level="$metal_md_level" "$mdadm_raid_devices" "${oval_raid_parts[@]}" || metal_die -b "Failed to make filesystem on /dev/md/ROOT"

crucible/scripts/lsnics.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+#
+# MIT License
+#
+# (C) Copyright 2023 Hewlett Packard Enterprise Development LP
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+# OTHER DEALINGS IN THE SOFTWARE.
+#
+set -e
+
+function usage() {
+cat << 'EOF'
+Prints PCI Vendor and Device IDs for network devices.
+EOF
+}
+
+while getopts "hCPp:" o; do
+    case "${o}" in
+        h)
+            usage
+            exit 0
+            ;;
+        *)
+            :
+            ;;
+    esac
+done
+shift $((OPTIND-1))
+
+mapfile -t nics < <(ls -1d /sys/bus/pci/drivers/*/*/net/*)
+
+if [ "${#nics[@]}" -eq 0 ]; then
+    echo >&2 'No NICs detected in /sys/bus/pci/drivers/'
+    exit 1
+fi
+
+printf '% -6s % -4s % -4s \n' 'Name' 'VID' 'DID'
+for nic in "${nics[@]}"; do
+    DID="$(awk -F: '/PCI_ID/{gsub("PCI_ID=","");print $NF}' "$(dirname "$(dirname "${nics[0]}")")/uevent")"
+    VID="$(awk -F: '/PCI_ID/{gsub("PCI_ID=","");print $1}' "$(dirname "$(dirname "${nics[0]}")")/uevent")"
+    printf '% -6s % -4s % -4s \n' "$(basename "$nic")" "${VID}" "${DID}"
+done