diff --git a/.github/workflows/k8s-ci.yml b/.github/workflows/k8s-ci.yml index cb678561e..63f315e69 100644 --- a/.github/workflows/k8s-ci.yml +++ b/.github/workflows/k8s-ci.yml @@ -7,6 +7,11 @@ jobs: k8s-ci: runs-on: ubuntu-latest steps: + - name: Bind mount /dev/sda1 to /nix + run: | + sudo mkdir -p /nix + sudo mount --bind /mnt /nix + lsblk - uses: actions/checkout@v4 with: submodules: recursive @@ -23,18 +28,21 @@ jobs: id: build run: | TAG=$(nix-shell ./shell.nix --run './scripts/python/generate-test-tag.sh') - BIN=$(mktemp -p . -d -t test-bin-XXXXXX) + TEST_DIR=$(realpath $(mktemp -d ./test-dir-XXXXXX)) nix-shell ./shell.nix --run "./scripts/python/tag-chart.sh $TAG" - RUSTFLAGS="-C debuginfo=0 -C strip=debuginfo" ./scripts/release.sh --tag $TAG --build-bins --build-binary-out $BIN --no-static-linking --skip-publish --debug + RUSTFLAGS="-C debuginfo=0 -C strip=debuginfo" ./scripts/release.sh --tag $TAG --build-binary-out $TEST_DIR --no-static-linking --skip-publish --debug echo "tag=$TAG" >> $GITHUB_OUTPUT - echo "bin=$BIN" >> $GITHUB_OUTPUT + echo "bin=$TEST_DIR" >> $GITHUB_OUTPUT - name: BootStrap k8s cluster run: | nix-shell ./scripts/k8s/shell.nix --run "./scripts/k8s/deployer.sh start --label" - name: Load images to Kind cluster run: nix-shell ./scripts/k8s/shell.nix --run "./scripts/k8s/load-images-to-kind.sh --tag ${{ steps.build.outputs.tag }} --trim-debug-suffix" - name: Run Pytests - run: nix-shell ./shell.nix --run './scripts/python/test.sh' + run: | + export UPGRADE_TARGET_VERSION=${{ steps.build.outputs.tag }} + export TEST_DIR=${{ steps.build.outputs.bin }} + nix-shell ./shell.nix --run "./scripts/python/test.sh" - name: The job has failed if: ${{ failure() }} run: | diff --git a/.gitignore b/.gitignore index 026122377..6398c18f6 100644 --- a/.gitignore +++ b/.gitignore @@ -22,5 +22,6 @@ __pycache__ /kubectl-plugin # Pytest assets -/test-bin-* +/test-dir-* tests/bdd/venv +pytest.log \ No newline at end of file diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 000000000..f9405c8c8 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,7 @@ +[pytest] +log_cli = true +log_cli_level = INFO +log_cli_format = %(asctime)s [%(levelname)s] %(message)s +log_cli_date_format = %Y-%m-%d %H:%M:%S +log_file = pytest.log +log_file_level = DEBUG diff --git a/scripts/helm/install.sh b/scripts/helm/install.sh index 66d6e5d6f..3963ac9c1 100755 --- a/scripts/helm/install.sh +++ b/scripts/helm/install.sh @@ -2,28 +2,54 @@ set -e +repo_add() { + local -r url=$1 + local -r preferred_name=$2 + + local repo + if [ "$(helm repo ls -o yaml | yq "contains([{\"url\": \"$url\"}])")" = "true" ]; then + repo=$(helm repo ls -o yaml | yq ".[] | select(.url == \"$url\") | .name") + else + helm repo add "$preferred_name" "$url" > /dev/null + repo=$preferred_name + fi + + helm repo update > /dev/null || true + + echo "$repo" +} + + TIMEOUT="5m" WAIT= DRY_RUN="" CHART= SCRIPT_DIR="$(dirname "$0")" CHART_DIR="$SCRIPT_DIR"/../../chart +CHART_SOURCE=$CHART_DIR DEP_UPDATE= RELEASE_NAME="mayastor" K8S_NAMESPACE="mayastor" FAIL_IF_INSTALLED= +HOSTED= +VERSION= +REGISTRY= +DEFAULT_REGISTRY="https://openebs.github.io/mayastor-extensions" help() { cat < How long to wait for helm to complete install (Default: $TIMEOUT). - --wait Wait for helm to complete install. - --dry-run Install helm with --dry-run. - --dep-update Run helm dependency update. - --fail-if-installed Fail with a status code 1 if the helm release '$RELEASE_NAME' already exists in the $K8S_NAMESPACE namespace. + -h, --help Display this text. + --timeout How long to wait for helm to complete install (Default: $TIMEOUT). + --wait Wait for helm to complete install. + --dry-run Install helm with --dry-run. + --dep-update Run helm dependency update. + --fail-if-installed Fail with a status code 1 if the helm release '$RELEASE_NAME' already exists in the $K8S_NAMESPACE namespace. + --hosted-chart Install a hosted chart instead of the local chart. + --version Set the version/version-range for the chart. Works only when used with the '--hosted' option. + --registry Set the registry URL for the hosted chart. Works only when used with the '--hosted' option. (Default: $DEFAULT_REGISTRY) Examples: $(basename "$0") @@ -40,10 +66,6 @@ die() { exit "${_return}" } -nvme_ana_check() { - cat /sys/module/nvme_core/parameters/multipath -} - while [ "$#" -gt 0 ]; do case $1 in -h|--help) @@ -67,6 +89,27 @@ while [ "$#" -gt 0 ]; do --fail-if-installed) FAIL_IF_INSTALLED="y" shift;; + --hosted-chart) + HOSTED=1 + shift;; + --version*) + if [ "$1" = "--version" ]; then + test $# -lt 2 && die "Missing value for the optional argument '$1'." + VERSION="$2" + shift + else + VERSION="${1#*=}" + fi + shift;; + --registry) + if [ "$1" = "--registry" ]; then + test $# -lt 2 && die "Missing value for the optional argument '$1'." + REGISTRY="$2" + shift + else + REGISTRY="${1#*=}" + fi + shift;; *) die "Unknown argument $1!" shift;; @@ -82,7 +125,19 @@ if [ -n "$WAIT" ]; then WAIT_ARG=" --wait --timeout $TIMEOUT" fi -if [ "$(helm ls -n openebs -o json | jq --arg release_name "$RELEASE_NAME" 'any(.[]; .name == $release_name)')" = "true" ]; then +VERSION_ARG= +if [ -n "$HOSTED" ]; then + if [ -n "$VERSION" ]; then + VERSION_ARG="--version $VERSION" + fi + if [ -z "$REGISTRY" ]; then + REGISTRY=$DEFAULT_REGISTRY + fi + CHART_SOURCE="$(repo_add "$REGISTRY" "mayastor")/mayastor" + DEP_UPDATE_ARG= +fi + +if [ "$(helm ls -n openebs -o yaml | yq "contains([{\"name\": \"$RELEASE_NAME\"}])")" = "true" ]; then already_exists_log= "Helm release $RELEASE_NAME already exists in namespace $K8S_NAMESPACE" if [ -n "$FAIL_IF_INSTALLED" ]; then die "ERROR: $already_exists_log" 1 @@ -91,11 +146,11 @@ if [ "$(helm ls -n openebs -o json | jq --arg release_name "$RELEASE_NAME" 'any( else echo "Installing Mayastor Chart" set -x - helm install "$RELEASE_NAME" "$CHART_DIR" -n "$K8S_NAMESPACE" --create-namespace \ + helm install "$RELEASE_NAME" "$CHART_SOURCE" -n "$K8S_NAMESPACE" --create-namespace \ --set="etcd.livenessProbe.initialDelaySeconds=5,etcd.readinessProbe.initialDelaySeconds=5,etcd.replicaCount=1" \ --set="obs.callhome.enabled=true,obs.callhome.sendReport=false,localpv-provisioner.analytics.enabled=false" \ --set="eventing.enabled=false" \ - $DRY_RUN $WAIT_ARG $DEP_UPDATE_ARG + $DRY_RUN $WAIT_ARG $DEP_UPDATE_ARG $VERSION_ARG set +x fi diff --git a/scripts/python/fmt.sh b/scripts/python/fmt.sh new file mode 100755 index 000000000..7b7609832 --- /dev/null +++ b/scripts/python/fmt.sh @@ -0,0 +1,122 @@ +#!/usr/bin/env bash + +SCRIPT_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]:-"$0"}")")" +ROOT_DIR="$SCRIPT_DIR/../.." + +# Imports +source "$ROOT_DIR"/scripts/utils/log.sh + +set -e + +fmt() { + local -r paths_glob=$1 + + bash -c " + black --quiet $paths_glob && + isort --profile=black --quiet $paths_glob && + autoflake --quiet -r -i --remove-unused-variables --remove-all-unused-imports --expand-star-imports $paths_glob" +} + +fmt_diff() { + local -r paths_glob=($(bash -c "echo $1")) + + for path in "${paths_glob[@]}"; do + find "$path" -type f -name "*.py" -exec bash -c ' + OUTPUT=$(diff -u --color=always <(cat {}) <(black -c "$(cat {})" | + isort --profile=black -d - | + autoflake --remove-unused-variables --remove-all-unused-imports --expand-star-imports -s -) + ); + if [ -n "$OUTPUT" ]; then + echo -e "Diff for file {}\n===================================================\n$OUTPUT\n" | cat + fi + ' \; + done +} + +fmt_check() { + local -r paths_glob=$1 + + local error= + bash -c " + black --quiet $paths_glob --check && + isort --profile=black --quiet $paths_glob --check 2> /dev/null && + autoflake --quiet -r --remove-unused-variables --remove-all-unused-imports --expand-star-imports $paths_glob --check > /dev/null + " || error=$? + + if [ -n "$error" ]; then + exit $error + fi + exit 0 +} + +CHECK= +DIFF= +PATHS_GLOB= +DEFAULT_PATHS_GLOB=$(realpath "$ROOT_DIR"/tests/bdd)/{common,features} + +# Print usage options for this script. +print_help() { + cat < Input a paths glob of directories and/or files which + would be parsed recursively. (default: "$DEFAULT_PATHS_GLOB") + +Examples: + $(basename "${0}") --check +EOF +} + +# Parse args. +while test $# -gt 0; do + arg="$1" + case "$arg" in + --check) + CHECK=1 + ;; + --diff) + DIFF=1 + ;; + --paths-glob*) + if [ "$arg" = "--paths-glob" ]; then + test $# -lt 2 && log_fatal "Missing value for the optional argument '$arg'." + PATHS_GLOB="$2" + shift + else + PATHS_GLOB="${arg#*=}" + fi + ;; + -h* | --help*) + print_help + exit 0 + ;; + *) + print_help + log_fatal "unexpected argument '$arg'" 1 + ;; + esac + shift +done + +if [ -z "$PATHS_GLOB" ]; then + PATHS_GLOB=$DEFAULT_PATHS_GLOB +fi + +if [ -n "$CHECK" ]; then + fmt_check "$PATHS_GLOB" +fi + +if [ -n "$DIFF" ]; then + fmt_diff "$PATHS_GLOB" +fi + +if [[ -z "$CHECK" ]] && [[ -z "$DIFF" ]]; then + fmt "$PATHS_GLOB" +fi diff --git a/scripts/python/generate-test-tag.sh b/scripts/python/generate-test-tag.sh index e5b3652c5..d943a35a7 100755 --- a/scripts/python/generate-test-tag.sh +++ b/scripts/python/generate-test-tag.sh @@ -3,6 +3,7 @@ SCRIPT_DIR="$(dirname "$(realpath "${BASH_SOURCE[0]:-"$0"}")")" ROOT_DIR="$SCRIPT_DIR/../.." +# Imports source "$ROOT_DIR"/scripts/utils/log.sh source "$ROOT_DIR"/scripts/utils/repo.sh diff --git a/shell.nix b/shell.nix index cab3ca6e1..d4aa69795 100644 --- a/shell.nix +++ b/shell.nix @@ -18,6 +18,8 @@ in mkShell { name = "extensions-shell"; buildInputs = [ + autoflake + black cacert cargo-expand cargo-udeps @@ -27,6 +29,7 @@ mkShell { cowsay git helm-docs + isort kubectl kubernetes-helm-wrapped llvmPackages.libclang diff --git a/tests/bdd/common/environment.py b/tests/bdd/common/environment.py new file mode 100644 index 000000000..f0d0b1257 --- /dev/null +++ b/tests/bdd/common/environment.py @@ -0,0 +1,17 @@ +import logging +import os + +logger = logging.getLogger(__name__) + + +def get_env(variable: str): + try: + value = os.getenv(variable) + if len(value) == 0: + raise ValueError("Env {variable} is empty") + logger.info(f"Found env {variable}={value}") + return value + + except Exception as e: + logger.error(f"Failed to get env {variable}: {e}") + return None diff --git a/tests/bdd/common/helm.py b/tests/bdd/common/helm.py new file mode 100644 index 000000000..b116de3db --- /dev/null +++ b/tests/bdd/common/helm.py @@ -0,0 +1,272 @@ +import json +import logging +import os +import subprocess +from enum import Enum +from shutil import which + +from common.environment import get_env +from common.repo import root_dir, run_script + +logger = logging.getLogger(__name__) + +helm_bin = which("helm") + + +def repo_ls(): + try: + result = subprocess.run( + [helm_bin, "repo", "ls", "-o", "json"], + capture_output=True, + check=True, + text=True, + ) + return json.loads(result.stdout.strip()) + + except subprocess.CalledProcessError as e: + logger.error( + f"Error: command 'helm repo ls -o json' failed with exit code {e.returncode}" + ) + logger.error(f"Error Output: {e.stderr}") + return None + + except Exception as e: + logger.error(f"An unexpected error occurred: {e}") + return None + + +def repo_add_mayastor(): + repos = repo_ls() + if repos is not None: + for r in repos: + if r["url"] == "https://openebs.github.io/mayastor-extensions": + return r["name"] + + try: + repo_name = "mayastor" + subprocess.run( + [ + helm_bin, + "repo", + "add", + repo_name, + "https://openebs.github.io/mayastor-extensions", + ], + capture_output=True, + check=True, + text=True, + ) + + subprocess.run( + [ + helm_bin, + "repo", + "update", + ], + capture_output=True, + check=True, + text=True, + ) + return repo_name + + except subprocess.CalledProcessError as e: + logger.error( + f"Error: command 'helm repo add mayastor https://openebs.github.io/mayastor-extensions' failed with exit code {e.returncode}" + ) + logger.error(f"Error Output: {e.stderr}") + return None + + except Exception as e: + logger.error(f"An unexpected error occurred: {e}") + return None + + +def latest_chart_so_far(version=None): + if version is None: + v = get_env("UPGRADE_TARGET_VERSION") + if v is None: + version = generate_test_tag() + else: + version = v + + repo_name = repo_add_mayastor() + assert repo_name is not None + + helm_search_command = [ + helm_bin, + "search", + "repo", + repo_name + "/mayastor", + "--version", + "<" + version, + "-o", + "json", + ] + try: + result = subprocess.run( + helm_search_command, + capture_output=True, + check=True, + text=True, + ) + result_chart_info = json.loads(result.stdout.strip()) + return result_chart_info[0]["version"] + + except subprocess.CalledProcessError as e: + logger.error( + f"Error: command {helm_search_command} failed with exit code {e.returncode}" + ) + logger.error(f"Error Output: {e.stderr}") + return None + + except Exception as e: + logger.error(f"An unexpected error occurred: {e}") + return None + + +class ChartSource(Enum): + HOSTED = [ + "/bin/bash", + "-c", + os.path.join(root_dir(), "scripts/helm/install.sh") + " --hosted-chart --wait", + ] + LOCAL = [ + "/bin/bash", + "-c", + os.path.join(root_dir(), "scripts/helm/install.sh") + " --dep-update --wait", + ] + + +class HelmReleaseClient: + """ + A client for interacting with Helm releases in a specified Kubernetes namespace. + + Attributes: + namespace (str): The Kubernetes namespace where the Helm releases are managed. + """ + + def __init__(self): + """ + Initializes the HelmReleaseClient. + """ + self.namespace = "mayastor" + + def get_metadata_mayastor(self): + command = [ + helm_bin, + "get", + "metadata", + "mayastor", + "-n", + self.namespace, + "-o", + "json", + ] + try: + result = subprocess.run( + command, + capture_output=True, + check=True, + text=True, + ) + return json.loads(result.stdout.strip()) + + except subprocess.CalledProcessError as e: + logger.error( + f"Error: command '{command}' failed with exit code {e.returncode}" + ) + logger.error(f"Error Output: {e.stderr}") + return None + + except Exception as e: + logger.error(f"An unexpected error occurred: {e}") + return None + + def list(self): + """ + Lists the deployed Helm releases in the specified namespace. + + Executes the 'helm ls' command to retrieve a list of deployed releases. + + Returns: + str: A newline-separated string of deployed release names, or None if an error occurs. + """ + try: + result = subprocess.run( + [ + helm_bin, + "ls", + "-n", + self.namespace, + "--deployed", + "--short", + ], + capture_output=True, + check=True, + text=True, + ) + return result.stdout.strip() + + except subprocess.CalledProcessError as e: + logger.error( + f"Error: command 'helm ls -n {self.namespace} --deployed --short' failed with exit code {e.returncode}" + ) + logger.error(f"Error Output: {e.stderr}") + return None + + except Exception as e: + logger.error(f"An unexpected error occurred: {e}") + return None + + def release_is_deployed(self, release_name: str): + releases = self.list() + if releases is not None: + for release in releases: + if release == release_name: + return True + return False + + def install_mayastor(self, source: ChartSource, version=None): + if self.release_is_deployed("mayastor"): + logger.error( + f"WARN: Helm release 'mayastor' already exists in the 'mayastor' namespace." + ) + return + + install_command = [] + if source == ChartSource.HOSTED: + install_command = source.value + if version is not None: + install_command[-1] = install_command[-1] + " --version " + version + logger.info( + f"Installing mayastor helm chart from hosted registry, version='{version}'" + ) + + if source == ChartSource.LOCAL: + install_command = source.value + logger.info("Installing mayastor helm chart from local directory") + + try: + result = subprocess.run( + install_command, + capture_output=True, + check=True, + text=True, + ) + logger.info("Installation succeeded") + return result.stdout.strip() + + except subprocess.CalledProcessError as e: + logger.error( + f"Error: command {install_command} failed with exit code {e.returncode}" + ) + logger.error(f"Error Output: {e.stderr}") + return None + + except Exception as e: + logger.error(f"An unexpected error occurred: {e}") + return None + + +def generate_test_tag(): + return run_script("scripts/python/generate-test-tag.sh") diff --git a/tests/bdd/common/kubectl_mayastor.py b/tests/bdd/common/kubectl_mayastor.py new file mode 100644 index 000000000..8ca4d2d8b --- /dev/null +++ b/tests/bdd/common/kubectl_mayastor.py @@ -0,0 +1,41 @@ +import logging +import os +import subprocess +from shutil import which + +from common.environment import get_env + +logger = logging.getLogger(__name__) + + +def get_bin_path(): + bins = get_env("TEST_DIR") + if bins: + return os.path.join(bins, "kubectl-plugin/bin/kubectl-mayastor") + logging.warning(f"Environmental variable 'BIN' is not set") + return which("kubectl-mayastor") + + +def kubectl_mayastor(args: list[str]): + command = [get_bin_path()] + command.extend(args) + logger.info(f"Running kubectl-mayastor command: {command}") + + try: + result = subprocess.run( + command, + capture_output=True, + check=True, + text=True, + ) + logger.info(f"kubectl-mayastor command succeeded") + return result.stdout.strip() + + except subprocess.CalledProcessError as e: + logger.error(f"Error: command '{command}' failed with exit code {e.returncode}") + logger.error(f"Error Output: {e.stderr}") + return None + + except Exception as e: + logger.error(f"An unexpected error occurred: {e}") + return None diff --git a/tests/bdd/common/repo.py b/tests/bdd/common/repo.py new file mode 100644 index 000000000..47da706b4 --- /dev/null +++ b/tests/bdd/common/repo.py @@ -0,0 +1,35 @@ +import logging +import os +import subprocess + +logger = logging.getLogger(__name__) + + +def root_dir(): + file_path = os.path.abspath(__file__) + return file_path.split("tests/bdd")[0] + + +def run_script(script: str): + script = os.path.join(root_dir(), script) + logger.info(f"Running script '{script}'") + command = ["/bin/bash", "-c", script] + try: + result = subprocess.run( + command, + capture_output=True, + check=True, + shell=True, + text=True, + ) + logger.info(f"Script succeeded") + return result.stdout.strip() + + except subprocess.CalledProcessError as e: + logger.error(f"Error: command {command} failed with exit code {e.returncode}") + logger.error(f"Error Output: {e.stderr}") + return None + + except Exception as e: + logger.error(f"An unexpected error occurred: {e}") + return None diff --git a/tests/bdd/features/test_upgrade.py b/tests/bdd/features/test_upgrade.py new file mode 100644 index 000000000..1061e48e1 --- /dev/null +++ b/tests/bdd/features/test_upgrade.py @@ -0,0 +1,84 @@ +"""Upgrade feature tests.""" + +import logging + +from common.environment import get_env +from common.helm import ChartSource, HelmReleaseClient, latest_chart_so_far +from common.kubectl_mayastor import kubectl_mayastor +from common.repo import run_script +from kubernetes import client, config +from pytest_bdd import given, scenario, then, when +from retrying import retry + +logger = logging.getLogger(__name__) + +helm = HelmReleaseClient() + + +@scenario("upgrade.feature", "upgrade command is issued") +def test_upgrade_command_is_issued(): + """upgrade command is issued.""" + + +@given("an installed mayastor helm chart") +def an_installed_mayastor_helm_chart(): + """an installed mayastor helm chart.""" + assert helm.install_mayastor(ChartSource.HOSTED, latest_chart_so_far()) is not None + + +@when("a kubectl mayastor upgrade command is issued") +def a_kubectl_mayastor_upgrade_command_is_issued(): + """a kubectl mayastor upgrade command is issued.""" + assert kubectl_mayastor(["upgrade"]) is not None + + +@then("the installed chart should be upgraded to the kubectl mayastor plugin's version") +def the_installed_chart_should_be_upgraded_to_the_kubectl_mayastor_plugins_version(): + """the installed chart should be upgraded to the kubectl mayastor plugin's version.""" + + upgrade_target_version = get_env("UPGRADE_TARGET_VERSION") + if upgrade_target_version is None: + upgrade_target_version = run_script("scripts/python/generate-test-tag.sh") + upgrade_target_version = upgrade_target_version.lstrip("v") + logger.info(f"Value of upgrade_target_version={upgrade_target_version}") + + @retry( + stop_max_attempt_number=450, + wait_fixed=2000, + ) + def helm_upgrade_succeeded(): + logger.info("Checking if helm upgrade succeeded...") + metadata = helm.get_metadata_mayastor() + logger.debug(f"helm get metadata output={metadata}") + logger.debug(f"upgrade_target_version={upgrade_target_version}") + if metadata: + assert metadata["version"] == upgrade_target_version + return + raise ValueError("helm get metadata returned a None") + + @retry( + stop_max_attempt_number=600, + wait_fixed=2000, + ) + def data_plane_upgrade_succeeded(): + logger.info("Checking if data-plane upgrade succeeded...") + config.load_kube_config() + v1 = client.CoreV1Api() + label_selector = "app=io-engine" + pods = v1.list_namespaced_pod( + namespace="mayastor", label_selector=label_selector + ) + switch = True + for pod in pods.items: + for i, container in enumerate(pod.spec.containers): + if container.name == "io-engine": + logger.info( + f"pod.metadata.name={pod.metadata.name}, pod.spec.containers[{i}].image={container.image}" + ) + switch = switch and container.image.endswith(":develop") + logger.info(f"Value of 'switch' after the AND={switch}") + break + assert switch + + helm_upgrade_succeeded() + data_plane_upgrade_succeeded() diff --git a/tests/bdd/features/upgrade.feature b/tests/bdd/features/upgrade.feature new file mode 100644 index 000000000..8d4d9b61f --- /dev/null +++ b/tests/bdd/features/upgrade.feature @@ -0,0 +1,8 @@ +Feature: Upgrade + + Background: + Given an installed mayastor helm chart + + Scenario: upgrade command is issued + When a kubectl mayastor upgrade command is issued + Then the installed chart should be upgraded to the kubectl mayastor plugin's version diff --git a/tests/bdd/requirements.txt b/tests/bdd/requirements.txt index ffc56cb63..f054f2bb6 100644 --- a/tests/bdd/requirements.txt +++ b/tests/bdd/requirements.txt @@ -1,2 +1,6 @@ -pytest-bdd==7.3.0 kubernetes==31.0.0 +pytest-bdd==7.3.0 +pytest==8.3.3 +requests==2.26.0 +retrying==1.3.4 +semver==3.0.2