tests/e2e: Migrate e2e test for s390x to GHA

This PR is to migrate the following e2e tests for s390x to GHA:

- confidential-containers-operator-main-ubuntu-20.04-s390x-containerd_kata-qemu-baseline
- confidential-containers-operator-main-ubuntu-20.04-s390x-containerd_kata-qemu-PR
- confidential-containers-operator-main-ubuntu-20.04-s390x-SE-daily

They are incorporated into the existing x86_64 test workflow `ccruntime_e2e.yaml`.

The workflows incorparated are triggered not only by PR, but also nightly at 2:00 UTC.

Fixes: #294

Signed-off-by: Hyounggyu Choi <[email protected]>

.github/workflows/ccruntime-nightly.yaml

@@ -0,0 +1,26 @@
+name: ccruntime e2e test nightly
+on:
+  schedule:
+    - cron: '0 2 * * *'
+  workflow_dispatch:
+
+jobs:
+  e2e-nightly:
+    uses: ./.github/workflows/ccruntime_e2e.yaml
+    with:
+      target-branch: ${{ github.ref_name }}
+
+  e2e-ibm-se-nightly:
+    runs-on: s390x
+    strategy:
+      fail-fast: false
+      matrix:
+        test_title:
+          - cc-operator-e2e-tests
+    steps:
+    - name: Fetch a test result for {{ matrix.test_title }}
+      run: |
+        file_name="${TEST_TITLE}-$(date +%Y-%m-%d).log"
+        /home/${USER}/script/handle_test_log.sh download $file_name
+      env:
+        TEST_TITLE: ${{ matrix.test_title }}

.github/workflows/ccruntime-pr.yaml

@@ -0,0 +1,19 @@
+name: ccruntime e2e test for PR
+on:
+  pull_request_target:
+    branches:
+      - 'main'
+    types:
+      - opened
+      - synchronize
+      - reopened
+      - labeled
+    paths-ignore:
+      - 'docs/**'
+
+jobs:
+  e2e-pr:
+    if: ${{ contains(github.event.pull_request.labels.*.name, 'ok-to-test') }}
+    uses: ./.github/workflows/ccruntime_e2e.yaml
+    with:
+      target-branch: ${{ github.event.pull_request.base.ref }}

.github/workflows/ccruntime_e2e.yaml

@@ -1,8 +1,10 @@
 name: ccruntime e2e tests
 on:
-  pull_request:
-    branches:
-      - main
+  workflow_call:
+    inputs:
+      target-branch:
+        required: true
+        type: string
 
 permissions:
   contents: read
@@ -13,12 +15,34 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        runtimeclass: ["kata-qemu", "kata-clh"]
-        instance: ["az-ubuntu-2004", "az-ubuntu-2204"]
+        runtimeclass:
+          - "kata-qemu"
+          - "kata-clh"
+        instance:
+          - "az-ubuntu-2004"
+          - "az-ubuntu-2204"
+          - "s390x"
+        exclude:
+          - runtimeclass: "kata-clh"
+            instance: "s390x"
     runs-on: ${{ matrix.instance }}
     steps:
+      - name: Take a pre-action for self-hosted runner
+        run: |
+          if [ -f ${HOME}/script/pre_action.sh ]; then
+            ${HOME}/script/pre_action.sh cc-operator
+          fi
+
       - uses: actions/checkout@v4
 
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          PUSHER_NAME: ${{ github.event.pusher.name }}
+          PUSHER_EMAIL: ${{ github.event.pusher.email }}
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
       - name: Install deps
         run: |
           sudo apt-get update -y
@@ -29,4 +53,17 @@ jobs:
         run: |
           cd tests/e2e
           export PATH="$PATH:/usr/local/bin"
-          ./run-local.sh -r "${{ matrix.runtimeclass }}" -u
+          args="-u"
+          if [ $RUNNING_INSTANCE = "s390x" ]; then
+            args=""
+          fi
+          ./run-local.sh -r "${{ matrix.runtimeclass }}" "${args}"
+        env:
+          RUNNING_INSTANCE: ${{ matrix.instance }}
+
+      - name: Take a post-action
+        if: always()
+        run: |
+          if [ -f ${HOME}/script/post_action.sh ]; then
+            ${HOME}/script/post_action.sh cc-operator
+          fi

tests/e2e/ansible/install_docker.yml

@@ -77,6 +77,43 @@
           - docker-ce-cli
         state: present
   when: docker_exist.rc != 0 and ansible_distribution == "CentOS"
+#
+# In order to prevent "systemd: docker.service Start request repeated too quickly"
+#
+- name: Set StartLimitBurst to 0 on Ubuntu for s390x
+  block:
+    - name: If a fragment path starts with `/lib`, then create /etc/systemd/system/docker.service.d
+      block:
+        - name: Get FragmentPath
+          shell: systemctl show -p FragmentPath docker.service | awk -F'=' '{print $2}'
+          register: fragment_path
+        - name: Copy fragment file to /etc/systemd/system/docker.service.d
+          copy:
+            src: "{{ fragment_path.stdout }}"
+            dest: "{{ fragment_path.stdout.replace('/lib', '/etc') }}"
+          when: fragment_path.stdout.find('/lib') != -1
+    - name: Configure `StartLimitBurst=0` in /etc/systemd/system/docker.service
+      block:
+        - name: Check if /etc/systemd/system/docker.service has StartLimitBurst=0
+          shell: grep -q 'StartLimitBurst' /etc/systemd/system/docker.service
+          register: start_limit_burst
+          ignore_errors: yes
+        - name: Replace a value of StartLimitBurst to 0
+          lineinfile:
+            path: /etc/systemd/system/docker.service
+            regexp: '^StartLimitBurst='
+            line: 'StartLimitBurst=0'
+          when: start_limit_burst.rc == 0
+        - name: Otherwise, insert `StartLimitBurst=0` just after a service section
+          lineinfile:
+            path: /etc/systemd/system/docker.service
+            insertafter: '\[Service\]'
+            line: 'StartLimitBurst=0'
+          when: start_limit_burst.rc != 0
+    - name: Reload systemd
+      systemd:
+        daemon_reload: yes
+  when: ansible_distribution == "Ubuntu" and ansible_architecture == 's390x'
 - name: Start docker service
   service:
     name: docker

tests/git-helper.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+#
+# Copyright Confidential Containers Contributors
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+function add_kata_bot_info() {
+	if [ -z $PUSHER_NAME ] || [ -z $PUSHER_EMAIL ]; then
+		>&2 echo "PUSHER_NAME and PUSHER_EMAIL must be set"
+		exit 1
+	fi
+	echo "Adding user name ${PUSER_NAME} and email ${PUSHER_EMAIL} to the local git repo"
+	git config user.name "${PUSHER_NAME}"
+	git config user.email "${PUSHER_EMAIL}"
+}
+
+function rebase_atop_of_the_latest_target_branch() {
+	if [ -n "${TARGET_BRANCH}" ]; then
+		echo "Rebasing atop of the latest ${TARGET_BRANCH}"
+		# Recover from any previous rebase left halfway
+		git rebase --abort 2> /dev/null || true
+		if ! git rebase origin/${TARGET_BRANCH}; then
+			echo "Rebase failed, exiting"
+			exit 1
+		fi
+	fi
+}
+
+function main() {
+	action="${1:-}"
+
+	add_kata_bot_info
+	case "${action}" in
+	rebase-atop-of-the-latest-target-branch) rebase_atop_of_the_latest_target_branch;;
+	*) >&2 echo "Invalid argument"; exit 2 ;;
+	esac
+}
+
+main "$@"