add resources for cluster_type: sno to deploy ACM

ansible/roles/create-inventory/defaults/main/networks.yml

@@ -1,5 +1,6 @@
 ---
-# Network configuration for bm cluster and rwn control-plane nodes
+# Index used to select mac address from ocpinventory control-plane interface.
+# Used for SNOs, MNO control-plane and worker nodes, and RWN control-plane nodes
 controlplane_network_interface_idx: 0
 # The assisted-installer actually does not need the exact nic name for our controlplane network. Eth0 is used
 # as a representative for the actual nic name. The actual nic is determined by the mac address which is

ansible/roles/create-inventory/templates/inventory-sno.j2

@@ -41,7 +41,7 @@ bmc_password={{ bmc_password }}
 {%- else -%}
 {%- set ip=(sno_foreman_data.results|selectattr('json.name', 'eq', sno.pm_addr | replace('mgmt-',''))|first).json.ip -%}
 {%- endif -%}
-{% if not loop.first %}# {% endif %}{{ sno.pm_addr.split('.')[0] | replace('mgmt-','') }} bmc_address={{ sno.pm_addr }} boot_iso={{ sno.pm_addr.split('.')[0] | replace('mgmt-','') }}.iso ip={{ ip }} vendor={{ hw_vendor[(sno.pm_addr.split('.')[0]).split('-')[-1]] }} lab_mac={{ (sno_foreman_data.results|selectattr('json.name', 'eq', sno.pm_addr | replace('mgmt-',''))|first) | json_query(mac_query) | join(', ') }} network_mac={{ sno.mac[0] }} install_disk={{ sno_install_disk }}
+{% if not loop.first %}# {% endif %}{{ sno.pm_addr.split('.')[0] | replace('mgmt-','') }} bmc_address={{ sno.pm_addr }} boot_iso={{ sno.pm_addr.split('.')[0] | replace('mgmt-','') }}.iso ip={{ ip }} vendor={{ hw_vendor[(sno.pm_addr.split('.')[0]).split('-')[-1]] }} lab_mac={{ (sno_foreman_data.results|selectattr('json.name', 'eq', sno.pm_addr | replace('mgmt-',''))|first) | json_query(mac_query) | join(', ') }} network_mac={{ sno.mac[controlplane_network_interface_idx] }} install_disk={{ sno_install_disk }}
 {% endfor %}
 
 [sno:vars]

ansible/roles/sno-post-cluster-install/defaults/main.yml → ansible/roles/sno-post-cluster-install/defaults/main/main.yml

@@ -5,6 +5,10 @@
 assisted_installer_host: "{{ groups['bastion'][0] }}"
 assisted_installer_port: 8090
 
+# Deploy openshift-gitops-operator
+setup_openshift_gitops: false
+gitops_channel: stable
+
 # Performance-addon-operator and du profile vars
 install_performance_addon_operator: false
 hugepages_count: 16
@@ -14,3 +18,11 @@ isolated_cpus: 2-47,50-95
 du_profile: false
 operator_index_name: redhat-operator-index
 operator_index_tag: v4.16
+
+# Minio deployment is an object storage pod for use with ACM
+setup_minio: false
+minio_image_tag: RELEASE.2022-04-01T03-41-39Z
+minio_pv_storageclass: localstorage2-sc
+
+# setup metal3
+sno_metal3: false

ansible/roles/sno-post-cluster-install/defaults/main/storage.yml

@@ -0,0 +1 @@
+../../../configure-local-storage/defaults/main.yml

ansible/roles/sno-post-cluster-install/tasks/main.yml

@@ -12,6 +12,27 @@
     state: directory
   loop: "{{ ai_cluster_ids | dict2items }}"
 
+- name: Create gitops directory per sno cluster
+  file:
+    path: "{{ bastion_cluster_config_dir }}/{{ item.key }}/gitops"
+    state: directory
+  loop: "{{ ai_cluster_ids | dict2items }}"
+  when: setup_openshift_gitops
+
+- name: Create localstorage directory per sno cluster
+  file:
+    path: "{{ bastion_cluster_config_dir }}/{{ item.key }}/localstorage"
+    state: directory
+  loop: "{{ ai_cluster_ids | dict2items }}"
+  when: sno_localstorage_configuration
+
+- name: Create minio directory per sno cluster
+  file:
+    path: "{{ bastion_cluster_config_dir }}/{{ item.key }}/minio"
+    state: directory
+  loop: "{{ ai_cluster_ids | dict2items }}"
+  when: setup_minio | bool
+
 - name: Create performance-profile directory per sno cluster
   file:
     path: "{{ bastion_cluster_config_dir }}/{{ item.key }}/performance-profile"
@@ -26,6 +47,47 @@
   loop: "{{ ai_cluster_ids | dict2items }}"
   when: install_performance_addon_operator or du_profile
 
+- name: Place templated gitops
+  template:
+    src: openshift-gitops-operator.yml.j2
+    dest: "{{ bastion_cluster_config_dir }}/{{ item.key }}/gitops/openshift-gitops-operator.yml"
+  loop: "{{ ai_cluster_ids | dict2items }}"
+  when: setup_openshift_gitops
+
+- name: Place templated localstorage
+  template:
+    src: localstorage.yml.j2
+    dest: "{{ bastion_cluster_config_dir }}/{{ item.key }}/localstorage/localstorage.yml"
+  loop: "{{ ai_cluster_ids | dict2items }}"
+  when: sno_localstorage_configuration
+
+- name: Place templated localvolume
+  template:
+    src: localvolume.yml.j2
+    dest: "{{ bastion_cluster_config_dir }}/{{ item.key }}/localstorage/localvolume.yml"
+  loop: "{{ ai_cluster_ids | dict2items }}"
+  when: sno_localstorage_configuration
+
+- name: Place templated localvolume2
+  template:
+    src: localvolume2.yml.j2
+    dest: "{{ bastion_cluster_config_dir }}/{{ item.key }}/localstorage/localvolume2.yml"
+  loop: "{{ ai_cluster_ids | dict2items }}"
+  when: localvolume2_configuration
+
+- name: Place templated minio
+  template:
+    src: minio.yml.j2
+    dest: "{{ bastion_cluster_config_dir }}/{{ item.key }}/minio/minio.yml"
+  loop: "{{ ai_cluster_ids | dict2items }}"
+  when: setup_minio | bool
+
+- name: Place metal3 manifest
+  template:
+    src: metal3.yml
+    dest: "{{ bastion_cluster_config_dir }}/metal3.yml"
+  when: sno_metal3 | bool
+
 - name: Get credentials (kubeconfig) from installed cluster(s)
   get_url:
     url: "http://{{ assisted_installer_host }}:{{ assisted_installer_port }}/api/assisted-install/v2/clusters/{{ item.value.cluster_id }}/downloads/credentials?file_name=kubeconfig"
@@ -133,6 +195,46 @@
   loop: "{{ ai_cluster_ids | dict2items }}"
   when: use_bastion_registry | default(false)
 
+- name: Install OpenShift-gitops-operator
+  shell: |
+    KUBECONFIG={{ bastion_cluster_config_dir }}/{{ item.key }}/kubeconfig oc apply -f {{ bastion_cluster_config_dir }}/{{ item.key }}/gitops/openshift-gitops-operator.yml
+  when: setup_openshift_gitops
+  loop: "{{ ai_cluster_ids | dict2items }}"
+
+- name: Setup nodes localstorage
+  when: sno_localstorage_configuration
+  block:
+  - name: Install local-storage operator
+    shell:
+      KUBECONFIG={{ bastion_cluster_config_dir }}/{{ item.key }}/kubeconfig oc apply -f {{ bastion_cluster_config_dir }}/{{ item.key }}/localstorage/localstorage.yml
+    loop: "{{ ai_cluster_ids | dict2items }}"
+
+  # The localvolume resource will not be instantly available, thus retry for around 2 minutes
+  - name: Create localvolume resource
+    shell:
+      KUBECONFIG={{ bastion_cluster_config_dir }}/{{ item.key }}/kubeconfig oc apply -f {{ bastion_cluster_config_dir }}/{{ item.key }}/localstorage/localvolume.yml
+    register: lv_result
+    until: not lv_result.failed
+    retries: 60
+    delay: 2
+    loop: "{{ ai_cluster_ids | dict2items }}"
+
+  - name: Create localvolume2 resource
+    shell:
+      KUBECONFIG={{ bastion_cluster_config_dir }}/{{ item.key }}/kubeconfig oc apply -f {{ bastion_cluster_config_dir }}/{{ item.key }}/localstorage/localvolume2.yml
+    register: lv_result
+    until: not lv_result.failed
+    retries: 60
+    delay: 2
+    when: localvolume2_configuration
+    loop: "{{ ai_cluster_ids | dict2items }}"
+
+- name: Setup Minio
+  when: setup_minio | bool
+  shell: |
+    KUBECONFIG={{ bastion_cluster_config_dir }}/{{ item.key }}/kubeconfig oc apply -f {{ bastion_cluster_config_dir }}/{{ item.key }}/minio/minio.yml
+  loop: "{{ ai_cluster_ids | dict2items }}"
+
 - name: Disable Network Diagnostics - Place DisableSnoNetworkDiag CR
   template:
     src: DisableSnoNetworkDiag.yaml
@@ -277,3 +379,9 @@
   until: result.stdout == "True"
   loop: "{{ ai_cluster_ids | dict2items }}"
   when: du_profile
+
+- name: Setup Metal3
+  when: sno_metal3 | bool
+  shell: |
+    KUBECONFIG={{ bastion_cluster_config_dir }}/{{ item.key }}/kubeconfig oc apply -f {{ bastion_cluster_config_dir }}/metal3.yml
+  loop: "{{ ai_cluster_ids | dict2items }}"

ansible/roles/sno-post-cluster-install/templates/localstorage.yml.j2

@@ -0,0 +1,32 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  annontations:
+    openshift.io/node-selector: ''
+  name: openshift-local-storage
+---
+apiVersion: operators.coreos.com/v1
+kind: OperatorGroup
+metadata:
+  name: local-operator-group
+  namespace: openshift-local-storage
+spec:
+  targetNamespaces:
+  - openshift-local-storage
+---
+apiVersion: operators.coreos.com/v1alpha1
+kind: Subscription
+metadata:
+  name: local-storage-operator
+  namespace: openshift-local-storage
+spec:
+  installPlanApproval: Automatic
+  name: local-storage-operator
+  channel: stable
+{% if use_bastion_registry | default(false) %}
+  source: {{ operator_index_name }}
+{% else %}
+  source: redhat-operators
+{% endif %}
+  sourceNamespace: openshift-marketplace

ansible/roles/sno-post-cluster-install/templates/localvolume.yml.j2

@@ -0,0 +1,15 @@
+---
+apiVersion: "local.storage.openshift.io/v1"
+kind: "LocalVolume"
+metadata:
+  name: "lv-worker-localstorage"
+  namespace: "openshift-local-storage"
+spec:
+  storageClassDevices:
+  - storageClassName: "localstorage-sc"
+    volumeMode: Filesystem
+    fsType: xfs
+    devicePaths:
+{% for i in range(sno_localstorage_lv_count) %}
+      - "/dev/vg_ls/lv_sno_tv{{ '%02d' % i }}"
+{% endfor %}

ansible/roles/sno-post-cluster-install/templates/localvolume2.yml.j2

@@ -0,0 +1,13 @@
+---
+apiVersion: "local.storage.openshift.io/v1"
+kind: "LocalVolume"
+metadata:
+  name: "localvolume2"
+  namespace: "openshift-local-storage"
+spec:
+  storageClassDevices:
+  - storageClassName: "localstorage2-sc"
+    volumeMode: Filesystem
+    fsType: xfs
+    devicePaths:
+    - {{ localvolume2_device }}

ansible/roles/sno-post-cluster-install/templates/metal3.yml

@@ -0,0 +1,8 @@
+apiVersion: metal3.io/v1alpha1
+kind: Provisioning
+metadata:
+  name: provisioning-configuration
+spec:
+  preProvisioningOSDownloadURLs: {}
+  provisioningNetwork: Disabled
+  watchAllNamespaces: true

ansible/roles/sno-post-cluster-install/templates/minio.yml.j2

@@ -0,0 +1,93 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: minio
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  labels:
+    app.kubernetes.io/name: minio
+  name: minio
+  namespace: minio
+spec:
+  storageClassName: {{ minio_pv_storageclass }}
+  accessModes:
+  - ReadWriteOnce
+  resources:
+    requests:
+      storage: 250Gi
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: minio
+  namespace: minio
+  labels:
+    app.kubernetes.io/name: minio
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: minio
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: minio
+    spec:
+      containers:
+      - command:
+        - /bin/sh
+        - -c
+        - mkdir -p /storage/thanos && mkdir -p /storage/dr4hub/velero && for i in $(seq -f "%05g" 1 4000); do mkdir -p "/storage/vm$i-ibu"; done && minio server /storage
+        env:
+        - name: MINIO_ACCESS_KEY
+          value: minio
+        - name: MINIO_SECRET_KEY
+          value: minio123
+{% if not use_bastion_registry | default(false) %}
+        image: quay.io/minio/minio:{{ minio_image_tag }}
+{% else %}
+        image: {{ registry_host }}:{{ registry_port }}/minio/minio:{{ minio_image_tag }}
+{% endif %}
+        name: minio
+        ports:
+        - containerPort: 9000
+          protocol: TCP
+        volumeMounts:
+        - mountPath: /storage
+          name: storage
+      volumes:
+      - name: storage
+        persistentVolumeClaim:
+          claimName: minio
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: minio
+  namespace: minio
+spec:
+  ports:
+  - port: 9000
+    protocol: TCP
+    targetPort: 9000
+  selector:
+    app.kubernetes.io/name: minio
+  type: ClusterIP
+---
+apiVersion: route.openshift.io/v1
+kind: Route
+metadata:
+  name: minio
+  namespace: minio
+spec:
+#  tls:
+#    termination: edge
+  to:
+    name: minio
+  port:
+    targetPort: 9000

ansible/roles/sno-post-cluster-install/templates/openshift-gitops-operator.yml.j2

@@ -0,0 +1,16 @@
+---
+apiVersion: operators.coreos.com/v1alpha1
+kind: Subscription
+metadata:
+  name: openshift-gitops-operator
+  namespace: openshift-operators
+spec:
+  name: openshift-gitops-operator
+  sourceNamespace: openshift-marketplace
+  channel: {{ gitops_channel }}
+  installPlanApproval: Automatic
+{% if use_bastion_registry | default(false) %}
+  source: {{ operator_index_name }}
+{% else %}
+  source: redhat-operators
+{% endif %}