Error: Failed to determine GroupVersionResource for manifest

[chrisadkin-zz]

Terraform version: 1.0
Kubernetes Alpha Provider version: 0.50
Kubernetes version: 1.20.7

Terraform configuration

provider "azuread" {
  alias   = "azure_ad"
}

provider "azurerm" {
  features {}
  alias   = "azure_rm"
}

data "azurerm_subscription" "primary" {
  provider = azurerm.azure_rm
}

resource "azuread_application" "auth" {
  display_name  = var.application_name
}

resource "azuread_service_principal" "auth" {
  application_id = azuread_application.auth.application_id
}

resource "azuread_service_principal_password" "auth" {
  service_principal_id = azuread_service_principal.auth.id
  value                = random_string.password.result
  end_date_relative    = "240h" 
}

resource "random_string" "password" {
  length               = var.password_length
  special              = var.password_special
  override_special     = var.password_override_special
}

resource "azurerm_role_assignment" "contributor" {
  provider             = azurerm.azure_rm
  scope                = data.azurerm_subscription.primary.id
  role_definition_name = "Contributor"
  principal_id         = azuread_service_principal.auth.id
}

resource "azurerm_role_assignment" "monitoring_metrics_publisher" {
  provider             = azurerm.azure_rm
  scope                = data.azurerm_subscription.primary.id
  role_definition_name = "Monitoring Metrics Publisher"
  principal_id         = azuread_service_principal.auth.id
}

provider "kubernetes-alpha" {
  config_path = "~/.kube/config"
}

resource "kubernetes_manifest" "data_controller_crd" { 
  provider = kubernetes-alpha

  manifest = {
    "apiVersion" = "apiextensions.k8s.io/v1beta1"
    "kind" = "CustomResourceDefinition"
    "metadata" = {
      "name" = "datacontrollers.arcdata.microsoft.com"
    }
    "spec" = {
      "additionalPrinterColumns" = [
        {
          "JSONPath" = ".status.state"
          "name" = "State"
          "type" = "string"
        },
      ]
      "group" = "arcdata.microsoft.com"
      "names" = {
        "kind" = "datacontroller"
        "plural" = "datacontrollers"
      }
      "scope" = "Namespaced"
      "subresources" = {
         "status" = {}
      }
      "version" = "v1alpha1"
    }
  }
}

resource "kubernetes_manifest" "sql_mi_crd" { 
  provider = kubernetes-alpha

  manifest = {
    "apiVersion" = "apiextensions.k8s.io/v1beta1"
    "kind" = "CustomResourceDefinition"
    "metadata" = {
      "name" = "sqlmanagedinstances.sql.arcdata.microsoft.com"
    }
    "spec" = {
      "additionalPrinterColumns" = [
        {
          "JSONPath" = ".status.state"
          "name" = "Status"
          "type" = "string"
        },
        {
          "JSONPath" = ".status.readyReplicas"
          "name" = "Replicas"
          "type" = "string"
        },
        {
          "JSONPath" = ".status.primaryEndpoint"
          "name" = "Primary-Endpoint"
          "type" = "string"
        },
        {
          "JSONPath" = ".metadata.creationTimestamp"
          "name" = "Age"
          "type" = "date"
        },
      ]
      "group" = "sql.arcdata.microsoft.com"
      "names" = {
        "kind" = "sqlmanagedinstance"
        "plural" = "sqlmanagedinstances"
        "shortNames" = [
          "sqlmi",
        ]
      }
      "scope" = "Namespaced"
      "subresources" = {
        "status" = {}
      }
      "version" = "v1alpha1"
    }
  }
}

resource "kubernetes_manifest" "sql_mi_restore_crd" { 
  provider = kubernetes-alpha

  manifest = {
    "apiVersion" = "apiextensions.k8s.io/v1beta1"
    "kind" = "CustomResourceDefinition"
    "metadata" = {
      "name" = "sqlmanagedinstancerestoretasks.tasks.sql.arcdata.microsoft.com"
    }
    "spec" = {
      "additionalPrinterColumns" = [
        {
          "JSONPath" = ".status.state"
          "name" = "Status"
          "type" = "string"
        },
        {
          "JSONPath" = ".metadata.creationTimestamp"
          "name" = "Age"
          "type" = "date"
        },
      ]
      "group" = "tasks.sql.arcdata.microsoft.com"
      "names" = {
        "kind" = "SqlManagedInstanceRestoreTask"
        "plural" = "sqlmanagedinstancerestoretasks"
        "shortNames" = [
          "sqlmirestoretask",
        ]
        "singular" = "sqlmanagedinstancerestoretask"
      }
      "scope" = "Namespaced"
      "subresources" = {
        "status" = {}
      }
      "version" = "v1alpha1"
    }
  }
}

resource "kubernetes_manifest" "postgres_sql_crd" { 
  provider = kubernetes-alpha

  manifest = {
    "apiVersion" = "apiextensions.k8s.io/v1beta1"
    "kind" = "CustomResourceDefinition"
    "metadata" = {
      "name" = "postgresqls.arcdata.microsoft.com"
    }
    "spec" = {
      "additionalPrinterColumns" = [
        {
          "JSONPath" = ".status.state"
          "name" = "State"
          "type" = "string"
        },
        {
          "JSONPath" = ".status.readyPods"
          "name" = "Ready-Pods"
          "type" = "string"
        },
        {
          "JSONPath" = ".status.primaryEndpoint"
          "name" = "Primary-Endpoint"
          "type" = "string"
        },
        {
          "JSONPath" = ".metadata.creationTimestamp"
          "name" = "Age"
          "type" = "date"
        },
      ]
      "group" = "arcdata.microsoft.com"
      "names" = {
        "kind" = "postgresql"
        "plural" = "postgresqls"
        "shortNames" = [
          "postgres",
        ]
      }
      "scope" = "Namespaced"
      "subresources" = {
        "status" = {}
      }
      "version" = "v1alpha1"
    }
  }
}

resource "kubernetes_namespace" "arc" {
  metadata {
    annotations = {
      name = "controller-namespace"
    }

    name = "arc"
  }

  depends_on = [
     kubernetes_manifest.data_controller_crd
    ,kubernetes_manifest.sql_mi_crd
    ,kubernetes_manifest.sql_mi_restore_crd
    ,kubernetes_manifest.postgres_sql_crd
  ]
}

resource "kubernetes_role" "role_bootstrapper" {
  metadata {
    name      = "role-bootstrapper"
    namespace = "arc"
  }

  rule {
    api_groups = [""]
    resources = ["pods", "configmaps", "services", "persistentvolumeclaims", "secrets", "serviceaccounts", "events"]
    verbs     = ["*"]
  }
  rule {
    api_groups = ["apps"]
    resources = ["replicasets", "statefulsets"]
    verbs     = ["*"]
  }
  rule {
    api_groups = ["rbac.authorization.k8s.io"]
    resources = ["roles", "rolebindings"]
    verbs     = ["*"]
  }
  rule {
    api_groups = ["sql.arcdata.microsoft.com", "tasks.sql.arcdata.microsoft.com", "arcdata.microsoft.com"]
    resources = ["*"]
    verbs     = ["*"]
  }

  depends_on = [
    kubernetes_namespace.arc 
  ]
}

resource "kubernetes_role_binding" "rb_bootstrapper" {
  metadata {
    name      = "rb-bootstrapper"
    namespace = "arc"
  }
  role_ref {
    api_group = "rbac.authorization.k8s.io"
    kind      = "Role"
    name      = "role-bootstrapper"
  }
  subject {
    kind      = "ServiceAccount"
    name      = "sa-bootstrapper"
  }

  depends_on = [
    kubernetes_role.role_bootstrapper
  ]
}

resource "kubernetes_deployment" "bootstrapper" {
  metadata {
    name = "bootstrapper"
    labels = {
      app = "bootstrapper"
    }
  }

  spec {
    replicas = 1

    selector {
      match_labels = {
        app = "bootstrtapper"
      }
    }

    template {
      metadata {
        labels = {
          app = "bootstrtapper"
        }
      }

      spec {
        node_selector = {
          "kubernetes.io/os" = "linux"
        }

        service_account_name = "sa-bootstrapper"

        image_pull_secrets {
          name = "arc-private-registry"
        }

        container {
          image = "microsoft.com/arcdata/arc-bootstrapper:latest"
          name  = "bootstrapper"
          image_pull_policy = "Always"
          security_context {
            run_as_user = 21000
          }
        }
      }
    }
  }

  depends_on = [
    kubernetes_role_binding.rb_bootstrapper
  ]
}

resource "kubernetes_secret" "controller-login-secret" {
  metadata {
    name      = "controller-login-secret"
    namespace = "arc"
  }

  binary_data = {
    "username" = "YXJjdXNlcg=="
    "password" = "T3NtaXVtNzY="
  }

  type = "kubernetes.io/basic-auth"

  depends_on = [
    kubernetes_deployment.bootstrapper 
  ]
}

resource "kubernetes_service_account" "sa-mssql-controller" {
  metadata {
    name = "sa-mssql-controller"
  }
  secret {
    name = "controller-login-secret"
  }

  depends_on = [
    kubernetes_secret.controller-login-secret 
  ]
}

resource "kubernetes_manifest" "arc" { 
  provider = kubernetes-alpha

  manifest = {
    "apiVersion" = "arcdata.microsoft.com/v1alpha1"
    "kind" = "datacontroller"
    "metadata" = {
      "name" = "arc"
      "namespace" = "arc"
    }
    "spec" = {
      "credentials" = {
        "controllerAdmin" = "controller-login-secret"
        "serviceAccount" = "sa-mssql-controller"
      }
      "docker" = {
        "imagePullPolicy" = "Always"
        "imageTag" = "latest"
        "registry" = "mcr.microsoft.com"
        "repository" = "arcdata"
      }
      "security" = {
        "allowDumps" = true
        "allowNodeMetricsCollection" = true
        "allowPodMetricsCollection" = true
        "allowRunAsRoot" = false
      }
      "services" = [
        {
          "name" = "controller"
          "port" = 30080
          "serviceType" = "LoadBalancer"
        },
        {
          "name" = "serviceProxy"
          "port" = 30777
          "serviceType" = "LoadBalancer"
        },
      ]
      "settings" = {
        "ElasticSearch" = {
          "vm.max_map_count" = "-1"
        }
        "azure" = {
          "connectionMode" = "indirect"
          "displayName" = "arc"
          "enableBilling" = "True"
          "location" = "eastus"
          "logs.rotation.days" = "7"
          "logs.rotation.size" = "5000"
          "resourceGroup" = "AzureArcTestEastUS"
          "subscription" = "XXX"
        }
      }
      "storage" = {
        "data" = {
          "accessMode" = "ReadWriteOnce"
          "className" = "portworx-sc"
          "size" = "15Gi"
        }
        "logs" = {
          "accessMode" = "ReadWriteOnce"
          "className" = "portworx-sc"
          "size" = "10Gi"
        }
      }
    }
  }

  depends_on = [
     kubernetes_service_account.sa-mssql-controller
  ]
}

Question

When the four custom resource definitions of:
- data_controller_crd
- sql_mi_crd
- sql_mi_restore_crd
- postgres_sql_crd

The configuration deploys without any issues, when the CRDs above are not already present in the cluster, I get:

 Error: Failed to determine GroupVersionResource for manifest

   with kubernetes_manifest.arc,
   on main.tf line 373, in resource "kubernetes_manifest" "arc":
  373: resource "kubernetes_manifest" "arc" {

 no matches for kind "datacontroller" in group "arcdata.microsoft.com"

Is this a bug or some sort of limitation that I'm running into ?

[chrisadkin-zz]

That should have read: when the four custom resource definitions have already been created, the configuration can be deployed without any problems, when they are not present - when you rely on the configuration to create them, this is when the GroupVersionResource error appears.

[avinashpancham]

Seems similar to #218. Still waiting for it to be solved

[Pluies]

As far as I can tell, #200, #218, #235, and #247 are all the same issue: can't define a new CRD and a new resource based on that CRD in kubernetes-alpha, or it will fail to plan.

This very issue was originally described in #72, and a massive refactor was supposed to fix it, but unfortunately this is still an issue on the latest version of the provider. 😢

@alexsomesan , given how often this issue pops up, should it be added to the Known Issues (#158) until it's resolved?