diff --git a/.config/.gdnsuppress b/.config/.gdnsuppress index 1077b13362..221145c1df 100644 --- a/.config/.gdnsuppress +++ b/.config/.gdnsuppress @@ -3,24 +3,22 @@ "suppressionSets": { "default": { "name": "default", - "createdDate": "2022-11-28 20:04:38Z", - "lastUpdatedDate": "2022-11-28 20:04:38Z" + "createdDate": "2024-11-21 01:58:07Z", + "lastUpdatedDate": "2024-11-21 01:58:07Z" } }, "results": { - "d7e55b5f3e54f9253a2fec595f97520ab0ffece607981d2db0fcfe4dae4cd490": { - "signature": "d7e55b5f3e54f9253a2fec595f97520ab0ffece607981d2db0fcfe4dae4cd490", + "9a675be4784d448ece481deb7f6f1429fd6606a08b9598f33f23c2123c581520": { + "signature": "9a675be4784d448ece481deb7f6f1429fd6606a08b9598f33f23c2123c581520", "alternativeSignatures": [], - "target": "**/testdata/dummy.pem", + "target": "keyvault/testdata/dummy.pem", + "line": 1, "memberOf": [ "default" ], "tool": "credscan", "ruleId": "CSCAN-GENERAL0020", - "justification": null, - "createdDate": "2022-11-28 20:04:38Z", - "expirationDate": null, - "type": null + "createdDate": "2024-11-21 01:58:07Z" } } } diff --git a/.config/credScanSuppressions.json b/.config/credScanSuppressions.json index 402a6cd9ea..8ca99ae249 100644 --- a/.config/credScanSuppressions.json +++ b/.config/credScanSuppressions.json @@ -1,4 +1,9 @@ { - "tool": "Credential Scanner", - "suppressions": [] + "tool": "Credential Scanner", + "suppressions": [ + { + "file": "keyvault/testdata/dummy.pem", + "_justification": "Fake credentials for testing" + } + ] } diff --git a/.pipelines/OneBranch.NonOfficial.CrossPlat.yaml b/.pipelines/OneBranch.NonOfficial.CrossPlat.yaml deleted file mode 100644 index 35f263ec6e..0000000000 --- a/.pipelines/OneBranch.NonOfficial.CrossPlat.yaml +++ /dev/null @@ -1,28 +0,0 @@ -trigger: none - -variables: - WindowsContainerImage: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest' # for Windows jobs - LinuxContainerImage: 'mcr.microsoft.com/onebranch/cbl-mariner/build:2.0' # for Linux jobs - -resources: - repositories: - - repository: templates - type: git - name: OneBranch.Pipelines/GovernedTemplates - ref: refs/heads/main - -extends: - template: v2/OneBranch.NonOfficial.CrossPlat.yml@templates - parameters: - featureFlags: - WindowsHostVersion: 1ESWindows2022 - stages: - - stage: build - jobs: - - job: main - pool: - type: windows - variables: - ob_outputDirectory: '$(Build.SourcesDirectory)\out' - steps: [] - diff --git a/.pipelines/OneBranch.Official.CrossPlat.yaml b/.pipelines/OneBranch.Official.CrossPlat.yaml deleted file mode 100644 index e67c7df267..0000000000 --- a/.pipelines/OneBranch.Official.CrossPlat.yaml +++ /dev/null @@ -1,27 +0,0 @@ -trigger: none - -variables: - WindowsContainerImage: 'onebranch.azurecr.io/windows/ltsc2022/vse2022:latest' # for Windows jobs - LinuxContainerImage: 'mcr.microsoft.com/onebranch/cbl-mariner/build:2.0' # for Linux jobs - -resources: - repositories: - - repository: templates - type: git - name: OneBranch.Pipelines/GovernedTemplates - ref: refs/heads/main - -extends: - template: v2/OneBranch.Official.CrossPlat.yml@templates - parameters: - featureFlags: - WindowsHostVersion: 1ESWindows2022 - stages: - - stage: build - jobs: - - job: main - pool: - type: windows - variables: - ob_outputDirectory: '$(Build.SourcesDirectory)\out' - steps: [] diff --git a/.pipelines/cni/k8s-e2e/k8s-e2e-job-template.yaml b/.pipelines/cni/k8s-e2e/k8s-e2e-job-template.yaml index 72c9092d76..fb7e79582e 100644 --- a/.pipelines/cni/k8s-e2e/k8s-e2e-job-template.yaml +++ b/.pipelines/cni/k8s-e2e/k8s-e2e-job-template.yaml @@ -8,11 +8,11 @@ parameters: jobs: - job: CNI_${{ parameters.os }} - condition: and( not(canceled()), not(failed()) ) displayName: CNI k8s E2E ${{ parameters.os }} dependsOn: ${{ parameters.dependsOn }} pool: name: $(BUILD_POOL_NAME_DEFAULT) + os: linux steps: - task: AzureCLI@2 inputs: diff --git a/.pipelines/cni/k8s-e2e/k8s-e2e.jobs.yaml b/.pipelines/cni/k8s-e2e/k8s-e2e.jobs.yaml index 158685826f..c43c8723bd 100644 --- a/.pipelines/cni/k8s-e2e/k8s-e2e.jobs.yaml +++ b/.pipelines/cni/k8s-e2e/k8s-e2e.jobs.yaml @@ -8,26 +8,17 @@ parameters: jobs: - job: CNI_${{ parameters.os }} - condition: and( not(canceled()), not(failed()) ) displayName: CNI k8s E2E ${{ parameters.os }} dependsOn: ${{ parameters.dependsOn }} pool: - isCustom: true - type: linux name: $(BUILD_POOL_NAME_DEFAULT) - variables: - ob_outputDirectory: $(Build.ArtifactStagingDirectory)/output - ob_git_checkout: true + os: linux steps: - - checkout: ACNReviewChanges - clean: true - - task: AzureCLI@2 inputs: azureSubscription: ${{ parameters.sub }} scriptLocation: "inlineScript" scriptType: "bash" - workingDirectory: $(ACN_DIR) addSpnToEnvironment: true inlineScript: | set -e @@ -42,9 +33,8 @@ jobs: # https://github.com/kubernetes/sig-release/blob/master/release-engineering/artifacts.md#content-of-kubernetes-test-system-archtargz-on-example-of-kubernetes-test-linux-amd64targz-directories-removed-from-list # explictly unzip and strip directories from ginkgo and e2e.test tar -xvzf kubernetes-test-linux-amd64.tar.gz --strip-components=3 kubernetes/test/bin/ginkgo kubernetes/test/bin/e2e.test - displayName: "Setup Environment" - retryCountOnTaskFailure: 5 + displayName: "Setup Environment" - ${{ if contains(parameters.os, 'windows') }}: - script: | set -e @@ -57,13 +47,11 @@ jobs: kubectl exec -i -n kube-system $pod -- powershell "Restart-Service kubeproxy" kubectl exec -i -n kube-system $pod -- powershell "Get-Service kubeproxy" done - workingDirectory: $(ACN_DIR) name: kubeproxy displayName: Restart Kubeproxy on Windows nodes retryCountOnTaskFailure: 3 - - ${{ if eq(parameters.datapath, true) }}: - - template: k8s-e2e.steps.yaml@ACNTools + - template: ../k8s-e2e/k8s-e2e-step-template.yaml parameters: testName: Datapath name: datapath @@ -72,9 +60,8 @@ jobs: os: ${{ parameters.os }} processes: 8 attempts: 10 - - ${{ if eq(parameters.dns, true) }}: - - template: k8s-e2e.steps.yaml@ACNTools + - template: ../k8s-e2e/k8s-e2e-step-template.yaml parameters: testName: DNS name: dns @@ -83,9 +70,8 @@ jobs: os: ${{ parameters.os }} processes: 8 attempts: 3 - - ${{ if eq(parameters.portforward, true) }}: - - template: k8s-e2e.steps.yaml@ACNTools + - template: ../k8s-e2e/k8s-e2e-step-template.yaml parameters: testName: Kubectl Portforward name: portforward @@ -94,9 +80,8 @@ jobs: os: ${{ parameters.os }} processes: 8 attempts: 3 - - ${{ if and( eq(parameters.service, true), contains(parameters.cni, 'cni') ) }}: - - template: k8s-e2e.steps.yaml@ACNTools + - template: ../k8s-e2e/k8s-e2e-step-template.yaml parameters: testName: Service Conformance name: service @@ -105,9 +90,8 @@ jobs: os: ${{ parameters.os }} processes: 8 attempts: 3 - - ${{ if and( eq(parameters.service, true), contains(parameters.cni, 'cilium') ) }}: - - template: k8s-e2e.steps.yaml@ACNTools + - template: ../k8s-e2e/k8s-e2e-step-template.yaml parameters: testName: Service Conformance|Cilium name: service @@ -116,9 +100,8 @@ jobs: os: ${{ parameters.os }} processes: 8 attempts: 3 - - ${{ if eq(parameters.hostport, true) }}: - - template: k8s-e2e.steps.yaml@ACNTools + - template: ../k8s-e2e/k8s-e2e-step-template.yaml parameters: testName: Host Port name: hostport @@ -127,9 +110,8 @@ jobs: os: ${{ parameters.os }} processes: 1 # Has a short serial test attempts: 3 - - ${{ if and(eq(parameters.hybridWin, true), eq(parameters.os, 'windows')) }}: - - template: k8s-e2e.steps.yaml@ACNTools + - template: ../k8s-e2e/k8s-e2e-step-template.yaml parameters: testName: Hybrid Network name: hybrid @@ -138,9 +120,8 @@ jobs: os: ${{ parameters.os }} processes: 8 attempts: 3 - - ${{ if and( eq(parameters.dualstack, true), eq(contains(parameters.cni, 'cilium'), false) ) }}: - - template: k8s-e2e.steps.yaml@ACNTools + - template: ../k8s-e2e/k8s-e2e-step-template.yaml parameters: testName: DualStack Test name: DualStack @@ -150,9 +131,8 @@ jobs: os: ${{ parameters.os }} processes: 8 attempts: 3 - - ${{ if and( eq(parameters.dualstack, true), contains(parameters.cni, 'cilium') ) }}: - - template: k8s-e2e.steps.yaml@ACNTools + - template: ../k8s-e2e/k8s-e2e-step-template.yaml parameters: testName: DualStack Test|Cilium name: DualStack diff --git a/.pipelines/containers/container-template.jobs.yaml b/.pipelines/containers/container-template.jobs.yaml new file mode 100644 index 0000000000..c817657377 --- /dev/null +++ b/.pipelines/containers/container-template.jobs.yaml @@ -0,0 +1,72 @@ +parameters: +- name: jobs + type: jobList + default: [] + +- name: arch + type: string + +- name: os + type: string + +- name: images + type: object + default: [] + + +jobs: +- ${{ each job in parameters.jobs }}: + - ${{ each image in parameters.images }}: + - job: ${{ replace(image, '-', '_') }}_${{ parameters.os }}_${{ parameters.arch }} + displayName: Build Image - (${{ image }} ${{ parameters.os }}/${{ parameters.arch }}) + steps: + - checkout: self + + - task: GoTool@0 + inputs: + version: $(GO_VERSION) + + - bash: | + set -e + + VAR_ID="${IMAGE}_${OS}_${ARCH}" + + BUILD_CONTEXT=$(make "$MAKE_BUILD_CONTEXT") + echo >&2 "##vso[task.setvariable variable=BUILD_CONTEXT_${VAR_ID};]$BUILD_CONTEXT" + + IMAGE_PLATFORM_TAG=$(make "$MAKE_IMAGE_TAG") + echo >&2 "##vso[task.setvariable variable=IMAGE_PLATFORM_TAG_${VAR_ID};]$IMAGE_PLATFORM_TAG" + + IMAGE_NAME_AND_TAG=$(make "$MAKE_IMAGE_NAME_AND_TAG") + echo >&2 "##vso[task.setvariable variable=IMAGE_NAME_AND_TAG_${VAR_ID};]$IMAGE_NAME_AND_TAG" + + DOCKERFILE_PATH=$(make "$MAKE_DOCKERFILE_PATH") + echo >&2 "##vso[task.setvariable variable=DOCKERFILE_PATH_${VAR_ID};]$DOCKERFILE_PATH" + + EXTRA_BUILD_ARGS=$(make "$MAKE_EXTRA_BUILD_ARGS") + echo >&2 "##vso[task.setvariable variable=EXTRA_BUILD_ARGS_${VAR_ID};]$EXTRA_BUILD_ARGS" + displayName: "Get Image Build Data" + env: + IMAGE: ${{ image }} + OS: ${{ parameters.os }} + ARCH: ${{ parameters.arch }} + PLATFORM: ${{ parameters.os }}/${{ parameters.arch }} + MAKE_IMAGE_TAG: ${{ image }}-image-tag + MAKE_BUILD_CONTEXT: ${{ image }}-image-build-context + MAKE_IMAGE_NAME_AND_TAG: ${{ image }}-image-name-and-tag + MAKE_DOCKERFILE_PATH: ${{ image }}-dockerfile-path + MAKE_EXTRA_BUILD_ARGS: ${{ image }}-docker-build-args + + - template: ./container-template.steps.yaml + parameters: + imageNameAndTag: $(IMAGE_NAME_AND_TAG_${{ image }}_${{ parameters.os }}_${{ parameters.arch }}) + dockerfilePath: $(DOCKERFILE_PATH_${{ image }}_${{ parameters.os }}_${{ parameters.arch }}) + buildContextPath: $(Build.SourcesDirectory) + targetVersion: $(IMAGE_PLATFORM_TAG_${{ image }}_${{ parameters.os }}_${{ parameters.arch }}) + targetOs: ${{ parameters.os }} + targetArch: ${{ parameters.arch }} + targetPlatform: "${{ parameters.os }}/${{ parameters.arch }}" + addBuildArgs: $(EXTRA_BUILD_ARGS_${{ image }}_${{ parameters.os }}_${{ parameters.arch }}) + buildOs: ${{ job.templatContext.buildOs }} + buildArch: ${{ job.templateContext.buildArch }} + buildPlatform: "${{ job.templatContext.buildOs }}/${{ job.templateContext.buildArch }}" diff --git a/.pipelines/containers/container-template.steps.yaml b/.pipelines/containers/container-template.steps.yaml index 3867ffa7e9..b986f037cc 100644 --- a/.pipelines/containers/container-template.steps.yaml +++ b/.pipelines/containers/container-template.steps.yaml @@ -1,37 +1,76 @@ parameters: - arch: "" - name: "" - os: "" +- name: imageNameAndTag + type: string + +- name: dockerfilePath + type: string + +- name: targetVersion + type: string + default: "" + +- name: buildContextPath + type: string + default: "" + +- name: targetPlatform + type: string + default: "" + +- name: targetOs + type: string + values: + - linux + - windows + +- name: targetArch + type: string + values: + - amd64 + - arm64 + +- name: buildOs + type: string + values: + - linux + - windows + +- name: buildArch + type: string + values: + - amd64 + - arm64 + +- name: buildPlatform + type: string + default: "" + +- name: addBuildArgs + type: object + default: "" + steps: -- task: AzureCLI@2 - displayName: "Login" - inputs: - azureSubscription: $(ACR_ARM_SERVICE_CONNECTION) - scriptLocation: "inlineScript" - scriptType: "bash" - inlineScript: | - az acr login -n $(ACR) - -- script: | - set -e - if [ "$IN_OS" = 'windows' ]; then export BUILDX_ACTION='--push'; fi - make "$IMGNAME" OS="$IN_OS" ARCH="$IN_ARCH" - name: image_build - displayName: Image Build - workingDirectory: $(ACN_DIR) +- task: 1ES.BuildContainerImage@1 retryCountOnTaskFailure: 3 - env: - IMGNAME: '${{ parameters.name }}-image' - IN_OS: '${{ parameters.os }}' - IN_ARCH: '${{ parameters.arch }}' - -- task: AzureCLI@2 - displayName: "Logout" inputs: - azureSubscription: $(ACR_ARM_SERVICE_CONNECTION) - scriptLocation: "inlineScript" - scriptType: "bash" - inlineScript: | - docker logout + image: ${{ parameters.imageNameAndTag }} + dockerfile: ${{ parameters.dockerfilePath }} + path: ${{ parameters.buildContextPath }} + buildArguments: | + --build-arg PLATFORM="${{ parameters.buildPlatform }}" + --build-arg ARCH="${{ parameters.buildArch }}" + --build-arg OS="${{ parameters.buildOs }}" + --build-arg VERSION="${{ parameters.targetVersion }}" + --platform "${{ parameters.targetPlatform }}" + --target "${{ parameters.targetOs }}" + ${{ parameters.addBuildArgs }} + enableNetwork: true + enablePull: false + enableCache: true + useBuildKit: true +- task: 1ES.PushContainerImage@1 + condition: ${{ eq(parameters.targetOs, 'windows') }} + inputs: + image: ${{ parameters.imageNameAndTag }} diff --git a/.pipelines/containers/containerize.jobs.yaml b/.pipelines/containers/containerize.jobs.yaml new file mode 100644 index 0000000000..1da57ee967 --- /dev/null +++ b/.pipelines/containers/containerize.jobs.yaml @@ -0,0 +1,85 @@ +parameters: +- name: supportedPlatforms + type: jobList + default: [] + +- name: targetOs + type: string + +- name: targetArch + type: string + + +- name: buildPlatform + type: string + default: '' + +- name: buildOs + type: string + values: + - linux + - windows + +- name: buildArch + type: string + values: + - amd64 + - arm64 + + +jobs: +- ${{ each supportedPlatformJob in parameters.supportedPlatforms }}: + - ${{ each imageName in supportedPlatformJob.images }}: + - job: ${{ replace(concat(imageName, '_', supportedPlatformJob.name), '-', '_') }} + displayName: Build Image - (${{ imageName }} ${{ parameters.targetOs }}/${{ parameters.targetArch }}) + steps: + - checkout: self + + - task: GoTool@0 + inputs: + version: $(GO_VERSION) + + - bash: | + set -e + + VAR_ID="${IMAGE}_${OS}_${ARCH}" + + BUILD_CONTEXT=$(make "$MAKE_BUILD_CONTEXT") + echo >&2 "##vso[task.setvariable variable=BUILD_CONTEXT_${VAR_ID};]$BUILD_CONTEXT" + + IMAGE_PLATFORM_TAG=$(make "$MAKE_IMAGE_TAG") + echo >&2 "##vso[task.setvariable variable=IMAGE_PLATFORM_TAG_${VAR_ID};]$IMAGE_PLATFORM_TAG" + + IMAGE_NAME_AND_TAG=$(make "$MAKE_IMAGE_NAME_AND_TAG") + echo >&2 "##vso[task.setvariable variable=IMAGE_NAME_AND_TAG_${VAR_ID};]$IMAGE_NAME_AND_TAG" + + DOCKERFILE_PATH=$(make "$MAKE_DOCKERFILE_PATH") + echo >&2 "##vso[task.setvariable variable=DOCKERFILE_PATH_${VAR_ID};]$DOCKERFILE_PATH" + + EXTRA_BUILD_ARGS=$(make "$MAKE_EXTRA_BUILD_ARGS") + echo >&2 "##vso[task.setvariable variable=EXTRA_BUILD_ARGS_${VAR_ID};]$EXTRA_BUILD_ARGS" + displayName: "Get Image Build Data" + env: + IMAGE: ${{ imageName }} + OS: ${{ parameters.targetOs }} + ARCH: ${{ parameters.targetArch }} + PLATFORM: ${{ parameters.targetOs }}/${{ parameters.targetArch }} + MAKE_IMAGE_TAG: ${{ imageName }}-image-tag + MAKE_BUILD_CONTEXT: ${{ imageName }}-image-build-context + MAKE_IMAGE_NAME_AND_TAG: ${{ imageName }}-image-name-and-tag + MAKE_DOCKERFILE_PATH: ${{ imageName }}-dockerfile-path + MAKE_EXTRA_BUILD_ARGS: ${{ imageName }}-docker-build-args + + - template: ./container-template.steps.yaml + parameters: + imageNameAndTag: $(IMAGE_NAME_AND_TAG_${{ imageName }}_${{ parameters.targetOs }}_${{ parameters.targetArch }}) + dockerfilePath: $(DOCKERFILE_PATH_${{ imageName }}_${{ parameters.targetOs }}_${{ parameters.targetArch }}) + buildContextPath: $(Build.SourcesDirectory) + targetVersion: $(IMAGE_PLATFORM_TAG_${{ imageName }}_${{ parameters.targetOs }}_${{ parameters.targetArch }}) + targetOs: ${{ parameters.targetOs }} + targetArch: ${{ parameters.targetArch }} + targetPlatform: "${{ parameters.targetOs }}/${{ parameters.targetArch }}" + addBuildArgs: $(EXTRA_BUILD_ARGS_${{ imageName }}_${{ parameters.targetOs }}_${{ parameters.targetArch }}) + buildOs: ${{ supportedPlatformJob.templatContext.buildOs }} + buildArch: ${{ supportedPlatformJob.templateContext.buildArch }} + buildPlatform: "${{ supportedPlatformJob.templatContext.buildOs }}/${{ supportedPlatformJob.templateContext.buildArch }}" diff --git a/.pipelines/containers/manifest-template.steps.yaml b/.pipelines/containers/manifest-template.steps.yaml index a8d1916460..82e69bd962 100644 --- a/.pipelines/containers/manifest-template.steps.yaml +++ b/.pipelines/containers/manifest-template.steps.yaml @@ -15,8 +15,7 @@ steps: - script: | set -e - make ${{ parameters.name }}-manifest-build PLATFORMS="${{ parameters.platforms }}" - workingDirectory: $(ACN_DIR) + make ${{ parameters.name }}-manifest-build PLATFORMS="${{ parameters.platforms }}" name: manifest_build displayName: Manifest Build retryCountOnTaskFailure: 3 @@ -35,19 +34,9 @@ steps: make ${{ parameters.name }}-skopeo-archive IMAGE_ARCHIVE_DIR=$(Build.ArtifactStagingDirectory)/images name: manifest_push displayName: Manifest Push - workingDirectory: $(ACN_DIR) retryCountOnTaskFailure: 3 - - task: AzureCLI@2 - displayName: "Logout" - inputs: - azureSubscription: $(ACR_ARM_SERVICE_CONNECTION) - scriptLocation: "inlineScript" - scriptType: "bash" - inlineScript: | - docker logout - - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: "Add SBOM Generator tool" + - task: 1ES.PublishPipelineArtifact@1 inputs: - BuildDropPath: "$(Build.ArtifactStagingDirectory)" + artifactName: "output" + targetPath: "$(Build.ArtifactStagingDirectory)" diff --git a/.pipelines/multitenancy/swiftv2-e2e.jobs.yaml b/.pipelines/multitenancy/swiftv2-e2e.jobs.yaml index b693827e30..3211a40a53 100644 --- a/.pipelines/multitenancy/swiftv2-e2e.jobs.yaml +++ b/.pipelines/multitenancy/swiftv2-e2e.jobs.yaml @@ -17,9 +17,8 @@ stages: - setup variables: commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] - ACN_DIR: $(Build.SourcesDirectory) jobs: - - template: ../templates/create-cluster-swiftv2.jobs.yaml@ACNTools + - template: ../templates/create-cluster-swiftv2.yaml parameters: name: ${{ parameters.name }} displayName: ${{ parameters.displayName }} @@ -39,7 +38,6 @@ stages: - publish - ${{ parameters.clusterName }} variables: - ACN_DIR: $(Build.SourcesDirectory) GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path GOBIN: "$(GOPATH)/bin" # Go binaries path modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking" @@ -48,22 +46,14 @@ stages: - job: ${{ parameters.name }} displayName: Swiftv2 Multitenancy E2E Test Suite - (${{ parameters.name }}) pool: - isCustom: true - type: linux name: $(BUILD_POOL_NAME_DEFAULT) + os: linux + image: acn-runner-vmss-image-1es-v3.0 demands: - - agent.os -equals Linux - Role -equals $(CUSTOM_E2E_ROLE) - variables: - ob_outputDirectory: $(Build.ArtifactStagingDirectory)/output - ob_git_checkout: true steps: - - checkout: ACNReviewChanges - clean: true - - - template: swiftv2-e2e.steps.yaml@ACNTools + - template: swiftv2-e2e.steps.yaml parameters: name: ${{ parameters.name }} clusterName: ${{ parameters.clusterName }}-$(commitID) os: linux - diff --git a/.pipelines/multitenancy/swiftv2-e2e.steps.yaml b/.pipelines/multitenancy/swiftv2-e2e.steps.yaml index 10a6a29982..7ef8ac6996 100644 --- a/.pipelines/multitenancy/swiftv2-e2e.steps.yaml +++ b/.pipelines/multitenancy/swiftv2-e2e.steps.yaml @@ -4,31 +4,25 @@ parameters: continueOnError: true steps: - - bash: | - go version - go env - mkdir -p '$(GOBIN)' - mkdir -p '$(GOPATH)/pkg' - mkdir -p '$(modulePath)' - echo '##vso[task.prependpath]$(GOBIN)' - echo '##vso[task.prependpath]$(GOROOT)/bin' - name: "GoEnv" - displayName: "Set up the Go environment" - - task: KubectlInstaller@0 inputs: kubectlVersion: latest - task: AzureCLI@2 + displayName: "Start Swiftv2 Pods" + continueOnError: ${{ parameters.continueOnError }} + env: + AZCLI: az + CLUSTER: ${{ parameters.clusterName }} + SUBNET_TOKEN: $(SUBNET_TOKEN) inputs: azureSubscription: $(ACN_TEST_SERVICE_CONNECTION) scriptLocation: "inlineScript" scriptType: "bash" - workingDirectory: $(ACN_DIR) addSpnToEnvironment: true inlineScript: | set -e - make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }} + make -C ./hack/aks set-kubeconf ls -lah pwd kubectl cluster-info @@ -58,11 +52,6 @@ steps: echo "Check pods after 4 minutes" kubectl get po -owide -A kubectl describe pni - name: "start_swiftv2_pods" - displayName: "Start Swiftv2 Pods" - continueOnError: ${{ parameters.continueOnError }} - env: - SUBNET_TOKEN: $(SUBNET_TOKEN) - script: | set -e @@ -70,9 +59,6 @@ steps: cd test/integration/swiftv2 echo "TestSwiftv2PodToPod and will run it after migration from scripts." go test -count=1 swiftv2_test.go -timeout 3m -tags swiftv2 -run ^TestSwiftv2PodToPod$ -tags=swiftv2,integration -v - workingDirectory: $(ACN_DIR) retryCountOnTaskFailure: 3 - name: "Swiftv2_Tests_future_version" displayName: "Swiftv2 Tests through code" continueOnError: ${{ parameters.continueOnError }} - diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml index 1bb07ef85c..28a6d19bec 100644 --- a/.pipelines/pipeline.yaml +++ b/.pipelines/pipeline.yaml @@ -433,10 +433,33 @@ stages: clusterName: "cildsovere2e" region: $(REGION_DUALSTACKOVERLAY_CLUSTER_TEST) steps: - - template: templates/delete-cluster.yaml + - template: templates/delete-cluster.steps.yaml parameters: name: $(name) clusterName: $(clusterName)-$(commitID) region: $(region) sub: $(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) svcConn: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + + - job: delete_test + displayName: Delete Cluster + pool: + name: "$(BUILD_POOL_NAME_DEFAULT)" + strategy: + matrix: + swiftv2_e2e: + name: swiftv2_e2e + clusterName: "mtacluster" + region: $(REGION_SWIFTV2_CLUSTER_TEST) + swiftv2_dummy_e2e: + name: swiftv2_dummy_e2e + clusterName: "swiftv2dummy" + region: $(REGION_SWIFTV2_CLUSTER_TEST) + steps: + - template: templates/delete-cluster.steps.yaml + parameters: + name: $(name) + clusterName: $(clusterName)-$(commitID) + region: $(region) + sub: $(SUB_AZURE_NETWORK_AGENT_TEST) + svcConn: $(ACN_TEST_SERVICE_CONNECTION) diff --git a/.pipelines/run-pipeline.yaml b/.pipelines/run-pipeline.yaml index 59c72c2d97..b2f47420bc 100644 --- a/.pipelines/run-pipeline.yaml +++ b/.pipelines/run-pipeline.yaml @@ -1,483 +1,375 @@ -parameters: - -- name: triggerBuildSourceBranch - type: string - default: '' - -- name: triggerBuildReason - type: string - default: '' - -- name: triggerBuildGitRef - type: string - default: '' - -- name: triggerBuildQueuedBy - type: string - default: '' - - stages: - - stage: setup - displayName: ACN - # Block build start until pre-build validation occurs. - dependsOn: pre_build - variables: - ACN_DIR: $(Build.SourcesDirectory) - jobs: - - job: env - displayName: Setup - pool: - isCustom: true - type: linux - name: "$(BUILD_POOL_NAME_DEFAULT)" - steps: - - checkout: ACNTools - clean: true - - - script: | - # To use the variables below, you must make the respective stage's dependsOn have - setup or it will not retain context of this stage - BUILD_NUMBER=$(Build.BuildNumber) - echo "##vso[task.setvariable variable=StorageID;isOutput=true]$(echo ${BUILD_NUMBER//./-})" - echo "##vso[task.setvariable variable=commitID;isOutput=true]$(echo $(make revision)-$(date "+%d%H%M"))" - echo "##vso[task.setvariable variable=Tag;isOutput=true]$(make version)" - echo "##vso[task.setvariable variable=npmVersion;isOutput=true]$(make npm-version)" - cat /etc/os-release - uname -a - sudo chown -R $(whoami):$(whoami) . - go version - go env - which go - echo $PATH - name: "EnvironmentalVariables" - displayName: "Set environmental variables" +- stage: setup + displayName: ACN + jobs: + - job: env + displayName: Setup + steps: + - script: | + # To use the variables below, you must make the respective stage's dependsOn have - setup or it will not retain context of this stage + BUILD_NUMBER=$(Build.BuildNumber) + echo "##vso[task.setvariable variable=StorageID;isOutput=true]$(echo ${BUILD_NUMBER//./-})" + echo "##vso[task.setvariable variable=commitID;isOutput=true]$(echo $(make revision)-$(date "+%d%H%M"))" + echo "##vso[task.setvariable variable=Tag;isOutput=true]$(make version)" + echo "##vso[task.setvariable variable=npmVersion;isOutput=true]$(make npm-version)" + cat /etc/os-release + uname -a + sudo chown -R $(whoami):$(whoami) . + go version + go env + which go + echo $PATH + echo "------" + echo $(Build.QueuedBy) + echo $(Build.Reason) # manual, PR, IndividualCI + echo $(Build.SourceBranch) + name: "EnvironmentalVariables" + displayName: "Set environmental variables" - - template: templates/unit-tests.stages.yaml@ACNTools +- template: templates/run-unit-tests.stages.yaml - - ${{ if not(contains(parameters.triggerBuildSourceBranch, 'refs/pull')) }}: - - stage: binaries - displayName: Build Binaries - dependsOn: - - setup - - test - variables: - ACN_DIR: $(Build.SourcesDirectory) - jobs: - - job: build - displayName: Build Binaries - variables: - STORAGE_ID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.StorageID'] ] - ob_outputDirectory: $(Build.ArtifactStagingDirectory) - ob_git_checkout: true - pool: - isCustom: true - type: linux - name: "$(BUILD_POOL_NAME_DEFAULT)" - steps: - - checkout: ACNReviewChanges - clean: true +#- ${{ if not(contains(variables['Build.SourceBranch'], 'refs/pull')) }}: +- stage: binaries + displayName: Build Binaries + dependsOn: + - setup + - test + jobs: + - job: build + displayName: Build Binaries + variables: + STORAGE_ID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.StorageID'] ] + steps: + - task: GoTool@0 + inputs: + version: $(GO_VERSION) - - script: | - make ipv6-hp-bpf-lib - make all-binaries-platforms - name: "BuildAllPlatformBinaries" - displayName: "Build all platform binaries" - workingDirectory: $(ACN_DIR) + - script: | + make ipv6-hp-bpf-lib + make all-binaries-platforms + displayName: "Build all platform binaries" - - script: | - mkdir -p ./output/bins - cd ./output - find . -name '*.tgz' -print -exec mv -t ./bins/ {} + - find . -name '*.zip' -print -exec mv -t ./bins/ {} + - shopt -s extglob - rm -rf !("bins") - name: "PrepareArtifacts" - displayName: "Prepare Artifacts" + - script: | + mkdir -p ./output/bins + cd ./output + find . -name '*.tgz' -print -exec mv -t ./bins/ {} + + find . -name '*.zip' -print -exec mv -t ./bins/ {} + + shopt -s extglob + rm -rf !("bins") + displayName: "Prepare Artifacts" - - task: CopyFiles@2 - inputs: - sourceFolder: "output" - targetFolder: $(Build.ArtifactStagingDirectory) - condition: succeeded() + - task: CopyFiles@2 + inputs: + sourceFolder: "output" + targetFolder: $(Build.ArtifactStagingDirectory) - - stage: containerize - displayName: Build Images - dependsOn: - - setup - - test - variables: - ACN_DIR: $(Build.SourcesDirectory) - jobs: - - job: containerize_amd64 - displayName: Build Images - pool: - isCustom: true - type: linux - name: "$(BUILD_POOL_NAME_LINUX_AMD64)" - strategy: - maxParallel: 4 - matrix: - azure_ipam_linux_amd64: - Suffix: azure_ipam_linux_amd64 - arch: amd64 - name: azure-ipam - os: linux - azure_ipam_windows_amd64: - Suffix: azure_ipam_windows_amd64 - arch: amd64 - name: azure-ipam - os: windows - cni_linux_amd64: - Suffix: cni_linux_amd64 - arch: amd64 - name: cni - os: linux - cni_windows_amd64: - Suffix: cni_windows_amd64 - arch: amd64 - name: cni - os: windows - cns_linux_amd64: - Suffix: cns_linux_amd64 - arch: amd64 - name: cns - os: linux - cns_windows_amd64: - Suffix: cns_windows_amd64 - arch: amd64 - name: cns - os: windows - ipv6_hp_bpf_linux_amd64: - Suffix: ipv6_hp_bpf_linux_amd64 - arch: amd64 - name: ipv6-hp-bpf - os: linux - npm_linux_amd64: - Suffix: npm_linux_amd64 - arch: amd64 - name: npm - os: linux - npm_windows_amd64: - Suffix: npm_windows_amd64 - arch: amd64 - name: npm - os: windows - variables: - ob_git_checkout: true - ob_artifactSuffix: $(Suffix) # this is needed to not collide published artifact containers - ob_outputDirectory: $(System.ArtifactStagingDirectory) - steps: - - checkout: ACNReviewChanges - clean: true + - task: 1ES.PublishPipelineArtifact@1 + inputs: + artifactName: "output" + targetPath: "$(Build.ArtifactStagingDirectory)" - - template: containers/container-template.steps.yaml@ACNTools - parameters: - arch: $(arch) - name: $(name) - os: $(os) +- stage: containerize + displayName: Build Images + dependsOn: + - setup + - test + templateContext: + authenticatedContainerRegistries: + - serviceConnection: $(ACR_SERVICECONNECTION) + jobs: + - template: containerize.jobs.yaml + parameters: + buildArch: amd64 + buildOs: linux + buildPlatform: linux/amd64 + supportedPlatforms: + - template: ./containers/containerize.jobs.yaml + parameters: + targetOs: linux + targetArch: amd64 + images: + - azure-ipam + - cni + - cns + - ipv6-hp-bpf + - npm + - template: ./containers/containerize.jobs.yaml + parameters: + targetOs: linux + targetArch: arm64 + images: + - azure-ipam + - cni + - cns + - ipv6-hp-bpf + - npm + - template: ./containers/containerize.jobs.yaml + parameters: + targetOs: windows + targetArch: amd64 + images: + - azure-ipam + - cni + - cns + - npm - - job: containerize_linux_arm64 - displayName: Build Images - pool: - isCustom: true - type: linux - name: "$(BUILD_POOL_NAME_LINUX_ARM64)" - strategy: - maxParallel: 4 - matrix: - azure_ipam_linux_arm64: - arch: arm64 - name: azure-ipam - os: linux - Suffix: azure-ipam-linux-arm64 - cni_linux_arm64: - arch: arm64 - name: cni - os: linux - Suffix: cni-linux-arm64 - cns_linux_arm64: - arch: arm64 - name: cns - os: linux - Suffix: cns - ipv6_hp_bpf_linux_arm64: - arch: arm64 - name: ipv6-hp-bpf - os: linux - Suffix: ipv6-hp-bpf-linux-arm64 - npm_linux_arm64: - arch: arm64 - name: npm - os: linux - Suffix: npm-linux-arm64 - variables: - STORAGE_ID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.StorageID'] ] - ob_outputDirectory: $(Build.ArtifactStagingDirectory) - ob_git_checkout: true - ob_artifactSuffix: $(Suffix) - steps: - - checkout: ACNReviewChanges - clean: true - - template: containers/container-template.steps.yaml@ACNTools - parameters: - arch: $(arch) - name: $(name) - os: $(os) +- stage: publish + displayName: Publish Multiarch Manifests + dependsOn: + - containerize + variables: + Packaging.EnableSBOMSigning: false + jobs: + - job: manifest + displayName: Compile Manifests + strategy: + maxParallel: 4 + matrix: + azure_ipam: + name: azure-ipam + platforms: linux/amd64 linux/arm64 windows/amd64 + cni: + name: cni + platforms: linux/amd64 linux/arm64 windows/amd64 + cns: + name: cns + platforms: linux/amd64 linux/arm64 windows/amd64 + ipv6_hp_bpf: + name: ipv6-hp-bpf + platforms: linux/amd64 linux/arm64 + npm: + name: npm + platforms: linux/amd64 linux/arm64 windows/amd64 + steps: + - task: GoTool@0 + inputs: + version: $(GO_VERSION) + - template: containers/manifest-template.steps.yaml + parameters: + name: $(name) + platforms: $(platforms) +# Cilium Podsubnet E2E tests +- template: singletenancy/cilium/cilium-e2e.jobs.yaml + parameters: + name: "cilium_e2e" + displayName: Cilium + clusterType: swift-byocni-nokubeproxy-up + clusterName: "ciliume2e" + vmSize: Standard_B2ms + k8sVersion: "" + dependsOn: "containerize" - - stage: publish - displayName: Publish Multiarch Manifests - dependsOn: - - containerize - variables: - Packaging.EnableSBOMSigning: false - ACN_DIR: $(Build.SourcesDirectory) - jobs: - - job: manifest - displayName: Compile Manifests - pool: - isCustom: true - type: linux - name: "$(BUILD_POOL_NAME_DEFAULT)" - strategy: - maxParallel: 4 - matrix: - azure_ipam: - name: azure-ipam - platforms: linux/amd64 linux/arm64 windows/amd64 - Suffix: azure-ipam - cni: - name: cni - platforms: linux/amd64 linux/arm64 windows/amd64 - Suffix: cni - cns: - name: cns - platforms: linux/amd64 linux/arm64 windows/amd64 - Suffix: cns - ipv6_hp_bpf: - name: ipv6-hp-bpf - platforms: linux/amd64 linux/arm64 - Suffix: ipv6-hp-bpf - npm: - name: npm - platforms: linux/amd64 linux/arm64 windows/amd64 - Suffix: npm - variables: - ob_outputDirectory: $(Build.ArtifactStagingDirectory) - ob_git_checkout: true - STORAGE_ID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.StorageID'] ] - # this is needed to not collide published artifact containers - #ob_artifactBaseName: drop_$(Job.StageName)_$(Job.JobName)_ - ob_artifactSuffix: $(Suffix) - #artifactName: ${{ ob_artifactBaseName }}${{ name }} - steps: - - checkout: ACNReviewChanges - clean: true +# Cilium Nodesubnet E2E tests +- template: singletenancy/cilium-nodesubnet/cilium-nodesubnet-e2e.jobs.yaml + parameters: + name: "cilium_nodesubnet_e2e" + displayName: Cilium NodeSubnet + clusterType: nodesubnet-byocni-nokubeproxy-up + clusterName: "cilndsubnete2e" + vmSize: Standard_B2s + k8sVersion: "" + dependsOn: "containerize" - - template: containers/manifest-template.steps.yaml@ACNTools - parameters: - name: $(name) - platforms: $(platforms) +# Cilium Overlay E2E tests +- template: singletenancy/cilium-overlay/cilium-overlay-e2e.jobs.yaml + parameters: + name: "cilium_overlay_e2e" + displayName: Cilium on AKS Overlay + clusterType: overlay-byocni-nokubeproxy-up + clusterName: "cilovere2e" + vmSize: Standard_B2ms + k8sVersion: "" + dependsOn: "containerize" - # Cilium Podsubnet E2E tests - - template: singletenancy/cilium/cilium-e2e.jobs.yaml@ACNTools - parameters: - name: "cilium_e2e" - displayName: Cilium - clusterType: swift-byocni-nokubeproxy-up - clusterName: "ciliume2e" - vmSize: Standard_B2ms - k8sVersion: "" - dependsOn: "containerize" - - # Cilium Overlay E2E tests - - template: singletenancy/cilium-overlay/cilium-overlay-e2e.jobs.yaml@ACNTools - parameters: - name: "cilium_overlay_e2e" - displayName: Cilium on AKS Overlay - clusterType: overlay-byocni-nokubeproxy-up - clusterName: "cilovere2e" - vmSize: Standard_B2ms - k8sVersion: "" - dependsOn: "containerize" - - # Cilium Dualstack Overlay E2E tests - - template: singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e.jobs.yaml@ACNTools - parameters: - name: "cilium_dualstackoverlay_e2e" - displayName: Cilium on AKS DualStack Overlay - os: linux - clusterType: dualstack-byocni-nokubeproxy-up - clusterName: "cildsovere2e" - vmSize: Standard_B2ms - k8sVersion: "" - dependsOn: "containerize" +# Cilium Dualstack Overlay E2E tests +- template: singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e.jobs.yaml + parameters: + name: "cilium_dualstackoverlay_e2e" + displayName: Cilium on AKS DualStack Overlay + os: linux + clusterType: dualstack-byocni-nokubeproxy-up + clusterName: "cildsovere2e" + vmSize: Standard_B2ms + k8sVersion: "" + dependsOn: "containerize" # Cilium Overlay with hubble E2E tests - - template: singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e.jobs.yaml@ACNTools - parameters: - name: "cilium_h_overlay_e2e" - displayName: Cilium on AKS Overlay with Hubble - clusterType: overlay-byocni-nokubeproxy-up - clusterName: "cilwhleovere2e" - vmSize: Standard_B2ms - k8sVersion: "" - dependsOn: "containerize" - testHubble: true +- template: singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e.jobs.yaml + parameters: + name: "cilium_h_overlay_e2e" + displayName: Cilium on AKS Overlay with Hubble + clusterType: overlay-byocni-nokubeproxy-up + clusterName: "cilwhleovere2e" + vmSize: Standard_B2ms + k8sVersion: "" + dependsOn: "containerize" + testHubble: true - # Azure Overlay E2E tests - - template: singletenancy/azure-cni-overlay/azure-cni-overlay-e2e.jobs.yaml@ACNTools - parameters: - name: "azure_overlay_e2e" - displayName: Azure Overlay - os: linux - clusterType: overlay-byocni-up - clusterName: "azovere2e" - vmSize: Standard_B2ms - k8sVersion: "" - dependsOn: "containerize" +# Azure Overlay E2E tests +- template: singletenancy/azure-cni-overlay/azure-cni-overlay-e2e.jobs.yaml + parameters: + name: "azure_overlay_e2e" + displayName: Azure Overlay + os: linux + clusterType: overlay-byocni-up + clusterName: "azovere2e" + vmSize: Standard_B2ms + k8sVersion: "" + dependsOn: "containerize" - # AKS Swift E2E tests - - template: singletenancy/aks-swift/aks-swift-e2e.jobs.yaml@ACNTools - parameters: - name: "aks_swift_e2e" - displayName: AKS Swift Ubuntu - os: linux - clusterType: swift-byocni-up - clusterName: "swifte2e" - vmSize: Standard_B2ms - k8sVersion: "" - dependsOn: "containerize" +# Azure Overlay E2E Stateless CNI tests +- template: singletenancy/azure-cni-overlay-stateless/azure-cni-overlay-stateless-e2e.jobs.yaml + parameters: + name: "azure_overlay_stateless_e2e" + displayName: Azure Stateless CNI Overlay + os: windows + clusterType: overlay-byocni-up + clusterName: "statelesswin" + vmSize: Standard_B2ms + dependsOn: "containerize" - # AKS Swift Vnet Scale E2E tests - - template: singletenancy/aks-swift/aks-swift-e2e.jobs.yaml@ACNTools - parameters: - name: "aks_swift_vnetscale_e2e" - displayName: AKS Swift Vnet Scale Ubuntu - os: linux - clusterType: vnetscale-swift-byocni-up - clusterName: "vscaleswifte2e" - vmSize: Standard_B2ms - k8sVersion: "1.28" - dependsOn: "containerize" +# AKS Swift E2E tests +- template: singletenancy/aks-swift/aks-swift-e2e.jobs.yaml + parameters: + name: "aks_swift_e2e" + displayName: AKS Swift Ubuntu + os: linux + clusterType: swift-byocni-up + clusterName: "swifte2e" + vmSize: Standard_B2ms + k8sVersion: "" + dependsOn: "containerize" - # CNIv1 E2E tests - - template: singletenancy/aks/aks-e2e.jobs.yaml@ACNTools - parameters: - name: "aks_ubuntu_22_linux_e2e" - displayName: AKS Ubuntu 22 - arch: "amd64" - os: "linux" - clusterType: linux-cniv1-up - clusterName: "ubuntu22e2e" - vmSize: Standard_B2s - k8sVersion: 1.25 - scaleup: 100 - dependsOn: "containerize" +# AKS Swift Vnet Scale E2E tests +- template: singletenancy/aks-swift/aks-swift-e2e.jobs.yaml + parameters: + name: "aks_swift_vnetscale_e2e" + displayName: AKS Swift Vnet Scale Ubuntu + os: linux + clusterType: vnetscale-swift-byocni-up + clusterName: "vscaleswifte2e" + vmSize: Standard_B2ms + k8sVersion: "1.30" + dependsOn: "containerize" - - template: singletenancy/aks/aks-e2e.jobs.yaml@ACNTools - parameters: - name: "aks_windows_22_e2e" - displayName: AKS Windows 2022 - arch: amd64 - os: windows - clusterType: windows-cniv1-up - clusterName: "win22e2e" - vmSize: Standard_B2ms - os_version: "ltsc2022" - scaleup: 50 - dependsOn: "containerize" +# CNIv1 E2E tests +- template: singletenancy/aks/aks-e2e.jobs.yaml + parameters: + name: "aks_ubuntu_22_linux_e2e" + displayName: AKS Ubuntu 22 + arch: "amd64" + os: "linux" + clusterType: linux-cniv1-up + clusterName: "ubuntu22e2e" + vmSize: Standard_B2s + k8sVersion: 1.25 + scaleup: 100 + dependsOn: "containerize" - # CNI dual stack overlay E2E tests - - template: singletenancy/dualstack-overlay/dualstackoverlay-e2e.jobs.yaml@ACNTools - parameters: - name: "dualstackoverlay_e2e" - displayName: AKS DualStack Overlay - os: linux - clusterType: dualstack-overlay-byocni-up - clusterName: "dsovere2e" - vmSize: Standard_B2ms - dependsOn: "containerize" +- template: singletenancy/aks/aks-e2e.jobs.yaml + parameters: + name: "aks_windows_22_e2e" + displayName: AKS Windows 2022 + arch: amd64 + os: windows + clusterType: windows-cniv1-up + clusterName: "win22e2e" + vmSize: Standard_B2ms + os_version: "ltsc2022" + scaleup: 50 + dependsOn: "containerize" - - stage: delete - displayName: Delete Clusters - condition: always() - dependsOn: - - setup - - azure_overlay_e2e - - aks_swift_e2e - - cilium_e2e - - cilium_overlay_e2e - - cilium_h_overlay_e2e - - aks_ubuntu_22_linux_e2e - - aks_swift_vnetscale_e2e - - aks_windows_22_e2e - - dualstackoverlay_e2e - - cilium_dualstackoverlay_e2e - variables: - ACN_DIR: $(Build.SourcesDirectory) - commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] - jobs: - - job: delete - displayName: Delete Cluster - pool: - isCustom: true - type: linux - name: "$(BUILD_POOL_NAME_DEFAULT)" - strategy: - matrix: - cilium_e2e: - name: cilium_e2e - clusterName: "ciliume2e" - Suffix: cilium_e2e - cilium_overlay_e2e: - name: cilium_overlay_e2e - clusterName: "cilovere2e" - Suffix: cilium_overlay_e2e - cilium_h_overlay_e2e: - name: cilium_h_overlay_e2e - clusterName: "cilwhleovere2e" - Suffix: cilium_h_overlay_e2e - azure_overlay_e2e: - name: azure_overlay_e2e - clusterName: "azovere2e" - Suffix: azure_overlay_e2e - aks_swift_e2e: - name: aks_swift_e2e - clusterName: "swifte2e" - Suffix: aks_swift_e2e - aks_swift_vnetscale_e2e: - name: aks_swift_vnetscale_e2e - clusterName: "vscaleswifte2e" - Suffix: aks_swift_vnetscale_e2e - aks_ubuntu_22_linux_e2e: - name: aks_ubuntu_22_linux_e2e - clusterName: "ubuntu22e2e" - Suffix: aks_ubuntu_22_linux_e2e - aks_windows_22_e2e: - name: aks_windows_22_e2e - clusterName: "win22e2e" - Suffix: aks_windows_22_e2e - dualstackoverlay_e2e: - name: dualstackoverlay_e2e - clusterName: "dsovere2e" - Suffix: dualstackoverlay_e2e - cilium_dualstackoverlay_e2e: - name: cilium_dualstackoverlay_e2e - clusterName: "cildsovere2e" - Suffix: cilium_dualstackoverlay_e2e - variables: - STORAGE_ID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.StorageID'] ] - ob_outputDirectory: $(Build.ArtifactStagingDirectory) - ob_git_checkout: true - ob_artifactSuffix: $(Suffix) # this is needed to not collide published artifact containers - steps: - - checkout: ACNReviewChanges - clean: true +# CNI dual stack overlay E2E tests +- template: singletenancy/dualstack-overlay/dualstackoverlay-e2e.jobs.yaml + parameters: + name: "dualstackoverlay_e2e" + displayName: AKS DualStack Overlay + os: linux + clusterType: dualstack-overlay-byocni-up + clusterName: "dsovere2e" + vmSize: Standard_B2ms + dependsOn: "containerize" - - template: templates/delete-cluster.steps.yaml@ACNTools - parameters: - name: $(name) - clusterName: $(clusterName)-$(commitID) - region: $(REGION_AKS_CLUSTER_TEST) +- stage: delete + displayName: Delete Clusters + condition: always() + dependsOn: + - setup + - azure_overlay_e2e + - azure_overlay_stateless_e2e + - aks_swift_e2e + - cilium_e2e + - cilium_nodesubnet_e2e + - cilium_overlay_e2e + - cilium_h_overlay_e2e + - aks_ubuntu_22_linux_e2e + - aks_swift_vnetscale_e2e + - aks_windows_22_e2e + - dualstackoverlay_e2e + - cilium_dualstackoverlay_e2e + variables: + commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] + jobs: + - job: delete_build + displayName: Delete Cluster + strategy: + maxParallel: 4 + matrix: + cilium_e2e: + name: cilium_e2e + clusterName: "ciliume2e" + region: $(REGION_AKS_CLUSTER_TEST) + cilium_nodesubnet_e2e: + name: cilium_nodesubnet_e2e + clusterName: "cilndsubnete2e" + region: $(REGION_AKS_CLUSTER_TEST) + cilium_overlay_e2e: + name: cilium_overlay_e2e + clusterName: "cilovere2e" + region: $(REGION_AKS_CLUSTER_TEST) + cilium_h_overlay_e2e: + name: cilium_h_overlay_e2e + clusterName: "cilwhleovere2e" + region: $(REGION_AKS_CLUSTER_TEST) + azure_overlay_e2e: + name: azure_overlay_e2e + clusterName: "azovere2e" + region: $(REGION_AKS_CLUSTER_TEST) + azure_overlay_stateless_e2e: + name: azure_overlay_stateless_e2e + clusterName: "statelesswin" + region: $(REGION_AKS_CLUSTER_TEST) + aks_swift_e2e: + name: aks_swift_e2e + clusterName: "swifte2e" + region: $(REGION_AKS_CLUSTER_TEST) + aks_swift_vnetscale_e2e: + name: aks_swift_vnetscale_e2e + clusterName: "vscaleswifte2e" + region: $(REGION_AKS_CLUSTER_TEST) + aks_ubuntu_22_linux_e2e: + name: aks_ubuntu_22_linux_e2e + clusterName: "ubuntu22e2e" + region: $(REGION_AKS_CLUSTER_TEST) + aks_windows_22_e2e: + name: aks_windows_22_e2e + clusterName: "win22e2e" + region: $(REGION_AKS_CLUSTER_TEST) + dualstackoverlay_e2e: + name: dualstackoverlay_e2e + clusterName: "dsovere2e" + region: $(REGION_DUALSTACKOVERLAY_CLUSTER_TEST) + cilium_dualstackoverlay_e2e: + name: cilium_dualstackoverlay_e2e + clusterName: "cildsovere2e" + region: $(REGION_DUALSTACKOVERLAY_CLUSTER_TEST) + steps: + - template: templates/delete-cluster.steps.yaml + parameters: + name: $(name) + clusterName: $(clusterName)-$(commitID) + region: $(region) + sub: $(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) + svcConn: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) diff --git a/.pipelines/singletenancy/aks-swift/aks-swift-e2e.jobs.yaml b/.pipelines/singletenancy/aks-swift/aks-swift-e2e.jobs.yaml index 928bcdec06..8c46964762 100644 --- a/.pipelines/singletenancy/aks-swift/aks-swift-e2e.jobs.yaml +++ b/.pipelines/singletenancy/aks-swift/aks-swift-e2e.jobs.yaml @@ -15,9 +15,8 @@ stages: - setup variables: commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] - ACN_DIR: $(Build.SourcesDirectory) jobs: - - template: ../../templates/create-cluster.jobs.yaml@ACNTools + - template: ../../templates/create-cluster.jobs.yaml parameters: name: ${{ parameters.name }} displayName: ${{ parameters.displayName }} @@ -35,7 +34,6 @@ stages: - publish - ${{ parameters.clusterName }} variables: - ACN_DIR: $(Build.SourcesDirectory) TAG: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.Tag'] ] CURRENT_VERSION: $[ stagedependencies.containerize.check_tag.outputs['CurrentTagManifests.currentTagManifests'] ] commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] @@ -48,26 +46,22 @@ stages: displayName: Singletenancy AKS Swift Suite - (${{ parameters.name }}) timeoutInMinutes: 120 pool: - isCustom: true - type: linux name: $(BUILD_POOL_NAME_DEFAULT) + os: linux + image: acn-runner-vmss-image-1es-v3.0 demands: - - agent.os -equals Linux - Role -equals $(CUSTOM_E2E_ROLE) - variables: - ob_outputDirectory: $(Build.ArtifactStagingDirectory)/output - ob_git_checkout: true steps: - - checkout: ACNReviewChanges + - checkout: self clean: true - - template: aks-swift-e2e.steps.yaml@ACNTools + - template: aks-swift-e2e.steps.yaml parameters: name: ${{ parameters.name }} clusterName: ${{ parameters.clusterName }}-$(commitID) scaleup: 100 - - template: ../../cni/k8s-e2e/k8s-e2e.jobs.yaml@ACNTools + - template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml parameters: sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) clusterName: ${{ parameters.clusterName }}-$(commitID) @@ -85,18 +79,8 @@ stages: - ${{ parameters.name }} - cni_linux condition: failed() - pool: - isCustom: true - type: linux - name: $(BUILD_POOL_NAME_DEFAULT) - variables: - ob_outputDirectory: $(ACN_DIR)/${{ parameters.clusterName }}-$(commitID)_FailedE2ELogs_Attempt_#$(System.StageAttempt) - ob_git_checkout: true steps: - - checkout: ACNReviewChanges - clean: true - - - template: ../../templates/log-template.steps.yaml@ACNTools + - template: ../../templates/log-template.steps.yaml parameters: clusterName: ${{ parameters.clusterName }}-$(commitID) os: ${{ parameters.os }} diff --git a/.pipelines/singletenancy/aks-swift/aks-swift-e2e.steps.yaml b/.pipelines/singletenancy/aks-swift/aks-swift-e2e.steps.yaml index f8177d1014..21a0ec3446 100644 --- a/.pipelines/singletenancy/aks-swift/aks-swift-e2e.steps.yaml +++ b/.pipelines/singletenancy/aks-swift/aks-swift-e2e.steps.yaml @@ -4,90 +4,93 @@ parameters: scaleup: "" steps: - - bash: | - go version - go env - mkdir -p '$(GOBIN)' - mkdir -p '$(GOPATH)/pkg' - mkdir -p '$(modulePath)' - echo '##vso[task.prependpath]$(GOBIN)' - echo '##vso[task.prependpath]$(GOROOT)/bin' - name: "GoEnv" - displayName: "Set up the Go environment" - - task: KubectlInstaller@0 inputs: kubectlVersion: latest - task: AzureCLI@2 + displayName: "Set Kubeconfig" + env: + AZCLI: az + CLUSTER: ${{ parameters.clusterName }} inputs: azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) scriptLocation: "inlineScript" scriptType: "bash" - workingDirectory: $(ACN_DIR) addSpnToEnvironment: true inlineScript: | set -e - make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }} - name: "kubeconfig" - displayName: "Set Kubeconfig" + make -C ./hack/aks set-kubeconf - script: | ls -lah pwd kubectl cluster-info kubectl get po -owide -A - sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux CNI_TYPE=cniv2 VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_AZURE_VNET=true CNS_VERSION=$(make cns-version) CNI_VERSION=$(make cni-version) CLEANUP=true - workingDirectory: $(ACN_DIR) + sudo -E env "PATH=$PATH" make test-load =true CNS_VERSION=$(make cns-version) CNI_VERSION=$(make cni-version) retryCountOnTaskFailure: 3 - name: "aksswifte2e" displayName: "Run AKS Swift E2E" + env: + SCALE_UP: 32 + OS_TYPE: linux + CNI_TYPE: cniv2 + VALIDATE_STATEFILE: true + INSTALL_CNS: true + INSTALL_AZURE_VNET: true + CLEANUP: true - task: AzureCLI@2 + displayName: "Restart Nodes" + env: + AZCLI: az + CLUSTER: ${{ parameters.clusterName }} + SCALE_UP: ${{ coalesce(parameters.scaleup, 32) }} + RESOURCE_GROUP: "MC_${{ parameters.clusterName }}_${{ parameters.clusterName }}_$(REGION_AKS_CLUSTER_TEST)" + REGION: $(REGION_AKS_CLUSTER_TEST) inputs: azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) scriptLocation: "inlineScript" scriptType: "bash" - workingDirectory: $(ACN_DIR) addSpnToEnvironment: true inlineScript: | set -e kubectl get po -owide -A - clusterName=${{ parameters.clusterName }} echo "Restarting nodes" - for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do - make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val} + for val in $(az vmss list -g "$RESOURCE_GROUP" --query "[].name" -o tsv); do + make -C ./hack/aks restart-vmss VMSS_NAME=${val} done - displayName: "Restart Nodes" - task: AzureCLI@2 + displayName: "Validate Node Restart" + retryCountOnTaskFailure: 3 + env: + ITERATIONS: 2 + SCALE_UP: ${{ parameters.scaleup }} + OS_TYPE: linux + RESTART_CASE: true + CNI_TYPE: cniv2 inputs: azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) scriptLocation: "inlineScript" scriptType: "bash" - workingDirectory: $(ACN_DIR) addSpnToEnvironment: true inlineScript: | set -e - cd test/integration/load - # Scale Cluster Up/Down to confirm functioning CNS - ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=linux go test -count 1 -timeout 30m -tags load -run ^TestLoad$ - kubectl get pods -owide -A + pushd test/integration/load + # Scale Cluster Up/Down to confirm functioning CNS + go test -count 1 -timeout 30m -tags load -run ^TestLoad$ + kubectl get pods -owide -A + popd - cd ../../.. echo "Validating Node Restart" - make test-validate-state OS_TYPE=linux RESTART_CASE=true CNI_TYPE=cniv2 + make test-validate-state kubectl delete ns load-test - displayName: "Validate Node Restart" - retryCountOnTaskFailure: 3 - script: | echo "Run wireserver and metadata connectivity Tests" bash test/network/wireserver_metadata_test.sh - workingDirectory: $(ACN_DIR) retryCountOnTaskFailure: 3 - name: "WireserverMetadataConnectivityTests" displayName: "Run Wireserver and Metadata Connectivity Tests" - script: | @@ -97,8 +100,5 @@ steps: if ! [ -z $(kubectl -n kube-system get ds azure-cns | grep non-existing) ]; then kubectl -n kube-system patch daemonset azure-cns --type json -p='[{"op": "remove", "path": "/spec/template/spec/nodeSelector/non-existing"}]' fi - workingDirectory: $(ACN_DIR) - retryCountOnTaskFailure: 3 - name: "testAsyncDelete" displayName: "Verify Async Delete when CNS is down" diff --git a/.pipelines/singletenancy/aks/aks-e2e.jobs.yaml b/.pipelines/singletenancy/aks/aks-e2e.jobs.yaml index acfa1149a8..f9dda9c1b3 100644 --- a/.pipelines/singletenancy/aks/aks-e2e.jobs.yaml +++ b/.pipelines/singletenancy/aks/aks-e2e.jobs.yaml @@ -19,9 +19,8 @@ stages: - setup variables: commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] - ACN_DIR: $(Build.SourcesDirectory) jobs: - - template: ../../templates/create-cluster.jobs.yaml@ACNTools + - template: ../../templates/create-cluster.jobs.yaml parameters: name: ${{ parameters.name }} displayName: ${{ parameters.displayName }} @@ -35,35 +34,30 @@ stages: - stage: ${{ parameters.name }} displayName: E2E - ${{ parameters.displayName }} - dependsOn: - - setup - - publish - - ${{ parameters.clusterName }} variables: - ACN_DIR: $(Build.SourcesDirectory) GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path GOBIN: "$(GOPATH)/bin" # Go binaries path modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking" commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] + dependsOn: + - setup + - publish + - ${{ parameters.clusterName }} jobs: - job: ${{ parameters.name }} displayName: Singletenancy AKS - (${{ parameters.name }}) timeoutInMinutes: 120 pool: - isCustom: true - type: linux name: $(BUILD_POOL_NAME_DEFAULT) + os: linux + image: acn-runner-vmss-image-1es-v3.0 demands: - - agent.os -equals Linux - Role -equals $(CUSTOM_E2E_ROLE) - variables: - ob_outputDirectory: $(Build.ArtifactStagingDirectory)/output - ob_git_checkout: true steps: - - checkout: ACNReviewChanges + - checkout: self clean: true - - template: aks-e2e.steps.yaml@ACNTools + - template: aks-e2e.steps.yaml parameters: name: ${{ parameters.name }} clusterName: ${{ parameters.clusterName }}-$(commitID) @@ -72,7 +66,7 @@ stages: os_version: ${{ parameters.os_version }} scaleup: ${{ parameters.scaleup }} - - template: ../../cni/k8s-e2e/k8s-e2e.jobs.yaml@ACNTools + - template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml parameters: sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) clusterName: ${{ parameters.clusterName }}-$(commitID) @@ -91,18 +85,9 @@ stages: - ${{ parameters.name }} - cni_${{ parameters.os }} condition: failed() - pool: - type: linux - variables: - ob_outputDirectory: $(ACN_DIR)/${{ parameters.clusterName }}-$(commitID)_FailedE2ELogs_Attempt_#$(System.StageAttempt) - ob_git_checkout: true steps: - - checkout: ACNTools - clean: true - - - template: ../../templates/log-template.steps.yaml@ACNTools + - template: ../../templates/log-template.steps.yaml parameters: clusterName: ${{ parameters.clusterName }}-$(commitID) os: ${{ parameters.os }} cni: cniv1 - diff --git a/.pipelines/singletenancy/aks/aks-e2e.steps.yaml b/.pipelines/singletenancy/aks/aks-e2e.steps.yaml index 85dd35dfbd..9892941854 100644 --- a/.pipelines/singletenancy/aks/aks-e2e.steps.yaml +++ b/.pipelines/singletenancy/aks/aks-e2e.steps.yaml @@ -7,81 +7,80 @@ parameters: scaleup: "" steps: - - bash: | - go version - go env - mkdir -p '$(GOBIN)' - mkdir -p '$(GOPATH)/pkg' - mkdir -p '$(modulePath)' - echo '##vso[task.prependpath]$(GOBIN)' - echo '##vso[task.prependpath]$(GOROOT)/bin' - name: "GoEnv" - displayName: "Set up the Go environment" - - task: KubectlInstaller@0 inputs: kubectlVersion: latest - task: AzureCLI@2 + displayName: "Deploy CNI" + env: + AZCLI: az + CLUSTER: ${{ parameters.clusterName }} + OS: ${{ parameters.os }} + ARCH: ${{ parameters.arch }} + OS_VERSION: linux inputs: azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) scriptLocation: "inlineScript" scriptType: "bash" - workingDirectory: $(ACN_DIR) addSpnToEnvironment: true inlineScript: | set -e - make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }} + make -C ./hack/aks set-kubeconf echo "Upload CNI" echo "Deploying on Linux nodes" - if [ "${{parameters.os}}" == "windows" ]; then - export CNI_IMAGE=$(make cni-image-name-and-tag OS='linux' ARCH=${{ parameters.arch }}) + if [ "$OS" == "windows" ]; then + export CNI_IMAGE=$(make cni-image-name-and-tag OS='linux') echo "CNI image: $CNI_IMAGE" envsubst '${CNI_IMAGE}' < ./test/integration/manifests/cni/cni-installer-v1.yaml | kubectl apply -f - kubectl rollout status daemonset/azure-cni -n kube-system echo "Deploying on windows nodes" - export CNI_IMAGE=$( make cni-image-name-and-tag OS='windows' ARCH=${{ parameters.arch }} OS_VERSION=${{ parameters.os_version }}) + export CNI_IMAGE=$( make cni-image-name-and-tag OS='windows') echo "CNI image: $CNI_IMAGE" envsubst '${CNI_IMAGE}' < ./test/integration/manifests/cni/cni-installer-v1-windows.yaml | kubectl apply -f - kubectl rollout status daemonset/azure-cni-windows -n kube-system else - export CNI_IMAGE=$(make cni-image-name-and-tag OS=${{ parameters.os }} ARCH=${{ parameters.arch }}) + export CNI_IMAGE=$(make cni-image-name-and-tag) echo "CNI image: $CNI_IMAGE" envsubst '${CNI_IMAGE}' < ./test/integration/manifests/cni/cni-installer-v1.yaml | kubectl apply -f - kubectl rollout status daemonset/azure-cni -n kube-system fi - name: "deployCNI" - displayName: "Deploy CNI" - task: AzureCLI@2 + displayName: "Restart Nodes" + env: + AZCLI: az + CLUSTER: ${{ parameters.clusterName }} + RESOURCE_GROUP: "MC_${{ parameters.clusterName }}_${{ parameters.clusterName }}_$(REGION_AKS_CLUSTER_TEST)" + REGION: $(REGION_AKS_CLUSTER_TEST) inputs: azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) scriptLocation: "inlineScript" scriptType: "bash" - workingDirectory: $(ACN_DIR) addSpnToEnvironment: true inlineScript: | set -e - clusterName=${{ parameters.clusterName }} echo "Restarting nodes" - for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do - make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val} + for val in $(az vmss list -g "$RESOURCE_GROUP" --query "[].name" -o tsv); do + make -C ./hack/aks restart-vmss VMSS_NAME=${val} done - displayName: "Restart Nodes" - script: | set -e - kubectl get pods -A -o wide - echo "Deploying test pods" + kubectl get pods -A -o wide pushd test/integration/load - ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=${{ parameters.os }} go test -count 1 -timeout 30m -tags load -run ^TestLoad$ + echo "Deploying test pods" + go test -count 1 -timeout 30m -tags load -run ^TestLoad$ popd - make test-validate-state OS_TYPE=${{ parameters.os }} CNI_TYPE=cniv1 - + make test-validate-state kubectl delete ns load-test - workingDirectory: $(ACN_DIR) displayName: "Validate State" retryCountOnTaskFailure: 3 - + env: + ITERATIONS: 2 + SCALE_UP: ${{ parameters.scaleup }} + OS_TYPE: ${{ parameters.os }} + RESTART_CASE: true + CNI_TYPE: cniv1 diff --git a/.pipelines/singletenancy/azure-cni-overlay-stateless/azure-cni-overlay-stateless-e2e.jobs.yaml b/.pipelines/singletenancy/azure-cni-overlay-stateless/azure-cni-overlay-stateless-e2e.jobs.yaml new file mode 100644 index 0000000000..2b1108e202 --- /dev/null +++ b/.pipelines/singletenancy/azure-cni-overlay-stateless/azure-cni-overlay-stateless-e2e.jobs.yaml @@ -0,0 +1,94 @@ +parameters: + name: "" + displayName: "" + clusterType: "" + clusterName: "" + vmSize: "" + k8sVersion: "" + dependsOn: "" + +stages: + - stage: ${{ parameters.clusterName }} + displayName: Create Cluster - ${{ parameters.displayName }} + dependsOn: + - ${{ parameters.dependsOn }} + - setup + variables: + commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] + jobs: + - template: ../../templates/create-cluster.jobs.yaml + parameters: + name: ${{ parameters.name }} + displayName: ${{ parameters.displayName }} + clusterType: ${{ parameters.clusterType }} + clusterName: ${{ parameters.clusterName }}-$(commitID) + vmSize: ${{ parameters.vmSize }} + k8sVersion: ${{ parameters.k8sVersion }} + dependsOn: ${{ parameters.dependsOn }} + region: $(REGION_AKS_CLUSTER_TEST) + + - template: ../../templates/add-windows-nodepool.jobs.yaml + parameters: + depend: ${{ parameters.name }} + clusterName: ${{ parameters.clusterName }}-$(commitID) + vmSize: ${{ parameters.vmSize }} + + - stage: ${{ parameters.name }} + displayName: E2E - ${{ parameters.displayName }} + dependsOn: + - setup + - publish + - ${{ parameters.clusterName }} + variables: + commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] + GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path + GOBIN: "$(GOPATH)/bin" # Go binaries path + modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking" + jobs: + - job: ${{ parameters.name }}_windows + displayName: Azure Stateless CNI Overlay Test Suite | Windows - (${{ parameters.name }}) + timeoutInMinutes: 120 + pool: + name: $(BUILD_POOL_NAME_DEFAULT) + os: linux + image: acn-runner-vmss-image-1es-v3.0 + demands: + - Role -equals $(CUSTOM_E2E_ROLE) + steps: + - checkout: self + clean: true + + - template: azure-cni-overlay-stateless-e2e.steps.yaml + parameters: + name: ${{ parameters.name }} + clusterName: ${{ parameters.clusterName }}-$(commitID) + os: windows + vmSizeWin: ${{ parameters.vmSize }} + + - template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml + parameters: + sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + clusterName: ${{ parameters.clusterName }}-$(commitID) + os: windows + dependsOn: ${{ parameters.name }}_windows + datapath: true + dns: true + portforward: true + hostport: true + service: true + hybridWin: true + + - job: failedE2ELogs_windows + displayName: "Windows Failure Logs" + dependsOn: + - ${{ parameters.name }}_windows + - cni_windows + condition: in(dependencies.${{ parameters.name }}_windows.result, 'Failed') + steps: + - template: ../../templates/log-template.steps.yaml + parameters: + clusterName: ${{ parameters.clusterName }}-$(commitID) + os: windows + cni: cniv2 + + diff --git a/.pipelines/singletenancy/azure-cni-overlay-stateless/azure-cni-overlay-stateless-e2e.steps.yaml b/.pipelines/singletenancy/azure-cni-overlay-stateless/azure-cni-overlay-stateless-e2e.steps.yaml new file mode 100644 index 0000000000..7942a3e92d --- /dev/null +++ b/.pipelines/singletenancy/azure-cni-overlay-stateless/azure-cni-overlay-stateless-e2e.steps.yaml @@ -0,0 +1,102 @@ +parameters: + name: "" + clusterName: "" + os: "" + +steps: + - task: KubectlInstaller@0 + inputs: + kubectlVersion: latest + + - task: AzureCLI@2 + displayName: "Set Kubeconfig" + env: + AZCLI: az + CLUSTER: ${{ parameters.clusterName }} + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + set -e + make -C ./hack/aks set-kubeconf + + - script: | + nodeList=`kubectl get node -owide | grep Windows | awk '{print $1}'` + for node in $nodeList; do + taint=`kubectl describe node $node | grep Taints | awk '{print $2}'` + if [ $taint == "node.cloudprovider.kubernetes.io/uninitialized=true:NoSchedule" ]; then + kubectl taint nodes $node node.cloudprovider.kubernetes.io/uninitialized=true:NoSchedule- + fi + done + sudo -E env "PATH=$PATH" make test-load CNS_VERSION=$(make cns-version) CNI_VERSION=$(make cni-version) + displayName: "Windows v4Overlay ControlPlane Scale Tests" + retryCountOnTaskFailure: 2 + env: + SCALE_UP: 32 + OS_TYPE: windows + CNI_TYPE: stateless + VALIDATE_STATEFILE: true + INSTALL_CNS: true + INSTALL_AZURE_VNET_STATELESS: true + VALIDATE_V4OVERLAY: true + CLEANUP: true + + - task: AzureCLI@2 + displayName: "Restart Nodes" + env: + AZCLI: az + CLUSTER: ${{ parameters.clusterName }} + SCALE_UP: ${{ coalesce(parameters.scaleup, 32) }} + RESOURCE_GROUP: "MC_${{ parameters.clusterName }}_${{ parameters.clusterName }}_$(REGION_AKS_CLUSTER_TEST)" + REGION: $(REGION_AKS_CLUSTER_TEST) + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + set -e + kubectl get po -owide -A + echo "Restarting nodes" + for val in $(az vmss list -g "$RESOURCE_GROUP" --query "[].name" -o tsv); do + make -C ./hack/aks restart-vmss VMSS_NAME=${val} + done + + - task: AzureCLI@2 + displayName: "Validate Node Restart" + retryCountOnTaskFailure: 3 + env: + AZCLI: az + CLUSTER: ${{ parameters.clusterName }} + REGION: $(REGION_AKS_CLUSTER_TEST) + SCALE_UP: ${{ parameters.scaleup }} + RESTART_CASE: true + CNI_TYPE: stateless + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + cd test/integration/load + make -C ./hack/aks set-kubeconf + make -C ./hack/aks azcfg + kubectl get pods -owide -A + echo "Validating Node Restart" + go test -timeout 30m -tags load -run ^TestValidateState$ + + - script: | + echo "Run wireserver and metadata connectivity Tests" + bash test/network/wireserver_metadata_test.sh + retryCountOnTaskFailure: 3 + displayName: "Run Wireserver and Metadata Connectivity Tests" + + - script: | + echo "IPv4 Overlay DataPath Test" + cd test/integration/datapath + sudo -E env "PATH=$PATH" go test -count=1 datapath_windows_test.go -timeout 3m -tags connection -restartKubeproxy true -run ^TestDatapathWin$ + displayName: "Windows v4Overlay Datapath Tests" + retryCountOnTaskFailure: 3 + diff --git a/.pipelines/singletenancy/azure-cni-overlay/add-windows-nodepool.jobs.yaml b/.pipelines/singletenancy/azure-cni-overlay/add-windows-nodepool.jobs.yaml new file mode 100644 index 0000000000..7d9c695058 --- /dev/null +++ b/.pipelines/singletenancy/azure-cni-overlay/add-windows-nodepool.jobs.yaml @@ -0,0 +1,69 @@ +parameters: + depend: "" + clusterName: "" # unique identifier + vmSize: "" + +jobs: +- job: windows_nodepool + displayName: Add Windows Nodepool + dependsOn: ${{ parameters.depend }} + pool: + name: $(BUILD_POOL_NAME_DEFAULT) + os: linux + image: acn-runner-vmss-image-1es-v3.0 + demands: + - Role -equals $(CUSTOM_E2E_ROLE) + timeoutInMinutes: 30 + steps: + - task: AzureCLI@2 + displayName: "Add windows node to cluster" + retryCountOnTaskFailure: 5 + env: + AZCLI: az + CLUSTER: ${{ parameters.clusterName }} + SUB: $(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) + VM_SIZE_WIN: ${{ parameters.vmSize }} + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + set -e + + windows_nodepool=$(az aks nodepool list \ + --resource-group "$CLUSTER" \ + --cluster-name "$CLUSTER" \ + --query "[?osType=='Windows']" \ + --output tsv) + + if [ -z "$windows_nodepool" ]; then + echo "No Windows node pool found in the AKS cluster." + + # wait for cluster to update + while true; do + cluster_state=$(az aks show \ + --resource-group "$CLUSTER" \ + --name "$CLUSTER" \ + --query provisioningState) + + if echo "$cluster_state" | grep -q "Updating"; then + echo "Cluster is updating. Sleeping for 30 seconds..." + sleep 30 + else + break + fi + done + # cluster state is always set and visible outside the loop + echo "Cluster state is: $cluster_state" + + make -C ./hack/aks set-kubeconf + make -C ./hack/aks windows-nodepool-up + echo "Windows node was successfully added" + kubectl cluster-info + kubectl get node -owide + kubectl get po -owide -A + else + echo "Windows node pool already exists in the AKS cluster." + fi + diff --git a/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e.jobs.yaml b/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e.jobs.yaml index 3af20d7bf7..73c304c7f6 100644 --- a/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e.jobs.yaml +++ b/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e.jobs.yaml @@ -15,9 +15,8 @@ stages: - setup variables: commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] - ACN_DIR: $(Build.SourcesDirectory) jobs: - - template: ../../templates/create-cluster.jobs.yaml@ACNTools + - template: ../../templates/create-cluster.jobs.yaml parameters: name: ${{ parameters.name }} displayName: ${{ parameters.displayName }} @@ -35,7 +34,6 @@ stages: - publish - ${{ parameters.clusterName }} variables: - ACN_DIR: $(Build.SourcesDirectory) commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path GOBIN: "$(GOPATH)/bin" # Go binaries path @@ -45,88 +43,50 @@ stages: displayName: Azure CNI Overlay Test Suite | Linux - (${{ parameters.name }}) timeoutInMinutes: 120 pool: - isCustom: true - type: linux name: $(BUILD_POOL_NAME_DEFAULT) + os: linux + image: acn-runner-vmss-image-1es-v3.0 demands: - - agent.os -equals Linux - Role -equals $(CUSTOM_E2E_ROLE) - variables: - ob_outputDirectory: $(Build.ArtifactStagingDirectory)/output - ob_git_checkout: true steps: - - checkout: ACNReviewChanges + - checkout: self clean: true - - template: azure-cni-overlay-e2e.steps.yaml@ACNTools + - template: azure-cni-overlay-e2e.steps.yaml parameters: name: ${{ parameters.name }} clusterName: ${{ parameters.clusterName }}-$(commitID) os: linux scaleup: 100 - - job: windows_nodepool - displayName: Add Windows Nodepool - dependsOn: ${{ parameters.name }}_linux - pool: - isCustom: true - type: linux - name: $(BUILD_POOL_NAME_DEFAULT) - demands: - - agent.os -equals Linux - - Role -equals $(CUSTOM_E2E_ROLE) - variables: - ob_outputDirectory: $(Build.ArtifactStagingDirectory)/output - ob_git_checkout: true - steps: - - checkout: ACNReviewChanges - clean: true - - - task: AzureCLI@2 - inputs: - azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) - scriptLocation: "inlineScript" - scriptType: "bash" - workingDirectory: $(ACN_DIR) - addSpnToEnvironment: true - inlineScript: | - set -e - make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID) - make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }}-$(commitID) VM_SIZE_WIN=${{ parameters.vmSize }} - echo "Windows node are successfully added to v4 Overlay Cluster" - kubectl cluster-info - kubectl get node -owide - kubectl get po -owide -A - name: "Add_Windows_Node" - displayName: "Add windows node on v4 overlay cluster" - + - template: ../../templates/add-windows-nodepool.jobs.yaml + parameters: + depend: ${{ parameters.name }}_linux + clusterName: ${{ parameters.clusterName }}-$(commitID) + vmSize: ${{ parameters.vmSize }} - job: ${{ parameters.name }}_windows displayName: Azure CNI Overlay Test Suite | Windows - (${{ parameters.name }}) timeoutInMinutes: 120 dependsOn: windows_nodepool pool: - isCustom: true - type: linux name: $(BUILD_POOL_NAME_DEFAULT) + os: linux + image: acn-runner-vmss-image-1es-v3.0 demands: - - agent.os -equals Linux - Role -equals $(CUSTOM_E2E_ROLE) - variables: - ob_outputDirectory: $(Build.ArtifactStagingDirectory)/output - ob_git_checkout: true steps: - - checkout: ACNReviewChanges + - checkout: self clean: true - - template: azure-cni-overlay-e2e.steps.yaml@ACNTools + - template: azure-cni-overlay-e2e.steps.yaml parameters: name: ${{ parameters.name }} clusterName: ${{ parameters.clusterName }}-$(commitID) os: windows scaleup: 50 - - template: ../../cni/k8s-e2e/k8s-e2e.jobs.yaml@ACNTools + - template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml parameters: sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) clusterName: ${{ parameters.clusterName }}-$(commitID) @@ -138,7 +98,7 @@ stages: hostport: true service: true - - template: ../../cni/k8s-e2e/k8s-e2e.jobs.yaml@ACNTools + - template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml parameters: sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) clusterName: ${{ parameters.clusterName }}-$(commitID) @@ -155,20 +115,10 @@ stages: displayName: "Linux Failure Logs" dependsOn: - ${{ parameters.name }}_linux - - CNI_linux - condition: failed() - pool: - isCustom: true - type: linux - name: $(BUILD_POOL_NAME_DEFAULT) - variables: - ob_outputDirectory: $(ACN_DIR)/${{ parameters.clusterName }}-$(commitID)_FailedE2ELogs_linux_Attempt_#$(System.StageAttempt) - ob_git_checkout: true + - cni_linux + condition: in(dependencies.${{ parameters.name }}_linux.result, 'Failed') steps: - - checkout: ACNReviewChanges - clean: true - - - template: ../../templates/log-template.steps.yaml@ACNTools + - template: ../../templates/log-template.steps.yaml parameters: clusterName: ${{ parameters.clusterName }}-$(commitID) os: linux @@ -178,22 +128,13 @@ stages: displayName: "Windows Failure Logs" dependsOn: - ${{ parameters.name }}_windows - - CNI_windows - condition: failed() - pool: - isCustom: true - type: linux - name: $(BUILD_POOL_NAME_DEFAULT) - variables: - ob_outputDirectory: $(ACN_DIR)/${{ parameters.clusterName }}-$(commitID)_FailedE2ELogs_windows_Attempt_#$(System.StageAttempt) - ob_git_checkout: true + - cni_windows + condition: in(dependencies.${{ parameters.name }}_windows.result, 'Failed') steps: - - checkout: ACNReviewChanges - clean: true - - - template: ../../templates/log-template.steps.yaml@ACNTools + - template: ../../templates/log-template.steps.yaml parameters: clusterName: ${{ parameters.clusterName }}-$(commitID) os: windows cni: cniv2 + diff --git a/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e.steps.yaml b/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e.steps.yaml index cf0a0310ff..519af2c9fd 100644 --- a/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e.steps.yaml +++ b/.pipelines/singletenancy/azure-cni-overlay/azure-cni-overlay-e2e.steps.yaml @@ -5,89 +5,93 @@ parameters: scaleup: "" steps: - - bash: | - go version - go env - mkdir -p '$(GOBIN)' - mkdir -p '$(GOPATH)/pkg' - mkdir -p '$(modulePath)' - echo '##vso[task.prependpath]$(GOBIN)' - echo '##vso[task.prependpath]$(GOROOT)/bin' - name: "GoEnv" - displayName: "Set up the Go environment" - - task: KubectlInstaller@0 inputs: kubectlVersion: latest - task: AzureCLI@2 + displayName: "Set Kubeconfig" + env: + AZCLI: az + CLUSTER: ${{ parameters.clusterName }} inputs: azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) scriptLocation: "inlineScript" scriptType: "bash" - workingDirectory: $(ACN_DIR) addSpnToEnvironment: true inlineScript: | set -e - make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }} - name: "kubeconfig" - displayName: "Set Kubeconfig" + make -C ./hack/aks set-kubeconf - ${{ if eq(parameters.os, 'linux') }}: - script: | echo "Start Integration Tests on Overlay Cluster" kubectl get po -owide -A - sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux CNI_TYPE=cniv2 VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_AZURE_CNI_OVERLAY=true VALIDATE_V4OVERLAY=true AZURE_IPAM_VERSION=$(make azure-ipam-version) CNS_VERSION=$(make cns-version) CNI_VERSION=$(make cni-version) CLEANUP=true - workingDirectory: $(ACN_DIR) + sudo -E env "PATH=$PATH" make test-load AZURE_IPAM_VERSION=$(make azure-ipam-version) CNS_VERSION=$(make cns-version) CNI_VERSION=$(make cni-version) retryCountOnTaskFailure: 2 - name: "integrationTest" displayName: "Run CNS Integration Tests on AKS Overlay" + env: + SCALE_UP: 32 + OS_TYPE: linux + CNI_TYPE: cniv2 + VALIDATE_STATEFILE: true + INSTALL_CNS: true + INSTALL_AZURE_CNI_OVERLAY: true + VALIDATE_V4OVERLAY: true + CLEANUP: true - task: AzureCLI@2 + displayName: "Restart Nodes" + env: + AZCLI: az + CLUSTER: ${{ parameters.clusterName }} + SCALE_UP: ${{ coalesce(parameters.scaleup, 32) }} + RESOURCE_GROUP: "MC_${{ parameters.clusterName }}_${{ parameters.clusterName }}_$(REGION_AKS_CLUSTER_TEST)" + REGION: $(REGION_AKS_CLUSTER_TEST) inputs: azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) scriptLocation: "inlineScript" scriptType: "bash" - workingDirectory: $(ACN_DIR) addSpnToEnvironment: true inlineScript: | set -e kubectl get po -owide -A - clusterName=${{ parameters.clusterName }} echo "Restarting nodes" - for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do - make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val} + for val in $(az vmss list -g "$RESOURCE_GROUP" --query "[].name" -o tsv); do + make -C ./hack/aks restart-vmss VMSS_NAME=${val} done - displayName: "Restart Nodes" - task: AzureCLI@2 + displayName: "Validate Node Restart" + retryCountOnTaskFailure: 3 + env: + ITERATIONS: 2 + SCALE_UP: ${{ parameters.scaleup }} + OS_TYPE: linux + RESTART_CASE: true + CNI_TYPE: cniv2 inputs: azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) scriptLocation: "inlineScript" scriptType: "bash" - workingDirectory: $(ACN_DIR) addSpnToEnvironment: true inlineScript: | set -e - cd test/integration/load - # Scale Cluster Up/Down to confirm functioning CNS - ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=linux go test -count 1 -timeout 30m -tags load -run ^TestLoad$ - kubectl get pods -owide -A + pushd test/integration/load + # Scale Cluster Up/Down to confirm functioning CNS + go test -count 1 -timeout 30m -tags load -run ^TestLoad$ + kubectl get pods -owide -A + popd - cd ../../.. echo "Validating Node Restart" - make test-validate-state OS_TYPE=linux RESTART_CASE=true CNI_TYPE=cniv2 + make test-validate-state kubectl delete ns load-test - displayName: "Validate Node Restart" - retryCountOnTaskFailure: 3 - script: | echo "Run wireserver and metadata connectivity Tests" bash test/network/wireserver_metadata_test.sh - workingDirectory: $(ACN_DIR) retryCountOnTaskFailure: 3 - name: "WireserverMetadataConnectivityTests" displayName: "Run Wireserver and Metadata Connectivity Tests" - ${{ if eq(parameters.os, 'windows') }}: @@ -99,56 +103,70 @@ steps: kubectl taint nodes $node node.cloudprovider.kubernetes.io/uninitialized=true:NoSchedule- fi done - sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=windows CNI_TYPE=cniv2 VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_AZURE_CNI_OVERLAY=true VALIDATE_V4OVERLAY=true CNS_VERSION=$(make cns-version) CNI_VERSION=$(make cni-version) CLEANUP=true - workingDirectory: $(ACN_DIR) - name: "WindowsOverlayControlPlaneScaleTests" + sudo -E env "PATH=$PATH" make test-load CNS_VERSION=$(make cns-version) CNI_VERSION=$(make cni-version) displayName: "Windows v4Overlay ControlPlane Scale Tests" retryCountOnTaskFailure: 2 + env: + SCALE_UP: 32 + OS_TYPE: windows + CNI_TYPE: cniv2 + VALIDATE_STATEFILE: true + INSTALL_CNS: true + INSTALL_AZURE_CNI_OVERLAY: true + VALIDATE_V4OVERLAY: true + CLEANUP: true - script: | echo "IPv4 Overlay DataPath Test" cd test/integration/datapath sudo -E env "PATH=$PATH" go test -count=1 datapath_windows_test.go -timeout 3m -tags connection -restartKubeproxy true -run ^TestDatapathWin$ - workingDirectory: $(ACN_DIR) - name: "WindowsV4OverlayDatapathTests" displayName: "Windows v4Overlay Datapath Tests" retryCountOnTaskFailure: 3 - task: AzureCLI@2 + displayName: "Restart Nodes" + env: + AZCLI: az + CLUSTER: ${{ parameters.clusterName }} + SCALE_UP: ${{ coalesce(parameters.scaleup, 32) }} + RESOURCE_GROUP: "MC_${{ parameters.clusterName }}_${{ parameters.clusterName }}_$(REGION_AKS_CLUSTER_TEST)" + REGION: $(REGION_AKS_CLUSTER_TEST) inputs: azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) scriptLocation: "inlineScript" scriptType: "bash" - workingDirectory: $(ACN_DIR) addSpnToEnvironment: true inlineScript: | set -e kubectl get po -owide -A - clusterName=${{ parameters.clusterName }} echo "Restarting nodes" - for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do - make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val} + for val in $(az vmss list -g "$RESOURCE_GROUP" --query "[].name" -o tsv); do + make -C ./hack/aks restart-vmss VMSS_NAME=${val} done - displayName: "Restart Nodes" - task: AzureCLI@2 + displayName: "Validate Node Restart" + retryCountOnTaskFailure: 3 + env: + ITERATIONS: 2 + SCALE_UP: ${{ parameters.scaleup }} + OS_TYPE: windows + RESTART_CASE: true + CNI_TYPE: cniv2 inputs: azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) scriptLocation: "inlineScript" scriptType: "bash" - workingDirectory: $(ACN_DIR) addSpnToEnvironment: true inlineScript: | set -e - cd test/integration/load - # Scale Cluster Up/Down to confirm functioning CNS - ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=windows go test -count 1 -timeout 30m -tags load -run ^TestLoad$ - kubectl get pods -owide -A + pushd test/integration/load + # Scale Cluster Up/Down to confirm functioning CNS + go test -count 1 -timeout 30m -tags load -run ^TestLoad$ + kubectl get pods -owide -A + popd - cd ../../.. echo "Validating Node Restart" - make test-validate-state OS_TYPE=windows RESTART_CASE=true CNI_TYPE=cniv2 + make test-validate-state kubectl delete ns load-test - displayName: "Validate Node Restart" - retryCountOnTaskFailure: 3 diff --git a/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e.jobs.yaml b/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e.jobs.yaml index cb08728d9e..edcb792f10 100644 --- a/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e.jobs.yaml +++ b/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e.jobs.yaml @@ -15,9 +15,8 @@ stages: - setup variables: commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] - ACN_DIR: $(Build.SourcesDirectory) jobs: - - template: ../../templates/create-cluster.jobs.yaml@ACNTools + - template: ../../templates/create-cluster.jobs.yaml parameters: name: ${{ parameters.name }} displayName: ${{ parameters.displayName }} @@ -39,32 +38,27 @@ stages: GOBIN: "$(GOPATH)/bin" # Go binaries path modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking" commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] - ACN_DIR: $(Build.SourcesDirectory) jobs: - job: ${{ parameters.name }} displayName: Cilium Dualstack Overlay Test Suite - (${{ parameters.name }}) timeoutInMinutes: 120 pool: - isCustom: true - type: linux name: $(BUILD_POOL_NAME_DEFAULT) + os: linux + image: acn-runner-vmss-image-1es-v3.0 demands: - - agent.os -equals Linux - Role -equals $(CUSTOM_E2E_ROLE) - variables: - ob_outputDirectory: $(Build.ArtifactStagingDirectory)/output - ob_git_checkout: true steps: - - checkout: ACNReviewChanges + - checkout: self clean: true - - template: cilium-dualstackoverlay-e2e.steps.yaml@ACNTools + - template: cilium-dualstackoverlay-e2e.steps.yaml parameters: name: ${{ parameters.name }} clusterName: ${{ parameters.clusterName }}-$(commitID) scaleup: 100 - - template: ../../cni/k8s-e2e/k8s-e2e.jobs.yaml@ACNTools + - template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml parameters: sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) clusterName: ${{ parameters.clusterName }}-$(commitID) @@ -82,18 +76,8 @@ stages: - ${{ parameters.name }} - cni_${{ parameters.os }} condition: failed() - pool: - isCustom: true - type: linux - name: $(BUILD_POOL_NAME_DEFAULT) - variables: - ob_outputDirectory: $(ACN_DIR)/${{ parameters.clusterName }}-$(commitID)_FailedE2ELogs_Attempt_#$(System.StageAttempt) - ob_git_checkout: true steps: - - checkout: ACNReviewChanges - clean: true - - - template: ../../templates/log-template.steps.yaml@ACNTools + - template: ../../templates/log-template.steps.yaml parameters: clusterName: ${{ parameters.clusterName }}-$(commitID) os: ${{ parameters.os }} diff --git a/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e.steps.yaml b/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e.steps.yaml index 8d0b262de6..6954e9fa65 100644 --- a/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e.steps.yaml +++ b/.pipelines/singletenancy/cilium-dualstack-overlay/cilium-dualstackoverlay-e2e.steps.yaml @@ -4,32 +4,23 @@ parameters: scaleup: "" steps: - - - bash: | - go version - go env - mkdir -p '$(GOBIN)' - mkdir -p '$(GOPATH)/pkg' - mkdir -p '$(modulePath)' - echo '##vso[task.prependpath]$(GOBIN)' - echo '##vso[task.prependpath]$(GOROOT)/bin' - name: "GoEnv" - displayName: "Set up the Go environment" - - task: KubectlInstaller@0 inputs: kubectlVersion: latest - task: AzureCLI@2 + displayName: "Install Cilium on AKS Dualstack Overlay" + env: + AZCLI: az + CLUSTER: ${{ parameters.clusterName }} inputs: azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) scriptLocation: "inlineScript" scriptType: "bash" - workingDirectory: $(ACN_DIR) addSpnToEnvironment: true inlineScript: | set -e - make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }} + make -C ./hack/aks set-kubeconf ls -lah pwd kubectl cluster-info @@ -49,18 +40,22 @@ steps: envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY},${IPV6_HP_BPF_VERSION}' < test/integration/manifests/cilium/v${DIR}/cilium-agent/templates/daemonset-dualstack.yaml | kubectl apply -f - envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/v${DIR}/cilium-operator/templates/deployment.yaml | kubectl apply -f - kubectl get po -owide -A - name: "installCilium" - displayName: "Install Cilium on AKS Dualstack Overlay" - - template: ../../templates/cilium-cli.steps.yaml@ACNTools + - template: ../../templates/cilium-cli.yaml - script: | echo "Start Azilium E2E Tests on Overlay Cluster" - sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux CNI_TYPE=cilium_dualstack VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_OVERLAY=true AZURE_IPAM_VERSION=$(make azure-ipam-version) CNS_VERSION=$(make cns-version) CLEANUP=true - workingDirectory: $(ACN_DIR) + sudo -E env "PATH=$PATH" make test-load AZURE_IPAM_VERSION=$(make azure-ipam-version) CNS_VERSION=$(make cns-version) retryCountOnTaskFailure: 3 - name: "aziliumTest" displayName: "Run Azilium E2E on AKS Overlay" + env: + SCALE_UP: 32 + OS_TYPE: linux + CNI_TYPE: cilium_dualstack + VALIDATE_STATEFILE: true + INSTALL_CNS: true + INSTALL_OVERLAY: true + CLEANUP: true - script: | kubectl get pods -A @@ -68,47 +63,54 @@ steps: # Ensure Cilium is ready Xm\Xs cilium status --wait --wait-duration 2m retryCountOnTaskFailure: 3 - name: "CiliumStatus" displayName: "Cilium Status" - task: AzureCLI@2 + displayName: "Restart Nodes" + env: + AZCLI: az + CLUSTER: ${{ parameters.clusterName }} + RESOURCE_GROUP: "MC_${{ parameters.clusterName }}_${{ parameters.clusterName }}_$(REGION_AKS_CLUSTER_TEST)" + REGION: $(REGION_AKS_CLUSTER_TEST) inputs: azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) scriptLocation: "inlineScript" scriptType: "bash" - workingDirectory: $(ACN_DIR) addSpnToEnvironment: true inlineScript: | set -e kubectl get po -owide -A - clusterName=${{ parameters.clusterName }} echo "Restarting nodes" - for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do - make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val} + for val in $(az vmss list -g "$RESOURCE_GROUP" --query "[].name" -o tsv); do + make -C ./hack/aks restart-vmss VMSS_NAME=${val} done - displayName: "Restart Nodes" - task: AzureCLI@2 + displayName: "Validate Node Restart" + retryCountOnTaskFailure: 3 + env: + ITERATIONS: 2 + SCALE_UP: ${{ parameters.scaleup }} + OS_TYPE: linux + RESTART_CASE: true + CNI_TYPE: cilium_dualstack inputs: azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) scriptLocation: "inlineScript" scriptType: "bash" - workingDirectory: $(ACN_DIR) addSpnToEnvironment: true inlineScript: | set -e - cd test/integration/load - # Scale Cluster Up/Down to confirm functioning CNS - ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=linux go test -count 1 -timeout 30m -tags load -run ^TestLoad$ - kubectl get pods -owide -A + pushd test/integration/load + # Scale Cluster Up/Down to confirm functioning CNS + go test -count 1 -timeout 30m -tags load -run ^TestLoad$ + kubectl get pods -owide -A + popd - cd ../../.. echo "Validating Node Restart" - make test-validate-state OS_TYPE=linux RESTART_CASE=true CNI_TYPE=cilium_dualstack + make test-validate-state kubectl delete ns load-test - displayName: "Validate Node Restart" - retryCountOnTaskFailure: 3 - script: | set -e @@ -118,7 +120,6 @@ steps: ns=`kubectl get ns | grep cilium-test | awk '{print $1}'` echo "##vso[task.setvariable variable=ciliumNamespace]$ns" retryCountOnTaskFailure: 3 - name: "ciliumConnectivityTests" displayName: "Run Cilium Connectivity Tests" - script: | @@ -129,9 +130,7 @@ steps: go test -count=1 datapath_linux_test.go -timeout 3m -tags connection -run ^TestDatapathLinux$ -tags=connection,integration -isDualStack=true echo "Dualstack Overlay Linux datapath IPv4 test" go test -count=1 datapath_linux_test.go -timeout 3m -tags connection -run ^TestDatapathLinux$ -tags=connection,integration - workingDirectory: $(ACN_DIR) retryCountOnTaskFailure: 3 - name: "DualStack_Overlay_Linux_Tests" displayName: "DualStack Overlay Linux Tests" - script: | @@ -139,23 +138,22 @@ steps: kubectl get pod -owide -A # Deleting echo-external-node deployment until cilium version matches TODO. https://github.com/cilium/cilium-cli/issues/67 is addressing the change. # Saves 17 minutes - kubectl delete deploy -n $(ciliumNamespace) echo-external-node + kubectl delete deploy -n "$CILIUM_NS" echo-external-node cd test/integration/load CNI_TYPE=cilium_dualstack go test -timeout 30m -tags load -run ^TestValidateState$ echo "delete cilium connectivity test resources and re-validate state" - kubectl delete ns $(ciliumNamespace) + kubectl delete ns "$CILIUM_NS" kubectl get pod -owide -A - CNI_TYPE=cilium_dualstack go test -timeout 30m -tags load -run ^TestValidateState$ - workingDirectory: $(ACN_DIR) - name: "validatePods" + go test -timeout 30m -tags load -run ^TestValidateState$ displayName: "Validate Pods" + env: + CILIUM_NS: $(ciliumNamespace) + CNI_TYPE: cilium_dualstack - script: | echo "Run wireserver and metadata connectivity Tests" bash test/network/wireserver_metadata_test.sh - workingDirectory: $(ACN_DIR) retryCountOnTaskFailure: 3 - name: "WireserverMetadataConnectivityTests" displayName: "Run Wireserver and Metadata Connectivity Tests" - script: | @@ -165,7 +163,5 @@ steps: if ! [ -z $(kubectl -n kube-system get ds azure-cns | grep non-existing) ]; then kubectl -n kube-system patch daemonset azure-cns --type json -p='[{"op": "remove", "path": "/spec/template/spec/nodeSelector/non-existing"}]' fi - workingDirectory: $(ACN_DIR) - retryCountOnTaskFailure: 3 - name: "testAsyncDelete" displayName: "Verify Async Delete when CNS is down" + diff --git a/.pipelines/singletenancy/cilium-nodesubnet/cilium-nodesubnet-e2e.jobs.yaml b/.pipelines/singletenancy/cilium-nodesubnet/cilium-nodesubnet-e2e.jobs.yaml new file mode 100644 index 0000000000..7076eea597 --- /dev/null +++ b/.pipelines/singletenancy/cilium-nodesubnet/cilium-nodesubnet-e2e.jobs.yaml @@ -0,0 +1,92 @@ +parameters: + dependsOn: "" + name: "cilium_nodesubnet_e2e" + clusterType: "nodesubnet-byocni-nokubeproxy-up" + clusterName: "cilndsubnete2e" + vmSize: "" + os: "linux" + arch: "" + osSKU: Ubuntu + hubbleEnabled: false + dualstackVersion: "" + cni: "cilium" + +stages: + - stage: ${{ parameters.clusterName }} + displayName: Create Cluster - ${{ parameters.displayName }} + dependsOn: + - ${{ parameters.dependsOn }} + - setup + variables: + commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] + jobs: + - template: ../../templates/create-cluster.jobs.yaml + parameters: + name: ${{ parameters.name }} + displayName: ${{ parameters.displayName }} + clusterType: ${{ parameters.clusterType }} + clusterName: ${{ parameters.clusterName }}-$(commitID) + vmSize: ${{ parameters.vmSize }} + region: $(REGION_AKS_CLUSTER_TEST) + + - stage: ${{ parameters.name }} + displayName: E2E - ${{ parameters.displayName }} + variables: + TAG: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.Tag'] ] + CURRENT_VERSION: $[ stagedependencies.containerize.check_tag.outputs['CurrentTagManifests.currentTagManifests'] ] + commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] + GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path + GOBIN: "$(GOPATH)/bin" # Go binaries path + modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking" + condition: and(succeeded(), eq(variables.TAG, variables.CURRENT_VERSION)) + dependsOn: + - setup + - publish + - ${{ parameters.clusterName }} + jobs: + - job: ${{ parameters.name }} + displayName: Nodesubnet with Cilium - (${{ parameters.name }}) + timeoutInMinutes: 120 + pool: + name: $(BUILD_POOL_NAME_DEFAULT) + os: linux + image: acn-runner-vmss-image-1es-v3.0 + demands: + - Role -equals $(CUSTOM_E2E_ROLE) + steps: + - checkout: self + clean: true + + - template: cilium-nodesubnet-e2e.steps.yaml + parameters: + name: ${{ parameters.name }} + clusterName: ${{ parameters.clusterName }}-$(commitID) + arch: ${{ parameters.arch }} + os: ${{ parameters.os }} + scaleup: ${{ parameters.scaleup }} + + - template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml + parameters: + sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + clusterName: ${{ parameters.clusterName }}-$(commitID) + os: ${{ parameters.os }} + datapath: true + dns: true + cni: cilium + portforward: true + service: true + dependsOn: ${{ parameters.name }} + + - job: failedE2ELogs + displayName: "Failure Logs" + dependsOn: + - ${{ parameters.name }} + - cni_${{ parameters.os }} + condition: failed() + steps: + - template: ../../templates/log-template.steps.yaml + parameters: + clusterName: ${{ parameters.clusterName }}-$(commitID) + os: ${{ parameters.os }} + cni: cilium + diff --git a/.pipelines/singletenancy/cilium-nodesubnet/cilium-nodesubnet-e2e.steps.yaml b/.pipelines/singletenancy/cilium-nodesubnet/cilium-nodesubnet-e2e.steps.yaml new file mode 100644 index 0000000000..f97ef66228 --- /dev/null +++ b/.pipelines/singletenancy/cilium-nodesubnet/cilium-nodesubnet-e2e.steps.yaml @@ -0,0 +1,75 @@ +parameters: + name: "" + clusterName: "" + scaleup: "" + +steps: + - task: AzureCLI@2 + displayName: 'Update IP configs' + env: + SCALE_UP: ${{ coalesce(parameters.scaleup, 32) }} + RESOURCE_GROUP: "MC_${{ parameters.clusterName }}_${{ parameters.clusterName }}_$(REGION_AKS_CLUSTER_TEST)" + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + set -e + SECONDARY_IP_COUNT=$((SCALE_UP * 2)) \ + go run $(Build.SourcesDirectory)/test/integration/cilium-nodesubnet/ipconfigupdate.go + + - task: KubectlInstaller@0 + inputs: + kubectlVersion: latest + + - task: AzureCLI@2 + env: + AZCLI: az + CLUSTER: ${{ parameters.clusterName }} + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + set -e + make -C ./hack/aks set-kubeconf + ls -lah + pwd + kubectl cluster-info + kubectl get po -owide -A + echo "install Cilium ${CILIUM_VERSION_TAG}" + export DIR=$(echo ${CILIUM_VERSION_TAG#v} | cut -d. -f1,2) + echo "installing files from ${DIR}" + echo "deploy Cilium ConfigMap" + kubectl apply -f test/integration/manifests/cilium/v${DIR}/cilium-config/cilium-config.yaml + # Passes Cilium image to daemonset and deployment + kubectl apply -f test/integration/manifests/cilium/v${DIR}/cilium-agent/files + kubectl apply -f test/integration/manifests/cilium/v${DIR}/cilium-operator/files + + envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/v${DIR}/cilium-agent/templates/daemonset.yaml | kubectl apply -f - + envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/v${DIR}/cilium-operator/templates/deployment.yaml | kubectl apply -f - + kubectl get po -owide -A + displayName: "Install Cilium" + + - template: ../../templates/cilium-cli.yaml + + - script: | + echo "Start Nodesubnet E2E Tests" + kubectl get po -owide -A + sudo -E env "PATH=$PATH" make test-load AZURE_IPAM_VERSION=$(make azure-ipam-version) CNS_VERSION=$(make cns-version) + retryCountOnTaskFailure: 3 + displayName: "Run NodeSubnet E2E" + env: + SCALE_UP: 32 + OS_TYPE: linux + VALIDATE_STATEFILE: true + INSTALL_CNS: true + INSTALL_CNS_NODESUBNET: true + CLEANUP: true + + - template: ../../templates/cilium-tests.yaml + parameters: + clusterName: ${{ parameters.clusterName }} + scaleup: ${{ parameters.scaleup }} diff --git a/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e.jobs.yaml b/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e.jobs.yaml index 0effe61217..dd4429234b 100644 --- a/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e.jobs.yaml +++ b/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e.jobs.yaml @@ -17,9 +17,8 @@ stages: - setup variables: commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] - ACN_DIR: $(Build.SourcesDirectory) jobs: - - template: ../../templates/create-cluster.jobs.yaml@ACNTools + - template: ../../templates/create-cluster.jobs.yaml parameters: name: ${{ parameters.name }} displayName: ${{ parameters.displayName }} @@ -37,7 +36,6 @@ stages: - publish - ${{ parameters.clusterName }} variables: - ACN_DIR: $(Build.SourcesDirectory) commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path GOBIN: "$(GOPATH)/bin" # Go binaries path @@ -47,27 +45,23 @@ stages: displayName: Cilium Overlay Test Suite - (${{ parameters.name }}) timeoutInMinutes: 120 pool: - isCustom: true - type: linux name: $(BUILD_POOL_NAME_DEFAULT) + os: linux + image: acn-runner-vmss-image-1es-v3.0 demands: - - agent.os -equals Linux - Role -equals $(CUSTOM_E2E_ROLE) - variables: - ob_outputDirectory: $(Build.ArtifactStagingDirectory)/output - ob_git_checkout: true steps: - - checkout: ACNReviewChanges + - checkout: self clean: true - - template: cilium-overlay-e2e.steps.yaml@ACNTools + - template: cilium-overlay-e2e.steps.yaml parameters: name: ${{ parameters.name }} clusterName: ${{ parameters.clusterName }}-$(commitID) testHubble: ${{ parameters.testHubble }} scaleup: 100 - - template: ../../cni/k8s-e2e/k8s-e2e.jobs.yaml@ACNTools + - template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml parameters: sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) clusterName: ${{ parameters.clusterName }}-$(commitID) @@ -85,18 +79,8 @@ stages: - ${{ parameters.name }} - cni_${{ parameters.os }} condition: failed() - pool: - isCustom: true - type: linux - name: $(BUILD_POOL_NAME_DEFAULT) - variables: - ob_outputDirectory: $(ACN_DIR)/${{ parameters.clusterName }}-$(commitID)_FailedE2ELogs_Attempt_#$(System.StageAttempt) - ob_git_checkout: true steps: - - checkout: ACNReviewChanges - clean: true - - - template: ../../templates/log-template.steps.yaml@ACNTools + - template: ../../templates/log-template.steps.yaml parameters: clusterName: ${{ parameters.clusterName }}-$(commitID) os: ${{ parameters.os }} diff --git a/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e.steps.yaml b/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e.steps.yaml index f52800e74f..719178f6a5 100644 --- a/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e.steps.yaml +++ b/.pipelines/singletenancy/cilium-overlay-withhubble/cilium-overlay-e2e.steps.yaml @@ -5,31 +5,23 @@ parameters: scaleup: "" steps: - - bash: | - go version - go env - mkdir -p '$(GOBIN)' - mkdir -p '$(GOPATH)/pkg' - mkdir -p '$(modulePath)' - echo '##vso[task.prependpath]$(GOBIN)' - echo '##vso[task.prependpath]$(GOROOT)/bin' - name: "GoEnv" - displayName: "Set up the Go environment" - - task: KubectlInstaller@0 inputs: kubectlVersion: latest - task: AzureCLI@2 + displayName: "Install Cilium on AKS Overlay" + env: + AZCLI: az + CLUSTER: ${{ parameters.clusterName }} inputs: azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) scriptLocation: "inlineScript" scriptType: "bash" - workingDirectory: $(ACN_DIR) addSpnToEnvironment: true inlineScript: | set -e - make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }} + make -C ./hack/aks set-kubeconf ls -lah export CILIUM_VERSION_TAG=${CILIUM_HUBBLE_VERSION_TAG} export DIR=$(echo ${CILIUM_VERSION_TAG#v} | cut -d. -f1,2) @@ -40,24 +32,29 @@ steps: envsubst '${CILIUM_IMAGE_REGISTRY},${CILIUM_VERSION_TAG}' < test/integration/manifests/cilium/v${DIR}/cilium-agent/templates/daemonset.yaml | kubectl apply -f - envsubst '${CILIUM_IMAGE_REGISTRY},${CILIUM_VERSION_TAG}' < test/integration/manifests/cilium/v${DIR}/cilium-operator/templates/deployment.yaml | kubectl apply -f - # Use different file directories for nightly and current cilium version - name: "installCilium" - displayName: "Install Cilium on AKS Overlay" - - template: ../../templates/cilium-cli.steps.yaml@ACNTools + - template: ../../templates/cilium-cli.yaml - script: | echo "Start Azilium E2E Tests on Overlay Cluster" if [ "$CILIUM_VERSION_TAG" = "cilium-nightly-pipeline" ] then - CNS=$(CNS_VERSION) IPAM=$(AZURE_IPAM_VERSION) && echo "Running nightly" + echo "Running nightly" else CNS=$(make cns-version) IPAM=$(make azure-ipam-version) fi - sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_OVERLAY=true AZURE_IPAM_VERSION=${IPAM} CNS_VERSION=${CNS} CLEANUP=true - workingDirectory: $(ACN_DIR) + sudo -E env "PATH=$PATH" make test-load AZURE_IPAM_VERSION=${IPAM} CNS_VERSION=${CNS} retryCountOnTaskFailure: 3 - name: "aziliumTest" displayName: "Run Azilium E2E on AKS Overlay" + env: + CNS: $(CNS_VERSION) + IPAM: $(AZURE_IPAM_VERSION) + SCALE_UP: 32 + OS_TYPE: linux + VALIDATE_STATEFILE: true + INSTALL_CNS: true + INSTALL_OVERLAY: true + CLEANUP: true - script: | kubectl get pods -A @@ -65,47 +62,54 @@ steps: # Ensure Cilium is ready Xm\Xs cilium status --wait --wait-duration 2m retryCountOnTaskFailure: 3 - name: "CiliumStatus" displayName: "Cilium Status" - task: AzureCLI@2 + displayName: "Restart Nodes" + env: + AZCLI: az + CLUSTER: ${{ parameters.clusterName }} + SCALE_UP: ${{ coalesce(parameters.scaleup, 32) }} + RESOURCE_GROUP: "MC_${{ parameters.clusterName }}_${{ parameters.clusterName }}_$(REGION_AKS_CLUSTER_TEST)" + REGION: $(REGION_AKS_CLUSTER_TEST) inputs: azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) scriptLocation: "inlineScript" scriptType: "bash" - workingDirectory: $(ACN_DIR) addSpnToEnvironment: true inlineScript: | set -e kubectl get po -owide -A - clusterName=${{ parameters.clusterName }} echo "Restarting nodes" - for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do - make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val} + for val in $(az vmss list -g "$RESOURCE_GROUP" --query "[].name" -o tsv); do + make -C ./hack/aks restart-vmss VMSS_NAME=${val} done - displayName: "Restart Nodes" - task: AzureCLI@2 + displayName: "Validate Node Restart" + retryCountOnTaskFailure: 3 + env: + ITERATIONS: 2 + SCALE_UP: ${{ parameters.scaleup }} + OS_TYPE: linux + RESTART_CASE: true inputs: azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) scriptLocation: "inlineScript" scriptType: "bash" - workingDirectory: $(ACN_DIR) addSpnToEnvironment: true inlineScript: | set -e cd test/integration/load # Scale Cluster Up/Down to confirm functioning CNS - ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=linux go test -count 1 -timeout 30m -tags load -run ^TestLoad$ + go test -count 1 -timeout 30m -tags load -run ^TestLoad$ kubectl get pods -owide -A cd ../../.. echo "Validating Node Restart" - make test-validate-state OS_TYPE=linux RESTART_CASE=true + make test-validate-state kubectl delete ns load-test - displayName: "Validate Node Restart" - retryCountOnTaskFailure: 3 - script: | set -e @@ -114,9 +118,7 @@ steps: cilium connectivity test --connect-timeout 4s --request-timeout 30s --test '!pod-to-pod-encryption,!node-to-node-encryption' --force-deploy ns=`kubectl get ns | grep cilium-test | awk '{print $1}'` echo "##vso[task.setvariable variable=ciliumNamespace]$ns" - workingDirectory: $(ACN_DIR) retryCountOnTaskFailure: 3 - name: "ciliumConnectivityTests" displayName: "Run Cilium Connectivity Tests" - ${{ if eq( parameters['testHubble'], true) }}: @@ -130,9 +132,7 @@ steps: kubectl get pods -Aowide echo "verify Hubble metrics endpoint is usable" go test ./test/integration/networkobservability -v -tags=networkobservability - workingDirectory: $(ACN_DIR) retryCountOnTaskFailure: 3 - name: "HubbleConnectivityTests" displayName: "Run Hubble Connectivity Tests" - script: | @@ -141,25 +141,26 @@ steps: kubectl get pod -owide -A if [ "$CILIUM_VERSION_TAG" = "cilium-nightly-pipeline" ]; then - echo "Check cilium identities in $(ciliumNamespace) namepsace during nightly run" + echo "Check cilium identities in "$CILIUM_NS" namepsace during nightly run" echo "expect the identities to be deleted when the namespace is deleted" kubectl get ciliumidentity | grep cilium-test fi make test-validate-state echo "delete cilium connectivity test resources and re-validate state" # TODO Delete this and the next 4 lines if connectivity no longer has bug - kubectl delete ns $(ciliumNamespace) + kubectl delete ns "$CILIUM_NS" kubectl get pod -owide -A make test-validate-state - workingDirectory: $(ACN_DIR) - name: "validatePods" displayName: "Validate Pods" + env: + CILIUM_NS: $(ciliumNamespace) + CNI_TYPE: cilium_dualstack - script: | if [ "$CILIUM_VERSION_TAG" = "cilium-nightly-pipeline" ]; then - kubectl get pod -owide -n $(ciliumNamespace) + kubectl get pod -owide -n "$CILIUM_NS" echo "wait for pod and cilium identity deletion in cilium-test namespace" while true; do - pods=$(kubectl get pods -n $(ciliumNamespace) --no-headers=true 2>/dev/null) + pods=$(kubectl get pods -n "$CILIUM_NS" --no-headers=true 2>/dev/null) if [[ -z "$pods" ]]; then echo "No pods found" break @@ -170,23 +171,22 @@ steps: echo "Verify cilium identities are deleted from cilium-test" checkIdentity="$(kubectl get ciliumidentity -o json | grep cilium-test | jq -e 'length == 0')" if [[ -n $checkIdentity ]]; then - echo "##[error]Cilium Identities still present in $(ciliumNamespace) namespace" + echo "##[error]Cilium Identities still present in "$CILIUM_NS" namespace" exit 1 else - printf -- "Identities deleted from $(ciliumNamespace) namespace\n" + printf -- "Identities deleted from "$CILIUM_NS" namespace\n" fi else echo "skip cilium identities check for PR pipeline" fi - name: "CiliumIdentities" displayName: "Verify Cilium Identities Deletion" + env: + CILIUM_NS: $(ciliumNamespace) - script: | echo "Run wireserver and metadata connectivity Tests" bash test/network/wireserver_metadata_test.sh - workingDirectory: $(ACN_DIR) retryCountOnTaskFailure: 3 - name: "WireserverMetadataConnectivityTests" displayName: "Run Wireserver and Metadata Connectivity Tests" - script: | @@ -196,17 +196,4 @@ steps: if ! [ -z $(kubectl -n kube-system get ds azure-cns | grep non-existing) ]; then kubectl -n kube-system patch daemonset azure-cns --type json -p='[{"op": "remove", "path": "/spec/template/spec/nodeSelector/non-existing"}]' fi - workingDirectory: $(ACN_DIR) - retryCountOnTaskFailure: 3 - name: "testAsyncDelete" displayName: "Verify Async Delete when CNS is down" - - - script: | - ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/test-output/ - echo $ARTIFACT_DIR - sudo rm -rf $ARTIFACT_DIR - sudo rm -rf test/integration/logs - workingDirectory: $(ACN_DIR) - name: "Cleanupartifactdir" - displayName: "Cleanup artifact dir" - condition: always() diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e.jobs.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e.jobs.yaml index a2485332c8..43fcd073e9 100644 --- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e.jobs.yaml +++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e.jobs.yaml @@ -16,9 +16,8 @@ stages: - setup variables: commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] - ACN_DIR: $(Build.SourcesDirectory) jobs: - - template: ../../templates/create-cluster.jobs.yaml@ACNTools + - template: ../../templates/create-cluster.jobs.yaml parameters: name: ${{ parameters.name }} displayName: ${{ parameters.displayName }} @@ -36,7 +35,6 @@ stages: - publish - ${{ parameters.clusterName }} variables: - ACN_DIR: $(Build.SourcesDirectory) commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path GOBIN: "$(GOPATH)/bin" # Go binaries path @@ -46,26 +44,22 @@ stages: displayName: Cilium Overlay Test Suite - (${{ parameters.name }}) timeoutInMinutes: 120 pool: - isCustom: true - type: linux name: $(BUILD_POOL_NAME_DEFAULT) + os: linux + image: acn-runner-vmss-image-1es-v3.0 demands: - - agent.os -equals Linux - Role -equals $(CUSTOM_E2E_ROLE) - variables: - ob_outputDirectory: $(Build.ArtifactStagingDirectory)/output - ob_git_checkout: true steps: - - checkout: ACNReviewChanges + - checkout: self clean: true - - template: cilium-overlay-e2e.steps.yaml@ACNTools + - template: cilium-overlay-e2e.steps.yaml parameters: name: ${{ parameters.name }} clusterName: ${{ parameters.clusterName }}-$(commitID) scaleup: 100 - - template: ../../cni/k8s-e2e/k8s-e2e.jobs.yaml@ACNTools + - template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml parameters: sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) clusterName: ${{ parameters.clusterName }}-$(commitID) @@ -83,18 +77,8 @@ stages: - ${{ parameters.name }} - cni_${{ parameters.os }} condition: failed() - pool: - isCustom: true - type: linux - name: $(BUILD_POOL_NAME_DEFAULT) - variables: - ob_outputDirectory: $(ACN_DIR)/${{ parameters.clusterName }}-$(commitID)_FailedE2ELogs_Attempt_#$(System.StageAttempt) - ob_git_checkout: true steps: - - checkout: ACNReviewChanges - clean: true - - - template: ../../templates/log-template.steps.yaml@ACNTools + - template: ../../templates/log-template.steps.yaml parameters: clusterName: ${{ parameters.clusterName }}-$(commitID) os: ${{ parameters.os }} diff --git a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e.steps.yaml b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e.steps.yaml index dea722df9c..458fc0205d 100644 --- a/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e.steps.yaml +++ b/.pipelines/singletenancy/cilium-overlay/cilium-overlay-e2e.steps.yaml @@ -6,31 +6,18 @@ parameters: steps: - - bash: | - go version - go env - mkdir -p '$(GOBIN)' - mkdir -p '$(GOPATH)/pkg' - mkdir -p '$(modulePath)' - echo '##vso[task.prependpath]$(GOBIN)' - echo '##vso[task.prependpath]$(GOROOT)/bin' - name: "GoEnv" - displayName: "Set up the Go environment" - - - task: KubectlInstaller@0 - inputs: - kubectlVersion: latest - - task: AzureCLI@2 + env: + CLUSTER: ${{ parameters.clusterName }} + AZCLI: az inputs: azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) scriptLocation: "inlineScript" scriptType: "bash" - workingDirectory: $(ACN_DIR) addSpnToEnvironment: true inlineScript: | set -e - make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }} + make -C ./hack/aks set-kubeconf ls -lah pwd kubectl cluster-info @@ -61,10 +48,9 @@ steps: fi kubectl get po -owide -A - name: "installCilium" displayName: "Install Cilium on AKS Overlay" - - template: ../../templates/cilium-cli.steps.yaml@ACNTools + - template: ../../templates/cilium-cli.yaml - script: | echo "Start Azilium E2E Tests on Overlay Cluster" @@ -75,11 +61,16 @@ steps: CNS=$(make cns-version) IPAM=$(make azure-ipam-version) fi kubectl get po -owide -A - sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_OVERLAY=true AZURE_IPAM_VERSION=${IPAM} CNS_VERSION=${CNS} CLEANUP=true - workingDirectory: $(ACN_DIR) + sudo -E env "PATH=$PATH" make test-load AZURE_IPAM_VERSION=${IPAM} CNS_VERSION=${CNS} retryCountOnTaskFailure: 3 - name: "aziliumTest" displayName: "Run Azilium E2E on AKS Overlay" + env: + SCALE_UP: 32 + OS_TYPE: linux + VALIDATE_STATEFILE: true + INSTALL_CNS: true + INSTALL_INSTALL_OVERLAY: true + CLEANUP: true - script: | kubectl get po -owide -A @@ -95,39 +86,46 @@ steps: azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) scriptLocation: "inlineScript" scriptType: "bash" - workingDirectory: $(ACN_DIR) addSpnToEnvironment: true inlineScript: | set -e kubectl get po -owide -A - clusterName=${{ parameters.clusterName }} echo "Restarting nodes" - for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do - make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val} + for val in $(az vmss list -g "$RESOURCE_GROUP" --query "[].name" -o tsv); do + make -C ./hack/aks restart-vmss VMSS_NAME=${val} done displayName: "Restart Nodes" + env: + AZCLI: az + CLUSTER: ${{ parameters.clusterName }} + RESOURCE_GROUP: "MC_${{ parameters.clusterName }}_${{ parameters.clusterName }}_$(REGION_AKS_CLUSTER_TEST)" + REGION: $(REGION_AKS_CLUSTER_TEST) - task: AzureCLI@2 inputs: azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) scriptLocation: "inlineScript" scriptType: "bash" - workingDirectory: $(ACN_DIR) addSpnToEnvironment: true inlineScript: | set -e - cd test/integration/load - # Scale Cluster Up/Down to confirm functioning CNS - ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=linux go test -count 1 -timeout 30m -tags load -run ^TestLoad$ - kubectl get pods -owide -A + pushd test/integration/load + # Scale Cluster Up/Down to confirm functioning CNS + go test -count 1 -timeout 30m -tags load -run ^TestLoad$ + kubectl get pods -owide -A + popd - cd ../../.. echo "Validating Node Restart" - make test-validate-state OS_TYPE=linux RESTART_CASE=true + make test-validate-state kubectl delete ns load-test displayName: "Validate Node Restart" retryCountOnTaskFailure: 3 + env: + ITERATIONS: 2 + SCALE_UP: ${{ parameters.scaleup }} + OS_TYPE: linux + RESTART_CASE: true - script: | set -e @@ -142,7 +140,6 @@ steps: ns=`kubectl get ns | grep cilium-test | awk '{print $1}'` echo "##vso[task.setvariable variable=ciliumNamespace]$ns" retryCountOnTaskFailure: 3 - name: "ciliumConnectivityTests" displayName: "Run Cilium Connectivity Tests" - ${{ if eq( parameters['testHubble'], true) }}: @@ -156,7 +153,6 @@ steps: kubectl get pods -Aowide echo "verify Hubble metrics endpoint is usable" go test ./test/integration/networkobservability -count=1 -v -tags=networkobservability - workingDirectory: $(ACN_DIR) retryCountOnTaskFailure: 3 name: "HubbleConnectivityTests" displayName: "Run Hubble Connectivity Tests" @@ -166,25 +162,25 @@ steps: echo "validate pod IP assignment and check systemd-networkd restart" kubectl get pod -owide -A if [ "$CILIUM_VERSION_TAG" = "cilium-nightly-pipeline" ]; then - echo "Check cilium identities in $(ciliumNamespace) namepsace during nightly run" + echo "Check cilium identities in "$CILIUM_NS" namepsace during nightly run" echo "expect the identities to be deleted when the namespace is deleted" kubectl get ciliumidentity | grep cilium-test fi make test-validate-state - echo "delete cilium connectivity test resources and re-validate state" # TODO Delete this and the next 4 lines if connectivity no longer has bug - kubectl delete ns $(ciliumNamespace) + echo "delete cilium connectivity test resources and re-validate state" + kubectl delete ns "$CILIUM_NS" kubectl get pod -owide -A make test-validate-state - workingDirectory: $(ACN_DIR) - name: "validatePods" displayName: "Validate Pods" + env: + CILIUM_NS: $(ciliumNamespace) - script: | if [ "$CILIUM_VERSION_TAG" = "cilium-nightly-pipeline" ]; then - kubectl get pod -owide -n $(ciliumNamespace) - echo "wait for pod and cilium identity deletion in $(ciliumNamespace) namespace" + kubectl get pod -owide -n "$CILIUM_NS" + echo "wait for pod and cilium identity deletion in "$CILIUM_NS" namespace" while true; do - pods=$(kubectl get pods -n $(ciliumNamespace) --no-headers=true 2>/dev/null) + pods=$(kubectl get pods -n "$CILIUM_NS" --no-headers=true 2>/dev/null) if [[ -z "$pods" ]]; then echo "No pods found" break @@ -192,19 +188,20 @@ steps: sleep 2s done sleep 20s - echo "Verify cilium identities are deleted from $(ciliumNamespace)" + echo "Verify cilium identities are deleted from "$CILIUM_NS"" checkIdentity="$(kubectl get ciliumidentity -o json | grep cilium-test | jq -e 'length == 0')" if [[ -n $checkIdentity ]]; then - echo "##[error]Cilium Identities still present in $(ciliumNamespace) namespace" + echo "##[error]Cilium Identities still present in "$CILIUM_NS" namespace" exit 1 else - printf -- "Identities deleted from $(ciliumNamespace) namespace\n" + printf -- "Identities deleted from "$CILIUM_NS" namespace\n" fi else echo "skip cilium identities check for PR pipeline" fi - name: "CiliumIdentities" displayName: "Verify Cilium Identities Deletion" + env: + CILIUM_NS: $(ciliumNamespace) - script: | # TODO REMOVE THIS STEP, make test-load covers this set -e @@ -217,14 +214,12 @@ steps: kubectl get pod -owide -A echo "validate pods after CNS restart" make test-validate-state - workingDirectory: $(ACN_DIR) name: "restartCNS" displayName: "Restart CNS and validate pods" - script: | echo "Run wireserver and metadata connectivity Tests" bash test/network/wireserver_metadata_test.sh - workingDirectory: $(ACN_DIR) retryCountOnTaskFailure: 3 name: "WireserverMetadataConnectivityTests" displayName: "Run Wireserver and Metadata Connectivity Tests" @@ -240,7 +235,6 @@ steps: kubectl -n kube-system patch daemonset azure-cns --type json -p='[{"op": "remove", "path": "/spec/template/spec/nodeSelector/non-existing"}]' fi fi - workingDirectory: $(ACN_DIR) name: "testAsyncDelete" displayName: "Verify Async Delete when CNS is down" diff --git a/.pipelines/singletenancy/cilium/cilium-e2e.jobs.yaml b/.pipelines/singletenancy/cilium/cilium-e2e.jobs.yaml index d3eb5f84a0..d053a1f291 100644 --- a/.pipelines/singletenancy/cilium/cilium-e2e.jobs.yaml +++ b/.pipelines/singletenancy/cilium/cilium-e2e.jobs.yaml @@ -14,11 +14,14 @@ stages: dependsOn: - ${{ parameters.dependsOn }} - setup + pool: + name: $(BUILD_POOL_NAME_DEFAULT) + os: linux + image: acn-runner-vmss-image-1es-v3.0 variables: commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] - ACN_DIR: $(Build.SourcesDirectory) jobs: - - template: ../../templates/create-cluster.jobs.yaml@ACNTools + - template: ../../templates/create-cluster.jobs.yaml parameters: name: ${{ parameters.name }} displayName: ${{ parameters.displayName }} @@ -36,7 +39,6 @@ stages: - publish - ${{ parameters.clusterName }} variables: - ACN_DIR: $(Build.SourcesDirectory) TAG: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.Tag'] ] CURRENT_VERSION: $[ stagedependencies.containerize.check_tag.outputs['CurrentTagManifests.currentTagManifests'] ] commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] @@ -49,26 +51,22 @@ stages: displayName: Cilium Test Suite - (${{ parameters.name }}) timeoutInMinutes: 120 pool: - isCustom: true - type: linux name: $(BUILD_POOL_NAME_DEFAULT) + os: linux + image: acn-runner-vmss-image-1es-v3.0 demands: - - agent.os -equals Linux - Role -equals $(CUSTOM_E2E_ROLE) - variables: - ob_outputDirectory: $(Build.ArtifactStagingDirectory)/output - ob_git_checkout: true steps: - - checkout: ACNReviewChanges + - checkout: self clean: true - - template: cilium-e2e.steps.yaml@ACNTools + - template: cilium-e2e.steps.yaml parameters: name: ${{ parameters.name }} clusterName: ${{ parameters.clusterName }}-$(commitID) scaleup: 100 - - template: ../../cni/k8s-e2e/k8s-e2e.jobs.yaml@ACNTools + - template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml parameters: sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) clusterName: ${{ parameters.clusterName }}-$(commitID) @@ -82,24 +80,13 @@ stages: - job: failedE2ELogs displayName: "Failure Logs" - condition: failed() - pool: - isCustom: true - type: linux - name: $(BUILD_POOL_NAME_DEFAULT) dependsOn: - ${{ parameters.name }} - cni_${{ parameters.os }} - variables: - ob_outputDirectory: $(ACN_DIR)/${{ parameters.clusterName }}-$(commitID)_FailedE2ELogs_Attempt_#$(System.StageAttempt) - ob_git_checkout: true + condition: failed() steps: - - checkout: ACNReviewChanges - clean: true - - - template: ../../templates/log-template.steps.yaml@ACNTools + - template: ../../templates/log-template.steps.yaml parameters: clusterName: ${{ parameters.clusterName }}-$(commitID) os: ${{ parameters.os }} cni: cilium - diff --git a/.pipelines/singletenancy/cilium/cilium-e2e.steps.yaml b/.pipelines/singletenancy/cilium/cilium-e2e.steps.yaml index 12e6f4e2ad..72fe79a919 100644 --- a/.pipelines/singletenancy/cilium/cilium-e2e.steps.yaml +++ b/.pipelines/singletenancy/cilium/cilium-e2e.steps.yaml @@ -4,31 +4,23 @@ parameters: scaleup: "" steps: - - bash: | - go version - go env - mkdir -p '$(GOBIN)' - mkdir -p '$(GOPATH)/pkg' - mkdir -p '$(modulePath)' - echo '##vso[task.prependpath]$(GOBIN)' - echo '##vso[task.prependpath]$(GOROOT)/bin' - name: "GoEnv" - displayName: "Set up the Go environment" - - task: KubectlInstaller@0 inputs: kubectlVersion: latest - task: AzureCLI@2 + displayName: "Install Cilium" + env: + SCALE_UP: ${{ coalesce(parameters.scaleup, 32) }} + RESOURCE_GROUP: "MC_${{ parameters.clusterName }}_${{ parameters.clusterName }}_$(REGION_AKS_CLUSTER_TEST)" inputs: azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) scriptLocation: "inlineScript" scriptType: "bash" - workingDirectory: $(ACN_DIR) addSpnToEnvironment: true inlineScript: | set -e - make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }} + make -C ./hack/aks set-kubeconf ls -lah pwd kubectl cluster-info @@ -45,110 +37,26 @@ steps: envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/v${DIR}/cilium-agent/templates/daemonset.yaml | kubectl apply -f - envsubst '${CILIUM_VERSION_TAG},${CILIUM_IMAGE_REGISTRY}' < test/integration/manifests/cilium/v${DIR}/cilium-operator/templates/deployment.yaml | kubectl apply -f - kubectl get po -owide -A - name: "installCilium" - displayName: "Install Cilium" - - template: ../../templates/cilium-cli.steps.yaml@ACNTools + - template: ../../templates/cilium-cli.yaml - script: | echo "Start Azilium E2E Tests" kubectl get po -owide -A - sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_AZILIUM=true AZURE_IPAM_VERSION=$(make azure-ipam-version) CNS_VERSION=$(make cns-version) CLEANUP=true - workingDirectory: $(ACN_DIR) + sudo -E env "PATH=$PATH" make test-load AZURE_IPAM_VERSION=$(make azure-ipam-version) CNS_VERSION=$(make cns-version) retryCountOnTaskFailure: 3 - name: "aziliumTest" displayName: "Run Azilium E2E" + env: + SCALE_UP: 32 + OS_TYPE: linux + VALIDATE_STATEFILE: true + INSTALL_CNS: true + INSTALL_AZILIUM: true + CLEANUP: true + + + - template: ../../templates/cilium-tests.yaml + parameters: + clusterName: ${{ parameters.clusterName }} + scaleup: ${{ parameters.scaleup }} - - script: | - kubectl get po -owide -A - echo "Waiting < 2 minutes for cilium to be ready" - # Ensure Cilium is ready Xm\Xs - cilium status --wait --wait-duration 2m - retryCountOnTaskFailure: 3 - name: "CiliumStatus" - displayName: "Cilium Status" - - - task: AzureCLI@2 - inputs: - azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) - scriptLocation: "inlineScript" - scriptType: "bash" - workingDirectory: $(ACN_DIR) - addSpnToEnvironment: true - inlineScript: | - set -e - kubectl get po -owide -A - clusterName=${{ parameters.clusterName }} - echo "Restarting nodes" - for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do - make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val} - done - displayName: "Restart Nodes" - - - task: AzureCLI@2 - inputs: - azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) - scriptLocation: "inlineScript" - scriptType: "bash" - workingDirectory: $(ACN_DIR) - addSpnToEnvironment: true - inlineScript: | - set -e - cd test/integration/load - - # Scale Cluster Up/Down to confirm functioning CNS - ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=linux go test -count 1 -timeout 30m -tags load -run ^TestLoad$ - kubectl get pods -owide -A - - cd ../../.. - echo "Validating Node Restart" - make test-validate-state OS_TYPE=linux RESTART_CASE=true - kubectl delete ns load-test - displayName: "Validate Node Restart" - retryCountOnTaskFailure: 3 - - - script: | - set -e - echo "Run Cilium Connectivity Tests" - cilium status - cilium connectivity test --connect-timeout 4s --request-timeout 30s --test '!pod-to-pod-encryption,!node-to-node-encryption' --force-deploy - ns=`kubectl get ns | grep cilium-test | awk '{print $1}'` - echo "##vso[task.setvariable variable=ciliumNamespace]$ns" - retryCountOnTaskFailure: 3 - name: "ciliumConnectivityTests" - displayName: "Run Cilium Connectivity Tests" - - - script: | - echo "validate pod IP assignment and check systemd-networkd restart" - kubectl get pod -owide -A - # Deleting echo-external-node deployment until cilium version matches TODO. https://github.com/cilium/cilium-cli/issues/67 is addressing the change. - # Saves 17 minutes - kubectl delete deploy -n $(ciliumNamespace) echo-external-node - make test-validate-state - echo "delete cilium connectivity test resources and re-validate state" - kubectl delete ns $(ciliumNamespace) - kubectl get pod -owide -A - make test-validate-state - workingDirectory: $(ACN_DIR) - name: "validatePods" - displayName: "Validate Pods" - - - script: | - echo "Run wireserver and metadata connectivity Tests" - bash test/network/wireserver_metadata_test.sh - workingDirectory: $(ACN_DIR) - retryCountOnTaskFailure: 3 - name: "WireserverMetadataConnectivityTests" - displayName: "Run Wireserver and Metadata Connectivity Tests" - - - script: | - cd hack/scripts - chmod +x async-delete-test.sh - ./async-delete-test.sh - if ! [ -z $(kubectl -n kube-system get ds azure-cns | grep non-existing) ]; then - kubectl -n kube-system patch daemonset azure-cns --type json -p='[{"op": "remove", "path": "/spec/template/spec/nodeSelector/non-existing"}]' - fi - workingDirectory: $(ACN_DIR) - retryCountOnTaskFailure: 3 - name: "testAsyncDelete" - displayName: "Verify Async Delete when CNS is down" diff --git a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e.jobs.yaml b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e.jobs.yaml index c6a0442ba8..a3e23ca486 100644 --- a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e.jobs.yaml +++ b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e.jobs.yaml @@ -15,9 +15,8 @@ stages: - setup variables: commitID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.commitID'] ] - ACN_DIR: $(Build.SourcesDirectory) jobs: - - template: ../../templates/create-cluster.jobs.yaml@ACNTools + - template: ../../templates/create-cluster.jobs.yaml parameters: name: ${{ parameters.name }} displayName: ${{ parameters.displayName }} @@ -36,7 +35,6 @@ stages: - publish - ${{ parameters.clusterName }} variables: - ACN_DIR: $(Build.SourcesDirectory) GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path GOBIN: "$(GOPATH)/bin" # Go binaries path modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking" @@ -46,87 +44,45 @@ stages: displayName: DualStack Overlay Test Suite | Linux - (${{ parameters.name }}) timeoutInMinutes: 120 pool: - isCustom: true - type: linux name: $(BUILD_POOL_NAME_DEFAULT) + os: linux + image: acn-runner-vmss-image-1es-v3.0 demands: - agent.os -equals Linux - Role -equals $(CUSTOM_E2E_ROLE) - variables: - ob_outputDirectory: $(Build.ArtifactStagingDirectory)/output - ob_git_checkout: true steps: - - checkout: ACNReviewChanges - clean: true - - - template: dualstackoverlay-e2e.steps.yaml@ACNTools + - template: dualstackoverlay-e2e.steps.yaml parameters: name: ${{ parameters.name }} clusterName: ${{ parameters.clusterName }}-$(commitID) os: linux scaleup: 100 - - job: windows_nodepool - displayName: Add Windows Nodepool - dependsOn: ${{ parameters.name }}_linux - pool: - isCustom: true - type: linux - name: $(BUILD_POOL_NAME_DEFAULT) - demands: - - agent.os -equals Linux - - Role -equals $(CUSTOM_E2E_ROLE) - variables: - ob_outputDirectory: $(Build.ArtifactStagingDirectory)/output - ob_git_checkout: true - steps: - - checkout: ACNReviewChanges - clean: true - - - task: AzureCLI@2 - inputs: - azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) - scriptLocation: "inlineScript" - scriptType: "bash" - workingDirectory: $(ACN_DIR) - addSpnToEnvironment: true - inlineScript: | - set -e - make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(commitID) - make -C ./hack/aks windows-nodepool-up AZCLI=az SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }}-$(commitID) VM_SIZE_WIN=${{ parameters.vmSize }} - echo "Windows nodes have been successfully added to DualStack Overlay Cluster" - kubectl cluster-info - kubectl get node -owide - kubectl get po -owide -A - name: "Add_Windows_Node" - displayName: "Add windows node" + - template: ../../templates/add-windows-nodepool.jobs.yaml + parameters: + depend: ${{ parameters.name }}_linux + clusterName: ${{ parameters.clusterName }}-$(commitID) + vmSize: "Standard_D2_v3" - job: ${{ parameters.name }}_windows displayName: DualStack Overlay Test Suite | Windows - (${{ parameters.name }}) timeoutInMinutes: 120 dependsOn: windows_nodepool pool: - isCustom: true - type: linux name: $(BUILD_POOL_NAME_DEFAULT) + os: linux + image: acn-runner-vmss-image-1es-v3.0 demands: - - agent.os -equals Linux - Role -equals $(CUSTOM_E2E_ROLE) - variables: - ob_outputDirectory: $(Build.ArtifactStagingDirectory)/output - ob_git_checkout: true steps: - - checkout: ACNReviewChanges - clean: true - - - template: dualstackoverlay-e2e.steps.yaml@ACNTools + - template: dualstackoverlay-e2e.steps.yaml parameters: name: ${{ parameters.name }} clusterName: ${{ parameters.clusterName }}-$(commitID) os: windows scaleup: 50 - - template: ../../cni/k8s-e2e/k8s-e2e.jobs.yaml@ACNTools + - template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml parameters: sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) clusterName: ${{ parameters.clusterName }}-$(commitID) @@ -138,38 +94,27 @@ stages: hostport: true service: true - - template: ../../cni/k8s-e2e/k8s-e2e.jobs.yaml@ACNTools + - template: ../../cni/k8s-e2e/k8s-e2e-job-template.yaml parameters: sub: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) clusterName: ${{ parameters.clusterName }}-$(commitID) os: windows dependsOn: cni_linux - dualstack: true + # dualstack: true # Currently broken for scenario and blocking releases, HNS is investigating. Covered by go test in E2E step template dns: true portforward: true - service: true + # service: true # Currently broken for scenario and blocking releases, HNS is investigating. hostport: true hybridWin: true - job: failedE2ELogs_linux displayName: "Linux Failure Logs" dependsOn: - - CNI_linux # k8s tests - ${{ parameters.name }}_linux - cni_linux - condition: failed() - pool: - isCustom: true - type: linux - name: $(BUILD_POOL_NAME_DEFAULT) - variables: - ob_outputDirectory: $(ACN_DIR)/${{ parameters.clusterName }}-$(commitID)_FailedE2ELogs_linux_Attempt_#$(System.StageAttempt) - ob_git_checkout: true + condition: in(dependencies.${{ parameters.name }}_linux.result, 'Failed') steps: - - checkout: ACNReviewChanges - clean: true - - - template: ../../templates/log-template.steps.yaml@ACNTools + - template: ../../templates/log-template.steps.yaml parameters: clusterName: ${{ parameters.clusterName }}-$(commitID) os: linux @@ -178,22 +123,11 @@ stages: - job: failedE2ELogs_windows displayName: "Windows Failure Logs" dependsOn: - - CNI_windows # k8s tests - ${{ parameters.name }}_windows - cni_windows - condition: failed() - pool: - isCustom: true - type: linux - name: $(BUILD_POOL_NAME_DEFAULT) - variables: - ob_outputDirectory: $(ACN_DIR)/${{ parameters.clusterName }}-$(commitID)_FailedE2ELogs_windows_Attempt_#$(System.StageAttempt) - ob_git_checkout: true + condition: in(dependencies.${{ parameters.name }}_windows.result, 'Failed') steps: - - checkout: ACNReviewChanges - clean: true - - - template: ../../templates/log-template.steps.yaml@ACNTools + - template: ../../templates/log-template.steps.yaml parameters: clusterName: ${{ parameters.clusterName }}-$(commitID) os: windows diff --git a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e.steps.yaml b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e.steps.yaml index 7764b08f0e..09db069453 100644 --- a/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e.steps.yaml +++ b/.pipelines/singletenancy/dualstack-overlay/dualstackoverlay-e2e.steps.yaml @@ -6,44 +6,41 @@ parameters: scaleup: "" steps: - - bash: | - go version - go env - mkdir -p '$(GOBIN)' - mkdir -p '$(GOPATH)/pkg' - mkdir -p '$(modulePath)' - echo '##vso[task.prependpath]$(GOBIN)' - echo '##vso[task.prependpath]$(GOROOT)/bin' - name: "GoEnv" - displayName: "Set up the Go environment" - - task: KubectlInstaller@0 inputs: kubectlVersion: latest - task: AzureCLI@2 + displayName: "Set Kubeconfig" + env: + AZCLI: az + CLUSTER: ${{ parameters.clusterName }} inputs: azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) scriptLocation: "inlineScript" scriptType: "bash" - workingDirectory: $(ACN_DIR) addSpnToEnvironment: true inlineScript: | set -e - make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }} - name: "kubeconfig" - displayName: "Set Kubeconfig" + make -C ./hack/aks set-kubeconf - ${{ if eq(parameters.os, 'linux') }}: - script: | kubectl cluster-info kubectl get node kubectl get po -owide -A - sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=linux CNI_TYPE=dualstack VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_DUALSTACK_OVERLAY=true VALIDATE_DUALSTACK=true CNI_VERSION=$(make cni-version) CNS_VERSION=$(make cns-version) CLEANUP=true - workingDirectory: $(ACN_DIR) + sudo -E env "PATH=$PATH" make test-load =true CNI_VERSION=$(make cni-version) CNS_VERSION=$(make cns-version) retryCountOnTaskFailure: 3 - name: "integrationTest" displayName: "Run CNS Integration Tests on AKS DualStack Overlay" + env: + SCALE_UP: 32 + OS_TYPE: linux + CNI_TYPE: dualstack + VALIDATE_STATEFILE: true + INSTALL_CNS: true + INSTALL_DUALSTACK_OVERLAY: true + VALIDATE_DUALSTACK: true + CLEANUP: true - script: | set -e @@ -53,48 +50,54 @@ steps: go test -count=1 datapath_linux_test.go -timeout 3m -tags connection -run ^TestDatapathLinux$ -tags=connection,integration -isDualStack=true echo "Dualstack Overlay Linux datapath IPv4 test" go test -count=1 datapath_linux_test.go -timeout 3m -tags connection -run ^TestDatapathLinux$ -tags=connection,integration - workingDirectory: $(ACN_DIR) retryCountOnTaskFailure: 3 - name: "DualStack_Overlay_Linux_Tests" displayName: "DualStack Overlay Linux Tests" - task: AzureCLI@2 + displayName: "Restart Nodes" + env: + AZCLI: az + CLUSTER: ${{ parameters.clusterName }} + RESOURCE_GROUP: "MC_${{ parameters.clusterName }}_${{ parameters.clusterName }}_$(REGION_DUALSTACKOVERLAY_CLUSTER_TEST)" + REGION: $(REGION_DUALSTACKOVERLAY_CLUSTER_TEST) inputs: azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) scriptLocation: "inlineScript" scriptType: "bash" - workingDirectory: $(ACN_DIR) addSpnToEnvironment: true inlineScript: | set -e - clusterName=${{ parameters.clusterName }} echo "Restarting nodes" - for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_DUALSTACKOVERLAY_CLUSTER_TEST) --query "[].name" -o tsv); do - make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_DUALSTACKOVERLAY_CLUSTER_TEST) VMSS_NAME=${val} + for val in $(az vmss list -g "$RESOURCE_GROUP" --query "[].name" -o tsv); do + make -C ./hack/aks restart-vmss VMSS_NAME=${val} done - displayName: "Restart Nodes" - task: AzureCLI@2 + displayName: "Validate Node Restart" + retryCountOnTaskFailure: 3 + env: + ITERATIONS: 2 + SCALE_UP: ${{ parameters.scaleup }} + OS_TYPE: linux + RESTART_CASE: true + CNI_TYPE: dualstack inputs: azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) scriptLocation: "inlineScript" scriptType: "bash" - workingDirectory: $(ACN_DIR) addSpnToEnvironment: true inlineScript: | set -e - cd test/integration/load - # Scale Cluster Up/Down to confirm functioning CNS - ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=linux go test -count 1 -timeout 30m -tags load -run ^TestLoad$ - kubectl get pods -owide -A + pushd test/integration/load + # Scale Cluster Up/Down to confirm functioning CNS + go test -count 1 -timeout 30m -tags load -run ^TestLoad$ + kubectl get pods -owide -A + popd - cd ../../.. echo "Validating Node Restart" - make test-validate-state OS_TYPE=linux RESTART_CASE=true CNI_TYPE=dualstack + make test-validate-state kubectl delete ns load-test - displayName: "Validate Node Restart" - retryCountOnTaskFailure: 3 - ${{ if eq(parameters.os, 'windows') }}: - script: | @@ -105,55 +108,70 @@ steps: kubectl taint nodes $node node.cloudprovider.kubernetes.io/uninitialized=true:NoSchedule- fi done - sudo -E env "PATH=$PATH" make test-load SCALE_UP=32 OS_TYPE=windows CNI_TYPE=cniv2 VALIDATE_STATEFILE=true INSTALL_CNS=true INSTALL_DUALSTACK_OVERLAY=true VALIDATE_DUALSTACK=true CNI_VERSION=$(make cni-version) CNS_VERSION=$(make cns-version) CLEANUP=true - workingDirectory: $(ACN_DIR) - name: "WindowsDualStackOverlayControlPlaneScaleTests" + sudo -E env "PATH=$PATH" make test-load CNI_VERSION=$(make cni-version) CNS_VERSION=$(make cns-version) displayName: "Windows DualStack Overlay ControlPlane Scale Tests" retryCountOnTaskFailure: 3 + env: + SCALE_UP: 32 + OS_TYPE: windows + # TODO: Check if this is correct. This is the original CNI_TYPE, but the previous test uses "dualstack" + CNI_TYPE: cniv2 + VALIDATE_STATEFILE: true + INSTALL_CNS: true + INSTALL_DUALSTACK_OVERLAY: true + VALIDATE_DUALSTACK: true + CLEANUP: true - script: | echo "DualStack Overlay DataPath Test" cd test/integration/datapath sudo -E env "PATH=$PATH" go test -count=1 datapath_windows_test.go -timeout 3m -tags connection -restartKubeproxy true -run ^TestDatapathWin$ - workingDirectory: $(ACN_DIR) - name: "WindowsDualStackOverlayDatapathTests" displayName: "Windows DualStack Overlay Datapath Tests" retryCountOnTaskFailure: 3 - task: AzureCLI@2 + displayName: "Restart Nodes" + env: + AZCLI: az + CLUSTER: ${{ parameters.clusterName }} + RESOURCE_GROUP: "MC_${{ parameters.clusterName }}_${{ parameters.clusterName }}_$(REGION_DUALSTACKOVERLAY_CLUSTER_TEST)" + REGION: $(REGION_DUALSTACKOVERLAY_CLUSTER_TEST) inputs: azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) scriptLocation: "inlineScript" scriptType: "bash" - workingDirectory: $(ACN_DIR) addSpnToEnvironment: true inlineScript: | set -e clusterName=${{ parameters.clusterName }} echo "Restarting nodes" - for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_DUALSTACKOVERLAY_CLUSTER_TEST) --query "[].name" -o tsv); do - make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_DUALSTACKOVERLAY_CLUSTER_TEST) VMSS_NAME=${val} + for val in $(az vmss list -g "$RESOURCE_GROUP" --query "[].name" -o tsv); do + make -C ./hack/aks restart-vmss VMSS_NAME=${val} done - displayName: "Restart Nodes" - task: AzureCLI@2 + displayName: "Validate Node Restart" + retryCountOnTaskFailure: 3 + env: + ITERATIONS: 2 + SCALE_UP: ${{ parameters.scaleup }} + OS_TYPE: windows + RESTART_CASE: true + CNI_TYPE: cniv2 inputs: azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) scriptLocation: "inlineScript" scriptType: "bash" - workingDirectory: $(ACN_DIR) addSpnToEnvironment: true inlineScript: | set -e - cd test/integration/load - # Scale Cluster Up/Down to confirm functioning CNS - ITERATIONS=2 SCALE_UP=${{ parameters.scaleup }} OS_TYPE=windows go test -count 1 -timeout 30m -tags load -run ^TestLoad$ - kubectl get pods -owide -A + pushd test/integration/load + # Scale Cluster Up/Down to confirm functioning CNS + OS_TYPE=windows go test -count 1 -timeout 30m -tags load -run ^TestLoad$ + kubectl get pods -owide -A + popd - cd ../../.. echo "Validating Node Restart" - make test-validate-state OS_TYPE=windows RESTART_CASE=true CNI_TYPE=cniv2 + make test-validate-state kubectl delete ns load-test - displayName: "Validate Node Restart" - retryCountOnTaskFailure: 3 diff --git a/.pipelines/template.trigger.jobs.yaml b/.pipelines/template.trigger.jobs.yaml index ce7ba6354c..f5723a14be 100644 --- a/.pipelines/template.trigger.jobs.yaml +++ b/.pipelines/template.trigger.jobs.yaml @@ -53,7 +53,7 @@ jobs: TMPL_REPO_REF: '${{ parameters.mainRepoRef }}' TMPL_REPO_NAME: '${{ parameters.mainRepoName }}' TMPL_REPO_TYPE: '${{ parameters.mainRepoType }}' - ACCEPTED_REPO_REFS: '^refs/heads/feature/ob-onboard-0$' + ACCEPTED_REPO_REFS: '^refs/heads/feature/ob-onboard-2$' ACCEPTED_REPO_NAME: '^Azure/azure-container-networking$' ACCEPTED_REPO_TYPE: '^github$' diff --git a/.pipelines/templates/add-windows-nodepool-job.yaml b/.pipelines/templates/add-windows-nodepool-job.yaml index fe700371b4..9200169dcb 100644 --- a/.pipelines/templates/add-windows-nodepool-job.yaml +++ b/.pipelines/templates/add-windows-nodepool-job.yaml @@ -22,7 +22,7 @@ jobs: addSpnToEnvironment: true inlineScript: | set -e - + windows_nodepool=$(az aks nodepool list \ --resource-group ${{ parameters.clusterName }} \ --cluster-name ${{ parameters.clusterName }} \ @@ -38,7 +38,7 @@ jobs: --name "${{ parameters.clusterName }}" \ --resource-group "${{ parameters.clusterName }}" \ --query provisioningState) - + if echo "$cluster_state" | grep -q "Updating"; then echo "Cluster is updating. Sleeping for 30 seconds..." sleep 30 diff --git a/.pipelines/templates/add-windows-nodepool.jobs.yaml b/.pipelines/templates/add-windows-nodepool.jobs.yaml new file mode 100644 index 0000000000..2d4d302f8c --- /dev/null +++ b/.pipelines/templates/add-windows-nodepool.jobs.yaml @@ -0,0 +1,62 @@ +parameters: + depend: "" + clusterName: "" # unique identifier + vmSize: "" + +jobs: +- job: windows_nodepool + displayName: Add Windows Nodepool + dependsOn: ${{ parameters.depend }} + timeoutInMinutes: 30 + steps: + - task: AzureCLI@2 + displayName: "Add windows node to cluster" + retryCountOnTaskFailure: 5 + env: + AZCLI: az + SUB: $(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) + CLUSTER: ${{ parameters.clusterName }} + VM_SIZE_WIN: ${{ parameters.vmSize }} + inputs: + azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + scriptLocation: "inlineScript" + scriptType: "bash" + addSpnToEnvironment: true + inlineScript: | + set -e + + windows_nodepool=$(az aks nodepool list \ + --resource-group "$CLUSTER" \ + --cluster-name "$CLUSTER" \ + --query "[?osType=='Windows']" \ + --output tsv) + + if [ -z "$windows_nodepool" ]; then + echo "No Windows node pool found in the AKS cluster." + + # wait for cluster to update + while true; do + cluster_state=$(az aks show \ + --resource-group "$CLUSTER" \ + --name "$CLUSTER" \ + --query provisioningState) + + if echo "$cluster_state" | grep -q "Updating"; then + echo "Cluster is updating. Sleeping for 30 seconds..." + sleep 30 + else + break + fi + done + # cluster state is always set and visible outside the loop + echo "Cluster state is: $cluster_state" + + make -C ./hack/aks set-kubeconf + make -C ./hack/aks windows-nodepool-up + echo "Windows node was successfully added" + kubectl cluster-info + kubectl get node -owide + kubectl get po -owide -A + else + echo "Windows node pool already exists in the AKS cluster." + fi diff --git a/.pipelines/templates/create-cluster-swiftv2.jobs.yaml b/.pipelines/templates/create-cluster-swiftv2.jobs.yaml deleted file mode 100644 index f293007ec4..0000000000 --- a/.pipelines/templates/create-cluster-swiftv2.jobs.yaml +++ /dev/null @@ -1,47 +0,0 @@ -parameters: - os: linux - continueOnError: true - -jobs: - - job: ${{ parameters.name }} - displayName: Cluster - ${{ parameters.name }} - pool: - isCustom: true - type: linux - name: $(BUILD_POOL_NAME_DEFAULT) - variables: - ob_outputDirectory: $(Build.ArtifactStagingDirectory)/output - ob_git_checkout: true - steps: - - checkout: ACNReviewChanges - clean: true - - - task: AzureCLI@2 - inputs: - azureSubscription: $(ACN_TEST_SERVICE_CONNECTION) - scriptLocation: "inlineScript" - scriptType: "bash" - workingDirectory: $(ACN_DIR) - addSpnToEnvironment: true - inlineScript: | - set -e - echo "Check az version" - az version - if ${{ lower(contains(parameters.clusterType, 'dualstack')) }} - then - echo "Install az cli extension preview" - az extension add --name aks-preview - az extension update --name aks-preview - fi - mkdir -p ~/.kube/ - make -C ./hack/aks azcfg AZCLI=az REGION=${{ parameters.region }} - - make -C ./hack/aks ${{ parameters.clusterType }} \ - AZCLI=az REGION=${{ parameters.region }} SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) \ - CLUSTER=${{ parameters.clusterName }} \ - VM_SIZE=${{ parameters.vmSize }} OS=${{parameters.os}} - - echo "Cluster successfully created" - displayName: Cluster - ${{ parameters.clusterType }} - continueOnError: ${{ parameters.continueOnError }} - diff --git a/.pipelines/templates/create-cluster.jobs.yaml b/.pipelines/templates/create-cluster.jobs.yaml index 2bfa2f56f2..b6800e9556 100644 --- a/.pipelines/templates/create-cluster.jobs.yaml +++ b/.pipelines/templates/create-cluster.jobs.yaml @@ -14,45 +14,43 @@ parameters: jobs: - job: ${{ parameters.name }} displayName: Cluster - ${{ parameters.name }} - pool: - isCustom: true - type: linux - name: $(BUILD_POOL_NAME_DEFAULT) - variables: - ob_outputDirectory: $(Build.ArtifactStagingDirectory)/output - ob_git_checkout: true steps: - - checkout: ACNReviewChanges - clean: true + - bash: | + set -e + echo "Check az version" + az version + + echo "Install az cli extension preview" + az extension add --name aks-preview + az extension update --name aks-preview + displayName: "Install AzCLI Extentions" + condition: and(succeeded(), ${{ contains(parameters.clusterType, 'dualstack') }}) + - task: AzureCLI@2 + displayName: Cluster - ${{ parameters.clusterType }} + continueOnError: ${{ contains(parameters.clusterType, 'dualstack') }} + env: + AZCLI: az + REGION: ${{ parameters.region }} + CLUSTER: ${{ parameters.clusterName }} + CLUSTER_TYPE: ${{ parameters.clusterType }} + SUB: $(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) + VM_SIZE: ${{ parameters.vmSize }} + VM_SIZE_WIN: ${{ parameters.vmSizeWin }} + OS: ${{ parameters.os }} + OS_SKU_WIN: ${{ parameters.osSkuWin }} inputs: azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) scriptLocation: "inlineScript" scriptType: "bash" - workingDirectory: $(ACN_DIR) addSpnToEnvironment: true inlineScript: | set -e - echo "Check az version" - az version - if ${{ lower(contains(parameters.clusterType, 'dualstack')) }} - then - echo "Install az cli extension preview" - az extension add --name aks-preview - az extension update --name aks-preview - fi - mkdir -p ~/.kube/ - make -C ./hack/aks azcfg AZCLI=az REGION=${{ parameters.region }} - make -C ./hack/aks ${{ parameters.clusterType }} \ - AZCLI=az REGION=${{ parameters.region }} SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) \ - CLUSTER=${{ parameters.clusterName }} \ - VM_SIZE=${{ parameters.vmSize }} VM_SIZE_WIN=${{ parameters.vmSizeWin }} \ - OS_SKU_WIN=${{ parameters.osSkuWin }} OS=${{parameters.os}} \ - WINDOWS_USERNAME=${WINDOWS_USERNAME} WINDOWS_PASSWORD=${WINDOWS_PASSWORD} + mkdir -p ~/.kube/ + make -C ./hack/aks azcfg + make -C ./hack/aks "$CLUSTER_TYPE" + #WINDOWS_USERNAME=${WINDOWS_USERNAME} WINDOWS_PASSWORD=${WINDOWS_PASSWORD} echo "Cluster successfully created" - displayName: Cluster - ${{ parameters.clusterType }} - continueOnError: ${{ contains(parameters.clusterType, 'dualstack') }} - diff --git a/.pipelines/templates/delete-cluster.steps.yaml b/.pipelines/templates/delete-cluster.steps.yaml index 04cc02d7f3..7e74a2eeab 100644 --- a/.pipelines/templates/delete-cluster.steps.yaml +++ b/.pipelines/templates/delete-cluster.steps.yaml @@ -5,17 +5,20 @@ parameters: steps: - task: AzureCLI@2 + displayName: Delete - ${{ parameters.name }} + env: + AZCLI: az + CLUSTER: ${{ parameters.clusterName }} + REGION: ${{ parameters.region }} + SUB: ${{ parameters.sub }} inputs: - azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) + azureSubscription: ${{ parameters.svcConn }} scriptLocation: "inlineScript" scriptType: "bash" addSpnToEnvironment: true inlineScript: | echo "Deleting cluster" - make -C ./hack/aks azcfg AZCLI=az REGION=${{ parameters.region }} - make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }} - make -C ./hack/aks down AZCLI=az REGION=${{ parameters.region }} SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) CLUSTER=${{ parameters.clusterName }} + make -C ./hack/aks azcfg + make -C ./hack/aks set-kubeconf + make -C ./hack/aks down echo "Cluster and resources down" - name: delete - displayName: Delete - ${{ parameters.name }} - diff --git a/.pipelines/templates/delete-cluster.yaml b/.pipelines/templates/delete-cluster.yaml deleted file mode 100644 index f06f6d2173..0000000000 --- a/.pipelines/templates/delete-cluster.yaml +++ /dev/null @@ -1,20 +0,0 @@ -parameters: - name: "" - clusterName: "" - region: "" - -steps: - - task: AzureCLI@2 - inputs: - azureSubscription: ${{ parameters.svcConn }} - scriptLocation: "inlineScript" - scriptType: "bash" - addSpnToEnvironment: true - inlineScript: | - echo "Deleting cluster" - make -C ./hack/aks azcfg AZCLI=az REGION=${{ parameters.region }} - make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }} - make -C ./hack/aks down AZCLI=az REGION=${{ parameters.region }} SUB=${{ parameters.sub }} CLUSTER=${{ parameters.clusterName }} - echo "Cluster and resources down" - name: delete - displayName: Delete - ${{ parameters.name }} diff --git a/.pipelines/templates/log-template.steps.yaml b/.pipelines/templates/log-template.steps.yaml index 0079bcee42..11d7abf80f 100644 --- a/.pipelines/templates/log-template.steps.yaml +++ b/.pipelines/templates/log-template.steps.yaml @@ -23,21 +23,16 @@ parameters: jobName: "FailedE2ELogs" steps: - - task: KubectlInstaller@0 - inputs: - kubectlVersion: latest - - task: AzureCLI@2 inputs: azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION) scriptLocation: "inlineScript" scriptType: "bash" - workingDirectory: $(ACN_DIR) addSpnToEnvironment: true inlineScript: | make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }} - acnLogs=$OB_OUTPUTDIRECTORY + acnLogs=$(System.DefaultWorkingDirectory)/${{ parameters.clusterName }}_${{ parameters.logType }}_Attempt_#$(System.StageAttempt) mkdir -p $acnLogs echo "Root Directory created: $acnLogs" echo "##vso[task.setvariable variable=acnLogs]$acnLogs" @@ -100,7 +95,6 @@ steps: fi displayName: Failure Logs condition: always() - workingDirectory: $(ACN_DIR) - ${{ if eq(parameters.os, 'linux') }}: - bash: | @@ -173,7 +167,7 @@ steps: file="azure-cns.json" kubectl exec -i -n kube-system $pod -- cat /var/lib/azure-network/$file > $(acnLogs)/"$node"_logs/CNS-output/$file echo "CNS State, $file, captured: $(acnLogs)/"$node"_logs/CNS-output/$file" - if [[ $managed =~ true ]]; then + if [ $managed = "true" ]; then file="azure-endpoints.json" kubectl exec -i -n kube-system $pod -- cat /var/run/azure-cns/$file > $(acnLogs)/"$node"_logs/CNS-output/$file echo "CNS Managed State, $file, captured: $(acnLogs)/"$node"_logs/CNS-output/$file" @@ -196,7 +190,6 @@ steps: echo "Cilium, $file, captured: $(acnLogs)/"$node"_logs/Cilium-output/$file" done fi - workingDirectory: $(ACN_DIR) displayName: Linux Logs condition: always() @@ -272,13 +265,12 @@ steps: fi done fi - workingDirectory: $(ACN_DIR) displayName: Windows Logs condition: always() -# - publish: $(System.DefaultWorkingDirectory)/${{ parameters.clusterName }}_${{ parameters.logType }}_Attempt_#$(System.StageAttempt) -# condition: always() -# artifact: ${{ parameters.clusterName }}_${{ parameters.os }}${{ parameters.jobName }}_Attempt_#$(System.StageAttempt) -# name: acnLogs_${{ parameters.logType }} -# displayName: Publish Cluster logs - + - task: 1ES.PublishPipelineArtifact@1 + displayName: Publish Cluster logs + condition: always() + inputs: + targetPath: $(System.DefaultWorkingDirectory)/${{ parameters.clusterName }}_${{ parameters.logType }}_Attempt_#$(System.StageAttempt) + artifactName: ${{ parameters.clusterName }}_${{ parameters.os }}${{ parameters.jobName }}_Attempt_#$(System.StageAttempt) diff --git a/.pipelines/templates/run-unit-tests.stages.yaml b/.pipelines/templates/run-unit-tests.stages.yaml new file mode 100644 index 0000000000..7075cdfdac --- /dev/null +++ b/.pipelines/templates/run-unit-tests.stages.yaml @@ -0,0 +1,61 @@ +stages: +- stage: test + displayName: Test ACN + dependsOn: + - setup + jobs: + - job: test + displayName: Run Tests + variables: + STORAGE_ID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.StorageID'] ] + pool: + name: "$(BUILD_POOL_NAME_DEFAULT)" + os: linux + image: acn-runner-vmss-image-1es-v3.0 + steps: + - task: GoTool@0 + inputs: + version: $(GO_VERSION) + + - script: | + make tools + # run test, echo exit status code to fd 3, pipe output from test to tee, which splits output to stdout and go-junit-report (which converts test output to report.xml), stdout from tee is redirected to fd 4. Take output written to fd 3 (which is the exit code of test), redirect to stdout, pipe to read from stdout then exit with that status code. Read all output from fd 4 (output from tee) and write to top stdout + { { { { + sudo -E env "PATH=$PATH" make test-all; + echo $? >&3; + } | tee >(build/tools/bin/go-junit-report > report.xml) >&4; + } 3>&1; + } | { read xs; exit $xs; } + } 4>&1 + retryCountOnTaskFailure: 3 + name: "Test" + displayName: "Run Tests" + +- stage: test_windows + displayName: Test ACN Windows + dependsOn: + - setup + pool: + name: "$(BUILD_POOL_NAME_DEFAULT_WINDOWS)" + os: windows + image: acn-runner-vmss-image-windows-1es-v2.0 + jobs: + - job: test + displayName: Run Tests + variables: + STORAGE_ID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.StorageID'] ] + steps: + - task: GoTool@0 + inputs: + version: $(GO_VERSION) + + - script: | + cd npm/ + go test ./... + cd ../cni/ + go test ./... + cd ../platform/ + go test ./... + retryCountOnTaskFailure: 3 + displayName: "Run Windows Tests" + diff --git a/.pipelines/templates/run-unit-tests.yaml b/.pipelines/templates/run-unit-tests.yaml index cedcae057d..2d7baf921a 100644 --- a/.pipelines/templates/run-unit-tests.yaml +++ b/.pipelines/templates/run-unit-tests.yaml @@ -10,7 +10,12 @@ stages: STORAGE_ID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.StorageID'] ] pool: name: "$(BUILD_POOL_NAME_DEFAULT)" + os: linux steps: + - task: GoTool@0 + inputs: + version: $(GO_VERSION) + - script: | make tools # run test, echo exit status code to fd 3, pipe output from test to tee, which splits output to stdout and go-junit-report (which converts test output to report.xml), stdout from tee is redirected to fd 4. Take output written to fd 3 (which is the exit code of test), redirect to stdout, pipe to read from stdout then exit with that status code. Read all output from fd 4 (output from tee) and write to top stdout @@ -35,8 +40,13 @@ stages: variables: STORAGE_ID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.StorageID'] ] pool: - name: "$(BUILD_POOL_NAME_DEFAULT_WINDOWS_ALT)" + name: "$(BUILD_POOL_NAME_DEFAULT_WINDOWS)" + os: windows steps: + - task: GoTool@0 + inputs: + version: $(GO_VERSION) + - script: | cd npm/ go test ./... diff --git a/.pipelines/templates/unit-tests.stages.yaml b/.pipelines/templates/unit-tests.stages.yaml index 204cde4c2e..e023bc45ba 100644 --- a/.pipelines/templates/unit-tests.stages.yaml +++ b/.pipelines/templates/unit-tests.stages.yaml @@ -1,66 +1,49 @@ stages: - - stage: test - displayName: Test ACN - dependsOn: - - setup - jobs: - - job: test - displayName: Run Tests - variables: - STORAGE_ID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.StorageID'] ] - ob_git_checkout: true - pool: - isCustom: true - type: linux - name: "$(BUILD_POOL_NAME_DEFAULT)" - steps: - # Test changes under review - - checkout: ACNReviewChanges - clean: true +- stage: test + displayName: Test ACN + dependsOn: + - setup + jobs: + - job: test + displayName: Run Tests + variables: + STORAGE_ID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.StorageID'] ] + steps: + - script: | + cd azure-container-networking + make tools + # run test, echo exit status code to fd 3, pipe output from test to tee, which splits output to stdout and go-junit-report (which converts test output to report.xml), stdout from tee is redirected to fd 4. Take output written to fd 3 (which is the exit code of test), redirect to stdout, pipe to read from stdout then exit with that status code. Read all output from fd 4 (output from tee) and write to top stdout + { { { { + sudo -E env "PATH=$PATH" make test-all; + echo $? >&3; + } | tee >(build/tools/bin/go-junit-report > report.xml) >&4; + } 3>&1; + } | { read xs; exit $xs; } + } 4>&1 + retryCountOnTaskFailure: 3 + displayName: "Run Tests" - - script: | - cd azure-container-networking - make tools - # run test, echo exit status code to fd 3, pipe output from test to tee, which splits output to stdout and go-junit-report (which converts test output to report.xml), stdout from tee is redirected to fd 4. Take output written to fd 3 (which is the exit code of test), redirect to stdout, pipe to read from stdout then exit with that status code. Read all output from fd 4 (output from tee) and write to top stdout - { { { { - sudo -E env "PATH=$PATH" make test-all; - echo $? >&3; - } | tee >(build/tools/bin/go-junit-report > report.xml) >&4; - } 3>&1; - } | { read xs; exit $xs; } - } 4>&1 - retryCountOnTaskFailure: 3 - name: "Test" - displayName: "Run Tests" - - - stage: test_windows - displayName: Test ACN Windows - dependsOn: - - setup - jobs: - - job: test - displayName: Run Tests - variables: - STORAGE_ID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.StorageID'] ] - ob_git_checkout: true - pool: - isCustom: true - type: windows - name: "$(BUILD_POOL_NAME_DEFAULT_WINDOWS_ALT)" - steps: - # Test changes under review - - checkout: ACNReviewChanges - clean: true - - - script: | - cd azure-container-networking/ - cd npm/ - go test ./... - cd ../cni/ - go test ./... - cd ../platform/ - go test ./... - retryCountOnTaskFailure: 3 - name: "TestWindows" - displayName: "Run Windows Tests" +- stage: test_windows + displayName: Test ACN Windows + dependsOn: + - setup + jobs: + - job: test + displayName: Run Tests + variables: + STORAGE_ID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.StorageID'] ] + pool: + type: windows + name: "$(BUILD_POOL_NAME_DEFAULT_WINDOWS_ALT)" + steps: + - script: | + cd azure-container-networking/ + cd npm/ + go test ./... + cd ../cni/ + go test ./... + cd ../platform/ + go test ./... + retryCountOnTaskFailure: 3 + displayName: "Run Windows Tests" diff --git a/.pipelines/trigger.yaml b/.pipelines/trigger.yaml index 8bed06043a..7b2077c1c0 100644 --- a/.pipelines/trigger.yaml +++ b/.pipelines/trigger.yaml @@ -1,52 +1,82 @@ -trigger: none -#pr: -# branches: -# include: -# - master -# - release/* -# paths: -# exclude: -# - ".devcontainer" -# - ".hooks" -# - ".vscode" -# - ".github" -# - docs -# -# -#trigger: -# branches: -# include: -# - gh-readonly-queue/master/* -# tags: -# include: -# - "*" +pr: + branches: + include: + - master + - release/* + paths: + exclude: + - ".devcontainer" + - ".hooks" + - ".vscode" + - ".github" + - docs +trigger: + branches: + include: + - gh-readonly-queue/master/* + tags: + include: + - "*" + resources: repositories: - - repository: azure-container-networking - type: github - name: Azure/azure-container-networking - endpoint: 'Azure-ACN RO Service Connection' - ref: refs/heads/feature/ob-onboard-0 + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/stable -variables: - REPO_REF: $[ resources.repositories['azure-container-networking'].ref ] - REPO_COMMIT: $[ resources.repositories['azure-container-networking'].version ] - REPO_NAME: $[ resources.repositories['azure-container-networking'].name ] - REPO_TYPE: $[ resources.repositories['azure-container-networking'].type ] - CHANGESET_COMMIT: $[ resources.repositories['self'].version ] - -pool: - vmImage: ubuntu-latest +variables: + # Local analysis is only enabled on PR branches + Codeql.AnalyzeInPipeline: $[startsWith(variables['Build.SourceBranch'], 'refs/pull/')] +# Codeql.ADO.Build.Repository.Uri: $(Build.Repository.Uri) +# Codeql.ADO.Build.Repository.Name: $(Build.Repository.Name) +# Codeql.ADO.Build.SourceVersion: $(Build.SourceVersion) +# Codeql.ADO.Build.DefinitionName: $(Build.DefinitionName) +# Codeql.ADO.Build.BuildId: $(Build.BuildId) +# Codeql.ADO.System.CollectionUri: $(System.CollectionUri) +# Codeql.ADO.System.TeamProject: $(System.TeamProject) +# Codeql.ADO.System.JobId: $(System.JobId) +# Codeql.ADO.Build.Repository.LocalPath: $(Build.Repository.LocalPath) +# Codeql.ADO.Build.SourceBranch: $(Build.SourceBranch) +# Codeql.ADO.Build.Repository.Provider: $(Build.Repository.Provider) +# Codeql.ADO.Build.SourceBranchName: $(Build.SourceBranchName) +# Codeql.ADO.Build.SourcesDirectory: $(Build.SourcesDirectory) extends: - template: /.pipelines/template.trigger.jobs.yaml@azure-container-networking + template: v1/1ES.Unofficial.PipelineTemplate.yml@1ESPipelineTemplates parameters: - mainRepoRef: $(REPO_REF) - mainRepoCommit: $(REPO_COMMIT) - mainRepoName: $(REPO_NAME) - mainRepoType: $(REPO_TYPE) + pool: + name: $(BUILD_POOL_NAME_DEFAULT) + os: linux + image: acn-runner-vmss-image-1es-v3.0 + featureFlags: + automaticContainerPatching: true + sdl: + sourceAnalysisPool: + name: $(BUILD_POOL_NAME_DEFAULT_WINDOWS) + os: windows + image: acn-runner-vmss-image-windows-1es-v2.0 + globalSdl: + #baseline: + #baselineFile: $(Build.SourcesDirectory)\.config\.gdnbaselines + #suppressionSet: default + suppression: + suppressionFile: $(Build.SourcesDirectory)\.config\.gdnsuppress + suppressionSet: default + credscan: + enabled: true + suppressionsFileForArtifacts: $(Build.SourcesDirectory)/.config/credScanSuppressions.json + codeql: + # Enables CodeQL on the main branch + compiled: + enabled: true + # Enables CodeQL on PR branches + ${{ if startsWith(variables['Build.SourceBranch'], 'refs/pull/') }}: + enabledOnNonDefaultBranches: true + stages: + - template: /.pipelines/run-pipeline.yaml diff --git a/Makefile b/Makefile index b564ba6c6d..2ceb878651 100644 --- a/Makefile +++ b/Makefile @@ -337,11 +337,24 @@ acncli-image-name: # util target to print the CNI manager image name. acncli-image-name-and-tag: # util target to print the CNI manager image name and tag. @echo $(IMAGE_REGISTRY)/$(ACNCLI_IMAGE):$(ACNCLI_PLATFORM_TAG) -acncli-image: ## build cni-manager container image. +acncli-image-build-context: + $(eval BUILD_CONTEXT := $(REPO_ROOT)) + @echo "$(BUILD_CONTEXT)" + +acncli-dockerfile-path: + $(eval DOCKERFILE_PATH := $(ACNCLI_DIR)/Dockerfile) + @echo "$(DOCKERFILE_PATH)" + +acncli-docker-build-args: + $(eval DOCKER_BUILD_ARGS := '') + @echo $(DOCKER_BUILD_ARGS) + +acncli-image: acncli-dockerfile-path acncli-build-args ## build cni-manager container image. $(MAKE) container \ - DOCKERFILE=tools/acncli/Dockerfile \ + DOCKERFILE=$(DOCKERFILE_PATH) \ IMAGE=$(ACNCLI_IMAGE) \ - TAG=$(ACNCLI_PLATFORM_TAG) + TAG=$(ACNCLI_PLATFORM_TAG) \ + $(DOCKER_BUILD_ARGS) acncli-image-push: ## push cni-manager container image. $(MAKE) container-push \ @@ -359,18 +372,34 @@ acncli-image-pull: ## pull cni-manager container image. azure-ipam-image-name: # util target to print the azure-ipam image name. @echo $(AZURE_IPAM_IMAGE) +azure-ipam-image-tag: + @echo $(AZURE_IPAM_PLATFORM_TAG) + azure-ipam-image-name-and-tag: # util target to print the azure-ipam image name and tag. @echo $(IMAGE_REGISTRY)/$(AZURE_IPAM_IMAGE):$(AZURE_IPAM_PLATFORM_TAG) +azure-ipam-image-build-context: + $(eval BUILD_CONTEXT := $(REPO_ROOT)) + @echo "$(BUILD_CONTEXT)" + +azure-ipam-dockerfile-path: + $(eval DOCKERFILE_PATH := $(AZURE_IPAM_DIR)/Dockerfile) + @echo "$(DOCKERFILE_PATH)" + +azure-ipam-docker-build-args: + $(eval DOCKER_BUILD_ARGS := '') + @echo $(DOCKER_BUILD_ARGS) + azure-ipam-image: ## build azure-ipam container image. $(MAKE) container \ - DOCKERFILE=azure-ipam/Dockerfile \ + DOCKERFILE=$(DOCKERFILE_PATH) \ IMAGE=$(AZURE_IPAM_IMAGE) \ PLATFORM=$(PLATFORM) \ TAG=$(AZURE_IPAM_PLATFORM_TAG) \ TARGET=$(OS) \ OS=$(OS) \ - ARCH=$(ARCH) + ARCH=$(ARCH) \ + EXTRA_BUILD_ARGS=$(DOCKER_BUILD_ARGS) azure-ipam-image-push: ## push azure-ipam container image. $(MAKE) container-push \ @@ -387,14 +416,29 @@ azure-ipam-image-pull: ## pull azure-ipam container image. ipv6-hp-bpf-image-name: # util target to print the ipv6-hp-bpf image name. @echo $(IPV6_HP_BPF_IMAGE) +ipv6-hp-bpf-image-tag: + @echo $(IPV6_HP_BPF_IMAGE_PLATFORM_TAG) + ipv6-hp-bpf-image-name-and-tag: # util target to print the ipv6-hp-bpf image name and tag. @echo $(IMAGE_REGISTRY)/$(IPV6_HP_BPF_IMAGE):$(IPV6_HP_BPF_IMAGE_PLATFORM_TAG) +ipv6-hp-bpf-image-build-context: + $(eval BUILD_CONTEXT := $(REPO_ROOT)) + @echo "$(BUILD_CONTEXT)" + +ipv6-hp-bpf-dockerfile-path: + $(eval DOCKERFILE_PATH := $(IPV6_HP_BPF_DIR)/$(OS).Dockerfile) + @echo "$(DOCKERFILE_PATH)" + +ipv6-hp-bpf-docker-build-args: + $(eval DOCKER_BUILD_ARGS := '--build-arg DEBUG=$(DEBUG)') + @echo $(DOCKER_BUILD_ARGS) + ipv6-hp-bpf-image: ## build ipv6-hp-bpf container image. $(MAKE) container \ - DOCKERFILE=bpf-prog/ipv6-hp-bpf/$(OS).Dockerfile \ + DOCKERFILE=$(DOCKERFILE_PATH) \ IMAGE=$(IPV6_HP_BPF_IMAGE) \ - EXTRA_BUILD_ARGS='--build-arg OS=$(OS) --build-arg ARCH=$(ARCH) --build-arg DEBUG=$(DEBUG)'\ + EXTRA_BUILD_ARGS=$(DOCKER_BUILD_ARGS) \ PLATFORM=$(PLATFORM) \ TAG=$(IPV6_HP_BPF_IMAGE_PLATFORM_TAG) \ TARGET=$(OS) \ @@ -416,19 +460,34 @@ ipv6-hp-bpf-image-pull: ## pull ipv6-hp-bpf container image. cni-image-name: # util target to print the cni image name. @echo $(CNI_IMAGE) +cni-image-tag: + @echo $(CNI_PLATFORM_TAG) + cni-image-name-and-tag: # util target to print the cni image name and tag. @echo $(IMAGE_REGISTRY)/$(CNI_IMAGE):$(CNI_PLATFORM_TAG) +cni-image-build-context: + $(eval BUILD_CONTEXT := $(REPO_ROOT)) + @echo "$(BUILD_CONTEXT)" + +cni-dockerfile-path: + $(eval DOCKERFILE_PATH := $(REPO_ROOT)/cni/Dockerfile) + @echo "$(DOCKERFILE_PATH)" + +cni-docker-build-args: + $(eval DOCKER_BUILD_ARGS="--build-arg CNI_AI_PATH=$(CNI_AI_PATH) --build-arg CNI_AI_ID=$(CNI_AI_ID)") + @echo $(DOCKER_BUILD_ARGS) + cni-image: ## build cni container image. $(MAKE) container \ - DOCKERFILE=cni/Dockerfile \ + DOCKERFILE=$(DOCKERFILE_PATH) \ IMAGE=$(CNI_IMAGE) \ PLATFORM=$(PLATFORM) \ TAG=$(CNI_PLATFORM_TAG) \ TARGET=$(OS) \ OS=$(OS) \ ARCH=$(ARCH) \ - EXTRA_BUILD_ARGS='--build-arg CNI_AI_PATH=$(CNI_AI_PATH) --build-arg CNI_AI_ID=$(CNI_AI_ID)' + EXTRA_BUILD_ARGS=$(DOCKER_BUILD_ARGS) cni-image-push: ## push cni container image. $(MAKE) container-push \ @@ -446,14 +505,29 @@ cni-image-pull: ## pull cni container image. cns-image-name: # util target to print the CNS image name @echo $(CNS_IMAGE) +cns-image-tag: + @echo $(CNS_PLATFORM_TAG) + cns-image-name-and-tag: # util target to print the CNS image name and tag. @echo $(IMAGE_REGISTRY)/$(CNS_IMAGE):$(CNS_PLATFORM_TAG) +cns-image-build-context: + $(eval BUILD_CONTEXT := $(REPO_ROOT)) + @echo "$(BUILD_CONTEXT)" + +cns-dockerfile-path: + $(eval DOCKERFILE_PATH := $(REPO_ROOT)/cns/Dockerfile) + @echo "$(DOCKERFILE_PATH)" + +cns-docker-build-args: + $(eval DOCKER_BUILD_ARGS="--build-arg CNS_AI_PATH=$(CNS_AI_PATH) --build-arg CNS_AI_ID=$(CNS_AI_ID)") + @echo $(DOCKER_BUILD_ARGS) + cns-image: ## build cns container image. $(MAKE) container \ - DOCKERFILE=cns/Dockerfile \ + DOCKERFILE=$(DOCKERFILE_PATH) \ IMAGE=$(CNS_IMAGE) \ - EXTRA_BUILD_ARGS='--build-arg CNS_AI_PATH=$(CNS_AI_PATH) --build-arg CNS_AI_ID=$(CNS_AI_ID)' \ + EXTRA_BUILD_ARGS=$(DOCKER_BUILD_ARGS) \ PLATFORM=$(PLATFORM) \ TAG=$(CNS_PLATFORM_TAG) \ TARGET=$(OS) \ @@ -475,14 +549,29 @@ cns-image-pull: ## pull cns container image. npm-image-name: # util target to print the NPM image name @echo $(NPM_IMAGE) +npm-image-tag: # util target to print the NPM image name and tag. + @echo $(NPM_PLATFORM_TAG) + npm-image-name-and-tag: # util target to print the NPM image name and tag. @echo $(IMAGE_REGISTRY)/$(NPM_IMAGE):$(NPM_PLATFORM_TAG) +npm-image-build-context: + $(eval BUILD_CONTEXT := $(REPO_ROOT)) + @echo "$(BUILD_CONTEXT)" + +npm-dockerfile-path: + $(eval DOCKERFILE_PATH := $(REPO_ROOT)/npm/$(OS).Dockerfile) + @echo "$(DOCKERFILE_PATH)" + +npm-docker-build-args: + $(eval DOCKER_BUILD_ARGS="--build-arg NPM_AI_PATH=$(NPM_AI_PATH) --build-arg NPM_AI_ID=$(NPM_AI_ID)") + @echo $(DOCKER_BUILD_ARGS) + npm-image: ## build the npm container image. $(MAKE) container-$(CONTAINER_BUILDER) \ - DOCKERFILE=npm/$(OS).Dockerfile \ + DOCKERFILE=$(DOCKERFILE_PATH) \ IMAGE=$(NPM_IMAGE) \ - EXTRA_BUILD_ARGS='--build-arg NPM_AI_PATH=$(NPM_AI_PATH) --build-arg NPM_AI_ID=$(NPM_AI_ID)' \ + EXTRA_BUILD_ARGS=$(DOCKER_BUILD_ARGS) \ PLATFORM=$(PLATFORM) \ TAG=$(NPM_PLATFORM_TAG) \ TARGET=$(OS) \