diff --git a/gitlab_templates/gitlab_ci_templates/.gitlab-main-ci.yml b/gitlab_templates/gitlab_ci_templates/.gitlab-main-ci.yml new file mode 100644 index 0000000..c9e44b6 --- /dev/null +++ b/gitlab_templates/gitlab_ci_templates/.gitlab-main-ci.yml @@ -0,0 +1,425 @@ +default: + image: + name: ghcr.io/mojaloop/iac-aws-platform:$PLATFORM_IAC_IMAGE_VERSION + before_script: + - "which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )" + - export BUCKET=$(grep bucket backend.hcl | cut -f2 -d '=' | tr -d '"' | xargs) + - export TF_VAR_bucket=$(grep bucket backend.hcl | cut -f2 -d '=' | tr -d '"' | xargs) + - aws s3 sync s3://$BUCKET/bootstrap ./k8ss3bootstrap/ --sse || true + - export CI_IMAGE_PROJECT_DIR=/iac-run-dir + # Define the contents of the workbench-config file as environment variables for use in gitlab and optionally in terraform + - for var in $(jq -r 'to_entries[] | "\(.key)=\(.value)\n"' ./workbench-config.json); do export $var; done + - for var in $(jq -r 'to_entries[] | "TF_VAR_\(.key)=\(.value)\n"' ./workbench-config.json); do export $var; done + - export ENVIRONMENT=$environment + - test -n "$BUCKET" + - cd $CI_IMAGE_PROJECT_DIR + - aws s3 sync s3://${BUCKET}/${environment}/ . --sse || true + - eval `ssh-agent` + - mkdir -p ~/.ssh + - chmod 700 ~/.ssh + - test -f terraform/ssh_provisioner_key && chmod 0600 terraform/ssh_provisioner_key && ssh-add terraform/ssh_provisioner_key + - chmod +x ./searchandreplace.sh + - if [ -d ${CI_PROJECT_DIR}/iac-sync-dir ]; then cp -r ${CI_PROJECT_DIR}/iac-sync-dir/. .; else echo "no local files to copy"; fi + - cp $CI_PROJECT_DIR/workbench-config.json . + - ./searchandreplace.sh + - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' + - export ANSIBLE_FEATURE_TOGGLES="-e kube_version=v${k8s_api_version} -e dashboard_enabled=True -e kube_proxy_mode=iptables -e artifacts_dir=inventory/artifacts/cluster -e cloud_provider=aws -e ansible_user=ubuntu -b --become-user=root --flush-cache -e kubeconfig_localhost=true" + +variables: + GIT_SSL_NO_VERIFY: "overwritten" + PLATFORM_IAC_IMAGE_VERSION: "overwritten" + K3S_IAC_IMAGE_VERSION: "overwritten" + +stages: + - Validate + - Destroy + - "Deploy AWS infrastructure" + - "Run Kubespray" + - "Run k3s" + - "Deploy Base Platform" + - "Deploy Platform" + - "Run Tests" + - "Maintain Platform" + - "Deploy All" + +Validate and Plan: + stage: Validate + script: + - cd terraform + - terraform init -backend-config ${CI_PROJECT_DIR}/backend.hcl + - terraform validate + - terraform plan -out=plan.cache + - terraform show -json plan.cache | jq -r '([.resource_changes[]?.change.actions?]|flatten)|{"create":(map(select(.=="create"))|length),"update":(map(select(.=="update"))|length),"delete":(map(select(.=="delete"))|length)}' > ${CI_PROJECT_DIR}/plan.json + artifacts: + reports: + terraform: plan.json + when: manual + +Destroy AWS: + stage: Destroy + script: + - cd terraform + - aws s3 sync s3://${BUCKET}/${environment}/terraform/oauth-apps/ ./oauth-apps/ || true + - terraform init -backend-config ${CI_PROJECT_DIR}/backend.hcl + - terraform validate + - terraform destroy -auto-approve -refresh=false + when: manual + +Destroy Mojaloop K8s Stack: + stage: Destroy + script: + - cd terraform/apps/wso2/config + - terraform init -backend-config ${CI_PROJECT_DIR}/backend.hcl + - terraform validate + - terraform destroy -auto-approve -var="project_root_path=$CI_IMAGE_PROJECT_DIR" || true + - cd $CI_IMAGE_PROJECT_DIR/terraform/k8s-setup/mojaloop-roles + - terraform init -backend-config ${CI_PROJECT_DIR}/backend.hcl + - terraform validate + - terraform destroy -auto-approve -var="project_root_path=$CI_IMAGE_PROJECT_DIR" || true + - cd $CI_IMAGE_PROJECT_DIR/terraform/k8s-setup + - terraform init -backend-config ${CI_PROJECT_DIR}/backend.hcl + - terraform validate + - terraform destroy -auto-approve -var="project_root_path=$CI_IMAGE_PROJECT_DIR" || true + #- kubectl --kubeconfig=../../admin-gateway.conf delete namespace mojaloop || true + + when: manual + +Destroy Add-On Apps: + stage: Destroy + script: + - aws s3 sync s3://${BUCKET}/${environment}/k3saddons/pm4ml-certoutput/ terraform/k8s-setup/addons/pm4ml-certoutput + - aws s3 sync s3://${client}${environment}k3s-pm4mladdons-state/k3s ./k3sstate --sse || true + - cp $CI_PROJECT_DIR/ext-pm4ml-certs.yaml terraform/k8s-setup/addons || true + - for var in $(jq -r '.internal_pm4ml_configs | to_entries[] .value.DFSP_NAME' ./workbench-config.json); do touch terraform/k8s-setup/addons/pm4ml-certoutput/$var-client-key.pem terraform/k8s-setup/addons/pm4ml-certoutput/$var-ca-cert.pem terraform/k8s-setup/addons/pm4ml-certoutput/$var-client-cert.pem; done + - cd terraform/k8s-setup/addons + - terraform init -backend-config ${CI_PROJECT_DIR}/backend.hcl + - terraform validate + - terraform destroy -auto-approve -var="project_root_path=$CI_IMAGE_PROJECT_DIR" + - aws s3 rm --recursive s3://$BUCKET/$environment/k8s-cluster + - aws s3 rm s3://$BUCKET/$environment/k3saddons/onboarding_pm4ml_output.json + - aws s3 rm s3://$BUCKET/$environment/k3saddons/ansible_pm4ml_output.yaml + when: manual + +Destroy of Internal PM4MLs Setup: + stage: Destroy + script: + - cd terraform/k8s-setup/pm4mls + - terraform init -backend-config ${CI_PROJECT_DIR}/backend.hcl + - terraform validate + - terraform destroy -auto-approve -var="project_root_path=$CI_IMAGE_PROJECT_DIR" + - aws s3 rm s3://$BUCKET/$environment/k3saddons/ansible_internal_pm4ml_output.json + - aws s3 rm s3://$BUCKET/$environment/k3saddons/onboarding_internal_pm4ml_output.json + - aws s3 rm s3://$BUCKET/$environment/k3saddons/ansible_external_pm4ml_output.yaml + when: manual + +Destroy Vault: + stage: Destroy + script: + - cd terraform/k8s-setup/vault-deploy + - terraform init -backend-config ${CI_PROJECT_DIR}/backend.hcl + - terraform validate + - terraform destroy -auto-approve -var="aws_secret_key=$AWS_SECRET_ACCESS_KEY" -var="aws_access_key=$AWS_ACCESS_KEY_ID" -var="project_root_path=$CI_IMAGE_PROJECT_DIR" + when: manual + +Destroy Support Services: + stage: Destroy + script: + - cd terraform/k8s-setup/support-svcs + - aws s3 sync s3://${BUCKET}/${environment}/terraform/oauth-apps/ ./oauth-apps/ || true + - terraform init -backend-config ${CI_PROJECT_DIR}/backend.hcl + - terraform validate + - terraform destroy -auto-approve -var="project_root_path=$CI_IMAGE_PROJECT_DIR" + ##kill force kill efs provsioner + - kubectl --kubeconfig=../../../admin-gateway.conf delete pods --all -n wso2 --grace-period=0 --force + - kubectl --kubeconfig=../../../admin-gateway.conf delete namespace wso2 || true + ##kill force wso2-bizops + - kubectl --kubeconfig=../../../admin-gateway.conf delete pods --all -n wso2-bizops --grace-period=0 --force + - kubectl --kubeconfig=../../../admin-gateway.conf delete namespace wso2-bizops || true + when: manual + +Destroy Stateful Services: + stage: Destroy + script: + - cd terraform/k8s-setup/state-setup + - terraform init -backend-config ${CI_PROJECT_DIR}/backend.hcl + - terraform validate + - terraform destroy -auto-approve -var="project_root_path=$CI_IMAGE_PROJECT_DIR" || true + when: manual + +Destroy Environment: + stage: Destroy + script: + - aws s3 sync s3://${BUCKET}/${environment}/k3saddons/pm4ml-certoutput/ terraform/k8s-setup/addons/pm4ml-certoutput + - aws s3 sync s3://${client}${environment}k3s-pm4mladdons-state/k3s ./k3sstate --sse || true + - aws s3 sync s3://${BUCKET}/${environment}/terraform/oauth-apps/ ./oauth-apps/ || true + - for var in $(jq -r '.internal_pm4ml_configs | to_entries[] .value.DFSP_NAME' ./workbench-config.json); do touch terraform/k8s-setup/addons/pm4ml-certoutput/$var-client-key.pem terraform/k8s-setup/addons/pm4ml-certoutput/$var-ca-cert.pem terraform/k8s-setup/addons/pm4ml-certoutput/$var-client-cert.pem; done + - cp $CI_PROJECT_DIR/ext-pm4ml-certs.yaml terraform/k8s-setup/addons || true + - cp $CI_PROJECT_DIR/teardown.sh $CI_IMAGE_PROJECT_DIR/teardown.sh || true + - $CI_IMAGE_PROJECT_DIR/teardown.sh + when: manual + +Destroy k3s Cluster: + stage: "Destroy" + image: + name: ghcr.io/pm4ml/k3sbootstrap:$K3S_IAC_IMAGE_VERSION + before_script: + - export BUCKET=$(grep bucket backend.hcl | cut -f2 -d '=' | tr -d '"' | xargs) + - export TF_VAR_bucket=$(grep bucket backend.hcl | cut -f2 -d '=' | tr -d '"') + - aws s3 sync s3://$BUCKET/bootstrap ./k8ss3bootstrap/ --sse || true + - export CI_IMAGE_PROJECT_DIR=/k3s-boot + - export peer_vpc_id=$(terraform output -state=k8ss3bootstrap/terraform.tfstate vpc_id | tr -d '"') + - for var in $(jq -r 'to_entries[] | "\(.key)=\(.value)\n"' ./workbench-config.json); do export $var; done + - for var in $(jq -r 'to_entries[] | "TF_VAR_\(.key)=\(.value)\n"' ./workbench-config.json); do export $var; done + - sed -i "s/##accesskeyid##/${AWS_ACCESS_KEY_ID}/g" $CI_IMAGE_PROJECT_DIR/aws_conf + - sed -i "s|##accesssecret##|${AWS_SECRET_ACCESS_KEY}|g" $CI_IMAGE_PROJECT_DIR/aws_conf + - sed -i "s/##region##/${region}/g" $CI_IMAGE_PROJECT_DIR/aws_conf + - mkdir -p ~/.aws && cp $CI_IMAGE_PROJECT_DIR/aws_conf ~/.aws/credentials + - "which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )" + - eval `ssh-agent` + - mkdir -p ~/.ssh + - chmod 700 ~/.ssh + - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' + - if [ -d ./k3s-sync-dir ]; then cp -r ./k3s-sync-dir/. /k3s-boot; else echo "not local files to copy"; fi + - aws s3 sync s3://$BUCKET/$environment ./k8ss3/ --sse || true + - export domain=$(terraform output -state=k8ss3/terraform.tfstate public_subdomain | tr -d '"') + + script: + - export + - cd /k3s-boot + - sed -i "s/##client##/${client}/g" .env + - sed -i "s/##domain##/${domain}/g" .env + - sed -i "s/##environment##/${environment}/g" .env + - sed -i "s/##region##/${region}/g" .env + - make backend -- -auto-approve || true + - make destroy -- -auto-approve || true + - aws s3 rm --recursive s3://$BUCKET/$environment/k3saddons || true + - aws s3 rm --recursive s3://${client}${environment}k3s-pm4mladdons-state/k3s || true + - $CI_IMAGE_PROJECT_DIR/delete_int_k3s_db_row.sh ${client}${environment}k3s-pm4mladdons-lock ${client}${environment}k3s-pm4mladdons-state/k3s $region + when: manual + +Undeploy PM4ML Charts: + stage: "Destroy" + image: + name: ghcr.io/pm4ml/k3sbootstrap:$K3S_IAC_IMAGE_VERSION + before_script: + - export BUCKET=$(grep bucket backend.hcl | cut -f2 -d '=' | tr -d '"' | xargs) + - export TF_VAR_bucket=$(grep bucket backend.hcl | cut -f2 -d '=' | tr -d '"' | xargs) + - aws s3 sync s3://$BUCKET/bootstrap ./k8ss3bootstrap/ --sse || true + - export CI_IMAGE_PROJECT_DIR=/k3s-boot + - export peer_vpc_id=$(terraform output -state=k8ss3bootstrap/terraform.tfstate vpc_id | tr -d '"') + - for var in $(jq -r 'to_entries[] | "\(.key)=\(.value)\n"' ./workbench-config.json); do export $var; done + - for var in $(jq -r 'to_entries[] | "TF_VAR_\(.key)=\(.value)\n"' ./workbench-config.json); do export $var; done + - sed -i "s/##accesskeyid##/${AWS_ACCESS_KEY_ID}/g" $CI_IMAGE_PROJECT_DIR/aws_conf + - sed -i "s|##accesssecret##|${AWS_SECRET_ACCESS_KEY}|g" $CI_IMAGE_PROJECT_DIR/aws_conf + - sed -i "s/##region##/${region}/g" $CI_IMAGE_PROJECT_DIR/aws_conf + - mkdir -p ~/.aws && cp $CI_IMAGE_PROJECT_DIR/aws_conf ~/.aws/credentials + - "which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )" + - eval `ssh-agent` + - mkdir -p ~/.ssh + - chmod 700 ~/.ssh + - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' + - if [ -d ./k3s-sync-dir ]; then cp -r ./k3s-sync-dir/. /k3s-boot; else echo "not local files to copy"; fi + - aws s3 sync s3://$BUCKET/$environment ./k8ss3/ --sse || true + - export domain=$(terraform output -state=k8ss3/terraform.tfstate public_subdomain | tr -d '"') + + script: + - aws s3 cp s3://$BUCKET/$environment/k3saddons/ssh-key /k3s-boot/ssh-key --sse + - chmod 400 /k3s-boot/ssh-key + - cd /k3s-boot + - sed -i "s/##client##/${client}/g" .env + - sed -i "s/##domain##/${domain}/g" .env + - sed -i "s/##environment##/${environment}/g" .env + - sed -i "s/##region##/${region}/g" .env + - sed -i "s/##peer_vpc_id##/${peer_vpc_id}/g" .env + #TODO: static value needs addressing + - sed -i "s/##pm4ml_client_cert_local_dir##/\/k3s-boot\/pm4ml-certoutput\//g" .env + - aws s3 cp s3://$BUCKET/$environment/k3saddons/ansible_internal_pm4ml_output.yaml ./ansible_pm4ml_output.yaml --sse + - make backend -- -auto-approve || true + - make apply -- -auto-approve || true + - make uninstall-pm4ml + when: manual + +Deploy AWS Infrastructure: + stage: "Deploy AWS infrastructure" + script: + - cd terraform + - aws s3 sync s3://${BUCKET}/${environment}/terraform/oauth-apps/ ./oauth-apps/ || true + - terraform init -backend-config ${CI_PROJECT_DIR}/backend.hcl + - terraform apply -auto-approve || true + - aws s3 sync $CI_IMAGE_PROJECT_DIR/kubespray-inventory s3://${BUCKET}/${environment}/kubespray-inventory/ --sse || true + - aws s3 cp $CI_IMAGE_PROJECT_DIR/terraform/ssh_provisioner_key s3://${BUCKET}/${environment}/terraform/ --sse || true + - aws s3 sync ./oauth-apps s3://${BUCKET}/${environment}/terraform/oauth-apps/ --sse || true + when: manual + + +Create k3s Cluster: + stage: "Run k3s" + image: + name: ghcr.io/pm4ml/k3sbootstrap:$K3S_IAC_IMAGE_VERSION + before_script: + - export BUCKET=$(grep bucket backend.hcl | cut -f2 -d '=' | tr -d '"' | xargs) + - export TF_VAR_bucket=$(grep bucket backend.hcl | cut -f2 -d '=' | tr -d '"' | xargs) + - aws s3 sync s3://$BUCKET/bootstrap ./k8ss3bootstrap/ --sse || true + - export CI_IMAGE_PROJECT_DIR=/k3s-boot + - export peer_vpc_id=$(terraform output -state=k8ss3bootstrap/terraform.tfstate vpc_id | tr -d '"') + - for var in $(jq -r 'to_entries[] | "\(.key)=\(.value)\n"' ./workbench-config.json); do export $var; done + - for var in $(jq -r 'to_entries[] | "TF_VAR_\(.key)=\(.value)\n"' ./workbench-config.json); do export $var; done + - sed -i "s/##accesskeyid##/${AWS_ACCESS_KEY_ID}/g" $CI_IMAGE_PROJECT_DIR/aws_conf + - sed -i "s|##accesssecret##|${AWS_SECRET_ACCESS_KEY}|g" $CI_IMAGE_PROJECT_DIR/aws_conf + - sed -i "s/##region##/${region}/g" $CI_IMAGE_PROJECT_DIR/aws_conf + - mkdir -p ~/.aws && cp $CI_IMAGE_PROJECT_DIR/aws_conf ~/.aws/credentials + - "which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )" + - eval `ssh-agent` + - mkdir -p ~/.ssh + - chmod 700 ~/.ssh + - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' + - if [ -d ./k3s-sync-dir ]; then cp -r ./k3s-sync-dir/. /k3s-boot; else echo "not local files to copy"; fi + - aws s3 sync s3://$BUCKET/$environment ./k8ss3/ --sse || true + - export domain=$(terraform output -state=k8ss3/terraform.tfstate public_subdomain | tr -d '"') + + script: + - export + - cd /k3s-boot + - sed -i "s/##client##/${client}/g" .env + - sed -i "s/##domain##/${domain}/g" .env + - sed -i "s/##environment##/${environment}/g" .env + - sed -i "s/##region##/${region}/g" .env + - sed -i "s/##peer_vpc_id##/${peer_vpc_id}/g" .env + - make backend -- -auto-approve + - make apply -- -auto-approve + - make k3s + - make kubeconfig + - aws s3 cp /k3s-boot/kubeconfig s3://$BUCKET/$environment/k3saddons/ --sse + - aws s3 cp /k3s-boot/ssh-key s3://${BUCKET}/$environment/k3saddons/ --sse + - aws s3 sync s3://$BUCKET/$environment/k3saddons /k3s-boot/k3saddons --sse + - cp -r /k3s-boot/k3saddons/* /k3s-boot/ + - if [ ! -f /k3s-boot/vault-keys.json ]; then make vault; else echo "found vault, skipping make vault"; fi + - aws s3 cp /k3s-boot/vault-keys.json s3://$BUCKET/$environment/k3saddons/ --sse + - make wireguard + - aws s3 cp /k3s-boot/wireguard.private.key s3://$BUCKET/$environment/k3saddons/ --sse + - aws s3 cp /k3s-boot/wireguard.public.key s3://$BUCKET/$environment/k3saddons/ --sse + - make monitoring + when: manual + +Create Gateway Cluster: + stage: "Run Kubespray" + script: + - cd /kubespray + - ansible-playbook -i $CI_IMAGE_PROJECT_DIR/kubespray-inventory/hosts-gateway -e @$CI_IMAGE_PROJECT_DIR/kubespray-inventory/extra-vars.json ./cluster.yml $ANSIBLE_FEATURE_TOGGLES + - aws s3 cp inventory/artifacts/cluster/admin.conf s3://$BUCKET/$environment/admin-gateway.conf --sse + when: manual + +1. Deploy Vault: + stage: "Deploy Base Platform" + script: + - cd terraform/k8s-setup/vault-deploy + - terraform init -backend-config ${CI_PROJECT_DIR}/backend.hcl + - terraform apply -auto-approve -var="aws_secret_key=$AWS_SECRET_ACCESS_KEY" -var="aws_access_key=$AWS_ACCESS_KEY_ID" -var="project_root_path=$CI_IMAGE_PROJECT_DIR" + - aws s3 cp $CI_IMAGE_PROJECT_DIR/vault_seal_key s3://${BUCKET}/${environment}/ --sse + when: manual + +2a. Deploy Stateful Services: + stage: "Deploy Base Platform" + script: + - cd terraform/k8s-setup/state-setup + - kubectl --kubeconfig=../../../admin-gateway.conf -n default wait --for=condition=ready certificate/wildcard-cert-internal --timeout=180s + - terraform init -backend-config ${CI_PROJECT_DIR}/backend.hcl + - terraform apply -auto-approve -var="project_root_path=$CI_IMAGE_PROJECT_DIR" + when: manual + +2b. Deploy Support Services: + stage: "Deploy Base Platform" + script: + - cd terraform/k8s-setup/support-svcs + - aws s3 sync s3://${BUCKET}/${environment}/terraform/oauth-apps/ ./oauth-apps/ || true + - kubectl --kubeconfig=../../../admin-gateway.conf -n default wait --for=condition=ready certificate/wildcard-cert-internal --timeout=180s + - terraform init -backend-config ${CI_PROJECT_DIR}/backend.hcl + - terraform apply -auto-approve -var="project_root_path=$CI_IMAGE_PROJECT_DIR" + - aws s3 sync ./oauth-apps s3://${BUCKET}/${environment}/terraform/oauth-apps/ --sse + when: manual + +1. Initial Install Base Apps: + stage: "Deploy Platform" + script: + - cd terraform/k8s-setup + - terraform init -backend-config ${CI_PROJECT_DIR}/backend.hcl + - terraform apply -auto-approve -var="project_root_path=$CI_IMAGE_PROJECT_DIR" + - cd $CI_IMAGE_PROJECT_DIR/terraform/k8s-setup/mojaloop-roles + - terraform init -backend-config ${CI_PROJECT_DIR}/backend.hcl + - terraform apply -auto-approve -var="project_root_path=$CI_IMAGE_PROJECT_DIR" + - chmod o-w $CI_IMAGE_PROJECT_DIR/terraform/apps/wso2/config/ + - cd $CI_IMAGE_PROJECT_DIR/terraform/apps/wso2/config/ + - terraform init -backend-config ${CI_PROJECT_DIR}/backend.hcl + - terraform apply -auto-approve -var="project_root_path=$CI_IMAGE_PROJECT_DIR" + - ansible-playbook publish.yaml + when: manual + +2a. Setup PM4MLs: + stage: "Deploy Platform" + script: + - cd terraform/k8s-setup/pm4mls + - terraform init -backend-config ${CI_PROJECT_DIR}/backend.hcl + - terraform apply -auto-approve -var="project_root_path=$CI_IMAGE_PROJECT_DIR" + - aws s3 cp $CI_IMAGE_PROJECT_DIR/terraform/k8s-setup/pm4mls/ansible_internal_pm4ml_output.yaml s3://${BUCKET}/${environment}/k3saddons/ --sse + - aws s3 cp $CI_IMAGE_PROJECT_DIR/terraform/k8s-setup/pm4mls/onboarding_internal_pm4ml_output.json s3://${BUCKET}/${environment}/k3saddons/ --sse + - aws s3 cp $CI_IMAGE_PROJECT_DIR/terraform/k8s-setup/pm4mls/ansible_external_pm4ml_output.yaml s3://${BUCKET}/${environment}/k3saddons/ --sse + - aws s3 cp $CI_IMAGE_PROJECT_DIR/terraform/k8s-setup/pm4mls/onboarding_external_pm4ml_output.json s3://${BUCKET}/${environment}/k3saddons/ --sse + #- aws s3 sync $CI_IMAGE_PROJECT_DIR/terraform/k8s-setup/pm4mls/secrets_chart/ s3://${BUCKET}/${environment}/k8s-cluster/secrets_chart/ --sse + when: manual + +2b. Install Internal PM4MLs: + stage: "Deploy Platform" + image: + name: ghcr.io/pm4ml/k3sbootstrap:$K3S_IAC_IMAGE_VERSION + before_script: + - export BUCKET=$(grep bucket backend.hcl | cut -f2 -d '=' | tr -d '"' | xargs) + - export TF_VAR_bucket=$(grep bucket backend.hcl | cut -f2 -d '=' | tr -d '"' | xargs) + - aws s3 sync s3://$BUCKET/bootstrap ./k8ss3bootstrap/ --sse || true + - export CI_IMAGE_PROJECT_DIR=/k3s-boot + - export peer_vpc_id=$(terraform output -state=k8ss3bootstrap/terraform.tfstate vpc_id | tr -d '"') + - for var in $(jq -r 'to_entries[] | "\(.key)=\(.value)\n"' ./workbench-config.json); do export $var; done + - for var in $(jq -r 'to_entries[] | "TF_VAR_\(.key)=\(.value)\n"' ./workbench-config.json); do export $var; done + - sed -i "s/##accesskeyid##/${AWS_ACCESS_KEY_ID}/g" $CI_IMAGE_PROJECT_DIR/aws_conf + - sed -i "s|##accesssecret##|${AWS_SECRET_ACCESS_KEY}|g" $CI_IMAGE_PROJECT_DIR/aws_conf + - sed -i "s/##region##/${region}/g" $CI_IMAGE_PROJECT_DIR/aws_conf + - mkdir -p ~/.aws && cp $CI_IMAGE_PROJECT_DIR/aws_conf ~/.aws/credentials + - "which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )" + - eval `ssh-agent` + - mkdir -p ~/.ssh + - chmod 700 ~/.ssh + - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' + - if [ -d ./k3s-sync-dir ]; then cp -r ./k3s-sync-dir/. /k3s-boot; else echo "not local files to copy"; fi + - aws s3 sync s3://$BUCKET/$environment ./k8ss3/ --sse || true + - export domain=$(terraform output -state=k8ss3/terraform.tfstate public_subdomain | tr -d '"') + script: + - aws s3 cp s3://$BUCKET/$environment/k3saddons/ssh-key /k3s-boot/ssh-key --sse + - chmod 400 /k3s-boot/ssh-key + - cd /k3s-boot + - sed -i "s/##client##/${client}/g" .env + - sed -i "s/##domain##/${domain}/g" .env + - sed -i "s/##environment##/${environment}/g" .env + - sed -i "s/##region##/${region}/g" .env + - sed -i "s/##peer_vpc_id##/${peer_vpc_id}/g" .env + #TODO: static value needs addressing + - sed -i "s/##pm4ml_client_cert_local_dir##/\/k3s-boot\/pm4ml-certoutput\//g" .env + - aws s3 cp s3://$BUCKET/$environment/k3saddons/ansible_internal_pm4ml_output.yaml ./ansible_pm4ml_output.yaml --sse + - make backend -- -auto-approve + - make apply -- -auto-approve + - make pm4ml + when: manual + +2c. Install Post Deployment: + stage: "Deploy Platform" + script: + #todo remove hardcoded values here + - aws s3 sync s3://${BUCKET}/${environment}/k3saddons/pm4ml-certoutput/ terraform/k8s-setup/addons/pm4ml-certoutput + - aws s3 sync s3://${client}${environment}k3s-pm4mladdons-state/k3s ./k3sstate --sse || true + - if [ -f k3sstate/terraform.tfstate ]; then export pm4ml_nat_ips_tmp=$(terraform output -state=k3sstate/terraform.tfstate nat_public_ips); else ls -la ./k3sstate; fi + - echo $pm4ml_nat_ips_tmp + - if [ $(echo $pm4ml_nat_ips_tmp | grep --quiet Warning) ]; then echo "nats not found"; else export TF_VAR_pm4ml_nat_ips=$pm4ml_nat_ips_tmp; fi + - cp $CI_PROJECT_DIR/ext-pm4ml-certs.yaml terraform/k8s-setup/addons || true + - cd terraform/k8s-setup/addons + - terraform init -backend-config ${CI_PROJECT_DIR}/backend.hcl + - terraform apply -auto-approve -var="project_root_path=$CI_IMAGE_PROJECT_DIR" + - aws s3 sync $CI_IMAGE_PROJECT_DIR/terraform/k8s-setup/addons/sim_tests s3://${BUCKET}/${environment}/k8s-cluster/sim_tests --sse + when: manual \ No newline at end of file diff --git a/gitlab_templates/gitlab_ci_templates/.gitlab-ops-ci.yml b/gitlab_templates/gitlab_ci_templates/.gitlab-ops-ci.yml new file mode 100644 index 0000000..c9dd0c7 --- /dev/null +++ b/gitlab_templates/gitlab_ci_templates/.gitlab-ops-ci.yml @@ -0,0 +1,287 @@ +variables: + GIT_SSL_NO_VERIFY: "overwritten" + PLATFORM_IAC_IMAGE_VERSION: "overwritten" + K3S_IAC_IMAGE_VERSION: "overwritten" + +#job template for 1. Run PM4ML GP Tests +.Run PM4ML GP Tests-job-template: + script: + - aws s3 sync s3://${BUCKET}/${environment}/k8s-cluster/sim_tests/ $CI_IMAGE_PROJECT_DIR/sim_tests --sse + #- aws s3 sync s3://${BUCKET}/${environment}/k8s-cluster/secrets_chart/ $CI_IMAGE_PROJECT_DIR/secrets_chart --sse + - aws s3 cp s3://${BUCKET}/${environment}/k3saddons/onboarding_internal_pm4ml_output.json $CI_IMAGE_PROJECT_DIR/sim_tests + - cd sim_tests + - newman run https://raw.githubusercontent.com/pm4ml/pm4ml-test-scripts/${TAG_PM4ML_GP}/postman/Deregister_Existing_Oracles.postman_collection.json --insecure -e Lab.postman_environment.json + - newman run https://raw.githubusercontent.com/mojaloop/postman/${TAG_ML_GP}/MojaloopHub_Setup.postman_collection.json --insecure -e Lab.postman_environment.json + #- newman run https://raw.githubusercontent.com/mojaloop/postman/${TAG_ML_GP}/MojaloopSims_Onboarding.postman_collection.json --ssl-client-cert-list test_cert_list.json --insecure -e Lab.postman_environment.json || true + - newman run https://raw.githubusercontent.com/pm4ml/pm4ml-test-scripts/${TAG_PM4ML_GP}/postman/Onboard-Generic-FSP-Central_Ledger.postman_collection.json --insecure -e Lab.postman_environment.json --ssl-client-cert-list test_cert_list.json -d onboarding_internal_pm4ml_output.json || true + - newman run https://raw.githubusercontent.com/pm4ml/pm4ml-test-scripts/${TAG_PM4ML_GP}/postman/Onboard-Generic-FSP-Sim_Backend.postman_collection.json --insecure -e Lab.postman_environment.json --ssl-client-cert-list test_cert_list.json -d onboarding_internal_pm4ml_output.json || true + - newman run https://raw.githubusercontent.com/pm4ml/pm4ml-test-scripts/${TAG_PM4ML_GP}/postman/Mojaloop-Payment-Manager-Golden-Path.postman_collection.json --insecure -e Lab.postman_environment.json --ssl-client-cert-list test_cert_list.json --folder feature-tests --reporters html,junit,cli --reporter-html-export ${CI_PROJECT_DIR}/gp_report.html --reporter-junit-export ${CI_PROJECT_DIR}/gp_report.xml --reporter-html-template ../tests/template-default-colored.hbs -x + after_script: + - curl https://slack.com/api/files.upload + -F file=@"gp_report.html" + -F channels="${SLACK_GITLABCI_ALERTS_CHANNEL_NAME}","${SLACK_GITLABCI_GP_REPORTS_CHANNEL_NAME}" + -F token="${SLACK_GITLABCI_ALERTS_TOKEN}" + -F title="Mojaloop PM4ML Goldenpath Test (Postman) Results.'${ENVIRONMENT}'-$(date +\"%Y-%m-%d-%H:%M:%S\")" + -F filetype="html" + artifacts: + paths: + - gp_report.html + reports: + junit: gp_report.xml + +1. Manual Run PM4ML GP Tests: + stage: "Run Tests" + extends: .Run PM4ML GP Tests-job-template + when: manual + except: + - schedules + +1. Scheduled Run PM4ML GP Tests: + stage: "Run Tests" + allow_failure: true + extends: .Run PM4ML GP Tests-job-template + only: + refs: + - schedules + variables: + - $SCHEDULE_PM4ML == "PM4ML_GP" + +2. Run Finance Portal V2 UI Tests: + stage: "Run Tests" + allow_failure: true + when: manual + script: +# - if [ -f terraform-k8s.tfstate ]; then export fin_portal_users=$(terraform output -json -state=terraform-k8s.tfstate finance_portal_users); else ls -la .; fi +# - if [ -f terraform-k8s.tfstate ]; then export fin_portal_url=$(terraform output -state=terraform-k8s.tfstate finance-portal-url); else ls -la .; fi +# - aws s3 sync s3://${BUCKET}/${environment}/k8s-cluster/sim_tests/ ${CI_IMAGE_PROJECT_DIR}/sim_tests --sse +# - aws s3 sync s3://${BUCKET}/${environment}/k8s-cluster/secrets_chart/ ${CI_IMAGE_PROJECT_DIR}/secrets_chart --sse +# - aws s3 cp s3://${BUCKET}/${environment}/k3saddons/onboarding_internal_pm4ml_output.json ${CI_IMAGE_PROJECT_DIR}/sim_tests +# - cd ${CI_IMAGE_PROJECT_DIR}/sim_tests +# - newman run https://raw.githubusercontent.com/pm4ml/pm4ml-test-scripts/${TAG_PM4ML_GP}/postman/Deregister_Existing_Oracles.postman_collection.json --insecure -e Lab.postman_environment.json +# - newman run https://raw.githubusercontent.com/mojaloop/postman/${TAG_ML_GP}/MojaloopHub_Setup.postman_collection.json --insecure -e Lab.postman_environment.json +# - newman run https://raw.githubusercontent.com/mojaloop/postman/${TAG_ML_GP}/MojaloopSims_Onboarding.postman_collection.json --ssl-client-cert-list test_cert_list.json --insecure -e Lab.postman_environment.json || true + - git clone -b MBP-137 https://github.com/pm4ml/mojaloop-payment-manager-ui.git ${CI_IMAGE_PROJECT_DIR}/mojaloop-payment-manager-ui + - exit_code=0 + - cd ${CI_IMAGE_PROJECT_DIR}/mojaloop-payment-manager-ui/ui-tests/tests + - echo ${SLACK_TOKEN} + - npm install testcafe-reporter-html + - echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list + - wget -q -O - https://dl.google.com/linux/linux_signing_key.pub | apt-key add - + - apt-get update + - apt-get install -y google-chrome-stable + - npm ci + - ENV="other" PM4ML_ENDPOINT=${PM4ML_ENDPOINT} SIM_CORE_CONNECTOR_ENDPOINT=${SIM_CORE_CONNECTOR_ENDPOINT} npm run test:headless | tee report.log || exit_code=$? +# - echo $fin_portal_users | jq -r '"ADMIN_USER_NAME=\(.portaladmin.username)\nADMIN_PASSWORD=\(.portaladmin.user_pass)\nUSER_NAME=\(.portaluser.username)\nPASSWORD=\(.portaluser.user_pass)"' > .env +# - echo "FINANCE_PORTAL_ENDPOINT=$fin_portal_url" >> .env +# - npm i +# - npm run test:headless || true + - cp ${CI_IMAGE_PROJECT_DIR}/mojaloop-payment-manager-ui/ui-tests/tests/report.log ${CI_PROJECT_DIR}/mojaloop-payment-manager-ui_report.log + after_script: + - curl https://slack.com/api/files.upload + -F file=@"mojaloop-payment-manager-ui_report.log" + -F channels="${SLACK_GITLABCI_ALERTS_CHANNEL_NAME}" + -F token="${SLACK_TOKEN}" + -F title="Run Mojaloop Payment Manager UI Tests" + -F filetype="log" + artifacts: + paths: + - mojaloop-payment-manager-ui_report.log + - exit $exit_code + +#job template for 4. Run Platform GP Tests: +.Run Platform GP Tests-job-template: + script: + - aws s3 sync s3://${BUCKET}/${TF_VAR_environment}/k8s-cluster/sim_tests/ $CI_IMAGE_PROJECT_DIR/sim_tests --sse + - aws s3 sync s3://${BUCKET}/${TF_VAR_environment}/k8s-cluster/secrets_chart/ $CI_IMAGE_PROJECT_DIR/secrets_chart --sse + - aws s3 cp s3://${BUCKET}/${TF_VAR_environment}/k3saddons/onboarding_internal_pm4ml_output.json $CI_IMAGE_PROJECT_DIR/sim_tests + - aws s3 cp s3://${BUCKET}/${TF_VAR_environment}/k3saddons/onboarding_external_pm4ml_output.json $CI_IMAGE_PROJECT_DIR/sim_tests + - cp $CI_PROJECT_DIR/tests/Myanmar-Golden-Path.postman_collection.json $CI_IMAGE_PROJECT_DIR/sim_tests + - cd sim_tests + - jq -s '[.[][]]' onboarding_external_pm4ml_output.json onboarding_internal_pm4ml_output.json > /tmp/onboarding_pm4ml_output.json + - newman run https://raw.githubusercontent.com/pm4ml/pm4ml-test-scripts/${TAG_PM4ML_GP}/postman/Deregister_Existing_Oracles.postman_collection.json --insecure -e Lab.postman_environment.json + - newman run https://raw.githubusercontent.com/pm4ml/pm4ml-test-scripts/${TAG_PM4ML_GP}/postman/Onboard-Generic-FSP-Central_Ledger.postman_collection.json --insecure -e Lab.postman_environment.json --ssl-client-cert-list test_cert_list.json -d /tmp/onboarding_pm4ml_output.json || true + - newman run https://raw.githubusercontent.com/pm4ml/pm4ml-test-scripts/${TAG_PM4ML_GP}/postman/Account_Alias_Oracle_Registration.postman_collection.json --insecure -e Lab.postman_environment.json --ssl-client-cert-list test_cert_list.json || true + - newman run https://raw.githubusercontent.com/pm4ml/pm4ml-test-scripts/${TAG_PM4ML_GP}/postman/Account_Alias_Oracle_Setup.postman_collection.json --insecure -e Lab.postman_environment.json --ssl-client-cert-list test_cert_list.json -d /tmp/onboarding_pm4ml_output.json || true + - newman run https://raw.githubusercontent.com/pm4ml/pm4ml-test-scripts/${TAG_PM4ML_GP}/postman/Onboard-Generic-FSP-Sim_Backend.postman_collection.json --insecure -e Lab.postman_environment.json --ssl-client-cert-list test_cert_list.json -d onboarding_internal_pm4ml_output.json || true + - newman run Myanmar-Golden-Path.postman_collection.json --ssl-client-cert-list test_cert_list.json --folder feature-tests --insecure -e Lab.postman_environment.json --reporters html,junit,cli --reporter-html-export ${CI_PROJECT_DIR}/Platform_report.html --reporter-junit-export ${CI_PROJECT_DIR}/alias_report.xml --reporter-html-template ../tests/template-default-colored.hbs -x + after_script: + - curl https://slack.com/api/files.upload + -F file=@"Platform_report.html" + -F channels="${SLACK_GITLABCI_ALERTS_CHANNEL_NAME}" + -F token="${SLACK_GITLABCI_ALERTS_TOKEN}" + -F title="Myanmar Goldenpath Test (Postman) Results" + -F filetype="html" + artifacts: + paths: + - Platform_report.html + +# - curl https://slack.com/api/files.upload +# -F file=@"alias_report.html" +# -F channels="${SLACK_GITLABCI_GP_REPORTS_CHANNEL_NAME}" +# -F token="${SLACK_GITLABCI_ALERTS_TOKEN}" +# -F title="Alias Goldenpath Test (Postman) Results.'${ENVIRONMENT}'-$(date +\"%Y-%m-%d-%H:%M:%S\")" +# -F filetype="html" +# artifacts: +# reports: +# junit: alias_report.xml + + +3. Manual Run Platform GP Tests: + stage: "Run Tests" + extends: .Run Platform GP Tests-job-template + when: manual + except: + - schedules + +3. Scheduled Run Platform GP Tests: + stage: "Run Tests" + allow_failure: true + extends: .Run Platform GP Tests-job-template + only: + refs: + - schedules + variables: + - $SCHEDULE_PLATFORM == "PLAT_GP" + +.ttk-snippets: + image: &ttk-snippets-image + name: mojaloop/ml-testing-toolkit:v13.5.1 + set-env-vars: &ttk-snippets-set-env-vars + - export TERRAFORM_VERSION=1.0.5 + - export GLIBC_VER=2.34-r0 + - apk --no-cache add binutils curl jq && curl -sL https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub -o /etc/apk/keys/sgerrand.rsa.pub && curl -sLO https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC_VER}/glibc-${GLIBC_VER}.apk && curl -sLO https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC_VER}/glibc-bin-${GLIBC_VER}.apk && apk add --no-cache glibc-${GLIBC_VER}.apk glibc-bin-${GLIBC_VER}.apk && curl -sL https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -o awscliv2.zip && unzip awscliv2.zip && aws/install && rm -rf awscliv2.zip aws /usr/local/aws-cli/v2/*/dist/aws_completer /usr/local/aws-cli/v2/*/dist/awscli/data/ac.index /usr/local/aws-cli/v2/*/dist/awscli/examples glibc-${GLIBC_VER}.apk glibc-bin-${GLIBC_VER}.apk /var/cache/apk/* + - wget -q https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip -O /tmp/terraform_${TERRAFORM_VERSION}_linux_amd64.zip && unzip /tmp/terraform_${TERRAFORM_VERSION}_linux_amd64.zip -d /usr/bin/ && rm /tmp/terraform_${TERRAFORM_VERSION}_linux_amd64.zip + - export BUCKET=$(grep bucket backend.hcl | cut -f2 -d '=' | tr -d '"' | xargs) + - for var in $(jq -r 'to_entries[] | "\(.key)=\(.value)\n"' ./workbench-config.json); do export $var; done + - for var in $(jq -r 'to_entries[] | "TF_VAR_\(.key)=\(.value)\n"' ./workbench-config.json); do export $var; done + - export TTK_TESTCASES_VERSION=${helm_mojaloop_version} + - export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + - export AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + - export AWS_DEFAULT_REGION=${region} + - aws s3 sync s3://$BUCKET/$environment ./k8ss3/ --sse || true + - export private_subdomain=$(terraform output -state=k8ss3/terraform.tfstate private_subdomain | tr -d '"') + - export ttkBackendUrl=http://ttkbackend.${private_subdomain}; + download-ttk-test-cases: &ttk-snippets-download-ttk-test-cases + - | + cd /opt/mojaloop-testing-toolkit; + export testCasesZipUrl=https://github.com/mojaloop/testing-toolkit-test-cases/archive/refs/tags/v${TTK_TESTCASES_VERSION}.zip; + echo "Downloading the test collection from $testCasesZipUrl"; + wget $testCasesZipUrl -O downloaded-test-collections.zip; + mkdir tmp_test_cases; + unzip -d tmp_test_cases -o downloaded-test-collections.zip; + rm downloaded-test-collections.zip + download-default-env: &ttk-snippets-download-default-env + - | + echo "Downloading the default environment file..."; + cd /opt/mojaloop-testing-toolkit; + wget -O user_config.json $ttkBackendUrl/api/config/user; + defaultEnvFileName=`node -pe 'JSON.parse(process.argv[1]).runtime.DEFAULT_ENVIRONMENT_FILE_NAME' "$(cat user_config.json)"`; + wget -O environment_http_response.json $ttkBackendUrl/api/samples/loadFolderWise?environment=examples/environments/$defaultEnvFileName; + node -pe 'JSON.stringify({inputValues: JSON.parse(process.argv[1]).body.environment})' "$(cat environment_http_response.json)" > cli-testcase-environment.json; + execute-provisioning-collection: &ttk-snippets-execute-provisioning-collection + - | + cd /opt/mojaloop-testing-toolkit; + export provisioningTestCasesPathInZip=testing-toolkit-test-cases-${TTK_TESTCASES_VERSION}/collections/hub/provisioning; + npm run cli -- -e cli-testcase-environment.json -i tmp_test_cases/$provisioningTestCasesPathInZip -u $ttkBackendUrl --report-format html --report-target file://${CI_PROJECT_DIR}/ttk-provisioning-report.html || true; + echo "Done"; + execute-golden-path-collection: &ttk-snippets-execute-golden-path-collection + - | + cd /opt/mojaloop-testing-toolkit; + export goldenPathTestCasesPathInZip=testing-toolkit-test-cases-${TTK_TESTCASES_VERSION}/collections/hub/golden_path; + npm run cli -- -e cli-testcase-environment.json -i tmp_test_cases/$goldenPathTestCasesPathInZip -u $ttkBackendUrl --report-format html --report-target file://${CI_PROJECT_DIR}/ttk-goldenpath-report.html; + echo "Done"; + execute-bulk-transfers-collection: &ttk-snippets-execute-bulk-transfers-collection + - | + cd /opt/mojaloop-testing-toolkit; + export bulkTransfersTestCasesPathInZip=testing-toolkit-test-cases-${TTK_TESTCASES_VERSION}/collections/hub/other_tests/bulk_transfers; + npm run cli -- -e cli-testcase-environment.json -i tmp_test_cases/$bulkTransfersTestCasesPathInZip -u $ttkBackendUrl --report-format html --report-target file://${CI_PROJECT_DIR}/ttk-bulk-transfers-report.html; + echo "Done"; + artifacts: &ttk-snippets-artifacts + when: always + paths: + - ${CI_PROJECT_DIR}/ttk-provisioning-report.html + - ${CI_PROJECT_DIR}/ttk-goldenpath-report.html + +4. Run TTK Tests: + stage: "Run Tests" + image: *ttk-snippets-image + before_script: + - echo "running with testing toolkit image" + script: + - *ttk-snippets-set-env-vars + - *ttk-snippets-download-ttk-test-cases + - *ttk-snippets-download-default-env + - *ttk-snippets-execute-provisioning-collection + - *ttk-snippets-execute-golden-path-collection + artifacts: *ttk-snippets-artifacts + when: manual + +5. Run RBAC Tests: + stage: "Run Tests" + script: + - helm --kubeconfig=admin-gateway.conf test bof --filter name=bof-rbac-tests -n mojaloop || true + - kubectl --kubeconfig=admin-gateway.conf -n mojaloop logs bof-rbac-tests + - exit `kubectl --kubeconfig=admin-gateway.conf -n mojaloop get pod bof-rbac-tests --output="jsonpath={.status.containerStatuses[].state.terminated.exitCode}"` + when: manual + +6. Run Report Tests: + stage: "Run Tests" + script: + - helm --kubeconfig=admin-gateway.conf test bof --filter name=bof-report-tests -n mojaloop || true + - kubectl --kubeconfig=admin-gateway.conf -n mojaloop logs bof-report-tests + - exit `kubectl --kubeconfig=admin-gateway.conf -n mojaloop get pod bof-report-tests --output="jsonpath={.status.containerStatuses[].state.terminated.exitCode}"` + when: manual + +7. Run MCM Tests: + stage: "Run Tests" + script: + - helm --kubeconfig=admin-gateway.conf test connection-manager -n mcm || true + - kubectl --kubeconfig=admin-gateway.conf -n mcm logs connection-manager-api-test + - exit `kubectl --kubeconfig=admin-gateway.conf -n mcm get pod connection-manager-api-test --output="jsonpath={.status.containerStatuses[].state.terminated.exitCode}"` + when: manual + +8. Run PM4ML TTK Tests: + stage: "Run Tests" + image: *ttk-snippets-image + before_script: + - echo "running with testing toolkit image" + script: + - *ttk-snippets-set-env-vars + - | + echo "Downloading the test collection..."; + cd /opt/mojaloop-testing-toolkit; + export testCasesZipUrl=https://github.com/pm4ml/pm4ml-test-scripts/archive/refs/tags/v${TTK_TESTCASES_VERSION}.zip; + wget $testCasesZipUrl -O downloaded-test-collections.zip; + mkdir tmp_test_cases; + unzip -d tmp_test_cases -o downloaded-test-collections.zip; + - | + cd /opt/mojaloop-testing-toolkit + export pm4mlgoldenPathTestCasesPathInZip=pm4ml-test-scripts-${ttk_pm4ml_gp_version}/ttk/collections/feature_tests; + # npm run cli -- -e $CI_PROJECT_DIR/tests/ttk-prod-dev-env.json -i tmp_test_cases/$pm4mlgoldenPathTestCasesPathInZip -u $ttkBackendUrl --report-format html --report-target s3://${BUCKET}/${environment}/pm4ml-ttk-goldenpath-report.html --slack-webhook-url=https://hooks.slack.com/services/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 --extra-summary-information="Test Suite:PM4ML GP TTK,Job ID: $CI_JOB_ID"; + npm run cli -- -e $CI_PROJECT_DIR/tests/ttk-prod-dev-env.json -i tmp_test_cases/$pm4mlgoldenPathTestCasesPathInZip -u $ttkBackendUrl --report-format html --report-target file://${CI_PROJECT_DIR}/ttk-pm4ml-report.html --extra-summary-information="Test Suite:PM4ML GP TTK,Job ID: $CI_JOB_ID"; + # Use the following syntax instead of the above line for slack notification and comment out artifacts. We can not have both with the current TTK CLI options. + # npm run cli -- -e $CI_PROJECT_DIR/tests/ttk-prod-dev-env.json -i tmp_test_cases/$pm4mlgoldenPathTestCasesPathInZip -u $ttkBackendUrl --report-format html --report-target s3://${BUCKET}/${environment}/pm4ml-ttk-goldenpath-report.html --slack-webhook-url=${SLACK_WEBHOOK_URL_HERE} --extra-summary-information="Test Suite:PM4ML GP TTK,Job ID: $CI_JOB_ID"; + echo "Done"; + artifacts: &ttk-snippets-artifacts + when: always + paths: + - ${CI_PROJECT_DIR}/ttk-pm4ml-report.html +# artifacts: +# when: always +# paths: +# - ${CI_PROJECT_DIR}/pm4ml-ttk-goldenpath-report.html + when: manual + +9. Run TTK Bulk Tests: + stage: "Run Tests" + image: *ttk-snippets-image + before_script: + - echo "running with testing toolkit image" + script: + - *ttk-snippets-set-env-vars + - *ttk-snippets-download-ttk-test-cases + - *ttk-snippets-download-default-env + - *ttk-snippets-execute-provisioning-collection + - *ttk-snippets-execute-bulk-transfers-collection + artifacts: *ttk-snippets-artifacts + when: manual \ No newline at end of file diff --git a/gitlab_templates/gitlab_ci_templates/.gitlab-tests-ci.yml b/gitlab_templates/gitlab_ci_templates/.gitlab-tests-ci.yml new file mode 100644 index 0000000..c9dd0c7 --- /dev/null +++ b/gitlab_templates/gitlab_ci_templates/.gitlab-tests-ci.yml @@ -0,0 +1,287 @@ +variables: + GIT_SSL_NO_VERIFY: "overwritten" + PLATFORM_IAC_IMAGE_VERSION: "overwritten" + K3S_IAC_IMAGE_VERSION: "overwritten" + +#job template for 1. Run PM4ML GP Tests +.Run PM4ML GP Tests-job-template: + script: + - aws s3 sync s3://${BUCKET}/${environment}/k8s-cluster/sim_tests/ $CI_IMAGE_PROJECT_DIR/sim_tests --sse + #- aws s3 sync s3://${BUCKET}/${environment}/k8s-cluster/secrets_chart/ $CI_IMAGE_PROJECT_DIR/secrets_chart --sse + - aws s3 cp s3://${BUCKET}/${environment}/k3saddons/onboarding_internal_pm4ml_output.json $CI_IMAGE_PROJECT_DIR/sim_tests + - cd sim_tests + - newman run https://raw.githubusercontent.com/pm4ml/pm4ml-test-scripts/${TAG_PM4ML_GP}/postman/Deregister_Existing_Oracles.postman_collection.json --insecure -e Lab.postman_environment.json + - newman run https://raw.githubusercontent.com/mojaloop/postman/${TAG_ML_GP}/MojaloopHub_Setup.postman_collection.json --insecure -e Lab.postman_environment.json + #- newman run https://raw.githubusercontent.com/mojaloop/postman/${TAG_ML_GP}/MojaloopSims_Onboarding.postman_collection.json --ssl-client-cert-list test_cert_list.json --insecure -e Lab.postman_environment.json || true + - newman run https://raw.githubusercontent.com/pm4ml/pm4ml-test-scripts/${TAG_PM4ML_GP}/postman/Onboard-Generic-FSP-Central_Ledger.postman_collection.json --insecure -e Lab.postman_environment.json --ssl-client-cert-list test_cert_list.json -d onboarding_internal_pm4ml_output.json || true + - newman run https://raw.githubusercontent.com/pm4ml/pm4ml-test-scripts/${TAG_PM4ML_GP}/postman/Onboard-Generic-FSP-Sim_Backend.postman_collection.json --insecure -e Lab.postman_environment.json --ssl-client-cert-list test_cert_list.json -d onboarding_internal_pm4ml_output.json || true + - newman run https://raw.githubusercontent.com/pm4ml/pm4ml-test-scripts/${TAG_PM4ML_GP}/postman/Mojaloop-Payment-Manager-Golden-Path.postman_collection.json --insecure -e Lab.postman_environment.json --ssl-client-cert-list test_cert_list.json --folder feature-tests --reporters html,junit,cli --reporter-html-export ${CI_PROJECT_DIR}/gp_report.html --reporter-junit-export ${CI_PROJECT_DIR}/gp_report.xml --reporter-html-template ../tests/template-default-colored.hbs -x + after_script: + - curl https://slack.com/api/files.upload + -F file=@"gp_report.html" + -F channels="${SLACK_GITLABCI_ALERTS_CHANNEL_NAME}","${SLACK_GITLABCI_GP_REPORTS_CHANNEL_NAME}" + -F token="${SLACK_GITLABCI_ALERTS_TOKEN}" + -F title="Mojaloop PM4ML Goldenpath Test (Postman) Results.'${ENVIRONMENT}'-$(date +\"%Y-%m-%d-%H:%M:%S\")" + -F filetype="html" + artifacts: + paths: + - gp_report.html + reports: + junit: gp_report.xml + +1. Manual Run PM4ML GP Tests: + stage: "Run Tests" + extends: .Run PM4ML GP Tests-job-template + when: manual + except: + - schedules + +1. Scheduled Run PM4ML GP Tests: + stage: "Run Tests" + allow_failure: true + extends: .Run PM4ML GP Tests-job-template + only: + refs: + - schedules + variables: + - $SCHEDULE_PM4ML == "PM4ML_GP" + +2. Run Finance Portal V2 UI Tests: + stage: "Run Tests" + allow_failure: true + when: manual + script: +# - if [ -f terraform-k8s.tfstate ]; then export fin_portal_users=$(terraform output -json -state=terraform-k8s.tfstate finance_portal_users); else ls -la .; fi +# - if [ -f terraform-k8s.tfstate ]; then export fin_portal_url=$(terraform output -state=terraform-k8s.tfstate finance-portal-url); else ls -la .; fi +# - aws s3 sync s3://${BUCKET}/${environment}/k8s-cluster/sim_tests/ ${CI_IMAGE_PROJECT_DIR}/sim_tests --sse +# - aws s3 sync s3://${BUCKET}/${environment}/k8s-cluster/secrets_chart/ ${CI_IMAGE_PROJECT_DIR}/secrets_chart --sse +# - aws s3 cp s3://${BUCKET}/${environment}/k3saddons/onboarding_internal_pm4ml_output.json ${CI_IMAGE_PROJECT_DIR}/sim_tests +# - cd ${CI_IMAGE_PROJECT_DIR}/sim_tests +# - newman run https://raw.githubusercontent.com/pm4ml/pm4ml-test-scripts/${TAG_PM4ML_GP}/postman/Deregister_Existing_Oracles.postman_collection.json --insecure -e Lab.postman_environment.json +# - newman run https://raw.githubusercontent.com/mojaloop/postman/${TAG_ML_GP}/MojaloopHub_Setup.postman_collection.json --insecure -e Lab.postman_environment.json +# - newman run https://raw.githubusercontent.com/mojaloop/postman/${TAG_ML_GP}/MojaloopSims_Onboarding.postman_collection.json --ssl-client-cert-list test_cert_list.json --insecure -e Lab.postman_environment.json || true + - git clone -b MBP-137 https://github.com/pm4ml/mojaloop-payment-manager-ui.git ${CI_IMAGE_PROJECT_DIR}/mojaloop-payment-manager-ui + - exit_code=0 + - cd ${CI_IMAGE_PROJECT_DIR}/mojaloop-payment-manager-ui/ui-tests/tests + - echo ${SLACK_TOKEN} + - npm install testcafe-reporter-html + - echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list + - wget -q -O - https://dl.google.com/linux/linux_signing_key.pub | apt-key add - + - apt-get update + - apt-get install -y google-chrome-stable + - npm ci + - ENV="other" PM4ML_ENDPOINT=${PM4ML_ENDPOINT} SIM_CORE_CONNECTOR_ENDPOINT=${SIM_CORE_CONNECTOR_ENDPOINT} npm run test:headless | tee report.log || exit_code=$? +# - echo $fin_portal_users | jq -r '"ADMIN_USER_NAME=\(.portaladmin.username)\nADMIN_PASSWORD=\(.portaladmin.user_pass)\nUSER_NAME=\(.portaluser.username)\nPASSWORD=\(.portaluser.user_pass)"' > .env +# - echo "FINANCE_PORTAL_ENDPOINT=$fin_portal_url" >> .env +# - npm i +# - npm run test:headless || true + - cp ${CI_IMAGE_PROJECT_DIR}/mojaloop-payment-manager-ui/ui-tests/tests/report.log ${CI_PROJECT_DIR}/mojaloop-payment-manager-ui_report.log + after_script: + - curl https://slack.com/api/files.upload + -F file=@"mojaloop-payment-manager-ui_report.log" + -F channels="${SLACK_GITLABCI_ALERTS_CHANNEL_NAME}" + -F token="${SLACK_TOKEN}" + -F title="Run Mojaloop Payment Manager UI Tests" + -F filetype="log" + artifacts: + paths: + - mojaloop-payment-manager-ui_report.log + - exit $exit_code + +#job template for 4. Run Platform GP Tests: +.Run Platform GP Tests-job-template: + script: + - aws s3 sync s3://${BUCKET}/${TF_VAR_environment}/k8s-cluster/sim_tests/ $CI_IMAGE_PROJECT_DIR/sim_tests --sse + - aws s3 sync s3://${BUCKET}/${TF_VAR_environment}/k8s-cluster/secrets_chart/ $CI_IMAGE_PROJECT_DIR/secrets_chart --sse + - aws s3 cp s3://${BUCKET}/${TF_VAR_environment}/k3saddons/onboarding_internal_pm4ml_output.json $CI_IMAGE_PROJECT_DIR/sim_tests + - aws s3 cp s3://${BUCKET}/${TF_VAR_environment}/k3saddons/onboarding_external_pm4ml_output.json $CI_IMAGE_PROJECT_DIR/sim_tests + - cp $CI_PROJECT_DIR/tests/Myanmar-Golden-Path.postman_collection.json $CI_IMAGE_PROJECT_DIR/sim_tests + - cd sim_tests + - jq -s '[.[][]]' onboarding_external_pm4ml_output.json onboarding_internal_pm4ml_output.json > /tmp/onboarding_pm4ml_output.json + - newman run https://raw.githubusercontent.com/pm4ml/pm4ml-test-scripts/${TAG_PM4ML_GP}/postman/Deregister_Existing_Oracles.postman_collection.json --insecure -e Lab.postman_environment.json + - newman run https://raw.githubusercontent.com/pm4ml/pm4ml-test-scripts/${TAG_PM4ML_GP}/postman/Onboard-Generic-FSP-Central_Ledger.postman_collection.json --insecure -e Lab.postman_environment.json --ssl-client-cert-list test_cert_list.json -d /tmp/onboarding_pm4ml_output.json || true + - newman run https://raw.githubusercontent.com/pm4ml/pm4ml-test-scripts/${TAG_PM4ML_GP}/postman/Account_Alias_Oracle_Registration.postman_collection.json --insecure -e Lab.postman_environment.json --ssl-client-cert-list test_cert_list.json || true + - newman run https://raw.githubusercontent.com/pm4ml/pm4ml-test-scripts/${TAG_PM4ML_GP}/postman/Account_Alias_Oracle_Setup.postman_collection.json --insecure -e Lab.postman_environment.json --ssl-client-cert-list test_cert_list.json -d /tmp/onboarding_pm4ml_output.json || true + - newman run https://raw.githubusercontent.com/pm4ml/pm4ml-test-scripts/${TAG_PM4ML_GP}/postman/Onboard-Generic-FSP-Sim_Backend.postman_collection.json --insecure -e Lab.postman_environment.json --ssl-client-cert-list test_cert_list.json -d onboarding_internal_pm4ml_output.json || true + - newman run Myanmar-Golden-Path.postman_collection.json --ssl-client-cert-list test_cert_list.json --folder feature-tests --insecure -e Lab.postman_environment.json --reporters html,junit,cli --reporter-html-export ${CI_PROJECT_DIR}/Platform_report.html --reporter-junit-export ${CI_PROJECT_DIR}/alias_report.xml --reporter-html-template ../tests/template-default-colored.hbs -x + after_script: + - curl https://slack.com/api/files.upload + -F file=@"Platform_report.html" + -F channels="${SLACK_GITLABCI_ALERTS_CHANNEL_NAME}" + -F token="${SLACK_GITLABCI_ALERTS_TOKEN}" + -F title="Myanmar Goldenpath Test (Postman) Results" + -F filetype="html" + artifacts: + paths: + - Platform_report.html + +# - curl https://slack.com/api/files.upload +# -F file=@"alias_report.html" +# -F channels="${SLACK_GITLABCI_GP_REPORTS_CHANNEL_NAME}" +# -F token="${SLACK_GITLABCI_ALERTS_TOKEN}" +# -F title="Alias Goldenpath Test (Postman) Results.'${ENVIRONMENT}'-$(date +\"%Y-%m-%d-%H:%M:%S\")" +# -F filetype="html" +# artifacts: +# reports: +# junit: alias_report.xml + + +3. Manual Run Platform GP Tests: + stage: "Run Tests" + extends: .Run Platform GP Tests-job-template + when: manual + except: + - schedules + +3. Scheduled Run Platform GP Tests: + stage: "Run Tests" + allow_failure: true + extends: .Run Platform GP Tests-job-template + only: + refs: + - schedules + variables: + - $SCHEDULE_PLATFORM == "PLAT_GP" + +.ttk-snippets: + image: &ttk-snippets-image + name: mojaloop/ml-testing-toolkit:v13.5.1 + set-env-vars: &ttk-snippets-set-env-vars + - export TERRAFORM_VERSION=1.0.5 + - export GLIBC_VER=2.34-r0 + - apk --no-cache add binutils curl jq && curl -sL https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub -o /etc/apk/keys/sgerrand.rsa.pub && curl -sLO https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC_VER}/glibc-${GLIBC_VER}.apk && curl -sLO https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC_VER}/glibc-bin-${GLIBC_VER}.apk && apk add --no-cache glibc-${GLIBC_VER}.apk glibc-bin-${GLIBC_VER}.apk && curl -sL https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -o awscliv2.zip && unzip awscliv2.zip && aws/install && rm -rf awscliv2.zip aws /usr/local/aws-cli/v2/*/dist/aws_completer /usr/local/aws-cli/v2/*/dist/awscli/data/ac.index /usr/local/aws-cli/v2/*/dist/awscli/examples glibc-${GLIBC_VER}.apk glibc-bin-${GLIBC_VER}.apk /var/cache/apk/* + - wget -q https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip -O /tmp/terraform_${TERRAFORM_VERSION}_linux_amd64.zip && unzip /tmp/terraform_${TERRAFORM_VERSION}_linux_amd64.zip -d /usr/bin/ && rm /tmp/terraform_${TERRAFORM_VERSION}_linux_amd64.zip + - export BUCKET=$(grep bucket backend.hcl | cut -f2 -d '=' | tr -d '"' | xargs) + - for var in $(jq -r 'to_entries[] | "\(.key)=\(.value)\n"' ./workbench-config.json); do export $var; done + - for var in $(jq -r 'to_entries[] | "TF_VAR_\(.key)=\(.value)\n"' ./workbench-config.json); do export $var; done + - export TTK_TESTCASES_VERSION=${helm_mojaloop_version} + - export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} + - export AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} + - export AWS_DEFAULT_REGION=${region} + - aws s3 sync s3://$BUCKET/$environment ./k8ss3/ --sse || true + - export private_subdomain=$(terraform output -state=k8ss3/terraform.tfstate private_subdomain | tr -d '"') + - export ttkBackendUrl=http://ttkbackend.${private_subdomain}; + download-ttk-test-cases: &ttk-snippets-download-ttk-test-cases + - | + cd /opt/mojaloop-testing-toolkit; + export testCasesZipUrl=https://github.com/mojaloop/testing-toolkit-test-cases/archive/refs/tags/v${TTK_TESTCASES_VERSION}.zip; + echo "Downloading the test collection from $testCasesZipUrl"; + wget $testCasesZipUrl -O downloaded-test-collections.zip; + mkdir tmp_test_cases; + unzip -d tmp_test_cases -o downloaded-test-collections.zip; + rm downloaded-test-collections.zip + download-default-env: &ttk-snippets-download-default-env + - | + echo "Downloading the default environment file..."; + cd /opt/mojaloop-testing-toolkit; + wget -O user_config.json $ttkBackendUrl/api/config/user; + defaultEnvFileName=`node -pe 'JSON.parse(process.argv[1]).runtime.DEFAULT_ENVIRONMENT_FILE_NAME' "$(cat user_config.json)"`; + wget -O environment_http_response.json $ttkBackendUrl/api/samples/loadFolderWise?environment=examples/environments/$defaultEnvFileName; + node -pe 'JSON.stringify({inputValues: JSON.parse(process.argv[1]).body.environment})' "$(cat environment_http_response.json)" > cli-testcase-environment.json; + execute-provisioning-collection: &ttk-snippets-execute-provisioning-collection + - | + cd /opt/mojaloop-testing-toolkit; + export provisioningTestCasesPathInZip=testing-toolkit-test-cases-${TTK_TESTCASES_VERSION}/collections/hub/provisioning; + npm run cli -- -e cli-testcase-environment.json -i tmp_test_cases/$provisioningTestCasesPathInZip -u $ttkBackendUrl --report-format html --report-target file://${CI_PROJECT_DIR}/ttk-provisioning-report.html || true; + echo "Done"; + execute-golden-path-collection: &ttk-snippets-execute-golden-path-collection + - | + cd /opt/mojaloop-testing-toolkit; + export goldenPathTestCasesPathInZip=testing-toolkit-test-cases-${TTK_TESTCASES_VERSION}/collections/hub/golden_path; + npm run cli -- -e cli-testcase-environment.json -i tmp_test_cases/$goldenPathTestCasesPathInZip -u $ttkBackendUrl --report-format html --report-target file://${CI_PROJECT_DIR}/ttk-goldenpath-report.html; + echo "Done"; + execute-bulk-transfers-collection: &ttk-snippets-execute-bulk-transfers-collection + - | + cd /opt/mojaloop-testing-toolkit; + export bulkTransfersTestCasesPathInZip=testing-toolkit-test-cases-${TTK_TESTCASES_VERSION}/collections/hub/other_tests/bulk_transfers; + npm run cli -- -e cli-testcase-environment.json -i tmp_test_cases/$bulkTransfersTestCasesPathInZip -u $ttkBackendUrl --report-format html --report-target file://${CI_PROJECT_DIR}/ttk-bulk-transfers-report.html; + echo "Done"; + artifacts: &ttk-snippets-artifacts + when: always + paths: + - ${CI_PROJECT_DIR}/ttk-provisioning-report.html + - ${CI_PROJECT_DIR}/ttk-goldenpath-report.html + +4. Run TTK Tests: + stage: "Run Tests" + image: *ttk-snippets-image + before_script: + - echo "running with testing toolkit image" + script: + - *ttk-snippets-set-env-vars + - *ttk-snippets-download-ttk-test-cases + - *ttk-snippets-download-default-env + - *ttk-snippets-execute-provisioning-collection + - *ttk-snippets-execute-golden-path-collection + artifacts: *ttk-snippets-artifacts + when: manual + +5. Run RBAC Tests: + stage: "Run Tests" + script: + - helm --kubeconfig=admin-gateway.conf test bof --filter name=bof-rbac-tests -n mojaloop || true + - kubectl --kubeconfig=admin-gateway.conf -n mojaloop logs bof-rbac-tests + - exit `kubectl --kubeconfig=admin-gateway.conf -n mojaloop get pod bof-rbac-tests --output="jsonpath={.status.containerStatuses[].state.terminated.exitCode}"` + when: manual + +6. Run Report Tests: + stage: "Run Tests" + script: + - helm --kubeconfig=admin-gateway.conf test bof --filter name=bof-report-tests -n mojaloop || true + - kubectl --kubeconfig=admin-gateway.conf -n mojaloop logs bof-report-tests + - exit `kubectl --kubeconfig=admin-gateway.conf -n mojaloop get pod bof-report-tests --output="jsonpath={.status.containerStatuses[].state.terminated.exitCode}"` + when: manual + +7. Run MCM Tests: + stage: "Run Tests" + script: + - helm --kubeconfig=admin-gateway.conf test connection-manager -n mcm || true + - kubectl --kubeconfig=admin-gateway.conf -n mcm logs connection-manager-api-test + - exit `kubectl --kubeconfig=admin-gateway.conf -n mcm get pod connection-manager-api-test --output="jsonpath={.status.containerStatuses[].state.terminated.exitCode}"` + when: manual + +8. Run PM4ML TTK Tests: + stage: "Run Tests" + image: *ttk-snippets-image + before_script: + - echo "running with testing toolkit image" + script: + - *ttk-snippets-set-env-vars + - | + echo "Downloading the test collection..."; + cd /opt/mojaloop-testing-toolkit; + export testCasesZipUrl=https://github.com/pm4ml/pm4ml-test-scripts/archive/refs/tags/v${TTK_TESTCASES_VERSION}.zip; + wget $testCasesZipUrl -O downloaded-test-collections.zip; + mkdir tmp_test_cases; + unzip -d tmp_test_cases -o downloaded-test-collections.zip; + - | + cd /opt/mojaloop-testing-toolkit + export pm4mlgoldenPathTestCasesPathInZip=pm4ml-test-scripts-${ttk_pm4ml_gp_version}/ttk/collections/feature_tests; + # npm run cli -- -e $CI_PROJECT_DIR/tests/ttk-prod-dev-env.json -i tmp_test_cases/$pm4mlgoldenPathTestCasesPathInZip -u $ttkBackendUrl --report-format html --report-target s3://${BUCKET}/${environment}/pm4ml-ttk-goldenpath-report.html --slack-webhook-url=https://hooks.slack.com/services/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 --extra-summary-information="Test Suite:PM4ML GP TTK,Job ID: $CI_JOB_ID"; + npm run cli -- -e $CI_PROJECT_DIR/tests/ttk-prod-dev-env.json -i tmp_test_cases/$pm4mlgoldenPathTestCasesPathInZip -u $ttkBackendUrl --report-format html --report-target file://${CI_PROJECT_DIR}/ttk-pm4ml-report.html --extra-summary-information="Test Suite:PM4ML GP TTK,Job ID: $CI_JOB_ID"; + # Use the following syntax instead of the above line for slack notification and comment out artifacts. We can not have both with the current TTK CLI options. + # npm run cli -- -e $CI_PROJECT_DIR/tests/ttk-prod-dev-env.json -i tmp_test_cases/$pm4mlgoldenPathTestCasesPathInZip -u $ttkBackendUrl --report-format html --report-target s3://${BUCKET}/${environment}/pm4ml-ttk-goldenpath-report.html --slack-webhook-url=${SLACK_WEBHOOK_URL_HERE} --extra-summary-information="Test Suite:PM4ML GP TTK,Job ID: $CI_JOB_ID"; + echo "Done"; + artifacts: &ttk-snippets-artifacts + when: always + paths: + - ${CI_PROJECT_DIR}/ttk-pm4ml-report.html +# artifacts: +# when: always +# paths: +# - ${CI_PROJECT_DIR}/pm4ml-ttk-goldenpath-report.html + when: manual + +9. Run TTK Bulk Tests: + stage: "Run Tests" + image: *ttk-snippets-image + before_script: + - echo "running with testing toolkit image" + script: + - *ttk-snippets-set-env-vars + - *ttk-snippets-download-ttk-test-cases + - *ttk-snippets-download-default-env + - *ttk-snippets-execute-provisioning-collection + - *ttk-snippets-execute-bulk-transfers-collection + artifacts: *ttk-snippets-artifacts + when: manual \ No newline at end of file diff --git a/gitlab_templates/switch-iac/.gitlab-ci.yml b/gitlab_templates/switch-iac/.gitlab-ci.yml index 967ea46..ea797a3 100644 --- a/gitlab_templates/switch-iac/.gitlab-ci.yml +++ b/gitlab_templates/switch-iac/.gitlab-ci.yml @@ -1,809 +1,9 @@ -default: - image: - name: ghcr.io/mojaloop/iac-aws-platform:2.1.3 - before_script: - - "which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )" - - export BUCKET=$(grep bucket backend.hcl | cut -f2 -d '=' | tr -d '"' | xargs) - - export TF_VAR_bucket=$(grep bucket backend.hcl | cut -f2 -d '=' | tr -d '"' | xargs) - - aws s3 sync s3://$BUCKET/bootstrap ./k8ss3bootstrap/ --sse || true - - export CI_IMAGE_PROJECT_DIR=/iac-run-dir - # Define the contents of the workbench-config file as environment variables for use in gitlab and optionally in terraform - - for var in $(jq -r 'to_entries[] | "\(.key)=\(.value)\n"' ./workbench-config.json); do export $var; done - - for var in $(jq -r 'to_entries[] | "TF_VAR_\(.key)=\(.value)\n"' ./workbench-config.json); do export $var; done - - export ENVIRONMENT=$environment - - test -n "$BUCKET" - - cd $CI_IMAGE_PROJECT_DIR - - aws s3 sync s3://${BUCKET}/${environment}/ . --sse || true - - eval `ssh-agent` - - mkdir -p ~/.ssh - - chmod 700 ~/.ssh - - test -f terraform/ssh_provisioner_key && chmod 0600 terraform/ssh_provisioner_key && ssh-add terraform/ssh_provisioner_key - - chmod +x ./searchandreplace.sh - - if [ -d ${CI_PROJECT_DIR}/iac-sync-dir ]; then cp -r ${CI_PROJECT_DIR}/iac-sync-dir/. .; else echo "no local files to copy"; fi - - cp $CI_PROJECT_DIR/workbench-config.json . - - ./searchandreplace.sh - - - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' - - export ANSIBLE_FEATURE_TOGGLES="-e kube_version=v${k8s_api_version} -e dashboard_enabled=True -e kube_proxy_mode=iptables -e artifacts_dir=inventory/artifacts/cluster -e cloud_provider=aws -e ansible_user=ubuntu -b --become-user=root --flush-cache -e kubeconfig_localhost=true" +include: + - remote: 'https://raw.githubusercontent.com/mojaloop/iac-aws-platform/v2.1.7/gitlab_templates/gitlab_ci_templates/.gitlab-main-ci.yml' + - remote: 'https://raw.githubusercontent.com/mojaloop/iac-aws-platform/v2.1.7/gitlab_templates/gitlab_ci_templates/.gitlab-ops-ci.yml' + - remote: 'https://raw.githubusercontent.com/mojaloop/iac-aws-platform/v2.1.7/gitlab_templates/gitlab_ci_templates/.gitlab-tests-ci.yml' variables: - #ANSIBLE_FEATURE_TOGGLES: "-e kube_version=v${k8s_api_version} -e dashboard_enabled=True -e kube_proxy_mode=iptables -e artifacts_dir=inventory/artifacts/cluster -e cloud_provider=aws -e ansible_user=ubuntu -b --become-user=root --flush-cache -e kubeconfig_localhost=true" GIT_SSL_NO_VERIFY: "true" - -stages: - - Validate - - Destroy - - "Deploy AWS infrastructure" - - "Run Kubespray" - - "Run k3s" - - "Deploy Base Platform" - - "Deploy Platform" - - "Run Tests" - - "Maintain Platform" - -Validate and Plan: - stage: Validate - script: - - cd terraform - - terraform init -backend-config ${CI_PROJECT_DIR}/backend.hcl - - terraform validate - - terraform plan -out=plan.cache - - terraform show -json plan.cache | jq -r '([.resource_changes[]?.change.actions?]|flatten)|{"create":(map(select(.=="create"))|length),"update":(map(select(.=="update"))|length),"delete":(map(select(.=="delete"))|length)}' > ${CI_PROJECT_DIR}/plan.json - artifacts: - reports: - terraform: plan.json - when: manual - -Destroy AWS: - stage: Destroy - script: - - cd terraform - - aws s3 sync s3://${BUCKET}/${environment}/terraform/oauth-apps/ ./oauth-apps/ || true - - terraform init -backend-config ${CI_PROJECT_DIR}/backend.hcl - - terraform validate - - terraform destroy -auto-approve -refresh=false - when: manual - -Destroy Mojaloop K8s Stack: - stage: Destroy - script: - - cd terraform/apps/wso2/config - - terraform init -backend-config ${CI_PROJECT_DIR}/backend.hcl - - terraform validate - - terraform destroy -auto-approve -var="project_root_path=$CI_IMAGE_PROJECT_DIR" || true - - cd $CI_IMAGE_PROJECT_DIR/terraform/k8s-setup/mojaloop-roles - - terraform init -backend-config ${CI_PROJECT_DIR}/backend.hcl - - terraform validate - - terraform destroy -auto-approve -var="project_root_path=$CI_IMAGE_PROJECT_DIR" || true - - cd $CI_IMAGE_PROJECT_DIR/terraform/k8s-setup - - terraform init -backend-config ${CI_PROJECT_DIR}/backend.hcl - - terraform validate - - terraform destroy -auto-approve -var="project_root_path=$CI_IMAGE_PROJECT_DIR" || true - #- kubectl --kubeconfig=../../admin-gateway.conf delete namespace mojaloop || true - - when: manual - -Destroy Add-On Apps: - stage: Destroy - script: - - aws s3 sync s3://${BUCKET}/${environment}/k3saddons/pm4ml-certoutput/ terraform/k8s-setup/addons/pm4ml-certoutput - - aws s3 sync s3://${client}${environment}k3s-pm4mladdons-state/k3s ./k3sstate --sse || true - - cp $CI_PROJECT_DIR/ext-pm4ml-certs.yaml terraform/k8s-setup/addons || true - - for var in $(jq -r '.internal_pm4ml_configs | to_entries[] .value.DFSP_NAME' ./workbench-config.json); do touch terraform/k8s-setup/addons/pm4ml-certoutput/$var-client-key.pem terraform/k8s-setup/addons/pm4ml-certoutput/$var-ca-cert.pem terraform/k8s-setup/addons/pm4ml-certoutput/$var-client-cert.pem; done - - cd terraform/k8s-setup/addons - - terraform init -backend-config ${CI_PROJECT_DIR}/backend.hcl - - terraform validate - - terraform destroy -auto-approve -var="project_root_path=$CI_IMAGE_PROJECT_DIR" - - aws s3 rm --recursive s3://$BUCKET/$environment/k8s-cluster - - aws s3 rm s3://$BUCKET/$environment/k3saddons/onboarding_pm4ml_output.json - - aws s3 rm s3://$BUCKET/$environment/k3saddons/ansible_pm4ml_output.yaml - when: manual - -Destroy of Internal PM4MLs Setup: - stage: Destroy - script: - - cd terraform/k8s-setup/pm4mls - - terraform init -backend-config ${CI_PROJECT_DIR}/backend.hcl - - terraform validate - - terraform destroy -auto-approve -var="project_root_path=$CI_IMAGE_PROJECT_DIR" - - aws s3 rm s3://$BUCKET/$environment/k3saddons/ansible_internal_pm4ml_output.json - - aws s3 rm s3://$BUCKET/$environment/k3saddons/onboarding_internal_pm4ml_output.json - - aws s3 rm s3://$BUCKET/$environment/k3saddons/ansible_external_pm4ml_output.yaml - when: manual - -Destroy Vault: - stage: Destroy - script: - - cd terraform/k8s-setup/vault-deploy - - terraform init -backend-config ${CI_PROJECT_DIR}/backend.hcl - - terraform validate - - terraform destroy -auto-approve -var="aws_secret_key=$AWS_SECRET_ACCESS_KEY" -var="aws_access_key=$AWS_ACCESS_KEY_ID" -var="project_root_path=$CI_IMAGE_PROJECT_DIR" - when: manual - -Destroy Support Services: - stage: Destroy - script: - - cd terraform/k8s-setup/support-svcs - - aws s3 sync s3://${BUCKET}/${environment}/terraform/oauth-apps/ ./oauth-apps/ || true - - terraform init -backend-config ${CI_PROJECT_DIR}/backend.hcl - - terraform validate - - terraform destroy -auto-approve -var="project_root_path=$CI_IMAGE_PROJECT_DIR" - ##kill force kill efs provsioner - - kubectl --kubeconfig=../../../admin-gateway.conf delete pods --all -n wso2 --grace-period=0 --force - - kubectl --kubeconfig=../../../admin-gateway.conf delete namespace wso2 || true - ##kill force wso2-bizops - - kubectl --kubeconfig=../../../admin-gateway.conf delete pods --all -n wso2-bizops --grace-period=0 --force - - kubectl --kubeconfig=../../../admin-gateway.conf delete namespace wso2-bizops || true - when: manual - -Destroy Stateful Services: - stage: Destroy - script: - - cd terraform/k8s-setup/state-setup - - terraform init -backend-config ${CI_PROJECT_DIR}/backend.hcl - - terraform validate - - terraform destroy -auto-approve -var="project_root_path=$CI_IMAGE_PROJECT_DIR" || true - when: manual - -Destroy Environment: - stage: Destroy - script: - - aws s3 sync s3://${BUCKET}/${environment}/k3saddons/pm4ml-certoutput/ terraform/k8s-setup/addons/pm4ml-certoutput - - aws s3 sync s3://${client}${environment}k3s-pm4mladdons-state/k3s ./k3sstate --sse || true - - aws s3 sync s3://${BUCKET}/${environment}/terraform/oauth-apps/ ./oauth-apps/ || true - - for var in $(jq -r '.internal_pm4ml_configs | to_entries[] .value.DFSP_NAME' ./workbench-config.json); do touch terraform/k8s-setup/addons/pm4ml-certoutput/$var-client-key.pem terraform/k8s-setup/addons/pm4ml-certoutput/$var-ca-cert.pem terraform/k8s-setup/addons/pm4ml-certoutput/$var-client-cert.pem; done - - cp $CI_PROJECT_DIR/ext-pm4ml-certs.yaml terraform/k8s-setup/addons || true - - cp $CI_PROJECT_DIR/teardown.sh $CI_IMAGE_PROJECT_DIR/teardown.sh || true - - $CI_IMAGE_PROJECT_DIR/teardown.sh - when: manual - -Destroy k3s Cluster: - stage: "Destroy" - image: - name: ghcr.io/pm4ml/k3sbootstrap:0.16.3 - before_script: - - export BUCKET=$(grep bucket backend.hcl | cut -f2 -d '=' | tr -d '"' | xargs) - - export TF_VAR_bucket=$(grep bucket backend.hcl | cut -f2 -d '=' | tr -d '"') - - aws s3 sync s3://$BUCKET/bootstrap ./k8ss3bootstrap/ --sse || true - - export CI_IMAGE_PROJECT_DIR=/k3s-boot - - export peer_vpc_id=$(terraform output -state=k8ss3bootstrap/terraform.tfstate vpc_id | tr -d '"') - - for var in $(jq -r 'to_entries[] | "\(.key)=\(.value)\n"' ./workbench-config.json); do export $var; done - - for var in $(jq -r 'to_entries[] | "TF_VAR_\(.key)=\(.value)\n"' ./workbench-config.json); do export $var; done - - sed -i "s/##accesskeyid##/${AWS_ACCESS_KEY_ID}/g" $CI_IMAGE_PROJECT_DIR/aws_conf - - sed -i "s|##accesssecret##|${AWS_SECRET_ACCESS_KEY}|g" $CI_IMAGE_PROJECT_DIR/aws_conf - - sed -i "s/##region##/${region}/g" $CI_IMAGE_PROJECT_DIR/aws_conf - - mkdir -p ~/.aws && cp $CI_IMAGE_PROJECT_DIR/aws_conf ~/.aws/credentials - - "which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )" - - eval `ssh-agent` - - mkdir -p ~/.ssh - - chmod 700 ~/.ssh - - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' - - if [ -d ./k3s-sync-dir ]; then cp -r ./k3s-sync-dir/. /k3s-boot; else echo "not local files to copy"; fi - - aws s3 sync s3://$BUCKET/$environment ./k8ss3/ --sse || true - - export domain=$(terraform output -state=k8ss3/terraform.tfstate public_subdomain | tr -d '"') - - script: - - export - - cd /k3s-boot - - sed -i "s/##client##/${client}/g" .env - - sed -i "s/##domain##/${domain}/g" .env - - sed -i "s/##environment##/${environment}/g" .env - - sed -i "s/##region##/${region}/g" .env - - make backend -- -auto-approve || true - - make destroy -- -auto-approve || true - - aws s3 rm --recursive s3://$BUCKET/$environment/k3saddons || true - - aws s3 rm --recursive s3://${client}${environment}k3s-pm4mladdons-state/k3s || true - - $CI_IMAGE_PROJECT_DIR/delete_int_k3s_db_row.sh ${client}${environment}k3s-pm4mladdons-lock ${client}${environment}k3s-pm4mladdons-state/k3s $region - when: manual - -Undeploy PM4ML Charts: - stage: "Destroy" - image: - name: ghcr.io/pm4ml/k3sbootstrap:0.16.3 - before_script: - - export BUCKET=$(grep bucket backend.hcl | cut -f2 -d '=' | tr -d '"' | xargs) - - export TF_VAR_bucket=$(grep bucket backend.hcl | cut -f2 -d '=' | tr -d '"' | xargs) - - aws s3 sync s3://$BUCKET/bootstrap ./k8ss3bootstrap/ --sse || true - - export CI_IMAGE_PROJECT_DIR=/k3s-boot - - export peer_vpc_id=$(terraform output -state=k8ss3bootstrap/terraform.tfstate vpc_id | tr -d '"') - - for var in $(jq -r 'to_entries[] | "\(.key)=\(.value)\n"' ./workbench-config.json); do export $var; done - - for var in $(jq -r 'to_entries[] | "TF_VAR_\(.key)=\(.value)\n"' ./workbench-config.json); do export $var; done - - sed -i "s/##accesskeyid##/${AWS_ACCESS_KEY_ID}/g" $CI_IMAGE_PROJECT_DIR/aws_conf - - sed -i "s|##accesssecret##|${AWS_SECRET_ACCESS_KEY}|g" $CI_IMAGE_PROJECT_DIR/aws_conf - - sed -i "s/##region##/${region}/g" $CI_IMAGE_PROJECT_DIR/aws_conf - - mkdir -p ~/.aws && cp $CI_IMAGE_PROJECT_DIR/aws_conf ~/.aws/credentials - - "which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )" - - eval `ssh-agent` - - mkdir -p ~/.ssh - - chmod 700 ~/.ssh - - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' - - if [ -d ./k3s-sync-dir ]; then cp -r ./k3s-sync-dir/. /k3s-boot; else echo "not local files to copy"; fi - - aws s3 sync s3://$BUCKET/$environment ./k8ss3/ --sse || true - - export domain=$(terraform output -state=k8ss3/terraform.tfstate public_subdomain | tr -d '"') - - script: - - aws s3 cp s3://$BUCKET/$environment/k3saddons/ssh-key /k3s-boot/ssh-key --sse - - chmod 400 /k3s-boot/ssh-key - - cd /k3s-boot - - sed -i "s/##client##/${client}/g" .env - - sed -i "s/##domain##/${domain}/g" .env - - sed -i "s/##environment##/${environment}/g" .env - - sed -i "s/##region##/${region}/g" .env - - sed -i "s/##peer_vpc_id##/${peer_vpc_id}/g" .env - #TODO: static value needs addressing - - sed -i "s/##pm4ml_client_cert_local_dir##/\/k3s-boot\/pm4ml-certoutput\//g" .env - - aws s3 cp s3://$BUCKET/$environment/k3saddons/ansible_internal_pm4ml_output.yaml ./ansible_pm4ml_output.yaml --sse - - make backend -- -auto-approve || true - - make apply -- -auto-approve || true - - make uninstall-pm4ml - #- aws s3 rm --recursive s3://$BUCKET/$PM4ML_ENVIRONMENT/pm4ml-certoutput - when: manual - -Deploy AWS Infrastructure: - stage: "Deploy AWS infrastructure" - script: - - cd terraform - - aws s3 sync s3://${BUCKET}/${environment}/terraform/oauth-apps/ ./oauth-apps/ || true - - terraform init -backend-config ${CI_PROJECT_DIR}/backend.hcl - - terraform apply -auto-approve || true - - aws s3 sync $CI_IMAGE_PROJECT_DIR/kubespray-inventory s3://${BUCKET}/${environment}/kubespray-inventory/ --sse || true - - aws s3 cp $CI_IMAGE_PROJECT_DIR/terraform/ssh_provisioner_key s3://${BUCKET}/${environment}/terraform/ --sse || true - - aws s3 sync ./oauth-apps s3://${BUCKET}/${environment}/terraform/oauth-apps/ --sse || true - when: manual - -Create k3s Cluster: - stage: "Run k3s" - image: - name: ghcr.io/pm4ml/k3sbootstrap:0.16.3 - before_script: - - export BUCKET=$(grep bucket backend.hcl | cut -f2 -d '=' | tr -d '"' | xargs) - - export TF_VAR_bucket=$(grep bucket backend.hcl | cut -f2 -d '=' | tr -d '"' | xargs) - - aws s3 sync s3://$BUCKET/bootstrap ./k8ss3bootstrap/ --sse || true - - export CI_IMAGE_PROJECT_DIR=/k3s-boot - - export peer_vpc_id=$(terraform output -state=k8ss3bootstrap/terraform.tfstate vpc_id | tr -d '"') - - for var in $(jq -r 'to_entries[] | "\(.key)=\(.value)\n"' ./workbench-config.json); do export $var; done - - for var in $(jq -r 'to_entries[] | "TF_VAR_\(.key)=\(.value)\n"' ./workbench-config.json); do export $var; done - - sed -i "s/##accesskeyid##/${AWS_ACCESS_KEY_ID}/g" $CI_IMAGE_PROJECT_DIR/aws_conf - - sed -i "s|##accesssecret##|${AWS_SECRET_ACCESS_KEY}|g" $CI_IMAGE_PROJECT_DIR/aws_conf - - sed -i "s/##region##/${region}/g" $CI_IMAGE_PROJECT_DIR/aws_conf - - mkdir -p ~/.aws && cp $CI_IMAGE_PROJECT_DIR/aws_conf ~/.aws/credentials - - "which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )" - - eval `ssh-agent` - - mkdir -p ~/.ssh - - chmod 700 ~/.ssh - - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' - - if [ -d ./k3s-sync-dir ]; then cp -r ./k3s-sync-dir/. /k3s-boot; else echo "not local files to copy"; fi - - aws s3 sync s3://$BUCKET/$environment ./k8ss3/ --sse || true - - export domain=$(terraform output -state=k8ss3/terraform.tfstate public_subdomain | tr -d '"') - - script: - - export - - cd /k3s-boot - - sed -i "s/##client##/${client}/g" .env - - sed -i "s/##domain##/${domain}/g" .env - - sed -i "s/##environment##/${environment}/g" .env - - sed -i "s/##region##/${region}/g" .env - - sed -i "s/##peer_vpc_id##/${peer_vpc_id}/g" .env - - make backend -- -auto-approve - - make apply -- -auto-approve - - make k3s - - make kubeconfig - - aws s3 cp /k3s-boot/kubeconfig s3://$BUCKET/$environment/k3saddons/ --sse - - aws s3 cp /k3s-boot/ssh-key s3://${BUCKET}/$environment/k3saddons/ --sse - - aws s3 sync s3://$BUCKET/$environment/k3saddons /k3s-boot/k3saddons --sse - - cp -r /k3s-boot/k3saddons/* /k3s-boot/ - - if [ ! -f /k3s-boot/vault-keys.json ]; then make vault; else echo "found vault, skipping make vault"; fi - - aws s3 cp /k3s-boot/vault-keys.json s3://$BUCKET/$environment/k3saddons/ --sse - - make wireguard - - aws s3 cp /k3s-boot/wireguard.private.key s3://$BUCKET/$environment/k3saddons/ --sse - - aws s3 cp /k3s-boot/wireguard.public.key s3://$BUCKET/$environment/k3saddons/ --sse - - make monitoring - when: manual - -Create Gateway Cluster: - stage: "Run Kubespray" - script: - - cd /kubespray - - ansible-playbook -i $CI_IMAGE_PROJECT_DIR/kubespray-inventory/hosts-gateway -e @$CI_IMAGE_PROJECT_DIR/kubespray-inventory/extra-vars.json ./cluster.yml $ANSIBLE_FEATURE_TOGGLES - - aws s3 cp inventory/artifacts/cluster/admin.conf s3://$BUCKET/$environment/admin-gateway.conf --sse - when: manual - -1. Deploy Vault: - stage: "Deploy Base Platform" - script: - - cd terraform/k8s-setup/vault-deploy - - terraform init -backend-config ${CI_PROJECT_DIR}/backend.hcl - - terraform apply -auto-approve -var="aws_secret_key=$AWS_SECRET_ACCESS_KEY" -var="aws_access_key=$AWS_ACCESS_KEY_ID" -var="project_root_path=$CI_IMAGE_PROJECT_DIR" - - aws s3 cp $CI_IMAGE_PROJECT_DIR/vault_seal_key s3://${BUCKET}/${environment}/ --sse - when: manual - -2a. Deploy Stateful Services: - stage: "Deploy Base Platform" - script: - - cd terraform/k8s-setup/state-setup - - kubectl --kubeconfig=../../../admin-gateway.conf -n default wait --for=condition=ready certificate/wildcard-cert-internal --timeout=180s - - terraform init -backend-config ${CI_PROJECT_DIR}/backend.hcl - - terraform apply -auto-approve -var="project_root_path=$CI_IMAGE_PROJECT_DIR" - when: manual - -2b. Deploy Support Services: - stage: "Deploy Base Platform" - script: - - cd terraform/k8s-setup/support-svcs - - aws s3 sync s3://${BUCKET}/${environment}/terraform/oauth-apps/ ./oauth-apps/ || true - - terraform init -backend-config ${CI_PROJECT_DIR}/backend.hcl - - terraform apply -auto-approve -var="project_root_path=$CI_IMAGE_PROJECT_DIR" - when: manual - -1. Initial Install Base Apps: - stage: "Deploy Platform" - script: - - cd terraform/k8s-setup - - terraform init -backend-config ${CI_PROJECT_DIR}/backend.hcl - - terraform apply -auto-approve -var="project_root_path=$CI_IMAGE_PROJECT_DIR" - - cd $CI_IMAGE_PROJECT_DIR/terraform/k8s-setup/mojaloop-roles - - terraform init -backend-config ${CI_PROJECT_DIR}/backend.hcl - - terraform apply -auto-approve -var="project_root_path=$CI_IMAGE_PROJECT_DIR" - - chmod o-w $CI_IMAGE_PROJECT_DIR/terraform/apps/wso2/config/ - - cd $CI_IMAGE_PROJECT_DIR/terraform/apps/wso2/config/ - - terraform init -backend-config ${CI_PROJECT_DIR}/backend.hcl - - terraform apply -auto-approve -var="project_root_path=$CI_IMAGE_PROJECT_DIR" - - ansible-playbook publish.yaml - when: manual - -2a. Setup PM4MLs: - stage: "Deploy Platform" - script: - - cd terraform/k8s-setup/pm4mls - - terraform init -backend-config ${CI_PROJECT_DIR}/backend.hcl - - terraform apply -auto-approve -var="project_root_path=$CI_IMAGE_PROJECT_DIR" - - aws s3 cp $CI_IMAGE_PROJECT_DIR/terraform/k8s-setup/pm4mls/ansible_internal_pm4ml_output.yaml s3://${BUCKET}/${environment}/k3saddons/ --sse - - aws s3 cp $CI_IMAGE_PROJECT_DIR/terraform/k8s-setup/pm4mls/onboarding_internal_pm4ml_output.json s3://${BUCKET}/${environment}/k3saddons/ --sse - - aws s3 cp $CI_IMAGE_PROJECT_DIR/terraform/k8s-setup/pm4mls/ansible_external_pm4ml_output.yaml s3://${BUCKET}/${environment}/k3saddons/ --sse - - aws s3 cp $CI_IMAGE_PROJECT_DIR/terraform/k8s-setup/pm4mls/onboarding_external_pm4ml_output.json s3://${BUCKET}/${environment}/k3saddons/ --sse - #- aws s3 sync $CI_IMAGE_PROJECT_DIR/terraform/k8s-setup/pm4mls/secrets_chart/ s3://${BUCKET}/${environment}/k8s-cluster/secrets_chart/ --sse - when: manual - -2b. Install Internal PM4MLs: - stage: "Deploy Platform" - image: - name: ghcr.io/pm4ml/k3sbootstrap:0.16.3 - before_script: - - export BUCKET=$(grep bucket backend.hcl | cut -f2 -d '=' | tr -d '"' | xargs) - - export TF_VAR_bucket=$(grep bucket backend.hcl | cut -f2 -d '=' | tr -d '"' | xargs) - - aws s3 sync s3://$BUCKET/bootstrap ./k8ss3bootstrap/ --sse || true - - export CI_IMAGE_PROJECT_DIR=/k3s-boot - - export peer_vpc_id=$(terraform output -state=k8ss3bootstrap/terraform.tfstate vpc_id | tr -d '"') - - for var in $(jq -r 'to_entries[] | "\(.key)=\(.value)\n"' ./workbench-config.json); do export $var; done - - for var in $(jq -r 'to_entries[] | "TF_VAR_\(.key)=\(.value)\n"' ./workbench-config.json); do export $var; done - - sed -i "s/##accesskeyid##/${AWS_ACCESS_KEY_ID}/g" $CI_IMAGE_PROJECT_DIR/aws_conf - - sed -i "s|##accesssecret##|${AWS_SECRET_ACCESS_KEY}|g" $CI_IMAGE_PROJECT_DIR/aws_conf - - sed -i "s/##region##/${region}/g" $CI_IMAGE_PROJECT_DIR/aws_conf - - mkdir -p ~/.aws && cp $CI_IMAGE_PROJECT_DIR/aws_conf ~/.aws/credentials - - "which ssh-agent || ( apt-get update -y && apt-get install openssh-client -y )" - - eval `ssh-agent` - - mkdir -p ~/.ssh - - chmod 700 ~/.ssh - - '[[ -f /.dockerenv ]] && echo -e "Host *\n\tStrictHostKeyChecking no\n\n" > ~/.ssh/config' - - if [ -d ./k3s-sync-dir ]; then cp -r ./k3s-sync-dir/. /k3s-boot; else echo "not local files to copy"; fi - - aws s3 sync s3://$BUCKET/$environment ./k8ss3/ --sse || true - - export domain=$(terraform output -state=k8ss3/terraform.tfstate public_subdomain | tr -d '"') - script: - - aws s3 cp s3://$BUCKET/$environment/k3saddons/ssh-key /k3s-boot/ssh-key --sse - - chmod 400 /k3s-boot/ssh-key - - cd /k3s-boot - - sed -i "s/##client##/${client}/g" .env - - sed -i "s/##domain##/${domain}/g" .env - - sed -i "s/##environment##/${environment}/g" .env - - sed -i "s/##region##/${region}/g" .env - - sed -i "s/##peer_vpc_id##/${peer_vpc_id}/g" .env - #TODO: static value needs addressing - - sed -i "s/##pm4ml_client_cert_local_dir##/\/k3s-boot\/pm4ml-certoutput\//g" .env - - aws s3 cp s3://$BUCKET/$environment/k3saddons/ansible_internal_pm4ml_output.yaml ./ansible_pm4ml_output.yaml --sse - - make backend -- -auto-approve - - make apply -- -auto-approve - - make pm4ml - #- aws s3 sync /k3s-boot/pm4ml-certoutput s3://$BUCKET/$environment/k3saddons/pm4ml-certoutput --sse - when: manual - -2c. Install Post Deployment: - stage: "Deploy Platform" - script: - #todo remove hardcoded values here - - aws s3 sync s3://${BUCKET}/${environment}/k3saddons/pm4ml-certoutput/ terraform/k8s-setup/addons/pm4ml-certoutput - - aws s3 sync s3://${client}${environment}k3s-pm4mladdons-state/k3s ./k3sstate --sse || true - - if [ -f k3sstate/terraform.tfstate ]; then export pm4ml_nat_ips_tmp=$(terraform output -state=k3sstate/terraform.tfstate nat_public_ips); else ls -la ./k3sstate; fi - - echo $pm4ml_nat_ips_tmp - - if [ $(echo $pm4ml_nat_ips_tmp | grep --quiet Warning) ]; then echo "nats not found"; else export TF_VAR_pm4ml_nat_ips=$pm4ml_nat_ips_tmp; fi - - cp $CI_PROJECT_DIR/ext-pm4ml-certs.yaml terraform/k8s-setup/addons || true - - cd terraform/k8s-setup/addons - #- wget -q https://releases.hashicorp.com/terraform/1.1.8/terraform_1.1.8_linux_amd64.zip -O /tmp/terraform_${TERRAFORM_VERSION}_linux_amd64.zip && unzip /tmp/terraform_${TERRAFORM_VERSION}_linux_amd64.zip - #- export TF_CLI_CONFIG_FILE=./restapi.tfrc - - terraform init -backend-config ${CI_PROJECT_DIR}/backend.hcl - - terraform apply -auto-approve -var="project_root_path=$CI_IMAGE_PROJECT_DIR" - #- aws s3 sync $CI_IMAGE_PROJECT_DIR/terraform/k8s-setup/addons/secrets_chart s3://${BUCKET}/${environment}/k8s-cluster/secrets_chart --sse - - aws s3 sync $CI_IMAGE_PROJECT_DIR/terraform/k8s-setup/addons/sim_tests s3://${BUCKET}/${environment}/k8s-cluster/sim_tests --sse - #- find . -type f -iname '*_results' -exec cat '{}' \; - when: manual - - -#job template for 1. Run PM4ML GP Tests -.Run PM4ML GP Tests-job-template: - script: - - aws s3 sync s3://${BUCKET}/${environment}/k8s-cluster/sim_tests/ $CI_IMAGE_PROJECT_DIR/sim_tests --sse - #- aws s3 sync s3://${BUCKET}/${environment}/k8s-cluster/secrets_chart/ $CI_IMAGE_PROJECT_DIR/secrets_chart --sse - - aws s3 cp s3://${BUCKET}/${environment}/k3saddons/onboarding_internal_pm4ml_output.json $CI_IMAGE_PROJECT_DIR/sim_tests - - cd sim_tests - - newman run https://raw.githubusercontent.com/pm4ml/pm4ml-test-scripts/${TAG_PM4ML_GP}/postman/Deregister_Existing_Oracles.postman_collection.json --insecure -e Lab.postman_environment.json - - newman run https://raw.githubusercontent.com/mojaloop/postman/${TAG_ML_GP}/MojaloopHub_Setup.postman_collection.json --insecure -e Lab.postman_environment.json - #- newman run https://raw.githubusercontent.com/mojaloop/postman/${TAG_ML_GP}/MojaloopSims_Onboarding.postman_collection.json --ssl-client-cert-list test_cert_list.json --insecure -e Lab.postman_environment.json || true - - newman run https://raw.githubusercontent.com/pm4ml/pm4ml-test-scripts/${TAG_PM4ML_GP}/postman/Onboard-Generic-FSP-Central_Ledger.postman_collection.json --insecure -e Lab.postman_environment.json --ssl-client-cert-list test_cert_list.json -d onboarding_internal_pm4ml_output.json || true - - newman run https://raw.githubusercontent.com/pm4ml/pm4ml-test-scripts/${TAG_PM4ML_GP}/postman/Onboard-Generic-FSP-Sim_Backend.postman_collection.json --insecure -e Lab.postman_environment.json --ssl-client-cert-list test_cert_list.json -d onboarding_internal_pm4ml_output.json || true - - newman run https://raw.githubusercontent.com/pm4ml/pm4ml-test-scripts/${TAG_PM4ML_GP}/postman/Mojaloop-Payment-Manager-Golden-Path.postman_collection.json --insecure -e Lab.postman_environment.json --ssl-client-cert-list test_cert_list.json --folder feature-tests --reporters html,junit,cli --reporter-html-export ${CI_PROJECT_DIR}/gp_report.html --reporter-junit-export ${CI_PROJECT_DIR}/gp_report.xml --reporter-html-template ../tests/template-default-colored.hbs -x - after_script: - - curl https://slack.com/api/files.upload - -F file=@"gp_report.html" - -F channels="${SLACK_GITLABCI_ALERTS_CHANNEL_NAME}","${SLACK_GITLABCI_GP_REPORTS_CHANNEL_NAME}" - -F token="${SLACK_GITLABCI_ALERTS_TOKEN}" - -F title="Mojaloop PM4ML Goldenpath Test (Postman) Results.'${ENVIRONMENT}'-$(date +\"%Y-%m-%d-%H:%M:%S\")" - -F filetype="html" - artifacts: - paths: - - gp_report.html - reports: - junit: gp_report.xml - -1. Manual Run PM4ML GP Tests: - stage: "Run Tests" - extends: .Run PM4ML GP Tests-job-template - when: manual - except: - - schedules - -1. Scheduled Run PM4ML GP Tests: - stage: "Run Tests" - allow_failure: true - extends: .Run PM4ML GP Tests-job-template - only: - refs: - - schedules - variables: - - $SCHEDULE_PM4ML == "PM4ML_GP" - -2. Run Finance Portal V2 UI Tests: - stage: "Run Tests" - allow_failure: true - when: manual - script: - - if [ -f terraform-k8s.tfstate ]; then export fin_portal_users=$(terraform output -json -state=terraform-k8s.tfstate finance_portal_users); else ls -la .; fi - - if [ -f terraform-k8s.tfstate ]; then export fin_portal_url=$(terraform output -state=terraform-k8s.tfstate finance-portal-url); else ls -la .; fi - - aws s3 sync s3://${BUCKET}/${environment}/k8s-cluster/sim_tests/ ${CI_IMAGE_PROJECT_DIR}/sim_tests --sse - - aws s3 sync s3://${BUCKET}/${environment}/k8s-cluster/secrets_chart/ ${CI_IMAGE_PROJECT_DIR}/secrets_chart --sse - - aws s3 cp s3://${BUCKET}/${environment}/k3saddons/onboarding_internal_pm4ml_output.json ${CI_IMAGE_PROJECT_DIR}/sim_tests - - cd ${CI_IMAGE_PROJECT_DIR}/sim_tests - - newman run https://raw.githubusercontent.com/pm4ml/pm4ml-test-scripts/${TAG_PM4ML_GP}/postman/Deregister_Existing_Oracles.postman_collection.json --insecure -e Lab.postman_environment.json - - newman run https://raw.githubusercontent.com/mojaloop/postman/${TAG_ML_GP}/MojaloopHub_Setup.postman_collection.json --insecure -e Lab.postman_environment.json - - newman run https://raw.githubusercontent.com/mojaloop/postman/${TAG_ML_GP}/MojaloopSims_Onboarding.postman_collection.json --ssl-client-cert-list test_cert_list.json --insecure -e Lab.postman_environment.json || true - - git clone https://github.com/mojaloop/finance-portal-v2-ui.git ${CI_IMAGE_PROJECT_DIR}/finance-portal-v2-ui - - cd ${CI_IMAGE_PROJECT_DIR}/finance-portal-v2-ui/tests/e2e-ui-tests - - echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list - - wget -q -O - https://dl.google.com/linux/linux_signing_key.pub | apt-key add - - - apt-get update - - apt-get install -y google-chrome-stable - - echo $fin_portal_users | jq -r '"ADMIN_USER_NAME=\(.portaladmin.username)\nADMIN_PASSWORD=\(.portaladmin.user_pass)\nUSER_NAME=\(.portaluser.username)\nPASSWORD=\(.portaluser.user_pass)"' > .env - - echo "FINANCE_PORTAL_ENDPOINT=$fin_portal_url" >> .env - - npm i - - npm run test:headless || true - - cp ${CI_IMAGE_PROJECT_DIR}/finance-portal-v2-ui/tests/e2e-ui-tests/src/reports/report.html ${CI_PROJECT_DIR}/fin_port_report.html - after_script: - - curl https://slack.com/api/files.upload - -F file=@"${CI_IMAGE_PROJECT_DIR}/finance-portal-v2-ui/tests/e2e-ui-tests/src/reports/report.html" - -F channels="${SLACK_GITLABCI_ALERTS_CHANNEL_NAME}" - -F token="${SLACK_GITLABCI_ALERTS_TOKEN}" - -F title="Run Finance Portal V2 UI Tests" - -F filetype="html" - artifacts: - paths: - - fin_port_report.html - #reports: - #junit: $CI_IMAGE_PROJECT_DIR/finance-portal-v2-ui/tests/e2e-ui-tests/src/reports/report.html - -#job template for 4. Run Platform GP Tests: -.Run Platform GP Tests-job-template: - script: - - aws s3 sync s3://${BUCKET}/${TF_VAR_environment}/k8s-cluster/sim_tests/ $CI_IMAGE_PROJECT_DIR/sim_tests --sse - - aws s3 sync s3://${BUCKET}/${TF_VAR_environment}/k8s-cluster/secrets_chart/ $CI_IMAGE_PROJECT_DIR/secrets_chart --sse - - aws s3 cp s3://${BUCKET}/${TF_VAR_environment}/k3saddons/onboarding_internal_pm4ml_output.json $CI_IMAGE_PROJECT_DIR/sim_tests - - aws s3 cp s3://${BUCKET}/${TF_VAR_environment}/k3saddons/onboarding_external_pm4ml_output.json $CI_IMAGE_PROJECT_DIR/sim_tests - - cp $CI_PROJECT_DIR/tests/Myanmar-Golden-Path.postman_collection.json $CI_IMAGE_PROJECT_DIR/sim_tests - - cd sim_tests - - jq -s '[.[][]]' onboarding_external_pm4ml_output.json onboarding_internal_pm4ml_output.json > /tmp/onboarding_pm4ml_output.json - - newman run https://raw.githubusercontent.com/pm4ml/pm4ml-test-scripts/${TAG_PM4ML_GP}/postman/Deregister_Existing_Oracles.postman_collection.json --insecure -e Lab.postman_environment.json - # - newman run https://raw.githubusercontent.com/pm4ml/pm4ml-test-scripts/${TAG_PM4ML_GP}/postman/Onboard-Generic-FSP-Central_Ledger.postman_collection.json --insecure -e Lab.postman_environment.json --ssl-client-cert-list test_cert_list.json -d /tmp/onboarding_pm4ml_output.json || true - - newman run https://raw.githubusercontent.com/pm4ml/pm4ml-test-scripts/${TAG_PM4ML_GP}/postman/Account_Alias_Oracle_Registration.postman_collection.json --insecure -e Lab.postman_environment.json --ssl-client-cert-list test_cert_list.json || true - # - newman run https://raw.githubusercontent.com/pm4ml/pm4ml-test-scripts/${TAG_PM4ML_GP}/postman/Account_Alias_Oracle_Setup.postman_collection.json --insecure -e Lab.postman_environment.json --ssl-client-cert-list test_cert_list.json -d /tmp/onboarding_pm4ml_output.json || true - - newman run https://raw.githubusercontent.com/pm4ml/pm4ml-test-scripts/${TAG_PM4ML_GP}/postman/Onboard-Generic-FSP-Sim_Backend.postman_collection.json --insecure -e Lab.postman_environment.json --ssl-client-cert-list test_cert_list.json -d onboarding_internal_pm4ml_output.json || true - - newman run Myanmar-Golden-Path.postman_collection.json --ssl-client-cert-list test_cert_list.json --folder loan-repayment --insecure -e Lab.postman_environment.json --reporters html,junit,cli --reporter-html-export ${CI_PROJECT_DIR}/alias_report.html --reporter-junit-export ${CI_PROJECT_DIR}/alias_report.xml --reporter-html-template ../tests/template-default-colored.hbs -x - after_script: -# - curl https://slack.com/api/files.upload -# -F file=@"Platform_report.html" -# -F channels="${SLACK_GITLABCI_ALERTS_CHANNEL_NAME}" -# -F token="${SLACK_GITLABCI_ALERTS_TOKEN}" -# -F title="Myanmar Goldenpath Test (Postman) Results" -# -F filetype="html" -# artifacts: -# reports: -# junit: alias_report.xml - - - curl https://slack.com/api/files.upload - -F file=@"alias_report.html" - -F channels="${SLACK_GITLABCI_GP_REPORTS_CHANNEL_NAME}" - -F token="${SLACK_GITLABCI_ALERTS_TOKEN}" - -F title="Alias Goldenpath Test (Postman) Results.'${ENVIRONMENT}'-$(date +\"%Y-%m-%d-%H:%M:%S\")" - -F filetype="html" - artifacts: - reports: - junit: alias_report.xml - - -3. Manual Run Platform GP Tests: - stage: "Run Tests" - extends: .Run Platform GP Tests-job-template - when: manual - except: - - schedules - -3. Scheduled Run Platform GP Tests: - stage: "Run Tests" - allow_failure: true - extends: .Run Platform GP Tests-job-template - only: - refs: - - schedules - variables: - - $SCHEDULE_PLATFORM == "PLAT_GP" - -.ttk-snippets: - image: &ttk-snippets-image - name: mojaloop/ml-testing-toolkit:v13.5.1 - set-env-vars: &ttk-snippets-set-env-vars - - export TERRAFORM_VERSION=1.0.5 - - export GLIBC_VER=2.34-r0 - - apk --no-cache add binutils curl jq && curl -sL https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub -o /etc/apk/keys/sgerrand.rsa.pub && curl -sLO https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC_VER}/glibc-${GLIBC_VER}.apk && curl -sLO https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC_VER}/glibc-bin-${GLIBC_VER}.apk && apk add --no-cache glibc-${GLIBC_VER}.apk glibc-bin-${GLIBC_VER}.apk && curl -sL https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip -o awscliv2.zip && unzip awscliv2.zip && aws/install && rm -rf awscliv2.zip aws /usr/local/aws-cli/v2/*/dist/aws_completer /usr/local/aws-cli/v2/*/dist/awscli/data/ac.index /usr/local/aws-cli/v2/*/dist/awscli/examples glibc-${GLIBC_VER}.apk glibc-bin-${GLIBC_VER}.apk /var/cache/apk/* - - wget -q https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip -O /tmp/terraform_${TERRAFORM_VERSION}_linux_amd64.zip && unzip /tmp/terraform_${TERRAFORM_VERSION}_linux_amd64.zip -d /usr/bin/ && rm /tmp/terraform_${TERRAFORM_VERSION}_linux_amd64.zip - - export BUCKET=$(grep bucket backend.hcl | cut -f2 -d '=' | tr -d '"' | xargs) - - for var in $(jq -r 'to_entries[] | "\(.key)=\(.value)\n"' ./workbench-config.json); do export $var; done - - for var in $(jq -r 'to_entries[] | "TF_VAR_\(.key)=\(.value)\n"' ./workbench-config.json); do export $var; done - - export TTK_TESTCASES_VERSION=${helm_mojaloop_version} - - export AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID} - - export AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} - - export AWS_DEFAULT_REGION=${region} - - aws s3 sync s3://$BUCKET/$environment ./k8ss3/ --sse || true - - export private_subdomain=$(terraform output -state=k8ss3/terraform.tfstate private_subdomain | tr -d '"') - - export ttkBackendUrl=http://ttkbackend.${private_subdomain}; - download-ttk-test-cases: &ttk-snippets-download-ttk-test-cases - - | - cd /opt/mojaloop-testing-toolkit; - export testCasesZipUrl=https://github.com/mojaloop/testing-toolkit-test-cases/archive/refs/tags/v${TTK_TESTCASES_VERSION}.zip; - echo "Downloading the test collection from $testCasesZipUrl"; - wget $testCasesZipUrl -O downloaded-test-collections.zip; - mkdir tmp_test_cases; - unzip -d tmp_test_cases -o downloaded-test-collections.zip; - rm downloaded-test-collections.zip - download-default-env: &ttk-snippets-download-default-env - - | - echo "Downloading the default environment file..."; - cd /opt/mojaloop-testing-toolkit; - wget -O user_config.json $ttkBackendUrl/api/config/user; - defaultEnvFileName=`node -pe 'JSON.parse(process.argv[1]).runtime.DEFAULT_ENVIRONMENT_FILE_NAME' "$(cat user_config.json)"`; - wget -O environment_http_response.json $ttkBackendUrl/api/samples/loadFolderWise?environment=examples/environments/$defaultEnvFileName; - node -pe 'JSON.stringify({inputValues: JSON.parse(process.argv[1]).body.environment})' "$(cat environment_http_response.json)" > cli-testcase-environment.json; - execute-provisioning-collection: &ttk-snippets-execute-provisioning-collection - - | - cd /opt/mojaloop-testing-toolkit; - export provisioningTestCasesPathInZip=testing-toolkit-test-cases-${TTK_TESTCASES_VERSION}/collections/hub/provisioning; - npm run cli -- -e cli-testcase-environment.json -i tmp_test_cases/$provisioningTestCasesPathInZip -u $ttkBackendUrl --report-format html --report-target file://${CI_PROJECT_DIR}/ttk-provisioning-report.html || true; - echo "Done"; - execute-golden-path-collection: &ttk-snippets-execute-golden-path-collection - - | - cd /opt/mojaloop-testing-toolkit; - export goldenPathTestCasesPathInZip=testing-toolkit-test-cases-${TTK_TESTCASES_VERSION}/collections/hub/golden_path; - npm run cli -- -e cli-testcase-environment.json -i tmp_test_cases/$goldenPathTestCasesPathInZip -u $ttkBackendUrl --report-format html --report-target file://${CI_PROJECT_DIR}/ttk-goldenpath-report.html; - echo "Done"; - execute-bulk-transfers-collection: &ttk-snippets-execute-bulk-transfers-collection - - | - cd /opt/mojaloop-testing-toolkit; - export bulkTransfersTestCasesPathInZip=testing-toolkit-test-cases-${TTK_TESTCASES_VERSION}/collections/hub/other_tests/bulk_transfers; - npm run cli -- -e cli-testcase-environment.json -i tmp_test_cases/$bulkTransfersTestCasesPathInZip -u $ttkBackendUrl --report-format html --report-target file://${CI_PROJECT_DIR}/ttk-bulk-transfers-report.html; - echo "Done"; - artifacts: &ttk-snippets-artifacts - when: always - paths: - - ${CI_PROJECT_DIR}/ttk-provisioning-report.html - - ${CI_PROJECT_DIR}/ttk-goldenpath-report.html - -4. Run TTK Tests: - stage: "Run Tests" - image: *ttk-snippets-image - before_script: - - export TTK_TESTCASES_VERSION=${helm_mojaloop_version} - script: - - *ttk-snippets-set-env-vars - - *ttk-snippets-download-ttk-test-cases - - *ttk-snippets-download-default-env - - *ttk-snippets-execute-provisioning-collection - - *ttk-snippets-execute-golden-path-collection - artifacts: *ttk-snippets-artifacts - when: manual - -5. Run RBAC Tests: - stage: "Run Tests" - script: - - helm --kubeconfig=admin-gateway.conf test bof --filter name=bof-rbac-tests -n mojaloop || true - - kubectl --kubeconfig=admin-gateway.conf -n mojaloop logs bof-rbac-tests - - exit `kubectl --kubeconfig=admin-gateway.conf -n mojaloop get pod bof-rbac-tests --output="jsonpath={.status.containerStatuses[].state.terminated.exitCode}"` - when: manual - -6. Run Report Tests: - stage: "Run Tests" - script: - - helm --kubeconfig=admin-gateway.conf test bof --filter name=bof-report-tests -n mojaloop || true - - kubectl --kubeconfig=admin-gateway.conf -n mojaloop logs bof-report-tests - - exit `kubectl --kubeconfig=admin-gateway.conf -n mojaloop get pod bof-report-tests --output="jsonpath={.status.containerStatuses[].state.terminated.exitCode}"` - when: manual - -7. Run MCM Tests: - stage: "Run Tests" - script: - - helm --kubeconfig=admin-gateway.conf test connection-manager -n mcm || true - - kubectl --kubeconfig=admin-gateway.conf -n mcm logs connection-manager-api-test - - exit `kubectl --kubeconfig=admin-gateway.conf -n mcm get pod connection-manager-api-test --output="jsonpath={.status.containerStatuses[].state.terminated.exitCode}"` - when: manual - -8. Run PM4ML TTK Tests: - stage: "Run Tests" - image: *ttk-snippets-image - before_script: - - export TTK_TESTCASES_VERSION=${helm_mojaloop_version} - script: - - *ttk-snippets-set-env-vars - - | - echo "Downloading the test collection..."; - cd /opt/mojaloop-testing-toolkit; - export testCasesZipUrl=https://github.com/pm4ml/pm4ml-test-scripts/archive/refs/tags/v${ttk_pm4ml_gp_version}.zip; - wget $testCasesZipUrl -O downloaded-test-collections.zip; - mkdir tmp_test_cases; - unzip -d tmp_test_cases -o downloaded-test-collections.zip; - - | - cd /opt/mojaloop-testing-toolkit - export pm4mlgoldenPathTestCasesPathInZip=pm4ml-test-scripts-${ttk_pm4ml_gp_version}/ttk/collections/feature_tests; - npm run cli -- -e $CI_PROJECT_DIR/tests/ttk-prod-dev-env.json -i tmp_test_cases/$pm4mlgoldenPathTestCasesPathInZip -u $ttkBackendUrl --report-format html --report-target file://${CI_PROJECT_DIR}/ttk-pm4ml-report.html --extra-summary-information="Test Suite:PM4ML GP TTK,Job ID: $CI_JOB_ID"; - # Use the following syntax instead of the above line for slack notification and comment out artifacts. We can not have both with the current TTK CLI options. - # npm run cli -- -e $CI_PROJECT_DIR/tests/ttk-prod-dev-env.json -i tmp_test_cases/$pm4mlgoldenPathTestCasesPathInZip -u $ttkBackendUrl --report-format html --report-target s3://${BUCKET}/${environment}/pm4ml-ttk-goldenpath-report.html --slack-webhook-url=${SLACK_WEBHOOK_URL_HERE} --extra-summary-information="Test Suite:PM4ML GP TTK,Job ID: $CI_JOB_ID"; - echo "Done"; - artifacts: &ttk-snippets-artifacts - when: always - paths: - - ${CI_PROJECT_DIR}/ttk-pm4ml-report.html - when: manual - -9. Run TTK Bulk Tests: - stage: "Run Tests" - image: *ttk-snippets-image - before_script: - - export TTK_TESTCASES_VERSION=${helm_mojaloop_version} - script: - - *ttk-snippets-set-env-vars - - *ttk-snippets-download-ttk-test-cases - - *ttk-snippets-download-default-env - - *ttk-snippets-execute-provisioning-collection - - *ttk-snippets-execute-bulk-transfers-collection - artifacts: *ttk-snippets-artifacts - when: manual - -1. Run External PM4ML Onboarding: - stage: "Maintain Platform" - script: - - aws s3 sync s3://${BUCKET}/${environment}/k8s-cluster/sim_tests/ $CI_IMAGE_PROJECT_DIR/sim_tests --sse - - aws s3 sync s3://${BUCKET}/${environment}/k8s-cluster/secrets_chart/ $CI_IMAGE_PROJECT_DIR/secrets_chart --sse - - aws s3 cp s3://${BUCKET}/${environment}/k3saddons/onboarding_external_pm4ml_output.json $CI_IMAGE_PROJECT_DIR/sim_tests - - cd sim_tests - - newman run https://raw.githubusercontent.com/pm4ml/pm4ml-test-scripts/${TAG_PM4ML_GP}/postman/Onboard-Generic-FSP-Central_Ledger.postman_collection.json --insecure -e Lab.postman_environment.json -d onboarding_external_pm4ml_output.json - when: manual - -2. Update oathkeeper rules: - stage: "Maintain Platform" - script: - - export public_subdomain=$(terraform output -state=terraform.tfstate public_subdomain | tr -d '"') - - helm -n mojaloop -i --kubeconfig=admin-gateway.conf --set-string base_domain=${public_subdomain} --set-string bof_release_name=bof --set-string moja_release_name=moja upgrade bof-oathkeeper-rules $CI_IMAGE_PROJECT_DIR/bof-custom-resources/oathkeeper-rules - when: manual - -3. Update mojaloop roles: - stage: "Maintain Platform" - script: - - | - public_subdomain=$(terraform output -state=terraform.tfstate public_subdomain | tr -d '"'); - validation_api_url=https://bofapi.${public_subdomain}/operator/validate/role-permissions; - echo ${validation_api_url}; - mojaloop_roles=`cat ./bof-custom-resources/role-permissions/mojaloop-roles.json`; - permission_exclusions=`cat ./bof-custom-resources/role-permissions/permission-exclusions.json`; - request_body='{ "rolePermissions":'${mojaloop_roles}',"permissionExclusions":'${permission_exclusions}'}'; - response_code=$(curl -s -X POST ${validation_api_url} -H 'Content-Type: application/json' -d "${request_body}" -w "%{http_code}" -o response_body.txt); - if [ $response_code != 200 ]; then echo "VALIDATION FAILED"; cat response_body.txt; exit 1; else echo "VALIDATION PASSED"; fi; - - cd $CI_IMAGE_PROJECT_DIR/terraform/k8s-setup/mojaloop-roles - - terraform init -backend-config ${CI_PROJECT_DIR}/backend.hcl - - terraform apply -auto-approve -var="project_root_path=$CI_IMAGE_PROJECT_DIR" - when: manual - -4. Update custom reports: - stage: "Maintain Platform" - script: - - cd $CI_IMAGE_PROJECT_DIR/terraform/k8s-setup/mojaloop-custom-reports - - terraform init -backend-config ${CI_PROJECT_DIR}/backend.hcl - - terraform apply -auto-approve -var="project_root_path=$CI_IMAGE_PROJECT_DIR" - when: manual - -11. Gateway K8S Cluste Deployments: - stage: "Run Tests" - script: - - aws s3 cp s3://${BUCKET}/${environment}/admin-gateway.conf ${CI_IMAGE_PROJECT_DIR}/reports/admin-gateway.conf - - cd ${CI_IMAGE_PROJECT_DIR}/reports - - export KUBECONFIG=admin-gateway.conf - - export REPORT_NAME=GW_K8S_CLUSTER_$(date +"%Y-%m-%d-%M").csv - - echo "Namespace,Helm Chart,Deployment Name,Docker image,Creation Timestamp,Number of Available Replicas,Available Status Change Timestamp,Is Available?" > ${CI_PROJECT_DIR}/gw-cluster-report.csv - - kubectl get deployments --all-namespaces -o=jsonpath="{range .items[*]}{.metadata.namespace}{','}{.metadata.labels.chart}{','}{.metadata.name}{','}{.spec.template.spec.containers..image}{','}{.metadata.creationTimestamp}{','}{.status.availableReplicas}{','}{.status.conditions[?(@.type=='Available')].lastUpdateTime}{','}{.status.conditions[?(@.type=='Available')].status}{'\n'}{end}" >> ${CI_PROJECT_DIR}/gw-cluster-report.csv -# - curl -F file=@"GW_K8S_CLUSTER_${REPORT_DATE}.csv" -F channels="${SLACK_CHANNEL}" -F token="${SLACK_API_TOKEN}" -F title="k8s Deployment Report for ${K8S_CLUSTER_NAME_DMZ_DEV} cluster on ${REPORT_DATE}" https://slack.com/api/files.upload - when: manual - artifacts: - paths: - - gw-cluster-report.csv - -12. Add-ons K8S Cluste Deployments: - stage: "Run Tests" - script: - - aws s3 cp s3://${BUCKET}/${environment}/admin-add-ons.conf ${CI_IMAGE_PROJECT_DIR}/reports/admin-add-ons.conf - - cd ${CI_IMAGE_PROJECT_DIR}/reports - - export KUBECONFIG=admin-add-ons.conf - - echo "Namespace,Helm Chart,Deployment Name,Docker image,Creation Timestamp,Number of Available Replicas,Available Status Change Timestamp,Is Available?" > ${CI_PROJECT_DIR}/add-ons-cluster-report.csv - - kubectl get deployments --all-namespaces -o=jsonpath="{range .items[*]}{.metadata.namespace}{','}{.metadata.labels.chart}{','}{.metadata.name}{','}{.spec.template.spec.containers..image}{','}{.metadata.creationTimestamp}{','}{.status.availableReplicas}{','}{.status.conditions[?(@.type=='Available')].lastUpdateTime}{','}{.status.conditions[?(@.type=='Available')].status}{'\n'}{end}" >> ${CI_PROJECT_DIR}/add-ons-cluster-report.csv -# - curl -F file=@"ADD-ONS_K8S_CLUSTER_${REPORT_DATE}.csv" -F channels="${SLACK_CHANNEL}" -F token="${SLACK_API_TOKEN}" -F title="k8s Deployment Report for ${K8S_CLUSTER_NAME_DMZ_DEV} cluster on ${REPORT_DATE}" https://slack.com/api/files.upload - when: manual - artifacts: - paths: - - add-ons-cluster-report.csv - -14. Automated Env Report : - stage: "Run Tests" - script: -# - aws s3 sync s3://${BUCKET}/${environment}/k8s-cluster/sim_tests/ $CI_IMAGE_PROJECT_DIR/sim_tests --sse - - cd /tmp && git clone https://github.com/ashokalinux/Env-template.git - - cd /tmp/Env-template - - export - - terraform init - - terraform plan -out=plan.env - - terraform apply "plan.env" - - pwd - - ls -R - - cp ./Auto/Env/Lab.auto_environment.html $CI_IMAGE_PROJECT_DIR - - ls -l $CI_IMAGE_PROJECT_DIR -# - aws s3 sync $CI_IMAGE_PROJECT_DIR/terraform/Auto s3://${BUCKET}/${environment}/Auto --sse - - aws s3 cp $CI_IMAGE_PROJECT_DIR/Lab.auto_environment.html s3://${BUCKET}/${environment}/Auto/Env/Lab.auto_environment.html --sse - when: manual - artifacts: - paths: - - Lab.auto_environment.html - - - -1. Setup MFI oracle: - stage: "Maintain Platform" - script: - - aws s3 sync s3://${BUCKET}/${environment}/k8s-cluster/sim_tests/ $CI_IMAGE_PROJECT_DIR/sim_tests --sse - # - aws s3 sync s3://${BUCKET}/${environment}/k8s-cluster/secrets_chart/ $CI_IMAGE_PROJECT_DIR/secrets_chart --sse - - aws s3 cp s3://${BUCKET}/${environment}/k3saddons/onboarding_internal_pm4ml_output.json $CI_IMAGE_PROJECT_DIR/sim_tests - - cd sim_tests - - newman run https://raw.githubusercontent.com/pm4ml/pm4ml-test-scripts/${TAG_PM4ML_GP}/postman/Deregister_Existing_Oracles.postman_collection.json --insecure -e Lab.postman_environment.json - - newman run https://raw.githubusercontent.com/pm4ml/pm4ml-test-scripts/${TAG_PM4ML_GP}/postman/Account_Alias_Oracle_Registration.postman_collection.json --insecure -e Lab.postman_environment.json --ssl-client-cert-list test_cert_list.json || true - - newman run https://raw.githubusercontent.com/pm4ml/pm4ml-test-scripts/${TAG_PM4ML_GP}/postman/Account_Alias_Oracle_Setup.postman_collection.json --insecure -e Lab.postman_environment.json --ssl-client-cert-list test_cert_list.json -d onboarding_internal_pm4ml_output.json || true -# - newman run https://raw.githubusercontent.com/pm4ml/pm4ml-test-scripts/${TAG_PM4ML_GP}/postman/Onboard-Generic-FSP-Central_Ledger.postman_collection.json --insecure -e Lab.postman_environment.json -d onboarding_internal_pm4ml_output.json - when: manual + PLATFORM_IAC_IMAGE_VERSION: "2.1.7" + K3S_IAC_IMAGE_VERSION: "0.17.0" \ No newline at end of file