Skip to content

Commit

Permalink
Finalizing The Terraform/Manifests Secrets Additions (#1602)
Browse files Browse the repository at this point in the history
* fixing some variables that weren't quite right

* a few more tweaks

* upadting dependencies

* Moving changes to production and dev code

* adding mocks and making my named dependency more descriptive

* adding plan to mocks and formatting file

* formatting

* adding changes to other enviornmebnts

* removing old variable that doesn't do anything

* removing another bad var
  • Loading branch information
P0NDER0SA authored Oct 22, 2024
1 parent a80b657 commit df9f8c2
Show file tree
Hide file tree
Showing 13 changed files with 282 additions and 15 deletions.
33 changes: 33 additions & 0 deletions .github/workflows/merge_to_main_production.yml
Original file line number Diff line number Diff line change
Expand Up @@ -693,6 +693,39 @@ jobs:
cd env/${{env.ENVIRONMENT}}/newrelic
terragrunt apply --terragrunt-non-interactive -auto-approve
terragrunt-apply-manifest_secrets:
if: |
always() &&
!contains(needs.*.result, 'failure') &&
!contains(needs.*.result, 'cancelled')
needs: [terragrunt-apply-rds, terragrunt-apply-elasticache, terragrunt-apply-eks, terragrunt-apply-ecr, terragrunt-apply-lambda-api, terragrunt-apply-heartbeat, terragrunt-apply-database-tools, terragrunt-apply-quicksight, terragrunt-apply-lambda-google-cidr, terragrunt-apply-ses_to_sqs_email_callbacks, terragrunt-apply-sns_to_sqs_sms_callbacks, terragrunt-apply-pinpoint_to_sqs_sms_callbacks, terragrunt-apply-system_status, terragrunt-apply-system_status_static_site, terragrunt-apply-newrelic]
runs-on: ubuntu-latest

steps:
- name: Checkout
uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0

- name: setup-terraform
uses: ./.github/actions/setup-terraform
with:
role_to_assume: arn:aws:iam::${{env.ACCOUNT_ID}}:role/notification-terraform-apply
role_session_name: NotifyTerraformApply

- name: Install 1Pass CLI
run: |
curl -o 1pass.deb https://downloads.1password.com/linux/debian/amd64/stable/1password-cli-amd64-latest.deb
sudo dpkg -i 1pass.deb
sudo mkdir -p aws
cd aws
op read op://ppnxsriom3alsxj4ogikyjxlzi/"TERRAFORM_SECRETS_${{env.ENVIRONMENT}}"/notesPlain > ${{env.ENVIRONMENT}}.tfvars
- name: terragrunt apply manifest_secrets
run: |
cd env/${{env.ENVIRONMENT}}/manifest_secrets
terragrunt apply --terragrunt-non-interactive -auto-approve
bump-version-and-push-tag:
if: |
always() &&
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/merge_to_main_staging.yml
Original file line number Diff line number Diff line change
Expand Up @@ -765,7 +765,7 @@ jobs:
always() &&
!contains(needs.*.result, 'failure') &&
!contains(needs.*.result, 'cancelled')
needs: [terragrunt-apply-rds]
needs: [terragrunt-apply-rds, terragrunt-apply-elasticache, terragrunt-apply-eks, terragrunt-apply-ecr, terragrunt-apply-lambda-api, terragrunt-apply-lambda-admin-pr, terragrunt-apply-performance-test, terragrunt-apply-heartbeat, terragrunt-apply-database-tools, terragrunt-apply-quicksight, terragrunt-apply-lambda-google-cidr, terragrunt-apply-ses_to_sqs_email_callbacks, terragrunt-apply-sns_to_sqs_sms_callbacks, terragrunt-apply-pinpoint_to_sqs_sms_callbacks, terragrunt-apply-system_status, terragrunt-apply-system_status_static_site, terragrunt-apply-newrelic]
runs-on: ubuntu-latest

steps:
Expand Down
31 changes: 31 additions & 0 deletions .github/workflows/terragrunt_create_dev_environment.yml
Original file line number Diff line number Diff line change
Expand Up @@ -772,6 +772,37 @@ jobs:
# cd env/${{env.ENVIRONMENT}}/newrelic
# terragrunt apply --terragrunt-non-interactive -auto-approve

terragrunt-apply-manifest_secrets:
if: |
always() &&
!contains(needs.*.result, 'failure') &&
!contains(needs.*.result, 'cancelled')
needs: [terragrunt-apply-common, terragrunt-apply-eks, terragrunt-apply-rds, terragrunt-apply-elasticache, terragrunt-apply-ecr, terragrunt-apply-ecr-us-east, terragrunt-apply-lambda-admin-pr, terragrunt-apply-performance-test, terragrunt-apply-heartbeat, terragrunt-apply-database-tools, terragrunt-apply-lambda-google-cidr, terragrunt-apply-ses_to_sqs_email_callbacks, terragrunt-apply-sns_to_sqs_sms_callbacks, terragrunt-apply-pinpoint_to_sqs_sms_callbacks, terragrunt-apply-system_status, terragrunt-apply-system_status_static_site]
runs-on: ubuntu-latest

steps:
- name: Checkout
uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0

- name: setup-terraform
uses: ./.github/actions/setup-terraform
with:
role_to_assume: arn:aws:iam::${{env.ACCOUNT_ID}}:role/notification-terraform-apply
role_session_name: NotifyTerraformApply

- name: Install 1Pass CLI
run: |
curl -o 1pass.deb https://downloads.1password.com/linux/debian/amd64/stable/1password-cli-amd64-latest.deb
sudo dpkg -i 1pass.deb
sudo mkdir -p aws
cd aws
op read op://4eyyuwddp6w4vxlabrr2i2duxm/"TERRAFORM_SECRETS_${{env.ENVIRONMENT}}"/notesPlain > ${{env.ENVIRONMENT}}.tfvars
- name: terragrunt apply manifest_secrets
run: |
cd env/${{env.ENVIRONMENT}}/manifest_secrets
terragrunt apply --terragrunt-non-interactive -auto-approve
deploy-application:
if: |
always() &&
Expand Down
40 changes: 39 additions & 1 deletion .github/workflows/terragrunt_plan_dev.yml
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,8 @@ jobs:
system_status: ${{ steps.filter.outputs.system_status }}
system_status_static_site: ${{ steps.filter.outputs.system_status_static_site }}
newrelic: ${{ steps.filter.outputs.newrelic }}

manifest_secrets: ${{ steps.filter.outputs.manifest_secrets }}

steps:
- uses: dorny/paths-filter@4512585405083f25c027a35db413c2b3b9006d50 # v2.11.1
id: filter
Expand Down Expand Up @@ -112,6 +113,9 @@ jobs:
newrelic:
- 'aws/newrelic/**'
- 'env/${{env.ENVIRONMENT}}/newrelic/**'
manifest_secrets:
- 'aws/manifest_secrets/**'
- 'env/${{env.ENVIRONMENT}}/manifest_secrets/**'
terragrunt-plan-common:
runs-on: ubuntu-latest
Expand Down Expand Up @@ -867,3 +871,37 @@ jobs:
terragrunt: "true"
terraform-init: |
-upgrade
terragrunt-plan-manifest_secrets:
if: |
always() &&
!contains(needs.*.result, 'failure') &&
!contains(needs.*.result, 'cancelled')
needs: [terragrunt-plan-rds, terragrunt-plan-elasticache, terragrunt-plan-eks, terragrunt-plan-lambda-api, terragrunt-plan-lambda-admin-pr, terragrunt-plan-performance-test, terragrunt-plan-heartbeat, terragrunt-plan-database-tools, terragrunt-plan-quicksight, terragrunt-plan-lambda-google-cidr, terragrunt-plan-ses_to_sqs_email_callbacks, terragrunt-plan-sns_to_sqs_sms_callbacks, terragrunt-plan-pinpoint_to_sqs_sms_callbacks, terragrunt-plan-system_status, terragrunt-plan-system_status_static_site, terragrunt-plan-newrelic]
runs-on: ubuntu-latest
env:
COMPONENT: "manifest_secrets"
steps:
- name: Checkout
uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
- uses: ./.github/actions/setup-terraform
with:
role_to_assume: arn:aws:iam::${{env.ACCOUNT_ID}}:role/notification-terraform-apply
role_session_name: NotifyTerraformApply
- name: Install 1Pass CLI and Download TFVars
run: |
curl -o 1pass.deb https://downloads.1password.com/linux/debian/amd64/stable/1password-cli-amd64-latest.deb
sudo dpkg -i 1pass.deb
sudo mkdir -p aws && cd aws
op read op://4eyyuwddp6w4vxlabrr2i2duxm/"TERRAFORM_SECRETS_${{env.ENVIRONMENT}}"/notesPlain > ${{env.ENVIRONMENT}}.tfvars
- name: Terragrunt plan ${{env.COMPONENT}}
uses: cds-snc/terraform-plan@7f4ce4a4bdffaba639d32a45272804e37a569408 # v3.0.6
with:
directory: "env/${{env.ENVIRONMENT}}/${{env.COMPONENT}}"
comment-delete: "true"
comment-title: "${{env.ENVIRONMENT}}: ${{env.COMPONENT}}"
github-token: "${{ secrets.GITHUB_TOKEN }}"
terragrunt: "true"
terraform-init: |
-upgrade
33 changes: 33 additions & 0 deletions .github/workflows/terragrunt_plan_production.yml
Original file line number Diff line number Diff line change
Expand Up @@ -706,3 +706,36 @@ jobs:
terragrunt: "true"
terraform-init: |
-upgrade
terragrunt-plan-manifest_secrets:
if: |
always() &&
!contains(needs.*.result, 'failure') &&
!contains(needs.*.result, 'cancelled')
needs: [terragrunt-plan-rds, terragrunt-plan-elasticache, terragrunt-plan-eks, terragrunt-plan-lambda-api, terragrunt-plan-heartbeat, terragrunt-plan-database-tools, terragrunt-plan-quicksight, terragrunt-plan-lambda-google-cidr, terragrunt-plan-ses_to_sqs_email_callbacks, terragrunt-plan-sns_to_sqs_sms_callbacks, terragrunt-plan-pinpoint_to_sqs_sms_callbacks, terragrunt-plan-system_status, terragrunt-plan-system_status_static_site, terragrunt-plan-newrelic]
runs-on: ubuntu-latest
env:
COMPONENT: "manifest_secrets"
steps:
- name: Checkout
uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
- uses: ./.github/actions/setup-terraform
with:
role_to_assume: arn:aws:iam::${{env.ACCOUNT_ID}}:role/notification-terraform-plan
role_session_name: NotifyTerraformPlan
- name: Install 1Pass CLI and Download TFVars
run: |
curl -o 1pass.deb https://downloads.1password.com/linux/debian/amd64/stable/1password-cli-amd64-latest.deb
sudo dpkg -i 1pass.deb
sudo mkdir -p aws && cd aws
op read op://ppnxsriom3alsxj4ogikyjxlzi/"TERRAFORM_SECRETS_${{env.ENVIRONMENT}}"/notesPlain > ${{env.ENVIRONMENT}}.tfvars
- name: Terragrunt plan ${{env.COMPONENT}}
uses: cds-snc/terraform-plan@7f4ce4a4bdffaba639d32a45272804e37a569408 # v3.0.6
with:
directory: "env/${{env.ENVIRONMENT}}/${{env.COMPONENT}}"
comment-delete: "true"
comment-title: "${{env.ENVIRONMENT}}: ${{env.COMPONENT}}"
github-token: "${{ secrets.GITHUB_TOKEN }}"
terragrunt: "true"
terraform-init: |
-upgrade
2 changes: 1 addition & 1 deletion .github/workflows/terragrunt_plan_staging.yml
Original file line number Diff line number Diff line change
Expand Up @@ -886,7 +886,7 @@ jobs:
always() &&
!contains(needs.*.result, 'failure') &&
!contains(needs.*.result, 'cancelled')
needs: [terragrunt-plan-rds]
needs: [terragrunt-plan-rds, terragrunt-plan-elasticache, terragrunt-plan-eks, terragrunt-plan-lambda-api, terragrunt-plan-lambda-admin-pr, terragrunt-plan-performance-test, terragrunt-plan-heartbeat, terragrunt-plan-database-tools, terragrunt-plan-quicksight, terragrunt-plan-lambda-google-cidr, terragrunt-plan-ses_to_sqs_email_callbacks, terragrunt-plan-sns_to_sqs_sms_callbacks, terragrunt-plan-pinpoint_to_sqs_sms_callbacks, terragrunt-plan-system_status, terragrunt-plan-system_status_static_site, terragrunt-plan-newrelic]
runs-on: ubuntu-latest
env:
COMPONENT: "manifest_secrets"
Expand Down
28 changes: 21 additions & 7 deletions aws/manifest_secrets/secrets.tf
Original file line number Diff line number Diff line change
Expand Up @@ -329,17 +329,19 @@ resource "aws_secretsmanager_secret_version" "manifest_aws_pinpoint_default_pool
}

resource "aws_secretsmanager_secret" "manifest_sqlalachemy_database_uri" {
name = "SQLALCHEMY_DATABASE_URI"
name = "MANIFEST_SQLALCHEMY_DATABASE_URI"
recovery_window_in_days = 0
}

# THESE BELOW ARE ARE DEPENDENT ON DYNAMICALLY GENERATED AWS INFORMATION

resource "aws_secretsmanager_secret_version" "manifest_sqlalachemy_database_uri" {
secret_id = aws_secretsmanager_secret.manifest_sqlalachemy_database_uri.id
secret_string = "postgresql://${var.app_db_user}:${var.app_db_user_password}@${var.database_read_write_proxy_endpoint}/${var.app_db_database_name}"
}

resource "aws_secretsmanager_secret" "manifest_sqlalachemy_database_reader_uri" {
name = "SQLALCHEMY_DATABASE_READER_URI"
name = "MANIFEST_SQLALCHEMY_DATABASE_READER_URI"
recovery_window_in_days = 0
}

Expand All @@ -355,23 +357,35 @@ resource "aws_secretsmanager_secret" "manifest_postgres_host" {

resource "aws_secretsmanager_secret_version" "manifest_postgres_host_version" {
secret_id = aws_secretsmanager_secret.manifest_postgres_host.id
secret_string = "notification-canada-ca-${var.env}-cluster.${var.postgres_rds_instance_id}.${var.region}.rds.amazonaws.com"
secret_string = var.postgres_cluster_endpoint
}

resource "aws_secretsmanager_secret" "manifest_postgres_sql" {
name = "MANIFEST_POSTGRES_SQL"
recovery_window_in_days = 0
}

resource "aws_secretsmanager_secret_version" "manifest_postgres_sql_version" {
secret_id = aws_secretsmanager_secret.manifest_postgres_sql.id
secret_string = "postgresql://${var.app_db_user}:${var.app_db_user_password}@${var.database_read_write_proxy_endpoint}/${var.app_db_database_name}"
}

resource "aws_secretsmanager_secret" "manifest_redis_publish_url" {
name = "MANIFEST_REDIS_PUBLISH_URL"
name = "MANIFEST_REDIS_PUBLISH_URL"
recovery_window_in_days = 0
}

resource "aws_secretsmanager_secret_version" "manifest_redis_publish_url" {
secret_id = aws_secretsmanager_secret.manifest_redis_publish_url.id
secret_string = "redis://notify-${var.env}-cluster-cache-az.${var.redis_cluster_security_group_id}.ng.0001.cac1.cache.amazonaws.com:6379"
secret_string = "redis://${var.redis_primary_endpoint_address}"
}

resource "aws_secretsmanager_secret" "manifest_redis_url" {
name = "MANIFEST_REDIS_URL"
name = "MANIFEST_REDIS_URL"
recovery_window_in_days = 0
}

resource "aws_secretsmanager_secret_version" "manifest_redis_url" {
secret_id = aws_secretsmanager_secret.manifest_redis_url.id
secret_string = "redis://notify-${var.env}-cluster-cache-az.${var.redis_cluster_security_group_id}.ng.0001.cac1.cache.amazonaws.com:6379"
secret_string = "redis://${var.redis_primary_endpoint_address}"
}
6 changes: 3 additions & 3 deletions aws/manifest_secrets/variables.tf
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,10 @@ variable "database_read_only_proxy_endpoint" {
type = string
}

variable "postgres_rds_instance_id" {
variable "postgres_cluster_endpoint" {
type = string
}

variable "redis_cluster_security_group_id" {
variable "redis_primary_endpoint_address" {
type = string
}
}
36 changes: 36 additions & 0 deletions env/dev/manifest_secrets/terragrunt.hcl
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
terraform {
source = "${get_env("ENVIRONMENT") == "production" ? "git::https://github.com/cds-snc/notification-terraform//aws/manifest_secrets?ref=v${get_env("INFRASTRUCTURE_VERSION")}" : "../../../aws//manifest_secrets"}"
}

dependencies {
paths = ["../rds", "../elasticache"]
}

dependency "rds" {
config_path = "../rds"
mock_outputs_allowed_terraform_commands = ["validate", "plan"]
mock_outputs = {
database_read_only_proxy_endpoint = "thisisamockstring_database_read_only_proxy_endpoint"
database_read_write_proxy_endpoint = "thisisamockstring_database_read_write_proxy_endpoint"
postgres_cluster_endpoint = "thisisamockstring_postgres_cluster_endpoint"
}
}

dependency "elasticache" {
config_path = "../elasticache"
mock_outputs_allowed_terraform_commands = ["validate", "plan"]
mock_outputs = {
redis_primary_endpoint_address = "thisisamockstring_redis_primary_endpoint_address"
}
}

include {
path = find_in_parent_folders()
}

inputs = {
database_read_only_proxy_endpoint = dependency.rds.outputs.database_read_only_proxy_endpoint
database_read_write_proxy_endpoint = dependency.rds.outputs.database_read_write_proxy_endpoint
postgres_cluster_endpoint = dependency.rds.outputs.postgres_cluster_endpoint
redis_primary_endpoint_address = dependency.elasticache.outputs.redis_primary_endpoint_address
}
36 changes: 36 additions & 0 deletions env/production/manifest_secrets/terragrunt.hcl
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
terraform {
source = "${get_env("ENVIRONMENT") == "production" ? "git::https://github.com/cds-snc/notification-terraform//aws/manifest_secrets?ref=v${get_env("INFRASTRUCTURE_VERSION")}" : "../../../aws//manifest_secrets"}"
}

dependencies {
paths = ["../rds", "../elasticache"]
}

dependency "rds" {
config_path = "../rds"
mock_outputs_allowed_terraform_commands = ["validate", "plan"]
mock_outputs = {
database_read_only_proxy_endpoint = "thisisamockstring_database_read_only_proxy_endpoint"
database_read_write_proxy_endpoint = "thisisamockstring_database_read_write_proxy_endpoint"
postgres_cluster_endpoint = "thisisamockstring_postgres_cluster_endpoint"
}
}

dependency "elasticache" {
config_path = "../elasticache"
mock_outputs_allowed_terraform_commands = ["validate", "plan"]
mock_outputs = {
redis_primary_endpoint_address = "thisisamockstring_redis_primary_endpoint_address"
}
}

include {
path = find_in_parent_folders()
}

inputs = {
database_read_only_proxy_endpoint = dependency.rds.outputs.database_read_only_proxy_endpoint
database_read_write_proxy_endpoint = dependency.rds.outputs.database_read_write_proxy_endpoint
postgres_cluster_endpoint = dependency.rds.outputs.postgres_cluster_endpoint
redis_primary_endpoint_address = dependency.elasticache.outputs.redis_primary_endpoint_address
}
36 changes: 36 additions & 0 deletions env/sandbox/manifest_secrets/terragrunt.hcl
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
terraform {
source = "${get_env("ENVIRONMENT") == "production" ? "git::https://github.com/cds-snc/notification-terraform//aws/manifest_secrets?ref=v${get_env("INFRASTRUCTURE_VERSION")}" : "../../../aws//manifest_secrets"}"
}

dependencies {
paths = ["../rds", "../elasticache"]
}

dependency "rds" {
config_path = "../rds"
mock_outputs_allowed_terraform_commands = ["validate", "plan"]
mock_outputs = {
database_read_only_proxy_endpoint = "thisisamockstring_database_read_only_proxy_endpoint"
database_read_write_proxy_endpoint = "thisisamockstring_database_read_write_proxy_endpoint"
postgres_cluster_endpoint = "thisisamockstring_postgres_cluster_endpoint"
}
}

dependency "elasticache" {
config_path = "../elasticache"
mock_outputs_allowed_terraform_commands = ["validate", "plan"]
mock_outputs = {
redis_primary_endpoint_address = "thisisamockstring_redis_primary_endpoint_address"
}
}

include {
path = find_in_parent_folders()
}

inputs = {
database_read_only_proxy_endpoint = dependency.rds.outputs.database_read_only_proxy_endpoint
database_read_write_proxy_endpoint = dependency.rds.outputs.database_read_write_proxy_endpoint
postgres_cluster_endpoint = dependency.rds.outputs.postgres_cluster_endpoint
redis_primary_endpoint_address = dependency.elasticache.outputs.redis_primary_endpoint_address
}
File renamed without changes.
Loading

0 comments on commit df9f8c2

Please sign in to comment.