Bugfix ignore regexes (#172)

Updated regex in template to be correct for the parameters,
Updated SourceCodeUrl to head of master branch
Fixed the interpretation of secret arns from lambda environment variable
GOOGLE_CREDENTIALS
GOOGLE_ADMIN
SCIM_ENDPOINT
SCIM_ACCESS_TOKEN
REGION
IDENTITY_STORE_ID
Fixed reading none-secret values that have default values from lambda environment variables
LOG_LEVEL
LOG_FORMAT
SYNC_METHOD
Change when log channel configured to respect log settings from above variables
By submitting this pull request, I confirm that you can use, modify, copy, and redistribute this contribution, under the terms of your choice.

update cleanup stage to remove the correct stack.

* Update template.yaml

Changed Source code Url to be head of master.

* Running Test this commit will be rolled back.

* Retrieve secret arns from env vars

Despite passing the secrets arns to the function via the env variables. This were never actually used instead using hard code secret short names.

* Update template.yaml

Remove redundant prefix from Env var names.

* Update secrets.go

Adding import for os module

* Update developer.yaml

Adding missing permission to CLI test role.

* Update Makefile

Copy arm64 executable to correct path for building SAM package

* Read secret arn from Env Var,

Actually read secret arn from Env Var, and use fall back to old static names if the Env is missing or empty.

* case-sensetivity env vars?

Checking whether case-sensetivity is impacts the import of envs

* Revert "case-sensetivity env vars?"

This reverts commit 96f9ccb.

* Update config.go

* Update config_test.go

* Revert "Update config.go"

This reverts commit 5b54099.

* Revert "Update config_test.go"

This reverts commit 29315a1.

* Bypass viper for lambda Env Vars

viper doesn't handle lambda Env Vars well, so conditionally overriding normal viper behaviour.

* Update root.go

Changed logConfig timing so log settings from Lambda Env Vars are respected.

Makefile

@@ -22,7 +22,7 @@ clean:
 
 build-SSOSyncFunction:
 	GOOS=linux GOARCH=arm64 go build -o bootstrap main.go
-	cp ./bootstrap $(ARTIFACTS_DIR)/.
+	cp dist/ssosync_linux_arm64/ssosync $(ARTIFACTS_DIR)/bootstrap
 
 .PHONY: install
 install:
@@ -49,6 +49,7 @@ publish:
 
 .PHONY: package
 package: build
+	cp dist/ssosync_linux_arm64/ssosync ./bootstrap
 	sam package --s3-bucket $(S3_BUCKET) --output-template-file $(PACKAGED_TEMPLATE) --s3-prefix $(S3_PREFIX)
 
 .PHONY: deploy

cicd/cloudformation/developer.yaml

@@ -29,6 +29,11 @@ Parameters:
     Type: String
     NoEcho: True
 
+  SecretsConfig:
+    Description:  Output from the secrets.yaml stack
+    Type: String
+    AllowedPattern: '(arn:aws:secretsmanager:((us(-gov)?|ap|ca|cn|eu|sa)-(central|(north|south)?(east|west)?)-\d):[0-9]{8,12}:secret:[a-zA-Z0-9/_+=.@-]{1,512})(,(arn:aws:secretsmanager:((us(-gov)?|ap|ca|cn|eu|sa)-(central|(north|south)?(east|west)?)-\d):[0-9]{8,12}:secret:[a-zA-Z0-9/_+=.@-]{1,512})){3}((,arn:aws:secretsmanager:((us(-gov)?|ap|ca|cn|eu|sa)-(central|(north|south)?(east|west)?)-\d):[0-9]{8,12}:secret:[a-zA-Z0-9/_+=.@-]{1,512})|(,"")){4}(,arn:aws:kms:((us(-gov)?|ap|ca|cn|eu|sa)-(central|(north|south)?(east|west)?)-\d):[0-9]{12}:key/[a-zA-Z0-9/_+=.@-]{1,512})'
+
 
 Metadata:
   AWS::CloudFormation::Interface:
@@ -53,6 +58,8 @@ Metadata:
         default: "Branch Name"
       GitHubOAuthToken:
         default: "GitHub OAuth Token"
+      SecretsConfig:
+        default: "TestConfig from the output of secrets.yaml"
 
 Resources:
   CodePipelineLogGroup:
@@ -184,7 +191,7 @@ Resources:
                 OAuthToken: !Ref GitHubOAuthToken
         - Name: App
           Actions:
-            - Name: Git
+            - Name: Git-Metadata
               Namespace: Git
               InputArtifacts:
                 - Name: Source
@@ -199,7 +206,7 @@ Resources:
               Configuration:
                 ProjectName: !Ref GitMetadata
                 PrimarySource: Source
-            - Name: Build
+            - Name: GoLang-Build
               InputArtifacts:
                 - Name: Source
               ActionTypeId:
@@ -214,7 +221,7 @@ Resources:
                 ProjectName: !Ref CodeBuildApp
                 PrimarySource: Source
                 EnvironmentVariables: '[{"name":"GitTag","value":"#{Git.Tag}","type":"PLAINTEXT"},{"name":"GitCommit","value":"#{Git.CommitId}","type":"PLAINTEXT"}]'
-            - Name: Package
+            - Name: SAM-Package-SAR-Stage
               ActionTypeId:
                 Category: Build
                 Owner: AWS
@@ -224,14 +231,15 @@ Resources:
               Configuration:
                 ProjectName: !Ref CodeBuildPackage
                 PrimarySource: Source
+                EnvironmentVariables: '[{"name":"GitTag","value":"#{Git.Tag}","type":"PLAINTEXT"},{"name":"GitVersionHash","value":"#{Git.CommitHash}","type":"PLAINTEXT"}]'
               OutputArtifacts:
                 - Name: Packaged
               InputArtifacts:
                 - Name: Built
                 - Name: Source
-        - Name: Prep
+        - Name: Deploy
           Actions:
-            - Name: Publish
+            - Name: Staging
               Namespace: RC
               ActionTypeId:
                 Category: Build
@@ -258,9 +266,9 @@ Resources:
               Configuration:
                 ActionMode: CREATE_UPDATE
                 Capabilities: CAPABILITY_IAM,CAPABILITY_AUTO_EXPAND,CAPABILITY_NAMED_IAM
-                StackName: SmokeTest
+                StackName: TestAccountExecution
                 RoleArn: !GetAtt [CloudFormationDeployerRole, Arn]
-                TemplateConfiguration: !Sub 'Tests::deploy/developer.json'
+                TemplateConfiguration: 'Tests::deploy/developer.json'
                 TemplatePath: !Sub 'Tests::deploy/stack.yml'
               InputArtifacts:
                 - Name: Tests
@@ -317,7 +325,7 @@ Resources:
                 Provider: CloudFormation
               Configuration:
                 ActionMode: DELETE_ONLY
-                StackName: SmokeTest
+                StackName: TestAccountExecution
                 RoleArn: !GetAtt [CloudFormationDeployerRole, Arn]
               InputArtifacts:
                 - Name: Tests
@@ -437,7 +445,7 @@ Resources:
       ServiceRole: !Ref CodeBuildPublishRole
       Source:
         Type: CODEPIPELINE
-        BuildSpec: "cicd/staging/build/buildspec.yml"
+        BuildSpec: "cicd/account_execution/staging/buildspec.yml"
       Environment:
         ComputeType: BUILD_GENERAL1_SMALL
         Image: aws/codebuild/standard:7.0
@@ -447,6 +455,24 @@ Resources:
             Value: !Sub ${ArtifactBucket}
           - Name: AppArn
             Value: !Sub "arn:aws:serverlessrepo:${AWS::Region}:${AWS::AccountId}:applications/SSOSync-Staging"
+          - Name: SecretSCIMEndpoint
+            Value: !Select [0, !Split [',', !Ref SecretsConfig]]
+          - Name: SecretSCIMAccessToken
+            Value: !Select [1, !Split [',', !Ref SecretsConfig]]
+          - Name: SecretRegion
+            Value: !Select [2, !Split [',', !Ref SecretsConfig]]
+          - Name: SecretIdentityStoreID
+            Value: !Select [3, !Split [',', !Ref SecretsConfig]]
+          - Name: SecretGoogleCredentials
+            Value: !Select [4, !Split [',', !Ref SecretsConfig]]
+          - Name: SecretGoogleAdminEmail
+            Value: !Select [5, !Split [',', !Ref SecretsConfig]]
+          - Name: SecretWIFClientLibraryConfig
+            Value: !Select [6, !Split [',', !Ref SecretsConfig]]
+          - Name: SecretWIFServiceAccountEmail
+            Value: !Select [7, !Split [',', !Ref SecretsConfig]]
+          - Name: KeyForSecrets
+            Value: !Select [8, !Split [',', !Ref SecretsConfig]]
       Artifacts:
         Name: SSOSync
         Type: CODEPIPELINE
@@ -470,7 +496,7 @@ Resources:
       ServiceRole: !Ref CodeBuildTestRole
       Source:
         Type: CODEPIPELINE
-        BuildSpec: "tests/smoke/cli/buildspec.yml"
+        BuildSpec: "tests/account_execution/cli/buildspec.yml"
       Environment:
         ComputeType: BUILD_GENERAL1_SMALL
         Image: aws/codebuild/standard:7.0
@@ -498,7 +524,7 @@ Resources:
       ServiceRole: !Ref CodeBuildTestRole
       Source:
         Type: CODEPIPELINE
-        BuildSpec: "tests/smoke/lambda/buildspec.yml"
+        BuildSpec: "tests/account_execution/lambda/buildspec.yml"
       Environment:
         ComputeType: BUILD_GENERAL1_SMALL
         Image: aws/codebuild/standard:7.0
@@ -664,6 +690,19 @@ Resources:
               Resource:
               - !Sub ${ArtifactBucket.Arn}/*
               - !Sub ${AppBucket.Arn}/*
+            - Action: 'serverlessrepo:*'
+              Effect: Allow
+              Resource:
+              - !Sub arn:aws:serverlessrepo:${AWS::Region}:${AWS::AccountId}:applications/*
+            - Action:
+              - 'ssm:GetParameters'
+              - 'ssm:PutParameter'
+              - 'ssm:DeleteParameter'
+              - 'ssm:DeleteParameters'
+              - 'ssm:DescribeParameters'
+              Effect: Allow
+              Resource:
+              - !Sub arn:aws:ssm:${AWS::Region}:${AWS::AccountId}:parameter/SSOSync/*
 
   CodeBuildPublishRole:
     Type: AWS::IAM::Role
@@ -758,6 +797,7 @@ Resources:
               Resource: '*'
             - Action:
                 - "identitystore:DeleteUser"
+                - "identitystore:DeleteGroup"
                 - "identitystore:CreateGroup"
                 - "identitystore:CreateGroupMembership"
                 - "identitystore:ListGroups"

cicd/cloudformation/secrets.yaml

@@ -253,7 +253,7 @@ Resources:
     DeletionPolicy: Retain
     UpdateReplacePolicy: Delete
     Properties:
-      Name: TestGoogleCredentials
+      Name: PipelineGoogleCredentials
       SecretString: !Ref GoogleCredentials
       KmsKeyId: !Ref KeyAlias
 
@@ -290,7 +290,7 @@ Resources:
     DeletionPolicy: Retain
     UpdateReplacePolicy: Delete
     Properties:
-      Name: TestGoogleAdminEmail
+      Name: PipelineGoogleAdminEmail
       SecretString: !Ref GoogleAdminEmail
       KmsKeyId: !Ref KeyAlias
 
@@ -327,7 +327,7 @@ Resources:
     DeletionPolicy: Retain
     UpdateReplacePolicy: Delete
     Properties:
-      Name: TestWIFServiceAccountEmail
+      Name: PipelineWIFServiceAccountEmail
       SecretString: !Ref WIFServiceAccountEmail
       KmsKeyId: !Ref KeyAlias
 
@@ -364,7 +364,7 @@ Resources:
     DeletionPolicy: Retain
     UpdateReplacePolicy: Delete
     Properties:
-      Name: TestWIFClientLibraryConfigSecret
+      Name: PipelineWIFClientLibraryConfigSecret
       SecretString: !Ref WIFClientLibraryConfig
       KmsKeyId: !Ref KeyAlias
 
@@ -400,7 +400,7 @@ Resources:
     DeletionPolicy: Retain
     UpdateReplacePolicy: Delete
     Properties:
-      Name: TestSCIMEndpointUrl
+      Name: PipelineSCIMEndpointUrl
       SecretString: !Ref SCIMEndpointUrl
       KmsKeyId: !Ref KeyAlias
 
@@ -435,7 +435,7 @@ Resources:
     DeletionPolicy: Retain
     UpdateReplacePolicy: Delete
     Properties:
-      Name: TestSCIMAccessToken
+      Name: PipelineSCIMAccessToken
       SecretString: !Ref SCIMEndpointAccessToken
       KmsKeyId: !Ref KeyAlias
 
@@ -470,7 +470,7 @@ Resources:
     DeletionPolicy: Retain
     UpdateReplacePolicy: Delete
     Properties:
-      Name: TestRegion
+      Name: PipelineRegion
       SecretString: !Select [1, !Split [".", !Ref SCIMEndpointUrl]]
       KmsKeyId: !Ref KeyAlias
 
@@ -505,7 +505,7 @@ Resources:
     DeletionPolicy: Retain
     UpdateReplacePolicy: Delete
     Properties:
-      Name: TestIdentityStoreId
+      Name: PipelineIdentityStoreId
       SecretString: !Ref IdentityStoreId
       KmsKeyId: !Ref KeyAlias
 

cmd/root.go

@@ -20,14 +20,14 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/awslabs/ssosync/internal"
-	"github.com/awslabs/ssosync/internal/config"
-	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-lambda-go/events"
-        "github.com/aws/aws-sdk-go/service/codepipeline"
 	"github.com/aws/aws-lambda-go/lambda"
+	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/session"
+        "github.com/aws/aws-sdk-go/service/codepipeline"
 	"github.com/aws/aws-sdk-go/service/secretsmanager"
+	"github.com/awslabs/ssosync/internal"
+	"github.com/awslabs/ssosync/internal/config"
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
@@ -66,14 +66,14 @@ Complete documentation is available at https://github.com/awslabs/ssosync`,
 // running inside of AWS Lambda, we use the Lambda
 // execution path.
 func Execute() {
-        if cfg.IsLambda {
-                log.Info("Executing as Lambda")
-        	lambda.Start(Handler) 
-	}
+    if cfg.IsLambda {
+        log.Info("Executing as Lambda")
+      	lambda.Start(Handler) 
+    }
 
-	if err := rootCmd.Execute(); err != nil {
-		log.Fatal(err)
-	}
+    if err := rootCmd.Execute(); err != nil {
+	log.Fatal(err)
+    }
 }
 
 func Handler(ctx context.Context, event events.CodePipelineEvent) (string, error) {
@@ -182,54 +182,70 @@ func initConfig() {
 		log.Fatalf(errors.Wrap(err, "cannot unmarshal config").Error())
 	}
 
-	// config logger
-	logConfig(cfg)
-
 	if cfg.IsLambda {
 		configLambda()
 	}
+
+	// config logger
+	logConfig(cfg)
+
 }
 
 func configLambda() {
 	s := session.Must(session.NewSession())
 	svc := secretsmanager.New(s)
 	secrets := config.NewSecrets(svc)
 
-	unwrap, err := secrets.GoogleAdminEmail()
+	unwrap, err := secrets.GoogleAdminEmail(os.Getenv("GOOGLE_ADMIN"))
 	if err != nil {
 		log.Fatalf(errors.Wrap(err, "cannot read config").Error())
 	}
 	cfg.GoogleAdmin = unwrap
 
-	unwrap, err = secrets.GoogleCredentials()
+	unwrap, err = secrets.GoogleCredentials(os.Getenv("GOOGLE_CREDENTIALS"))
 	if err != nil {
 		log.Fatalf(errors.Wrap(err, "cannot read config").Error())
 	}
 	cfg.GoogleCredentials = unwrap
 
-	unwrap, err = secrets.SCIMAccessToken()
+	unwrap, err = secrets.SCIMAccessToken(os.Getenv("SCIM_ACCESS_TOKEN"))
 	if err != nil {
 		log.Fatalf(errors.Wrap(err, "cannot read config").Error())
 	}
 	cfg.SCIMAccessToken = unwrap
 
-	unwrap, err = secrets.SCIMEndpointUrl()
+	unwrap, err = secrets.SCIMEndpointUrl(os.Getenv("SCIM_ENDPOINT"))
 	if err != nil {
 		log.Fatalf(errors.Wrap(err, "cannot read config").Error())
 	}
 	cfg.SCIMEndpoint = unwrap
 
-	unwrap, err = secrets.Region()
+	unwrap, err = secrets.Region(os.Getenv("REGION"))
 	if err != nil {
 		log.Fatalf(errors.Wrap(err, "cannot read config").Error())
 	}
 	cfg.Region = unwrap
 
-	unwrap, err = secrets.IdentityStoreID()
+	unwrap, err = secrets.IdentityStoreID(os.Getenv("IDENTITY_STORE_ID"))
 	if err != nil {
 		log.Fatalf(errors.Wrap(err, "cannot read config").Error())
 	}
 	cfg.IdentityStoreID = unwrap
+
+	unwrap = os.Getenv("LOG_LEVEL")
+        if len([]rune(unwrap)) != 0 {
+	   cfg.LogLevel = unwrap
+        }
+
+        unwrap = os.Getenv("LOG_FORMAT")
+        if len([]rune(unwrap)) != 0 {
+           cfg.LogFormat = unwrap
+        }
+
+	unwrap = os.Getenv("SYNC_METHOD")
+        if len([]rune(unwrap)) != 0 {
+           cfg.SyncMethod = unwrap
+        }
 }
 
 func addFlags(cmd *cobra.Command, cfg *config.Config) {