add module parameter :use_ssh_agent

README.md

@@ -194,6 +194,19 @@ Synced ./Puppetfile with 4 git repositories and 0 Forge modules in 1.1s with git
 
 Now also supports the r10k setting name `:default_branch => 'master'` See [#73](https://github.com/xorpaul/g10k/issues/73)
 
+- additionl Git attribute `:use_ssh_agent`:
+
+Normally g10k adds the SSH key specified in the g10k config for each SSH+Git module in your Puppetfile.
+If you don't want to use this SSH key, need a different key for a certain Git module or have the key encrypted in your SSH agent, then use this parameter to skip the `ssh-add` commands:
+
+```
+mod 'example_module',
+  :git => '[email protected]/foo/example-module.git',
+  :branch => 'foo',
+  :use_ssh_agent => true
+```
+See [#171](https://github.com/xorpaul/g10k/issues/171) for more details.
+
 - additional Forge attribute `:sha256sum`:
 
 For (some) increased security you can add a SHA256 sum for each Forge module, which g10k will verify after downloading the respective .tar.gz file:

config.go

@@ -196,7 +196,7 @@ func readPuppetfile(pf string, sshKey string, source string, branch string, forc
 	reForgeModule := regexp.MustCompile("^\\s*(?:mod)\\s+['\"]?([^'\"]+[-/][^'\"]+)['\"](?:\\s*)[,]?(.*)")
 	reForgeAttribute := regexp.MustCompile("\\s*['\"]?([^\\s'\"]+)\\s*['\"]?(?:=>)?\\s*['\"]?([^'\"]+)?")
 	reGitModule := regexp.MustCompile("^\\s*(?:mod)\\s+['\"]?([^'\"/]+)['\"]\\s*,(.*)")
-	reGitAttribute := regexp.MustCompile("\\s*:(git|commit|tag|branch|ref|link|ignore[-_]unreachable|fallback|install_path|default_branch|local)\\s*=>\\s*['\"]?([^'\"]+)['\"]?")
+	reGitAttribute := regexp.MustCompile("\\s*:(git|commit|tag|branch|ref|link|ignore[-_]unreachable|fallback|install_path|default_branch|local|use_ssh_agent)\\s*=>\\s*['\"]?([^'\"]+)['\"]?")
 	reUniqueGitAttribute := regexp.MustCompile("\\s*:(?:commit|tag|branch|ref|link)\\s*=>")
 	reDanglingAttribute := regexp.MustCompile("^\\s*:[^ ]+\\s*=>")
 	moduleDir := "modules"
@@ -392,6 +392,12 @@ func readPuppetfile(pf string, sshKey string, source string, branch string, forc
 						if local {
 							gm.local = true
 						}
+					} else if gitModuleAttribute == "use_ssh_agent" {
+						useSSHAgent, err := strconv.ParseBool(a[2])
+						if err != nil {
+							Fatalf("Error: Can not convert value " + a[2] + " of parameter " + gitModuleAttribute + " to boolean. In " + pf + " for module " + gitModuleName + " line: " + line)
+						}
+						gm.useSSHAgent = useSSHAgent
 					}
 
 				}

g10k.go

@@ -182,6 +182,7 @@ type GitModule struct {
 	installPath       string
 	local             bool
 	moduleDir         string
+	useSSHAgent       bool
 }
 
 // ForgeResult is returned by queryForgeAPI and contains if and which version of the Puppetlabs Forge module needs to be downloaded

g10k_puppetfile_test.go

@@ -98,7 +98,9 @@ func equalGitModule(a, b GitModule) bool {
 		a.ref != b.ref ||
 		a.link != b.link ||
 		a.ignoreUnreachable != b.ignoreUnreachable ||
-		a.installPath != b.installPath {
+		a.installPath != b.installPath ||
+		a.local != b.local ||
+		a.useSSHAgent != b.useSSHAgent {
 		return false
 	}
 	if len(a.fallback) != len(b.fallback) {
@@ -500,3 +502,24 @@ func TestReadPuppetfileGitDashNSlashNotation(t *testing.T) {
 		t.Errorf("Expected Puppetfile: %+v, but got Puppetfile: %+v", expected, got)
 	}
 }
+
+func TestReadPuppetfileSSHKeyAlreadyLoaded(t *testing.T) {
+	quiet = true
+	funcName := strings.Split(funcName(), ".")[len(strings.Split(funcName(), "."))-1]
+	got := readPuppetfile("tests/"+funcName, "", "test", "test", false, false)
+
+	fm := make(map[string]ForgeModule)
+	gm := make(map[string]GitModule)
+	gm["example_module"] = GitModule{git: "[email protected]/foo/example-module.git", branch: "foo", useSSHAgent: true}
+
+	expected := Puppetfile{source: "test", gitModules: gm, forgeModules: fm}
+	//fmt.Println(got)
+
+	if !equalPuppetfile(got, expected) {
+		fmt.Println("Expected:")
+		spew.Dump(expected)
+		fmt.Println("Got:")
+		spew.Dump(got)
+		t.Errorf("Expected Puppetfile: %+v, but got Puppetfile: %+v", expected, got)
+	}
+}

g10k_test.go

@@ -9,6 +9,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"reflect"
+	"runtime"
 	"strconv"
 	"strings"
 	"syscall"
@@ -3208,3 +3209,56 @@ func TestBranchFilterRegex(t *testing.T) {
 
 	purgeDir("/tmp/branchfilter", funcName)
 }
+
+func TestResolvePuppetfileUseSSHAgent(t *testing.T) {
+	quiet = true
+	funcName := strings.Split(funcName(), ".")[len(strings.Split(funcName(), "."))-1]
+	configFile = "tests/TestConfigUseSSHAgent.yaml"
+	config = readConfigfile(configFile)
+	if os.Getenv("TEST_FOR_CRASH_"+funcName) == "1" {
+		purgeDir("/tmp/example/", funcName)
+		purgeDir("/tmp/g10k/", funcName)
+		debug = true
+		branchParam = "use_ssh_agent"
+		resolvePuppetEnvironment(false, "")
+		return
+	}
+
+	cmd := exec.Command(os.Args[0], "-test.run="+funcName+"$")
+	cmd.Env = append(os.Environ(), "TEST_FOR_CRASH_"+funcName+"=1")
+	out, err := cmd.CombinedOutput()
+
+	exitCode := 0
+	if msg, ok := err.(*exec.ExitError); ok { // there is error code
+		exitCode = msg.Sys().(syscall.WaitStatus).ExitStatus()
+	}
+
+	if 1 != exitCode {
+		t.Errorf("terminated with %v, but we expected exit status %v Output: %s", exitCode, 1, string(out))
+	}
+	//fmt.Println(string(out))
+
+	sshAddCmd := "ssh-add"
+	if runtime.GOOS == "darwin" {
+		sshAddCmd = "ssh-add -K"
+	}
+
+	expectedLines := []string{
+		"DEBUG git repo url [email protected]:foo/git_module_with_ssh_agent.git with loaded SSH keys from ssh-agent",
+		"DEBUG git repo url [email protected]:foobar/github_module_without_ssh_add.git with SSH key tests/TestConfigUseSSHAgent.yaml",
+		"DEBUG git repo url [email protected]:bar/git_module_with_ssh_add.git with SSH key tests/TestConfigUseSSHAgent.yaml",
+		"DEBUG executeCommand(): Executing git clone --mirror [email protected]:foo/git_module_with_ssh_agent.git /tmp/g10k/modules/[email protected]_git_module_with_ssh_agent.git",
+		"DEBUG executeCommand(): Executing git clone --mirror [email protected]:foobar/github_module_without_ssh_add.git /tmp/g10k/modules/[email protected]_github_module_without_ssh_add.git",
+		"DEBUG executeCommand(): Executing ssh-agent bash -c '" + sshAddCmd + " tests/TestConfigUseSSHAgent.yaml; git clone --mirror [email protected]:bar/git_module_with_ssh_add.git /tmp/g10k/modules/[email protected]_git_module_with_ssh_add.git'",
+	}
+
+	for _, expectedLine := range expectedLines {
+		if !strings.Contains(string(out), expectedLine) {
+			t.Errorf("Could not find expected line '" + expectedLine + "' in debug output")
+		}
+	}
+
+	moduleParam = ""
+	debug = false
+
+}

git.go

@@ -58,7 +58,6 @@ func resolveGitRepositories(uniqueGitModules map[string]GitModule) {
 	wg.Add(len(uniqueGitModules))
 
 	for url, gm := range uniqueGitModules {
-		Debugf("git repo url " + url)
 		privateKey := gm.privateKey
 		go func(url string, gm GitModule, bar *uiprogress.Bar) {
 			// Try to receive from the concurrentGoroutines channel. When we have something,
@@ -69,8 +68,10 @@ func resolveGitRepositories(uniqueGitModules map[string]GitModule) {
 			defer bar.Incr()
 			defer wg.Done()
 
-			if len(gm.privateKey) > 0 {
-				Debugf("git repo url " + url + " with ssh key " + privateKey)
+			if gm.useSSHAgent {
+				Debugf("git repo url " + url + " with loaded SSH keys from ssh-agent")
+			} else if len(gm.privateKey) > 0 {
+				Debugf("git repo url " + url + " with SSH key " + privateKey)
 			} else {
 				Debugf("git repo url " + url + " without ssh key")
 			}
@@ -95,11 +96,12 @@ func resolveGitRepositories(uniqueGitModules map[string]GitModule) {
 }
 
 func doMirrorOrUpdate(gitModule GitModule, workDir string, retryCount int) bool {
+	//fmt.Printf("%+v\n", gitModule)
 	isControlRepo := strings.HasPrefix(workDir, config.EnvCacheDir)
 	isInModulesCacheDir := strings.HasPrefix(workDir, config.ModulesCacheDir)
 
 	needSSHKey := true
-	if len(gitModule.privateKey) == 0 || strings.Contains(gitModule.git, "github.com") {
+	if len(gitModule.privateKey) == 0 || strings.Contains(gitModule.git, "github.com") || gitModule.useSSHAgent == true {
 		if isControlRepo {
 			needSSHKey = true
 		} else {
@@ -109,7 +111,6 @@ func doMirrorOrUpdate(gitModule GitModule, workDir string, retryCount int) bool
 	er := ExecResult{}
 	gitCmd := "git clone --mirror " + gitModule.git + " " + workDir
 	if config.CloneGitModules && !isControlRepo && !isInModulesCacheDir {
-		//fmt.Printf("%+v\n", gitModule)
 		gitCmd = "git clone --single-branch --branch " + gitModule.tree + " " + gitModule.git + " " + workDir
 	}
 	if isDir(workDir) {

tests/TestConfigUseSSHAgent.yaml

@@ -0,0 +1,8 @@
+---
+:cachedir: '/tmp/g10k'
+
+sources:
+  example:
+    remote: 'https://github.com/xorpaul/g10k-environment.git'
+    basedir: '/tmp/example/'
+    private_key: 'tests/TestConfigUseSSHAgent.yaml' # just some existing file, we just want to check if the ssh-add command isn't used

tests/TestReadPuppetfileSSHKeyAlreadyLoaded

@@ -0,0 +1,5 @@
+mod 'example_module',
+  :git => '[email protected]/foo/example-module.git',
+  :branch => 'foo',
+  :use_ssh_agent => true
+