Directory artifact paths

agent/artifact_uploader.go

@@ -7,6 +7,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/fs"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -117,7 +118,86 @@ func (a *ArtifactUploader) Collect(ctx context.Context) (artifacts []*api.Artifa
 	// file paths are deduplicated after resolving globs etc
 	seenPaths := make(map[string]bool)
 
-	for _, globPath := range strings.Split(a.conf.Paths, ArtifactPathDelimiter) {
+	prospects := strings.Split(a.conf.Paths, ArtifactPathDelimiter)
+
+	// Handle directories first
+	for _, literalPath := range prospects {
+		// I think this is a bug; '  name.ext  ' is a valid filepath.
+		// Keeping it because it was handled in the globbing case.
+		literalPath = strings.TrimSpace(literalPath)
+		if literalPath == "" {
+			continue
+		}
+
+		a.logger.Debug("Searching for %s", literalPath)
+
+		absolutePath, err := filepath.Abs(literalPath)
+		if err != nil {
+			return nil, err
+		}
+
+		if _, ok := seenPaths[absolutePath]; ok {
+			a.logger.Debug("Skipping duplicate path %s", literalPath)
+			continue
+		}
+
+		if isDir(absolutePath) {
+			a.logger.Debug("Searching for %s", literalPath)
+
+			err := filepath.WalkDir(absolutePath, func(p string, d fs.DirEntry, error error) error {
+				if error != nil {
+					return error
+				}
+
+				if _, ok := seenPaths[p]; ok {
+					a.logger.Debug("Skipping duplicate path %s", p);
+					return nil
+				}
+
+				if d.IsDir() {
+					return nil
+				}
+
+				path := p
+
+				if a.conf.GlobResolveFollowSymlinks {
+					resolvedLink, err := filepath.EvalSymlinks(path)
+					if err != nil {
+						return err
+					}
+					path = resolvedLink
+				}
+
+				seenPaths[p] = true
+
+				path, err := filepath.Rel(wd, path)
+				if err != nil {
+					return err
+				}
+
+				if experiments.IsEnabled(`normalised-upload-paths`) {
+					// Convert any Windows paths to Unix/URI form
+					path = filepath.ToSlash(path)
+				}
+
+				// Build an artifact object using the paths we have.
+				artifact, err := a.build(path, p, literalPath)
+				if err != nil {
+					return err
+				}
+
+				artifacts = append(artifacts, artifact)
+
+
+				return nil
+			})
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	for _, globPath := range prospects {
 		globPath = strings.TrimSpace(globPath)
 		if globPath == "" {
 			continue

agent/artifact_uploader_test.go

@@ -80,12 +80,29 @@ func TestCollect(t *testing.T) {
 			Sha1Sum:      "bd4caf2e01e59777744ac1d52deafa01c2cb9bfd",
 			Sha256Sum:    "fc5e8608c7772e4ae834fbc47eec3d902099eb3599f5191e40d9e3d9b3764b0e",
 		},
+		{
+			Name:         "output.log",
+			Path:         []string{"test", "fixtures", "artifacts", "nested", "output.log"},
+			AbsolutePath: filepath.Join(root, "test", "fixtures", "artifacts", "nested", "output.log"),
+			GlobPath:     filepath.Join("test", "fixtures", "artifacts", "nested"),
+			FileSize:     14,
+			Sha1Sum:      "1ec2e13e3871a980eaee0f11c6f47ad1d1f97b3e",
+		},
+		{
+			Name:         "logfile.out",
+			Path:         []string{"test", "fixtures", "artifacts", "nested", "nested0", "logfile.out"},
+			AbsolutePath: filepath.Join(root, "test", "fixtures", "artifacts", "nested", "nested0", "logfile.out"),
+			GlobPath:     filepath.Join("test", "fixtures", "artifacts", "nested"),
+			FileSize:     19,
+			Sha1Sum:      "e30f8100faed6430e6fd617da8ea17100ac97188",
+		},
 	}
 
 	uploader := NewArtifactUploader(logger.Discard, nil, ArtifactUploaderConfig{
-		Paths: fmt.Sprintf("%s;%s",
+		Paths: fmt.Sprintf("%s;%s;%s",
 			filepath.Join("test", "fixtures", "artifacts", "**/*.jpg"),
 			filepath.Join(root, "test", "fixtures", "artifacts", "**/*.gif"),
+			filepath.Join("test", "fixtures", "artifacts", "nested"),
 		),
 	})
 
@@ -108,13 +125,13 @@ func TestCollect(t *testing.T) {
 	if err != nil {
 		t.Fatalf("[normalised-upload-paths disabled] uploader.Collect() error = %v", err)
 	}
-	assert.Equal(t, 5, len(artifactsWithoutExperimentEnabled))
+	assert.Equal(t, 7, len(artifactsWithoutExperimentEnabled))
 
 	artifactsWithExperimentEnabled, err := uploader.Collect(ctxExpEnabled)
 	if err != nil {
 		t.Fatalf("[normalised-upload-paths enabled] uploader.Collect() error = %v", err)
 	}
-	assert.Equal(t, 5, len(artifactsWithExperimentEnabled))
+	assert.Equal(t, 7, len(artifactsWithExperimentEnabled))
 
 	// These test cases use filepath.Join, which uses per-OS path separators;
 	// this is the behaviour without normalised-upload-paths.

test/fixtures/artifacts/nested/nested0/logfile.out

@@ -0,0 +1 @@
+logging out to file

test/fixtures/artifacts/nested/output.log

@@ -0,0 +1 @@
+logging output