Collect artifacts in directories.

agent/artifact_uploader.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/fs"
 	"os"
 	"path/filepath"
 	"runtime"
@@ -103,7 +104,80 @@ func (a *ArtifactUploader) Collect() (artifacts []*api.Artifact, err error) {
 	// file paths are deduplicated after resolving globs etc
 	seenPaths := make(map[string]bool)
 
-	for _, globPath := range strings.Split(a.conf.Paths, ArtifactPathDelimiter) {
+	prospects := strings.Split(a.conf.Paths, ArtifactPathDelimiter)
+
+	// Handle directories first
+	for _, literalPath := range prospects {
+		// I think this is a bug; '  name.ext  ' is a valid filepath.
+		// Keeping it because it was handled in the globbing case.
+		literalPath = strings.TrimSpace(literalPath)
+		if literalPath == "" {
+			continue
+		}
+
+		a.logger.Debug("Searching for %s", literalPath)
+
+		absolutePath, err := filepath.Abs(literalPath)
+		if err != nil {
+			return nil, err
+		}
+
+		if _, ok := seenPaths[absolutePath]; ok {
+			a.logger.Debug("Skipping duplicate path %s", literalPath)
+			continue
+		}
+
+		if isDir(absolutePath) {
+			err := filepath.WalkDir(absolutePath, func(p string, d fs.DirEntry, error error) error {
+				seenPaths[p] = true
+
+				if error != nil {
+					return error
+				}
+				if d.IsDir() {
+					return nil
+				} else if a.conf.FollowSymlinks {
+					// FIXME: Resolve symlink relative to cwd and set seenPaths
+				} else {
+					// If a glob is absolute, we need to make it relative to the root so that
+					// it can be combined with the download destination to make a valid path.
+					// This is possibly weird and crazy, this logic dates back to
+					// https://github.com/buildkite/agent/commit/8ae46d975aa60d1ae0e2cc0bff7a43d3bf960935
+					// from 2014, so I'm replicating it here to avoid breaking things
+					if runtime.GOOS == "windows" {
+						wd = filepath.VolumeName(p) + "/"
+					} else {
+						wd = "/"
+					}
+
+					path, err := filepath.Rel(wd, p)
+					if err != nil {
+						return err
+					}
+
+					if experiments.IsEnabled(`normalised-upload-paths`) {
+						// Convert any Windows paths to Unix/URI form
+						path = filepath.ToSlash(path)
+					}
+
+					// Build an artifact object using the paths we have.
+					artifact, err := a.build(path, p, literalPath)
+					if err != nil {
+						return err
+					}
+
+					artifacts = append(artifacts, artifact)
+				}
+
+				return nil
+			})
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	for _, globPath := range prospects {
 		globPath = strings.TrimSpace(globPath)
 		if globPath == "" {
 			continue
@@ -118,6 +192,7 @@ func (a *ArtifactUploader) Collect() (artifacts []*api.Artifact, err error) {
 			// Follow symbolic links for files & directories while expanding globs
 			globfunc = zglob.GlobFollowSymlinks
 		}
+
 		files, err := globfunc(globPath)
 		if err == os.ErrNotExist {
 			a.logger.Info("File not found: %s", globPath)