From 8f81fbe0f05736c0e1ee62254ccb274aca05db1d Mon Sep 17 00:00:00 2001 From: Gabriel Adrian Samfira Date: Mon, 14 Nov 2022 07:36:13 -0800 Subject: [PATCH 1/7] Fix tests on Windows Make sure all paths are properly formatted and the package works on Windows. --- copy/copy_test.go | 97 ++++++------------- copy/copy_unix_test.go | 58 ++++++++++++ copy/copy_windows.go | 4 + diff.go | 3 +- diff_containerd.go | 4 +- diskwriter.go | 43 +++------ diskwriter_freebsd.go | 9 ++ diskwriter_test.go | 61 ++---------- diskwriter_unix.go | 7 ++ diskwriter_windows.go | 79 ++++++++++++++++ followlinks.go | 13 ++- followlinks_test.go | 49 ++++++---- followlinks_unix.go | 14 +++ followlinks_windows.go | 15 +++ go.mod | 2 +- go.sum | 4 +- receive.go | 3 +- receive_test.go | 208 ++++++++++++++++++++++++++++------------- stat.go | 2 +- validator.go | 25 +++-- validator_test.go | 3 +- walker_test.go | 152 ++++++++++++++++++------------ 22 files changed, 533 insertions(+), 322 deletions(-) create mode 100644 copy/copy_unix_test.go create mode 100644 followlinks_unix.go create mode 100644 followlinks_windows.go diff --git a/copy/copy_test.go b/copy/copy_test.go index a7190bee..e77d54e7 100644 --- a/copy/copy_test.go +++ b/copy/copy_test.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path/filepath" + "runtime" "sort" "strings" "testing" @@ -13,10 +14,8 @@ import ( "github.com/containerd/continuity/fs/fstest" "github.com/pkg/errors" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/tonistiigi/fsutil" - "golang.org/x/sys/unix" ) // requiresRoot skips tests that require root @@ -44,7 +43,7 @@ func TestCopyDirectory(t *testing.T) { fstest.CreateDir("/home", 0755), ) - exp := "add:/etc,add:/etc/hosts,add:/etc/hosts.allow,add:/home,add:/usr,add:/usr/local,add:/usr/local/lib,add:/usr/local/lib/libnothing.so,add:/usr/local/lib/libnothing.so.2" + exp := filepath.FromSlash("add:/etc,add:/etc/hosts,add:/etc/hosts.allow,add:/home,add:/usr,add:/usr/local,add:/usr/local/lib,add:/usr/local/lib/libnothing.so,add:/usr/local/lib/libnothing.so.2") if err := testCopy(t, apply, exp); err != nil { t.Fatalf("Copy test failed: %+v", err) @@ -60,7 +59,7 @@ func TestCopyDirectoryWithLocalSymlink(t *testing.T) { fstest.Symlink("nothing.txt", "link-no-nothing.txt"), ) - exp := "add:/link-no-nothing.txt,add:/nothing.txt" + exp := filepath.FromSlash("add:/link-no-nothing.txt,add:/nothing.txt") if err := testCopy(t, apply, exp); err != nil { t.Fatalf("Copy test failed: %+v", err) @@ -84,49 +83,6 @@ func TestCopyToWorkDir(t *testing.T) { require.NoError(t, err) } -func TestCopyDevicesAndFifo(t *testing.T) { - requiresRoot(t) - - t1 := t.TempDir() - - err := mknod(filepath.Join(t1, "char"), unix.S_IFCHR|0444, int(unix.Mkdev(1, 9))) - require.NoError(t, err) - - err = mknod(filepath.Join(t1, "block"), unix.S_IFBLK|0441, int(unix.Mkdev(3, 2))) - require.NoError(t, err) - - err = mknod(filepath.Join(t1, "socket"), unix.S_IFSOCK|0555, 0) - require.NoError(t, err) - - err = unix.Mkfifo(filepath.Join(t1, "fifo"), 0555) - require.NoError(t, err) - - t2 := t.TempDir() - - err = Copy(context.TODO(), t1, ".", t2, ".") - require.NoError(t, err) - - fi, err := os.Lstat(filepath.Join(t2, "char")) - require.NoError(t, err) - assert.Equal(t, os.ModeCharDevice, fi.Mode()&os.ModeCharDevice) - assert.Equal(t, os.FileMode(0444), fi.Mode()&0777) - - fi, err = os.Lstat(filepath.Join(t2, "block")) - require.NoError(t, err) - assert.Equal(t, os.ModeDevice, fi.Mode()&os.ModeDevice) - assert.Equal(t, os.FileMode(0441), fi.Mode()&0777) - - fi, err = os.Lstat(filepath.Join(t2, "fifo")) - require.NoError(t, err) - assert.Equal(t, os.ModeNamedPipe, fi.Mode()&os.ModeNamedPipe) - assert.Equal(t, os.FileMode(0555), fi.Mode()&0777) - - fi, err = os.Lstat(filepath.Join(t2, "socket")) - require.NoError(t, err) - assert.NotEqual(t, os.ModeSocket, fi.Mode()&os.ModeSocket) // socket copied as stub - assert.Equal(t, os.FileMode(0555), fi.Mode()&0777) -} - func TestCopySingleFile(t *testing.T) { t1 := t.TempDir() @@ -168,7 +124,7 @@ func TestCopySingleFile(t *testing.T) { _, err = os.Stat(filepath.Join(t4, "a/b/c/foo2.txt")) require.NoError(t, err) - require.Equal(t, "add:/a/b/c/foo2.txt", ch.String()) + require.Equal(t, filepath.FromSlash("add:/a/b/c/foo2.txt"), ch.String()) } func TestCopyOverrideFile(t *testing.T) { @@ -221,7 +177,7 @@ func TestCopyDirectoryBasename(t *testing.T) { err = fstest.CheckDirectoryEqual(t1, t2) require.NoError(t, err) - require.Equal(t, "add:/foo,add:/foo/bar,add:/foo/bar/baz.txt", ch.String()) + require.Equal(t, filepath.FromSlash("add:/foo,add:/foo/bar,add:/foo/bar/baz.txt"), ch.String()) ch = &changeCollector{} err = Copy(context.TODO(), t1, "foo", t2, "foo", WithCopyInfo(CopyInfo{ @@ -230,7 +186,7 @@ func TestCopyDirectoryBasename(t *testing.T) { })) require.NoError(t, err) - require.Equal(t, "add:/foo/bar,add:/foo/bar/baz.txt", ch.String()) + require.Equal(t, filepath.FromSlash("add:/foo/bar,add:/foo/bar/baz.txt"), ch.String()) err = fstest.CheckDirectoryEqual(t1, t2) require.NoError(t, err) @@ -311,9 +267,12 @@ func TestCopyExistingDirDest(t *testing.T) { st, err := os.Lstat(filepath.Join(t2, "dir")) require.NoError(t, err) require.Equal(t, st.Mode()&os.ModePerm, os.FileMode(0700)) - uid, gid := getUIDGID(st) - require.Equal(t, 1, uid) - require.Equal(t, 1, gid) + var uid, gid int + if runtime.GOOS != "windows" { + uid, gid = getUIDGID(st) + require.Equal(t, 1, uid) + require.Equal(t, 1, gid) + } // verify that non-existing file was created _, err = os.Lstat(filepath.Join(t2, "dir/foo.txt")) @@ -323,9 +282,11 @@ func TestCopyExistingDirDest(t *testing.T) { st, err = os.Lstat(filepath.Join(t2, "dir/bar.txt")) require.NoError(t, err) require.Equal(t, os.FileMode(0644), st.Mode()&os.ModePerm) - uid, gid = getUIDGID(st) - require.Equal(t, 0, uid) - require.Equal(t, 0, gid) + if runtime.GOOS != "windows" { + uid, gid = getUIDGID(st) + require.Equal(t, 0, uid) + require.Equal(t, 0, gid) + } dt, err := os.ReadFile(filepath.Join(t2, "dir/bar.txt")) require.NoError(t, err) require.Equal(t, "bar-contents", string(dt)) @@ -376,7 +337,9 @@ func TestCopySymlinks(t *testing.T) { // verify that existing destination dir's metadata was not overwritten st, err := os.Lstat(filepath.Join(t2, "foo")) require.NoError(t, err) - require.Equal(t, os.FileMode(0644), st.Mode()&os.ModePerm) + if runtime.GOOS != "windows" { + require.Equal(t, os.FileMode(0644), st.Mode()&os.ModePerm) + } require.Equal(t, 0, int(st.Mode()&os.ModeSymlink)) dt, err := os.ReadFile(filepath.Join(t2, "foo")) require.NoError(t, err) @@ -439,37 +402,37 @@ func TestCopyIncludeExclude(t *testing.T) { name: "include bar", opts: []Opt{WithIncludePattern("bar")}, expectedResults: []string{"bar", "bar/foo", "bar/baz", "bar/baz/foo3"}, - expectedChanges: "add:/bar,add:/bar/baz,add:/bar/baz/foo3,add:/bar/foo", + expectedChanges: filepath.FromSlash("add:/bar,add:/bar/baz,add:/bar/baz/foo3,add:/bar/foo"), }, { name: "include *", opts: []Opt{WithIncludePattern("*")}, expectedResults: []string{"bar", "bar/foo", "bar/baz", "bar/baz/foo3", "foo2"}, - expectedChanges: "add:/bar,add:/bar/baz,add:/bar/baz/foo3,add:/bar/foo,add:/foo2", + expectedChanges: filepath.FromSlash("add:/bar,add:/bar/baz,add:/bar/baz/foo3,add:/bar/foo,add:/foo2"), }, { name: "include bar/foo", opts: []Opt{WithIncludePattern("bar/foo")}, expectedResults: []string{"bar", "bar/foo"}, - expectedChanges: "add:/bar/foo", + expectedChanges: filepath.FromSlash("add:/bar/foo"), }, { name: "include bar except bar/foo", opts: []Opt{WithIncludePattern("bar"), WithIncludePattern("!bar/foo")}, expectedResults: []string{"bar", "bar/baz", "bar/baz/foo3"}, - expectedChanges: "add:/bar,add:/bar/baz,add:/bar/baz/foo3", + expectedChanges: filepath.FromSlash("add:/bar,add:/bar/baz,add:/bar/baz/foo3"), }, { name: "include bar/foo and foo*", opts: []Opt{WithIncludePattern("bar/foo"), WithIncludePattern("foo*")}, expectedResults: []string{"bar", "bar/foo", "foo2"}, - expectedChanges: "add:/bar/foo,add:/foo2", + expectedChanges: filepath.FromSlash("add:/bar/foo,add:/foo2"), }, { name: "include b*", opts: []Opt{WithIncludePattern("b*")}, expectedResults: []string{"bar", "bar/foo", "bar/baz", "bar/baz/foo3"}, - expectedChanges: "add:/bar,add:/bar/baz,add:/bar/baz/foo3,add:/bar/foo", + expectedChanges: filepath.FromSlash("add:/bar,add:/bar/baz,add:/bar/baz/foo3,add:/bar/foo"), }, { name: "include bar/f*", @@ -530,25 +493,25 @@ func TestCopyIncludeExclude(t *testing.T) { name: "doublestar matching second item in path", opts: []Opt{WithIncludePattern("**/baz")}, expectedResults: []string{"bar", "bar/baz", "bar/baz/foo3"}, - expectedChanges: "add:/bar/baz,add:/bar/baz/foo3", + expectedChanges: filepath.FromSlash("add:/bar/baz,add:/bar/baz/foo3"), }, { name: "doublestar matching first item in path", opts: []Opt{WithIncludePattern("**/bar")}, expectedResults: []string{"bar", "bar/foo", "bar/baz", "bar/baz/foo3"}, - expectedChanges: "add:/bar,add:/bar/baz,add:/bar/baz/foo3,add:/bar/foo", + expectedChanges: filepath.FromSlash("add:/bar,add:/bar/baz,add:/bar/baz/foo3,add:/bar/foo"), }, { name: "doublestar exclude", opts: []Opt{WithIncludePattern("bar"), WithExcludePattern("**/foo3")}, expectedResults: []string{"bar", "bar/foo", "bar/baz"}, - expectedChanges: "add:/bar,add:/bar/baz,add:/bar/foo", + expectedChanges: filepath.FromSlash("add:/bar,add:/bar/baz,add:/bar/foo"), }, { name: "exclude bar/baz", opts: []Opt{WithExcludePattern("bar/baz")}, expectedResults: []string{"bar", "bar/foo", "foo2"}, - expectedChanges: "add:/bar,add:/bar/foo,add:/foo2", + expectedChanges: filepath.FromSlash("add:/bar,add:/bar/foo,add:/foo2"), }, } diff --git a/copy/copy_unix_test.go b/copy/copy_unix_test.go new file mode 100644 index 00000000..df710d4c --- /dev/null +++ b/copy/copy_unix_test.go @@ -0,0 +1,58 @@ +//go:build !windows +// +build !windows + +package fs + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" +) + +func TestCopyDevicesAndFifo(t *testing.T) { + requiresRoot(t) + + t1 := t.TempDir() + + err := mknod(filepath.Join(t1, "char"), unix.S_IFCHR|0444, int(unix.Mkdev(1, 9))) + require.NoError(t, err) + + err = mknod(filepath.Join(t1, "block"), unix.S_IFBLK|0441, int(unix.Mkdev(3, 2))) + require.NoError(t, err) + + err = mknod(filepath.Join(t1, "socket"), unix.S_IFSOCK|0555, 0) + require.NoError(t, err) + + err = unix.Mkfifo(filepath.Join(t1, "fifo"), 0555) + require.NoError(t, err) + + t2 := t.TempDir() + + err = Copy(context.TODO(), t1, ".", t2, ".") + require.NoError(t, err) + + fi, err := os.Lstat(filepath.Join(t2, "char")) + require.NoError(t, err) + assert.Equal(t, os.ModeCharDevice, fi.Mode()&os.ModeCharDevice) + assert.Equal(t, os.FileMode(0444), fi.Mode()&0777) + + fi, err = os.Lstat(filepath.Join(t2, "block")) + require.NoError(t, err) + assert.Equal(t, os.ModeDevice, fi.Mode()&os.ModeDevice) + assert.Equal(t, os.FileMode(0441), fi.Mode()&0777) + + fi, err = os.Lstat(filepath.Join(t2, "fifo")) + require.NoError(t, err) + assert.Equal(t, os.ModeNamedPipe, fi.Mode()&os.ModeNamedPipe) + assert.Equal(t, os.FileMode(0555), fi.Mode()&0777) + + fi, err = os.Lstat(filepath.Join(t2, "socket")) + require.NoError(t, err) + assert.NotEqual(t, os.ModeSocket, fi.Mode()&os.ModeSocket) // socket copied as stub + assert.Equal(t, os.FileMode(0555), fi.Mode()&0777) +} diff --git a/copy/copy_windows.go b/copy/copy_windows.go index 19a44a75..68d20d7d 100644 --- a/copy/copy_windows.go +++ b/copy/copy_windows.go @@ -13,6 +13,10 @@ const ( seTakeOwnershipPrivilege = "SeTakeOwnershipPrivilege" ) +func getUIDGID(fi os.FileInfo) (uid, gid int) { + return 0, 0 +} + func (c *copier) copyFileInfo(fi os.FileInfo, src, name string) error { if err := os.Chmod(name, fi.Mode()); err != nil { return errors.Wrapf(err, "failed to chmod %s", name) diff --git a/diff.go b/diff.go index a7405dc5..d9fa5a0c 100644 --- a/diff.go +++ b/diff.go @@ -4,6 +4,7 @@ import ( "context" "hash" "os" + "path/filepath" "github.com/pkg/errors" "github.com/tonistiigi/fsutil/types" @@ -32,7 +33,7 @@ func getWalkerFn(root string) walkerFn { } p := ¤tPath{ - path: path, + path: filepath.FromSlash(path), stat: stat, } diff --git a/diff_containerd.go b/diff_containerd.go index d8619abf..9b2050e4 100644 --- a/diff_containerd.go +++ b/diff_containerd.go @@ -5,6 +5,7 @@ import ( "context" "io" "os" + "path/filepath" "strings" "github.com/tonistiigi/fsutil/types" @@ -110,7 +111,7 @@ func doubleWalkDiff(ctx context.Context, changeFn ChangeFunc, a, b walkerFn, fil if filter != nil { filter(f2.path, &statCopy) } - f2copy = ¤tPath{path: f2.path, stat: &statCopy} + f2copy = ¤tPath{path: filepath.FromSlash(f2.path), stat: &statCopy} } k, p := pathChange(f1, f2copy) switch k { @@ -169,7 +170,6 @@ func pathChange(lower, upper *currentPath) (ChangeKind, string) { if upper == nil { return ChangeKindDelete, lower.path } - switch i := ComparePath(lower.path, upper.path); { case i < 0: // File in lower that is not in upper diff --git a/diskwriter.go b/diskwriter.go index b822644d..a9ba1de9 100644 --- a/diskwriter.go +++ b/diskwriter.go @@ -4,7 +4,6 @@ import ( "context" "hash" "io" - gofs "io/fs" "os" "path/filepath" "strconv" @@ -34,11 +33,10 @@ type DiskWriter struct { opt DiskWriterOpt dest string - ctx context.Context - cancel func() - eg *errgroup.Group - filter FilterFunc - dirModTimes map[string]int64 + ctx context.Context + cancel func() + eg *errgroup.Group + filter FilterFunc } func NewDiskWriter(ctx context.Context, dest string, opt DiskWriterOpt) (*DiskWriter, error) { @@ -53,32 +51,17 @@ func NewDiskWriter(ctx context.Context, dest string, opt DiskWriterOpt) (*DiskWr eg, ctx := errgroup.WithContext(ctx) return &DiskWriter{ - opt: opt, - dest: dest, - eg: eg, - ctx: ctx, - cancel: cancel, - filter: opt.Filter, - dirModTimes: map[string]int64{}, + opt: opt, + dest: dest, + eg: eg, + ctx: ctx, + cancel: cancel, + filter: opt.Filter, }, nil } func (dw *DiskWriter) Wait(ctx context.Context) error { - if err := dw.eg.Wait(); err != nil { - return err - } - return filepath.WalkDir(dw.dest, func(path string, d gofs.DirEntry, prevErr error) error { - if prevErr != nil { - return prevErr - } - if !d.IsDir() { - return nil - } - if mtime, ok := dw.dirModTimes[path]; ok { - return chtimes(path, mtime) - } - return nil - }) + return dw.eg.Wait() } func (dw *DiskWriter) HandleChange(kind ChangeKind, p string, fi os.FileInfo, err error) (retErr error) { @@ -164,7 +147,6 @@ func (dw *DiskWriter) HandleChange(kind ChangeKind, p string, fi os.FileInfo, er if err := os.Mkdir(newPath, fi.Mode()); err != nil { return errors.Wrapf(err, "failed to create dir %s", newPath) } - dw.dirModTimes[destPath] = statCopy.ModTime case fi.Mode()&os.ModeDevice != 0 || fi.Mode()&os.ModeNamedPipe != 0: if err := handleTarTypeBlockCharFifo(newPath, &statCopy); err != nil { return errors.Wrapf(err, "failed to create device %s", newPath) @@ -205,7 +187,8 @@ func (dw *DiskWriter) HandleChange(kind ChangeKind, p string, fi os.FileInfo, er return errors.Wrapf(err, "failed to remove %s", destPath) } } - if err := os.Rename(newPath, destPath); err != nil { + + if err := renameFile(newPath, destPath); err != nil { return errors.Wrapf(err, "failed to rename %s to %s", newPath, destPath) } } diff --git a/diskwriter_freebsd.go b/diskwriter_freebsd.go index ed6356fa..31f3ec79 100644 --- a/diskwriter_freebsd.go +++ b/diskwriter_freebsd.go @@ -4,6 +4,8 @@ package fsutil import ( + "os" + "github.com/tonistiigi/fsutil/types" "golang.org/x/sys/unix" ) @@ -15,3 +17,10 @@ func createSpecialFile(path string, mode uint32, stat *types.Stat) error { func mkdev(major int64, minor int64) uint64 { return unix.Mkdev(uint32(major), uint32(minor)) } + +func renameFile(src, dst string) error { + if err := os.Rename(src, dst); err != nil { + return errors.Wrapf(err, "failed to rename %s to %s", src, dst) + } + return nil +} diff --git a/diskwriter_test.go b/diskwriter_test.go index 4b6edb3b..7e069628 100644 --- a/diskwriter_test.go +++ b/diskwriter_test.go @@ -1,3 +1,6 @@ +//go:build !windows +// +build !windows + package fsutil import ( @@ -6,27 +9,15 @@ import ( "io" "os" "path/filepath" - "sync" "syscall" "testing" "time" - "github.com/opencontainers/go-digest" - "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/sys/unix" ) -// requiresRoot skips tests that require root -func requiresRoot(t *testing.T) { - t.Helper() - if os.Getuid() != 0 { - t.Skip("skipping test that requires root") - return - } -} - func TestWriterSimple(t *testing.T) { requiresRoot(t) @@ -54,7 +45,7 @@ func TestWriterSimple(t *testing.T) { err = Walk(context.Background(), dest, nil, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, string(b.Bytes()), `dir bar + assert.Equal(t, b.String(), `dir bar file bar/foo symlink:../foo bar/foo2 file foo @@ -91,7 +82,7 @@ func TestWriterFileToDir(t *testing.T) { err = Walk(context.Background(), dest, nil, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, string(b.Bytes()), `dir foo + assert.Equal(t, b.String(), `dir foo file foo/bar `) } @@ -124,7 +115,7 @@ func TestWriterDirToFile(t *testing.T) { err = Walk(context.Background(), dest, nil, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, string(b.Bytes()), `file foo + assert.Equal(t, b.String(), `file foo `) } @@ -153,7 +144,7 @@ func TestWalkerWriterSimple(t *testing.T) { err = Walk(context.Background(), dest, nil, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, string(b.Bytes()), `dir bar + assert.Equal(t, b.String(), `dir bar file bar/foo symlink:../foo bar/foo2 file foo @@ -294,41 +285,3 @@ func newWriteToFunc(baseDir string, delay time.Duration) WriteToFunc { return nil } } - -type notificationBuffer struct { - items map[string]digest.Digest - sync.Mutex -} - -func newNotificationBuffer() *notificationBuffer { - nb := ¬ificationBuffer{ - items: map[string]digest.Digest{}, - } - return nb -} - -type hashed interface { - Digest() digest.Digest -} - -func (nb *notificationBuffer) HandleChange(kind ChangeKind, p string, fi os.FileInfo, err error) (retErr error) { - nb.Lock() - defer nb.Unlock() - if kind == ChangeKindDelete { - delete(nb.items, p) - } else { - h, ok := fi.(hashed) - if !ok { - return errors.Errorf("invalid FileInfo: %s", p) - } - nb.items[p] = h.Digest() - } - return nil -} - -func (nb *notificationBuffer) Hash(p string) (digest.Digest, bool) { - nb.Lock() - v, ok := nb.items[p] - nb.Unlock() - return v, ok -} diff --git a/diskwriter_unix.go b/diskwriter_unix.go index 1d97d6f9..ccd554ab 100644 --- a/diskwriter_unix.go +++ b/diskwriter_unix.go @@ -51,3 +51,10 @@ func handleTarTypeBlockCharFifo(path string, stat *types.Stat) error { } return nil } + +func renameFile(src, dst string) error { + if err := os.Rename(src, dst); err != nil { + return errors.Wrapf(err, "failed to rename %s to %s", src, dst) + } + return nil +} diff --git a/diskwriter_windows.go b/diskwriter_windows.go index 036544f0..13dbcf19 100644 --- a/diskwriter_windows.go +++ b/diskwriter_windows.go @@ -1,8 +1,15 @@ +//go:build windows // +build windows package fsutil import ( + "fmt" + ioFS "io/fs" + "os" + "syscall" + + "github.com/Microsoft/go-winio" "github.com/pkg/errors" "github.com/tonistiigi/fsutil/types" ) @@ -16,3 +23,75 @@ func rewriteMetadata(p string, stat *types.Stat) error { func handleTarTypeBlockCharFifo(path string, stat *types.Stat) error { return errors.New("Not implemented on windows") } + +func getFileHandle(path string, info ioFS.FileInfo) (syscall.Handle, error) { + p, err := syscall.UTF16PtrFromString(path) + if err != nil { + return 0, errors.Wrap(err, "converting string to UTF-16") + } + attrs := uint32(syscall.FILE_FLAG_BACKUP_SEMANTICS) + if info.Mode()&os.ModeSymlink != 0 { + // Use FILE_FLAG_OPEN_REPARSE_POINT, otherwise CreateFile will follow symlink. + // See https://docs.microsoft.com/en-us/windows/desktop/FileIO/symbolic-link-effects-on-file-systems-functions#createfile-and-createfiletransacted + attrs |= syscall.FILE_FLAG_OPEN_REPARSE_POINT + } + h, err := syscall.CreateFile(p, 0, 0, nil, syscall.OPEN_EXISTING, attrs, 0) + if err != nil { + return 0, errors.Wrap(err, "getting file handle") + } + return h, nil +} + +func readlink(path string, info ioFS.FileInfo) ([]byte, error) { + h, err := getFileHandle(path, info) + if err != nil { + return nil, errors.Wrap(err, "getting file handle") + } + defer syscall.CloseHandle(h) + + rdbbuf := make([]byte, syscall.MAXIMUM_REPARSE_DATA_BUFFER_SIZE) + var bytesReturned uint32 + err = syscall.DeviceIoControl(h, syscall.FSCTL_GET_REPARSE_POINT, nil, 0, &rdbbuf[0], uint32(len(rdbbuf)), &bytesReturned, nil) + if err != nil { + return nil, errors.Wrap(err, "sending I/O control command") + } + return rdbbuf[:bytesReturned], nil +} + +func getReparsePoint(path string, info ioFS.FileInfo) (*winio.ReparsePoint, error) { + target, err := readlink(path, info) + if err != nil { + return nil, errors.Wrap(err, "fetching link") + } + rp, err := winio.DecodeReparsePoint(target) + if err != nil { + return nil, errors.Wrap(err, "decoding reparse point") + } + return rp, nil +} + +func renameFile(src, dst string) error { + info, err := os.Lstat(dst) + if err != nil { + if !os.IsNotExist(err) { + return errors.Wrap(err, "getting file info") + } + } + + if info != nil && info.Mode()&os.ModeSymlink != 0 { + dstInfoRp, err := getReparsePoint(dst, info) + if err != nil { + return errors.Wrap(err, "getting reparse point") + } + if dstInfoRp.IsMountPoint { + return fmt.Errorf("%s is a mount point", dst) + } + if err := os.Remove(dst); err != nil { + return errors.Wrapf(err, "removing %s", dst) + } + } + if err := os.Rename(src, dst); err != nil { + return errors.Wrapf(err, "failed to rename %s to %s", src, dst) + } + return nil +} diff --git a/followlinks.go b/followlinks.go index 136a9082..f6809f62 100644 --- a/followlinks.go +++ b/followlinks.go @@ -31,7 +31,13 @@ type symlinkResolver struct { } func (r *symlinkResolver) append(p string) error { - p = filepath.Join(".", p) + p = filepath.FromSlash(p) + if runtime.GOOS == "windows" && filepath.IsAbs(p) { + absParts := strings.SplitN(p, ":", 2) + if len(absParts) == 2 { + p = absParts[1] + } + } current := "." for { parts := strings.SplitN(p, string(filepath.Separator), 2) @@ -41,7 +47,6 @@ func (r *symlinkResolver) append(p string) error { if err != nil { return err } - p = "" if len(parts) == 2 { p = parts[1] @@ -76,7 +81,7 @@ func (r *symlinkResolver) readSymlink(p string, allowWildcard bool) ([]string, e if allowWildcard && containsWildcards(base) { fis, err := os.ReadDir(filepath.Dir(realPath)) if err != nil { - if errors.Is(err, os.ErrNotExist) { + if isNotFound(err) { return nil, nil } return nil, errors.Wrap(err, "readdir") @@ -96,7 +101,7 @@ func (r *symlinkResolver) readSymlink(p string, allowWildcard bool) ([]string, e fi, err := os.Lstat(realPath) if err != nil { - if errors.Is(err, os.ErrNotExist) { + if isNotFound(err) { return nil, nil } return nil, errors.WithStack(err) diff --git a/followlinks_test.go b/followlinks_test.go index 522763f6..4d32f1a9 100644 --- a/followlinks_test.go +++ b/followlinks_test.go @@ -1,6 +1,8 @@ package fsutil import ( + "path/filepath" + "runtime" "testing" "github.com/containerd/continuity/fs/fstest" @@ -24,7 +26,7 @@ func TestFollowLinks(t *testing.T) { out, err := FollowLinks(tmpDir, []string{"l2", "bar"}) require.NoError(t, err) - require.Equal(t, out, []string{"bar", "dir/foo", "dir/l1", "l2"}) + require.Equal(t, out, []string{"bar", filepath.FromSlash("dir/foo"), filepath.FromSlash("dir/l1"), "l2"}) } func TestFollowLinksLoop(t *testing.T) { @@ -46,9 +48,14 @@ func TestFollowLinksLoop(t *testing.T) { func TestFollowLinksAbsolute(t *testing.T) { tmpDir := t.TempDir() + abslutePathForBaz := "/foo/bar/baz" + if runtime.GOOS == "windows" { + abslutePathForBaz = "C:/foo/bar/baz" + } + apply := fstest.Apply( fstest.CreateDir("dir", 0700), - fstest.Symlink("/foo/bar/baz", "dir/l1"), + fstest.Symlink(abslutePathForBaz, "dir/l1"), fstest.CreateDir("foo", 0700), fstest.Symlink("../", "foo/bar"), fstest.CreateFile("baz", nil, 0600), @@ -58,14 +65,14 @@ func TestFollowLinksAbsolute(t *testing.T) { out, err := FollowLinks(tmpDir, []string{"dir/l1"}) require.NoError(t, err) - require.Equal(t, out, []string{"baz", "dir/l1", "foo/bar"}) + require.Equal(t, []string{"baz", filepath.FromSlash("dir/l1"), filepath.FromSlash("foo/bar")}, out) // same but a link outside root tmpDir = t.TempDir() apply = fstest.Apply( fstest.CreateDir("dir", 0700), - fstest.Symlink("/foo/bar/baz", "dir/l1"), + fstest.Symlink(abslutePathForBaz, "dir/l1"), fstest.CreateDir("foo", 0700), fstest.Symlink("../../../", "foo/bar"), fstest.CreateFile("baz", nil, 0600), @@ -75,7 +82,7 @@ func TestFollowLinksAbsolute(t *testing.T) { out, err = FollowLinks(tmpDir, []string{"dir/l1"}) require.NoError(t, err) - require.Equal(t, out, []string{"baz", "dir/l1", "foo/bar"}) + require.Equal(t, []string{"baz", filepath.FromSlash("dir/l1"), filepath.FromSlash("foo/bar")}, out) } func TestFollowLinksNotExists(t *testing.T) { @@ -84,7 +91,7 @@ func TestFollowLinksNotExists(t *testing.T) { out, err := FollowLinks(tmpDir, []string{"foo/bar/baz", "bar/baz"}) require.NoError(t, err) - require.Equal(t, out, []string{"bar/baz", "foo/bar/baz"}) + require.Equal(t, out, []string{filepath.FromSlash("bar/baz"), filepath.FromSlash("foo/bar/baz")}) // root works fine with empty directory out, err = FollowLinks(tmpDir, []string{"."}) @@ -95,7 +102,7 @@ func TestFollowLinksNotExists(t *testing.T) { out, err = FollowLinks(tmpDir, []string{"f*/foo/t*"}) require.NoError(t, err) - require.Equal(t, out, []string{"f*/foo/t*"}) + require.Equal(t, []string{filepath.FromSlash("f*/foo/t*")}, out) } func TestFollowLinksNormalized(t *testing.T) { @@ -104,12 +111,17 @@ func TestFollowLinksNormalized(t *testing.T) { out, err := FollowLinks(tmpDir, []string{"foo/bar/baz", "foo/bar"}) require.NoError(t, err) - require.Equal(t, out, []string{"foo/bar"}) + require.Equal(t, out, []string{filepath.FromSlash("foo/bar")}) + + rootPath := "/" + if runtime.GOOS == "windows" { + rootPath = "C:/" + } apply := fstest.Apply( fstest.CreateDir("dir", 0700), - fstest.Symlink("/foo", "dir/l1"), - fstest.Symlink("/", "dir/l2"), + fstest.Symlink(filepath.Join(rootPath, "foo"), "dir/l1"), + fstest.Symlink(rootPath, "dir/l2"), fstest.CreateDir("foo", 0700), fstest.CreateFile("foo/bar", nil, 0600), ) @@ -118,7 +130,7 @@ func TestFollowLinksNormalized(t *testing.T) { out, err = FollowLinks(tmpDir, []string{"dir/l1", "foo/bar"}) require.NoError(t, err) - require.Equal(t, out, []string{"dir/l1", "foo"}) + require.Equal(t, out, []string{filepath.FromSlash("dir/l1"), "foo"}) out, err = FollowLinks(tmpDir, []string{"dir/l2", "foo", "foo/bar"}) require.NoError(t, err) @@ -129,12 +141,17 @@ func TestFollowLinksNormalized(t *testing.T) { func TestFollowLinksWildcard(t *testing.T) { tmpDir := t.TempDir() + absolutePathForFoo := "/foo" + if runtime.GOOS == "windows" { + absolutePathForFoo = "C:/foo" + } + apply := fstest.Apply( fstest.CreateDir("dir", 0700), fstest.CreateDir("foo", 0700), - fstest.Symlink("/foo/bar1", "dir/l1"), - fstest.Symlink("/foo/bar2", "dir/l2"), - fstest.Symlink("/foo/bar3", "dir/anotherlink"), + fstest.Symlink(filepath.Join(absolutePathForFoo, "bar1"), "dir/l1"), + fstest.Symlink(filepath.Join(absolutePathForFoo, "bar2"), "dir/l2"), + fstest.Symlink(filepath.Join(absolutePathForFoo, "bar3"), "dir/anotherlink"), fstest.Symlink("../baz", "foo/bar2"), fstest.CreateFile("foo/bar1", nil, 0600), fstest.CreateFile("foo/bar3", nil, 0600), @@ -145,7 +162,7 @@ func TestFollowLinksWildcard(t *testing.T) { out, err := FollowLinks(tmpDir, []string{"dir/l*"}) require.NoError(t, err) - require.Equal(t, out, []string{"baz", "dir/l*", "foo/bar1", "foo/bar2"}) + require.Equal(t, []string{"baz", filepath.FromSlash("dir/l*"), filepath.FromSlash("foo/bar1"), filepath.FromSlash("foo/bar2")}, out) out, err = FollowLinks(tmpDir, []string{"dir"}) require.NoError(t, err) @@ -155,5 +172,5 @@ func TestFollowLinksWildcard(t *testing.T) { out, err = FollowLinks(tmpDir, []string{"dir", "dir/*link"}) require.NoError(t, err) - require.Equal(t, out, []string{"dir", "foo/bar3"}) + require.Equal(t, out, []string{"dir", filepath.FromSlash("foo/bar3")}) } diff --git a/followlinks_unix.go b/followlinks_unix.go new file mode 100644 index 00000000..41ae5e42 --- /dev/null +++ b/followlinks_unix.go @@ -0,0 +1,14 @@ +//go:build !windows +// +build !windows + +package fsutil + +import ( + "os" + + "github.com/pkg/errors" +) + +func isNotFound(err error) bool { + return errors.Is(err, os.ErrNotExist) +} diff --git a/followlinks_windows.go b/followlinks_windows.go new file mode 100644 index 00000000..3d954154 --- /dev/null +++ b/followlinks_windows.go @@ -0,0 +1,15 @@ +package fsutil + +import ( + "os" + + "github.com/pkg/errors" + "golang.org/x/sys/windows" +) + +func isNotFound(err error) bool { + if errors.Is(err, os.ErrNotExist) || errors.Is(err, windows.ERROR_INVALID_NAME) { + return true + } + return false +} diff --git a/go.mod b/go.mod index 80b0c6bf..09b5d035 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.18 require ( github.com/Microsoft/go-winio v0.5.2 - github.com/containerd/continuity v0.3.0 + github.com/containerd/continuity v0.3.1-0.20221025173834-5817935fcfbe github.com/gogo/protobuf v1.3.2 github.com/moby/patternmatcher v0.5.0 github.com/opencontainers/go-digest v1.0.0 diff --git a/go.sum b/go.sum index eb2b7c57..bccda3ad 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= -github.com/containerd/continuity v0.3.0 h1:nisirsYROK15TAMVukJOUyGJjz4BNQJBVsNvAXZJ/eg= -github.com/containerd/continuity v0.3.0/go.mod h1:wJEAIwKOm/pBZuBd0JmeTvnLquTB1Ag8espWhkykbPM= +github.com/containerd/continuity v0.3.1-0.20221025173834-5817935fcfbe h1:vrHfzd6wMSEZe3870709Tl6m6ZN3rhKmxxOTOWZFGIA= +github.com/containerd/continuity v0.3.1-0.20221025173834-5817935fcfbe/go.mod h1:wJEAIwKOm/pBZuBd0JmeTvnLquTB1Ag8espWhkykbPM= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/receive.go b/receive.go index 209d1d2f..c69fda9d 100644 --- a/receive.go +++ b/receive.go @@ -4,6 +4,7 @@ import ( "context" "io" "os" + "path/filepath" "sync" "github.com/pkg/errors" @@ -190,7 +191,7 @@ func (r *receiver) run(ctx context.Context) error { r.mu.Unlock() } i++ - cp := ¤tPath{path: p.Stat.Path, stat: p.Stat} + cp := ¤tPath{path: filepath.FromSlash(p.Stat.Path), stat: p.Stat} if err := r.orderValidator.HandleChange(ChangeKindAdd, cp.path, &StatInfo{cp.stat}, nil); err != nil { return err } diff --git a/receive_test.go b/receive_test.go index 0f9eb310..fe639c58 100644 --- a/receive_test.go +++ b/receive_test.go @@ -8,6 +8,7 @@ import ( "io" "os" "path/filepath" + "runtime" "sync" "testing" "time" @@ -98,42 +99,6 @@ func TestCopyWithSubDir(t *testing.T) { assert.Equal(t, "data1", string(dt)) } -func TestCopyDirectoryTimestamps(t *testing.T) { - d, err := tmpDir(changeStream([]string{ - "ADD foo dir", - "ADD foo/bar file data1", - })) - assert.NoError(t, err) - defer os.RemoveAll(d) - - timestamp := time.Unix(0, 0) - require.NoError(t, os.Chtimes(filepath.Join(d, "foo"), timestamp, timestamp)) - - dest := t.TempDir() - - eg, ctx := errgroup.WithContext(context.Background()) - s1, s2 := sockPairProto(ctx) - - eg.Go(func() error { - defer s1.(*fakeConnProto).closeSend() - return Send(ctx, s1, NewFS(d, nil), nil) - }) - eg.Go(func() error { - return Receive(ctx, s2, dest, ReceiveOpt{}) - }) - - err = eg.Wait() - assert.NoError(t, err) - - dt, err := os.ReadFile(filepath.Join(dest, "foo/bar")) - assert.NoError(t, err) - assert.Equal(t, "data1", string(dt)) - - stat, err := os.Stat(filepath.Join(dest, "foo")) - require.NoError(t, err) - assert.Equal(t, timestamp, stat.ModTime()) -} - func TestCopySwitchDirToFile(t *testing.T) { d, err := tmpDir(changeStream([]string{ "ADD foo file data1", @@ -143,10 +108,10 @@ func TestCopySwitchDirToFile(t *testing.T) { dest, err := tmpDir(changeStream([]string{ "ADD foo dir", - "ADD foo/bar dile data2", + "ADD foo/bar file data2", })) assert.NoError(t, err) - defer os.RemoveAll(d) + defer os.RemoveAll(dest) copy := func(src, dest string) (*changes, error) { ts := newNotificationBuffer() @@ -170,8 +135,13 @@ func TestCopySwitchDirToFile(t *testing.T) { NotifyHashed: chs.HandleChange, ContentHasher: simpleSHA256Hasher, Filter: func(_ string, s *types.Stat) bool { - s.Uid = uint32(os.Getuid()) - s.Gid = uint32(os.Getgid()) + if runtime.GOOS != "windows" { + // On Windows, Getuid() and Getgid() always return -1 + // See: https://pkg.go.dev/os#Getgid + // See: https://pkg.go.dev/os#Geteuid + s.Uid = uint32(os.Getuid()) + s.Gid = uint32(os.Getgid()) + } return true }, }) @@ -196,7 +166,7 @@ func TestCopySwitchDirToFile(t *testing.T) { err = Walk(context.Background(), dest, nil, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, string(b.Bytes()), `file foo + assert.Equal(t, b.String(), `file foo `) } @@ -239,8 +209,13 @@ func TestCopySimple(t *testing.T) { NotifyHashed: chs.HandleChange, ContentHasher: simpleSHA256Hasher, Filter: func(p string, s *types.Stat) bool { - s.Uid = uint32(os.Getuid()) - s.Gid = uint32(os.Getgid()) + if runtime.GOOS != "windows" { + // On Windows, Getuid() and Getgid() always return -1 + // See: https://pkg.go.dev/os#Getgid + // See: https://pkg.go.dev/os#Geteuid + s.Uid = uint32(os.Getuid()) + s.Gid = uint32(os.Getgid()) + } s.ModTime = tm.UnixNano() return true }, @@ -253,7 +228,18 @@ func TestCopySimple(t *testing.T) { err = Walk(context.Background(), dest, nil, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, string(b.Bytes()), `file foo + if runtime.GOOS == "windows" { + assert.Equal(t, b.String(), `file foo +file foo2 +dir zzz +file zzz\aa +dir zzz\bb +dir zzz\bb\cc +symlink:..\..\ zzz\bb\cc\dd +file zzz.aa +`) + } else { + assert.Equal(t, b.String(), `file foo file foo2 dir zzz file zzz/aa @@ -262,6 +248,7 @@ dir zzz/bb/cc symlink:../../ zzz/bb/cc/dd file zzz.aa `) + } dt, err := os.ReadFile(filepath.Join(dest, "zzz/aa")) assert.NoError(t, err) @@ -272,22 +259,33 @@ file zzz.aa assert.Equal(t, "dat2", string(dt)) fi, err := os.Stat(filepath.Join(dest, "foo2")) - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, tm, fi.ModTime()) - h, ok := ts.Hash("zzz/aa") + h, ok := ts.Hash(filepath.FromSlash("zzz/aa")) assert.True(t, ok) - assert.Equal(t, digest.Digest("sha256:99b6ef96ee0572b5b3a4eb28f00b715d820bfd73836e59cc1565e241f4d1bb2f"), h) + if runtime.GOOS == "windows" { + assert.Equal(t, digest.Digest("sha256:5da6b6a222dca8d9260384da15378d4389f7e16943e812e08c39759b8514b456"), h) + } else { + assert.Equal(t, digest.Digest("sha256:99b6ef96ee0572b5b3a4eb28f00b715d820bfd73836e59cc1565e241f4d1bb2f"), h) + } h, ok = ts.Hash("foo2") assert.True(t, ok) - assert.Equal(t, digest.Digest("sha256:dd2529f7749ba45ea55de3b2e10086d6494cc45a94e57650c2882a6a14b4ff32"), h) + if runtime.GOOS == "windows" { + assert.Equal(t, digest.Digest("sha256:cd83620e5308f6ddb9953a82b2c7450832eac78521dbf067d2882318cabc1311"), h) + } else { + assert.Equal(t, digest.Digest("sha256:dd2529f7749ba45ea55de3b2e10086d6494cc45a94e57650c2882a6a14b4ff32"), h) + } - h, ok = ts.Hash("zzz/bb/cc/dd") + h, ok = ts.Hash(filepath.FromSlash("zzz/bb/cc/dd")) assert.True(t, ok) - assert.Equal(t, digest.Digest("sha256:eca07e8f2d09bd574ea2496312e6de1685ef15b8e6a49a534ed9e722bcac8adc"), h) - - k, ok := chs.c["zzz/aa"] + if runtime.GOOS == "windows" { + assert.Equal(t, digest.Digest("sha256:47dc68d117ae85dc688103d6ba2cee54caabbbcf606e54ca62fda6a3d9deae19"), h) + } else { + assert.Equal(t, digest.Digest("sha256:eca07e8f2d09bd574ea2496312e6de1685ef15b8e6a49a534ed9e722bcac8adc"), h) + } + k, ok := chs.c[filepath.FromSlash("zzz/aa")] assert.Equal(t, ok, true) assert.Equal(t, k, ChangeKindAdd) @@ -317,8 +315,13 @@ file zzz.aa NotifyHashed: chs.HandleChange, ContentHasher: simpleSHA256Hasher, Filter: func(_ string, s *types.Stat) bool { - s.Uid = uint32(os.Getuid()) - s.Gid = uint32(os.Getgid()) + if runtime.GOOS != "windows" { + // On Windows, Getuid() and Getgid() always return -1 + // See: https://pkg.go.dev/os#Getgid + // See: https://pkg.go.dev/os#Geteuid + s.Uid = uint32(os.Getuid()) + s.Gid = uint32(os.Getgid()) + } s.ModTime = tm.UnixNano() return true }, @@ -330,8 +333,18 @@ file zzz.aa b = &bytes.Buffer{} err = Walk(context.Background(), dest, nil, bufWalk(b)) assert.NoError(t, err) - - assert.Equal(t, string(b.Bytes()), `file foo + if runtime.GOOS == "windows" { + assert.Equal(t, `file foo +dir zzz +file zzz\aa +dir zzz\bb +dir zzz\bb\cc +symlink:..\..\ zzz\bb\cc\dd +file zzz\bb\cc\foo +file zzz.aa +`, b.String()) + } else { + assert.Equal(t, `file foo dir zzz file zzz/aa dir zzz/bb @@ -339,20 +352,27 @@ dir zzz/bb/cc symlink:../../ zzz/bb/cc/dd file zzz/bb/cc/foo file zzz.aa -`) +`, b.String()) + } dt, err = os.ReadFile(filepath.Join(dest, "zzz/bb/cc/foo")) assert.NoError(t, err) assert.Equal(t, "data5", string(dt)) - h, ok = ts.Hash("zzz/bb/cc/dd") + h, ok = ts.Hash(filepath.FromSlash("zzz/bb/cc/dd")) assert.True(t, ok) - assert.Equal(t, digest.Digest("sha256:eca07e8f2d09bd574ea2496312e6de1685ef15b8e6a49a534ed9e722bcac8adc"), h) - - h, ok = ts.Hash("zzz/bb/cc/foo") + if runtime.GOOS == "windows" { + assert.Equal(t, digest.Digest("sha256:47dc68d117ae85dc688103d6ba2cee54caabbbcf606e54ca62fda6a3d9deae19"), h) + } else { + assert.Equal(t, digest.Digest("sha256:eca07e8f2d09bd574ea2496312e6de1685ef15b8e6a49a534ed9e722bcac8adc"), h) + } + h, ok = ts.Hash(filepath.FromSlash("zzz/bb/cc/foo")) assert.True(t, ok) - assert.Equal(t, digest.Digest("sha256:cd14a931fc2e123ded338093f2864b173eecdee578bba6ec24d0724272326c3a"), h) - + if runtime.GOOS == "windows" { + assert.Equal(t, digest.Digest("sha256:9184a7db8d056ee43838613279db9a7ab02272e50d5e20d253393521bb34aa46"), h) + } else { + assert.Equal(t, digest.Digest("sha256:cd14a931fc2e123ded338093f2864b173eecdee578bba6ec24d0724272326c3a"), h) + } _, ok = ts.Hash("foo2") assert.False(t, ok) @@ -360,15 +380,15 @@ file zzz.aa assert.Equal(t, ok, true) assert.Equal(t, k, ChangeKindDelete) - k, ok = chs.c["zzz/bb/cc/foo"] + k, ok = chs.c[filepath.FromSlash("zzz/bb/cc/foo")] assert.Equal(t, ok, true) assert.Equal(t, k, ChangeKindAdd) - _, ok = chs.c["zzz/aa"] + _, ok = chs.c[filepath.FromSlash("zzz/aa")] assert.Equal(t, ok, false) _, ok = chs.c["zzz.aa"] - assert.Equal(t, ok, false) + assert.Equal(t, false, ok) } func sockPairProto(ctx context.Context) (Stream, Stream) { @@ -490,8 +510,15 @@ func simpleSHA256Hasher(s *types.Stat) (hash.Hash, error) { // Unlike Linux, on FreeBSD's stat() call returns -1 in st_rdev for regular files ss.Devminor = 0 ss.Devmajor = 0 + if runtime.GOOS == "windows" { + // On Windows, Getuid() and Getgid() always return -1 + // See: https://pkg.go.dev/os#Getgid + // See: https://pkg.go.dev/os#Geteuid + ss.Uid = 0 + ss.Gid = 0 + } - if os.FileMode(ss.Mode)&os.ModeSymlink != 0 { + if os.FileMode(ss.Mode)&os.ModeSymlink != 0 && runtime.GOOS != "windows" { ss.Mode = ss.Mode | 0777 } @@ -502,3 +529,50 @@ func simpleSHA256Hasher(s *types.Stat) (hash.Hash, error) { h.Write(dt) return h, nil } + +type notificationBuffer struct { + items map[string]digest.Digest + sync.Mutex +} + +func newNotificationBuffer() *notificationBuffer { + nb := ¬ificationBuffer{ + items: map[string]digest.Digest{}, + } + return nb +} + +type hashed interface { + Digest() digest.Digest +} + +func (nb *notificationBuffer) HandleChange(kind ChangeKind, p string, fi os.FileInfo, err error) (retErr error) { + nb.Lock() + defer nb.Unlock() + if kind == ChangeKindDelete { + delete(nb.items, p) + } else { + h, ok := fi.(hashed) + if !ok { + return errors.Errorf("invalid FileInfo: %s", p) + } + nb.items[p] = h.Digest() + } + return nil +} + +func (nb *notificationBuffer) Hash(p string) (digest.Digest, bool) { + nb.Lock() + v, ok := nb.items[p] + nb.Unlock() + return v, ok +} + +// requiresRoot skips tests that require root +func requiresRoot(t *testing.T) { + t.Helper() + if os.Getuid() != 0 { + t.Skip("skipping test that requires root") + return + } +} diff --git a/stat.go b/stat.go index 2ab8da11..44441cb6 100644 --- a/stat.go +++ b/stat.go @@ -19,7 +19,7 @@ func mkstat(path, relpath string, fi os.FileInfo, inodemap map[uint64]string) (* relpath = filepath.ToSlash(relpath) stat := &types.Stat{ - Path: relpath, + Path: filepath.FromSlash(relpath), Mode: uint32(fi.Mode()), ModTime: fi.ModTime().UnixNano(), } diff --git a/validator.go b/validator.go index 9bd7d94d..ac3473f2 100644 --- a/validator.go +++ b/validator.go @@ -1,9 +1,9 @@ package fsutil import ( + "fmt" "os" - "path" - "runtime" + "path/filepath" "sort" "strings" "syscall" @@ -28,21 +28,20 @@ func (v *Validator) HandleChange(kind ChangeKind, p string, fi os.FileInfo, err if v.parentDirs == nil { v.parentDirs = make([]parent, 1, 10) } - if runtime.GOOS == "windows" { - p = strings.Replace(p, "\\", "", -1) - } - if p != path.Clean(p) { + + if p != filepath.Clean(p) { return errors.WithStack(&os.PathError{Path: p, Err: syscall.EINVAL, Op: "unclean path"}) } - if path.IsAbs(p) { + + if filepath.IsAbs(p) { return errors.WithStack(&os.PathError{Path: p, Err: syscall.EINVAL, Op: "absolute path"}) } - dir := path.Dir(p) - base := path.Base(p) + dir := filepath.Dir(p) + base := filepath.Base(p) if dir == "." { dir = "" } - if dir == ".." || strings.HasPrefix(p, "../") { + if dir == ".." || strings.HasPrefix(p, fmt.Sprintf("..%c", os.PathSeparator)) { return errors.WithStack(&os.PathError{Path: p, Err: syscall.EINVAL, Op: "escape check"}) } @@ -56,12 +55,12 @@ func (v *Validator) HandleChange(kind ChangeKind, p string, fi os.FileInfo, err } if dir != v.parentDirs[len(v.parentDirs)-1].dir || v.parentDirs[i].last >= base { - return errors.Errorf("changes out of order: %q %q", p, path.Join(v.parentDirs[i].dir, v.parentDirs[i].last)) + return errors.Errorf("changes out of order: %q %q", p, filepath.Join(v.parentDirs[i].dir, v.parentDirs[i].last)) } v.parentDirs[i].last = base if kind != ChangeKindDelete && fi.IsDir() { v.parentDirs = append(v.parentDirs, parent{ - dir: path.Join(dir, base), + dir: filepath.Join(dir, base), last: "", }) } @@ -76,7 +75,7 @@ func ComparePath(p1, p2 string) int { switch { case p1[i] == p2[i]: continue - case p2[i] != '/' && p1[i] < p2[i] || p1[i] == '/': + case p2[i] != os.PathSeparator && p1[i] < p2[i] || p1[i] == os.PathSeparator: return -1 default: return 1 diff --git a/validator_test.go b/validator_test.go index aabb0732..b3acf688 100644 --- a/validator_test.go +++ b/validator_test.go @@ -3,6 +3,7 @@ package fsutil import ( "fmt" "os" + "path/filepath" "strings" "testing" @@ -210,7 +211,7 @@ func parseChange(str string) *change { default: panic(errStr) } - c.path = f[1] + c.path = filepath.FromSlash(f[1]) st := &types.Stat{} switch f[2] { case "file": diff --git a/walker_test.go b/walker_test.go index bcc89bb7..419c0daf 100644 --- a/walker_test.go +++ b/walker_test.go @@ -28,7 +28,7 @@ func TestWalkerSimple(t *testing.T) { err = Walk(context.Background(), d, nil, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, string(b.Bytes()), `file foo + assert.Equal(t, b.String(), `file foo file foo2 `) @@ -48,9 +48,9 @@ func TestWalkerInclude(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, `dir bar + assert.Equal(t, filepath.FromSlash(`dir bar file bar/foo -`, string(b.Bytes())) +`), b.String()) b.Reset() err = Walk(context.Background(), d, &WalkOpt{ @@ -58,9 +58,9 @@ file bar/foo }, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, `dir bar + assert.Equal(t, filepath.FromSlash(`dir bar file bar/foo -`, string(b.Bytes())) +`), b.String()) b.Reset() err = Walk(context.Background(), d, &WalkOpt{ @@ -68,9 +68,9 @@ file bar/foo }, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, `dir bar + assert.Equal(t, filepath.FromSlash(`dir bar file bar/foo -`, string(b.Bytes())) +`), b.String()) b.Reset() err = Walk(context.Background(), d, &WalkOpt{ @@ -78,9 +78,9 @@ file bar/foo }, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, `dir bar + assert.Equal(t, filepath.FromSlash(`dir bar file bar/foo -`, string(b.Bytes())) +`), b.String()) b.Reset() err = Walk(context.Background(), d, &WalkOpt{ @@ -97,7 +97,7 @@ file bar/foo assert.NoError(t, err) assert.Equal(t, `file foo2 -`, string(b.Bytes())) +`, b.String()) b.Reset() err = Walk(context.Background(), d, &WalkOpt{ @@ -105,9 +105,9 @@ file bar/foo }, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, `dir bar + assert.Equal(t, filepath.FromSlash(`dir bar file bar/foo -`, string(b.Bytes())) +`), b.String()) b.Reset() err = Walk(context.Background(), d, &WalkOpt{ @@ -115,9 +115,9 @@ file bar/foo }, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, `dir bar + assert.Equal(t, filepath.FromSlash(`dir bar file bar/foo -`, string(b.Bytes())) +`), b.String()) b.Reset() err = Walk(context.Background(), d, &WalkOpt{ @@ -125,9 +125,9 @@ file bar/foo }, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, `dir bar + assert.Equal(t, filepath.FromSlash(`dir bar file bar/foo -`, string(b.Bytes())) +`), b.String()) } func TestWalkerExclude(t *testing.T) { @@ -145,24 +145,40 @@ func TestWalkerExclude(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, `file bar + assert.Equal(t, filepath.FromSlash(`file bar dir foo file foo/bar2 -`, string(b.Bytes())) +`), b.String()) } func TestWalkerFollowLinks(t *testing.T) { - d, err := tmpDir(changeStream([]string{ - "ADD bar file", - "ADD foo dir", - "ADD foo/l1 symlink /baz/one", - "ADD foo/l2 symlink /baz/two", - "ADD baz dir", - "ADD baz/one file", - "ADD baz/two symlink ../bax", - "ADD bax file", - "ADD bay file", // not included - })) + var d string + var err error + if runtime.GOOS == "windows" { + d, err = tmpDir(changeStream([]string{ + "ADD bar file", + "ADD foo dir", + "ADD foo/l1 symlink C:/baz/one", + "ADD foo/l2 symlink C:/baz/two", + "ADD baz dir", + "ADD baz/one file", + "ADD baz/two symlink ../bax", + "ADD bax file", + "ADD bay file", // not included + })) + } else { + d, err = tmpDir(changeStream([]string{ + "ADD bar file", + "ADD foo dir", + "ADD foo/l1 symlink /baz/one", + "ADD foo/l2 symlink /baz/two", + "ADD baz dir", + "ADD baz/one file", + "ADD baz/two symlink ../bax", + "ADD bax file", + "ADD bay file", // not included + })) + } assert.NoError(t, err) defer os.RemoveAll(d) b := &bytes.Buffer{} @@ -171,7 +187,18 @@ func TestWalkerFollowLinks(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, `file bar + if runtime.GOOS == "windows" { + assert.Equal(t, filepath.FromSlash(`file bar +file bax +dir baz +file baz/one +symlink:../bax baz/two +dir foo +symlink:C:/baz/one foo/l1 +symlink:C:/baz/two foo/l2 +`), b.String()) + } else { + assert.Equal(t, filepath.FromSlash(`file bar file bax dir baz file baz/one @@ -179,7 +206,8 @@ symlink:../bax baz/two dir foo symlink:/baz/one foo/l1 symlink:/baz/two foo/l2 -`, string(b.Bytes())) +`), b.String()) + } } func TestWalkerFollowLinksToRoot(t *testing.T) { @@ -198,12 +226,12 @@ func TestWalkerFollowLinksToRoot(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, `file bar + assert.Equal(t, filepath.FromSlash(`file bar file bax dir bay file bay/baz symlink:. foo -`, string(b.Bytes())) +`), b.String()) } func TestWalkerMap(t *testing.T) { @@ -227,10 +255,10 @@ func TestWalkerMap(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, `dir _foo + assert.Equal(t, filepath.FromSlash(`dir _foo file _foo/bar2 file _foo2 -`, string(b.Bytes())) +`), b.String()) } func TestWalkerMapSkipDir(t *testing.T) { @@ -261,10 +289,10 @@ func TestWalkerMapSkipDir(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, `dir includeDir + assert.Equal(t, filepath.FromSlash(`dir includeDir file includeDir/a.txt -`, string(b.Bytes())) - assert.Equal(t, []string{"excludeDir", "includeDir", "includeDir/a.txt"}, walked) +`), b.String()) + assert.Equal(t, []string{"excludeDir", "includeDir", filepath.FromSlash("includeDir/a.txt")}, walked) } func TestWalkerPermissionDenied(t *testing.T) { @@ -299,7 +327,7 @@ func TestWalkerPermissionDenied(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) assert.Equal(t, `dir foo -`, string(b.Bytes())) +`, b.String()) b.Reset() err = Walk(context.Background(), d, &WalkOpt{ @@ -307,7 +335,7 @@ func TestWalkerPermissionDenied(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) assert.Equal(t, `dir foo -`, string(b.Bytes())) +`, b.String()) b.Reset() err = Walk(context.Background(), d, &WalkOpt{ @@ -323,7 +351,7 @@ func TestWalkerPermissionDenied(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) assert.Equal(t, `dir foo -`, string(b.Bytes())) +`, b.String()) } func bufWalk(buf *bytes.Buffer) filepath.WalkFunc { @@ -521,7 +549,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - trimEqual(t, ` + trimEqual(t, filepath.FromSlash(` dir a dir a/b dir a/b/bar @@ -535,7 +563,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { dir foo/bar file foo/bar/bee file foo2 - `, string(b.Bytes())) + `), b.String()) b.Reset() err = Walk(context.Background(), d, &WalkOpt{ @@ -543,7 +571,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - trimEqual(t, ` + trimEqual(t, filepath.FromSlash(` dir a dir a/b dir a/b/bar @@ -554,7 +582,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { dir foo dir foo/bar file foo/bar/bee - `, string(b.Bytes())) + `), b.String()) b.Reset() err = Walk(context.Background(), d, &WalkOpt{ @@ -562,14 +590,14 @@ func TestWalkerDoublestarInclude(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - trimEqual(t, ` + trimEqual(t, filepath.FromSlash(` dir a dir a/b dir a/b/bar file a/b/bar/foo dir bar file bar/foo - `, string(b.Bytes())) + `), b.String()) b.Reset() err = Walk(context.Background(), d, &WalkOpt{ @@ -577,7 +605,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - trimEqual(t, ` + trimEqual(t, filepath.FromSlash(` dir a dir a/b dir a/b/bar @@ -590,7 +618,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { dir foo dir foo/bar file foo/bar/bee - `, string(b.Bytes())) + `), b.String()) b.Reset() err = Walk(context.Background(), d, &WalkOpt{ @@ -598,7 +626,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - trimEqual(t, ` + trimEqual(t, filepath.FromSlash(` dir a dir a/b dir a/b/bar @@ -606,7 +634,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { file a/b/bar/fop dir bar file bar/foo - `, string(b.Bytes())) + `), b.String()) b.Reset() err = Walk(context.Background(), d, &WalkOpt{ @@ -614,7 +642,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - trimEqual(t, ``, string(b.Bytes())) + trimEqual(t, ``, b.String()) b.Reset() err = Walk(context.Background(), d, &WalkOpt{ @@ -622,7 +650,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - trimEqual(t, ` + trimEqual(t, filepath.FromSlash(` dir a dir a/b dir a/b/bar @@ -634,7 +662,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { dir foo/bar file foo/bar/bee file foo2 - `, string(b.Bytes())) + `), b.String()) b.Reset() err = Walk(context.Background(), d, &WalkOpt{ @@ -642,7 +670,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - trimEqual(t, ` + trimEqual(t, filepath.FromSlash(` dir a dir a/b dir a/b/bar @@ -650,7 +678,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { file a/b/bar/fop dir bar file bar/foo - `, string(b.Bytes())) + `), b.String()) b.Reset() err = Walk(context.Background(), d, &WalkOpt{ @@ -658,14 +686,14 @@ func TestWalkerDoublestarInclude(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - trimEqual(t, ` + trimEqual(t, filepath.FromSlash(` dir a dir a/b dir a/b/bar file a/b/bar/foo dir bar file bar/foo - `, string(b.Bytes())) + `), b.String()) b.Reset() err = Walk(context.Background(), d, &WalkOpt{ @@ -673,11 +701,11 @@ func TestWalkerDoublestarInclude(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - trimEqual(t, ` + trimEqual(t, filepath.FromSlash(` dir foo dir foo/bar file foo/bar/bee - `, string(b.Bytes())) + `), b.String()) b.Reset() err = Walk(context.Background(), d, &WalkOpt{ @@ -685,12 +713,12 @@ func TestWalkerDoublestarInclude(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - trimEqual(t, ` + trimEqual(t, filepath.FromSlash(` dir a dir a/b dir a/b/baz dir baz - `, string(b.Bytes())) + `), b.String()) } func trimEqual(t assert.TestingT, expected, actual string, msgAndArgs ...interface{}) bool { From ac379c35c1179d38bbe271b63b8cda6a849a2c05 Mon Sep 17 00:00:00 2001 From: Gabriel Adrian Samfira Date: Thu, 17 Nov 2022 04:30:52 -0800 Subject: [PATCH 2/7] Copy timestamp --- copy/copy_windows.go | 13 ++++++++++++- copy/mkdir.go | 3 --- copy/mkdir_windows.go | 9 +++++++++ 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/copy/copy_windows.go b/copy/copy_windows.go index 68d20d7d..5e37e8e2 100644 --- a/copy/copy_windows.go +++ b/copy/copy_windows.go @@ -58,12 +58,23 @@ func (c *copier) copyFileInfo(fi os.FileInfo, src, name string) error { return err } + + if err := c.copyFileTimestamp(fi, name); err != nil { + return err + } return nil } func (c *copier) copyFileTimestamp(fi os.FileInfo, name string) error { - // TODO: copy windows specific metadata + if c.utime != nil { + return Utimes(name, c.utime) + } + if fi.Mode()&os.ModeSymlink == 0 { + if err := os.Chtimes(name, fi.ModTime(), fi.ModTime()); err != nil { + return errors.Wrap(err, "changing mtime") + } + } return nil } diff --git a/copy/mkdir.go b/copy/mkdir.go index 9553c08b..c2fdd8be 100644 --- a/copy/mkdir.go +++ b/copy/mkdir.go @@ -22,12 +22,10 @@ func MkdirAll(path string, perm os.FileMode, user Chowner, tm *time.Time) error for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator. i-- } - j := i for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element. j-- } - if j > 1 { // Create parent. err = MkdirAll(fixRootDirectory(path[:j-1]), perm, user, tm) @@ -35,7 +33,6 @@ func MkdirAll(path string, perm os.FileMode, user Chowner, tm *time.Time) error return err } } - dir, err1 := os.Lstat(path) if err1 == nil && dir.IsDir() { return nil diff --git a/copy/mkdir_windows.go b/copy/mkdir_windows.go index 6edb1f5f..988a9d50 100644 --- a/copy/mkdir_windows.go +++ b/copy/mkdir_windows.go @@ -28,6 +28,15 @@ func fixRootDirectory(p string) string { } func Utimes(p string, tm *time.Time) error { + info, err := os.Lstat(p) + if err != nil { + return errors.Wrap(err, "fetching file info") + } + if tm != nil && info.Mode()&os.ModeSymlink == 0 { + if err := os.Chtimes(p, *tm, *tm); err != nil { + return errors.Wrap(err, "changing times") + } + } return nil } From 7b64c8499fffcb038c84e1ca082d6bba48121d09 Mon Sep 17 00:00:00 2001 From: Gabriel Adrian Samfira Date: Thu, 17 Nov 2022 14:41:32 +0200 Subject: [PATCH 3/7] Update go.mod --- bench/go.mod | 2 +- bench/go.sum | 4 ++-- diskwriter_freebsd.go | 9 --------- 3 files changed, 3 insertions(+), 12 deletions(-) diff --git a/bench/go.mod b/bench/go.mod index 62d0af87..0f79e8d0 100644 --- a/bench/go.mod +++ b/bench/go.mod @@ -3,7 +3,7 @@ module github.com/tonistiigi/fsutil/bench go 1.18 require ( - github.com/containerd/continuity v0.3.0 + github.com/containerd/continuity v0.3.1-0.20221025173834-5817935fcfbe github.com/docker/docker v20.10.18+incompatible github.com/pkg/errors v0.9.1 github.com/tonistiigi/fsutil v0.0.0-00010101000000-000000000000 diff --git a/bench/go.sum b/bench/go.sum index d124a290..b3c855b9 100644 --- a/bench/go.sum +++ b/bench/go.sum @@ -140,8 +140,8 @@ github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe/go.mod h1:cE github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR3BEg7bDFaEddKm54WSmrol1fKWDU1nKYkgrcgZT7Y= github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ= github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM= -github.com/containerd/continuity v0.3.0 h1:nisirsYROK15TAMVukJOUyGJjz4BNQJBVsNvAXZJ/eg= -github.com/containerd/continuity v0.3.0/go.mod h1:wJEAIwKOm/pBZuBd0JmeTvnLquTB1Ag8espWhkykbPM= +github.com/containerd/continuity v0.3.1-0.20221025173834-5817935fcfbe h1:vrHfzd6wMSEZe3870709Tl6m6ZN3rhKmxxOTOWZFGIA= +github.com/containerd/continuity v0.3.1-0.20221025173834-5817935fcfbe/go.mod h1:wJEAIwKOm/pBZuBd0JmeTvnLquTB1Ag8espWhkykbPM= github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= github.com/containerd/fifo v0.0.0-20200410184934-f15a3290365b/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= diff --git a/diskwriter_freebsd.go b/diskwriter_freebsd.go index 31f3ec79..ed6356fa 100644 --- a/diskwriter_freebsd.go +++ b/diskwriter_freebsd.go @@ -4,8 +4,6 @@ package fsutil import ( - "os" - "github.com/tonistiigi/fsutil/types" "golang.org/x/sys/unix" ) @@ -17,10 +15,3 @@ func createSpecialFile(path string, mode uint32, stat *types.Stat) error { func mkdev(major int64, minor int64) uint64 { return unix.Mkdev(uint32(major), uint32(minor)) } - -func renameFile(src, dst string) error { - if err := os.Rename(src, dst); err != nil { - return errors.Wrapf(err, "failed to rename %s to %s", src, dst) - } - return nil -} From b38dde96a4e3cbc87c993047f90bd5c642e1f01e Mon Sep 17 00:00:00 2001 From: Gabriel Adrian Samfira Date: Thu, 17 Nov 2022 14:51:35 +0200 Subject: [PATCH 4/7] Add windows test workflow --- .github/workflows/ci.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b1bff2e4..5f66fe7b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,3 +95,17 @@ jobs: - name: Test run: vagrant ssh -- "cd /vagrant; go test -buildvcs=false ./..." + test-windows: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [windows-2019, windows-2022] + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-go@v2 + with: + go-version: 1.19 + - name: Test + run: | + go test -v ./... From 47cb68b2825e874b7dc8acf93eefd40f40850007 Mon Sep 17 00:00:00 2001 From: Gabriel Adrian Samfira Date: Thu, 17 Nov 2022 15:10:22 +0200 Subject: [PATCH 5/7] Use LF as a line ending --- followlinks_windows.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/followlinks_windows.go b/followlinks_windows.go index 3d954154..443ebd7f 100644 --- a/followlinks_windows.go +++ b/followlinks_windows.go @@ -1,15 +1,15 @@ -package fsutil - -import ( - "os" - - "github.com/pkg/errors" - "golang.org/x/sys/windows" -) - -func isNotFound(err error) bool { - if errors.Is(err, os.ErrNotExist) || errors.Is(err, windows.ERROR_INVALID_NAME) { - return true - } - return false -} +package fsutil + +import ( + "os" + + "github.com/pkg/errors" + "golang.org/x/sys/windows" +) + +func isNotFound(err error) bool { + if errors.Is(err, os.ErrNotExist) || errors.Is(err, windows.ERROR_INVALID_NAME) { + return true + } + return false +} From e66cb41804515dce8410e59f1792b302e387151b Mon Sep 17 00:00:00 2001 From: Gabriel Adrian Samfira Date: Fri, 18 Nov 2022 12:56:16 -0800 Subject: [PATCH 6/7] Fix copyFileInfo on Windows The c.chowner function was being ignored. This change makes use of the supplied chowner and calls Chown() on the file. If no chowner is specified, we copy over the DACL and owner information from the source. --- copy/copy_windows.go | 69 +++++++++++++++++++++++++++----------------- 1 file changed, 42 insertions(+), 27 deletions(-) diff --git a/copy/copy_windows.go b/copy/copy_windows.go index 5e37e8e2..58f822d0 100644 --- a/copy/copy_windows.go +++ b/copy/copy_windows.go @@ -17,46 +17,61 @@ func getUIDGID(fi os.FileInfo) (uid, gid int) { return 0, 0 } -func (c *copier) copyFileInfo(fi os.FileInfo, src, name string) error { - if err := os.Chmod(name, fi.Mode()); err != nil { - return errors.Wrapf(err, "failed to chmod %s", name) - } - - // Copy file ownership and ACL - // We need SeRestorePrivilege and SeTakeOwnershipPrivilege in order - // to restore security info on a file, especially if we're trying to - // apply security info which includes SIDs not necessarily present on - // the host. - privileges := []string{winio.SeRestorePrivilege, seTakeOwnershipPrivilege} - if err := winio.EnableProcessPrivileges(privileges); err != nil { - return err - } - defer winio.DisableProcessPrivileges(privileges) - +func getFileSecurityInfo(name string) (*windows.SID, *windows.ACL, error) { secInfo, err := windows.GetNamedSecurityInfo( - src, windows.SE_FILE_OBJECT, + name, windows.SE_FILE_OBJECT, windows.OWNER_SECURITY_INFORMATION|windows.DACL_SECURITY_INFORMATION) if err != nil { - return err + return nil, nil, errors.Wrap(err, "fetching security info") + } + sid, _, err := secInfo.Owner() + if err != nil { + return nil, nil, errors.Wrap(err, "fetching owner SID") } - dacl, _, err := secInfo.DACL() if err != nil { - return err + return nil, nil, errors.Wrap(err, "fetching dacl") } + return sid, dacl, nil +} - sid, _, err := secInfo.Owner() +func (c *copier) copyFileInfo(fi os.FileInfo, src, name string) error { + if err := os.Chmod(name, fi.Mode()); err != nil { + return errors.Wrapf(err, "failed to chmod %s", name) + } + + sid, dacl, err := getFileSecurityInfo(src) if err != nil { - return err + return errors.Wrap(err, "getting file info") } - if err := windows.SetNamedSecurityInfo( - name, windows.SE_FILE_OBJECT, - windows.OWNER_SECURITY_INFORMATION|windows.DACL_SECURITY_INFORMATION, - sid, nil, dacl, nil); err != nil { + if c.chown != nil { + // Use the defined chowner. + usr := &User{SID: sid.String()} + if err := Chown(name, usr, c.chown); err != nil { + return errors.Wrapf(err, "failed to chown %s", name) + } + return nil + } else { + // Copy file ownership and ACL from the source file. + // We need SeRestorePrivilege and SeTakeOwnershipPrivilege in order + // to restore security info on a file, especially if we're trying to + // apply security info which includes SIDs not necessarily present on + // the host. + privileges := []string{winio.SeRestorePrivilege, seTakeOwnershipPrivilege} + if err := winio.EnableProcessPrivileges(privileges); err != nil { + return err + } + defer winio.DisableProcessPrivileges(privileges) - return err + if err := windows.SetNamedSecurityInfo( + name, windows.SE_FILE_OBJECT, + windows.OWNER_SECURITY_INFORMATION|windows.DACL_SECURITY_INFORMATION, + sid, nil, dacl, nil); err != nil { + + return err + } } if err := c.copyFileTimestamp(fi, name); err != nil { From 45944776807ec82330f61049359fffebf45364e0 Mon Sep 17 00:00:00 2001 From: Gabriel Adrian Samfira Date: Thu, 9 Feb 2023 01:12:37 -0800 Subject: [PATCH 7/7] Undo dir mod time removal Mistakenly removed those tests and the mod time code. Signed-off-by: Gabriel Adrian Samfira --- chtimes_nolinux.go | 1 + copy/mkdir.go | 3 +++ diff.go | 3 +-- diskwriter.go | 40 +++++++++++++++++++++++++++++----------- receive_test.go | 36 ++++++++++++++++++++++++++++++++++++ 5 files changed, 70 insertions(+), 13 deletions(-) diff --git a/chtimes_nolinux.go b/chtimes_nolinux.go index a3ba0988..08251ec2 100644 --- a/chtimes_nolinux.go +++ b/chtimes_nolinux.go @@ -1,3 +1,4 @@ +//go:build !linux // +build !linux package fsutil diff --git a/copy/mkdir.go b/copy/mkdir.go index c2fdd8be..9553c08b 100644 --- a/copy/mkdir.go +++ b/copy/mkdir.go @@ -22,10 +22,12 @@ func MkdirAll(path string, perm os.FileMode, user Chowner, tm *time.Time) error for i > 0 && os.IsPathSeparator(path[i-1]) { // Skip trailing path separator. i-- } + j := i for j > 0 && !os.IsPathSeparator(path[j-1]) { // Scan backward over element. j-- } + if j > 1 { // Create parent. err = MkdirAll(fixRootDirectory(path[:j-1]), perm, user, tm) @@ -33,6 +35,7 @@ func MkdirAll(path string, perm os.FileMode, user Chowner, tm *time.Time) error return err } } + dir, err1 := os.Lstat(path) if err1 == nil && dir.IsDir() { return nil diff --git a/diff.go b/diff.go index d9fa5a0c..a7405dc5 100644 --- a/diff.go +++ b/diff.go @@ -4,7 +4,6 @@ import ( "context" "hash" "os" - "path/filepath" "github.com/pkg/errors" "github.com/tonistiigi/fsutil/types" @@ -33,7 +32,7 @@ func getWalkerFn(root string) walkerFn { } p := ¤tPath{ - path: filepath.FromSlash(path), + path: path, stat: stat, } diff --git a/diskwriter.go b/diskwriter.go index a9ba1de9..37c85f57 100644 --- a/diskwriter.go +++ b/diskwriter.go @@ -4,6 +4,7 @@ import ( "context" "hash" "io" + gofs "io/fs" "os" "path/filepath" "strconv" @@ -33,10 +34,11 @@ type DiskWriter struct { opt DiskWriterOpt dest string - ctx context.Context - cancel func() - eg *errgroup.Group - filter FilterFunc + ctx context.Context + cancel func() + eg *errgroup.Group + filter FilterFunc + dirModTimes map[string]int64 } func NewDiskWriter(ctx context.Context, dest string, opt DiskWriterOpt) (*DiskWriter, error) { @@ -51,17 +53,32 @@ func NewDiskWriter(ctx context.Context, dest string, opt DiskWriterOpt) (*DiskWr eg, ctx := errgroup.WithContext(ctx) return &DiskWriter{ - opt: opt, - dest: dest, - eg: eg, - ctx: ctx, - cancel: cancel, - filter: opt.Filter, + opt: opt, + dest: dest, + eg: eg, + ctx: ctx, + cancel: cancel, + filter: opt.Filter, + dirModTimes: map[string]int64{}, }, nil } func (dw *DiskWriter) Wait(ctx context.Context) error { - return dw.eg.Wait() + if err := dw.eg.Wait(); err != nil { + return err + } + return filepath.WalkDir(dw.dest, func(path string, d gofs.DirEntry, prevErr error) error { + if prevErr != nil { + return prevErr + } + if !d.IsDir() { + return nil + } + if mtime, ok := dw.dirModTimes[path]; ok { + return chtimes(path, mtime) + } + return nil + }) } func (dw *DiskWriter) HandleChange(kind ChangeKind, p string, fi os.FileInfo, err error) (retErr error) { @@ -147,6 +164,7 @@ func (dw *DiskWriter) HandleChange(kind ChangeKind, p string, fi os.FileInfo, er if err := os.Mkdir(newPath, fi.Mode()); err != nil { return errors.Wrapf(err, "failed to create dir %s", newPath) } + dw.dirModTimes[destPath] = statCopy.ModTime case fi.Mode()&os.ModeDevice != 0 || fi.Mode()&os.ModeNamedPipe != 0: if err := handleTarTypeBlockCharFifo(newPath, &statCopy); err != nil { return errors.Wrapf(err, "failed to create device %s", newPath) diff --git a/receive_test.go b/receive_test.go index fe639c58..e38ef200 100644 --- a/receive_test.go +++ b/receive_test.go @@ -99,6 +99,42 @@ func TestCopyWithSubDir(t *testing.T) { assert.Equal(t, "data1", string(dt)) } +func TestCopyDirectoryTimestamps(t *testing.T) { + d, err := tmpDir(changeStream([]string{ + "ADD foo dir", + "ADD foo/bar file data1", + })) + assert.NoError(t, err) + defer os.RemoveAll(d) + + timestamp := time.Unix(0, 0) + require.NoError(t, os.Chtimes(filepath.Join(d, "foo"), timestamp, timestamp)) + + dest := t.TempDir() + + eg, ctx := errgroup.WithContext(context.Background()) + s1, s2 := sockPairProto(ctx) + + eg.Go(func() error { + defer s1.(*fakeConnProto).closeSend() + return Send(ctx, s1, NewFS(d, nil), nil) + }) + eg.Go(func() error { + return Receive(ctx, s2, dest, ReceiveOpt{}) + }) + + err = eg.Wait() + assert.NoError(t, err) + + dt, err := os.ReadFile(filepath.Join(dest, "foo/bar")) + assert.NoError(t, err) + assert.Equal(t, "data1", string(dt)) + + stat, err := os.Stat(filepath.Join(dest, "foo")) + require.NoError(t, err) + assert.Equal(t, timestamp, stat.ModTime()) +} + func TestCopySwitchDirToFile(t *testing.T) { d, err := tmpDir(changeStream([]string{ "ADD foo file data1",