diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b1bff2e4..5f66fe7b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -95,3 +95,17 @@ jobs: - name: Test run: vagrant ssh -- "cd /vagrant; go test -buildvcs=false ./..." + test-windows: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [windows-2019, windows-2022] + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-go@v2 + with: + go-version: 1.19 + - name: Test + run: | + go test -v ./... diff --git a/bench/go.mod b/bench/go.mod index 62d0af87..0f79e8d0 100644 --- a/bench/go.mod +++ b/bench/go.mod @@ -3,7 +3,7 @@ module github.com/tonistiigi/fsutil/bench go 1.18 require ( - github.com/containerd/continuity v0.3.0 + github.com/containerd/continuity v0.3.1-0.20221025173834-5817935fcfbe github.com/docker/docker v20.10.18+incompatible github.com/pkg/errors v0.9.1 github.com/tonistiigi/fsutil v0.0.0-00010101000000-000000000000 diff --git a/bench/go.sum b/bench/go.sum index d124a290..b3c855b9 100644 --- a/bench/go.sum +++ b/bench/go.sum @@ -140,8 +140,8 @@ github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe/go.mod h1:cE github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR3BEg7bDFaEddKm54WSmrol1fKWDU1nKYkgrcgZT7Y= github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ= github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM= -github.com/containerd/continuity v0.3.0 h1:nisirsYROK15TAMVukJOUyGJjz4BNQJBVsNvAXZJ/eg= -github.com/containerd/continuity v0.3.0/go.mod h1:wJEAIwKOm/pBZuBd0JmeTvnLquTB1Ag8espWhkykbPM= +github.com/containerd/continuity v0.3.1-0.20221025173834-5817935fcfbe h1:vrHfzd6wMSEZe3870709Tl6m6ZN3rhKmxxOTOWZFGIA= +github.com/containerd/continuity v0.3.1-0.20221025173834-5817935fcfbe/go.mod h1:wJEAIwKOm/pBZuBd0JmeTvnLquTB1Ag8espWhkykbPM= github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= github.com/containerd/fifo v0.0.0-20200410184934-f15a3290365b/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= diff --git a/chtimes_nolinux.go b/chtimes_nolinux.go index a3ba0988..08251ec2 100644 --- a/chtimes_nolinux.go +++ b/chtimes_nolinux.go @@ -1,3 +1,4 @@ +//go:build !linux // +build !linux package fsutil diff --git a/copy/copy_test.go b/copy/copy_test.go index a7190bee..e77d54e7 100644 --- a/copy/copy_test.go +++ b/copy/copy_test.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path/filepath" + "runtime" "sort" "strings" "testing" @@ -13,10 +14,8 @@ import ( "github.com/containerd/continuity/fs/fstest" "github.com/pkg/errors" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/tonistiigi/fsutil" - "golang.org/x/sys/unix" ) // requiresRoot skips tests that require root @@ -44,7 +43,7 @@ func TestCopyDirectory(t *testing.T) { fstest.CreateDir("/home", 0755), ) - exp := "add:/etc,add:/etc/hosts,add:/etc/hosts.allow,add:/home,add:/usr,add:/usr/local,add:/usr/local/lib,add:/usr/local/lib/libnothing.so,add:/usr/local/lib/libnothing.so.2" + exp := filepath.FromSlash("add:/etc,add:/etc/hosts,add:/etc/hosts.allow,add:/home,add:/usr,add:/usr/local,add:/usr/local/lib,add:/usr/local/lib/libnothing.so,add:/usr/local/lib/libnothing.so.2") if err := testCopy(t, apply, exp); err != nil { t.Fatalf("Copy test failed: %+v", err) @@ -60,7 +59,7 @@ func TestCopyDirectoryWithLocalSymlink(t *testing.T) { fstest.Symlink("nothing.txt", "link-no-nothing.txt"), ) - exp := "add:/link-no-nothing.txt,add:/nothing.txt" + exp := filepath.FromSlash("add:/link-no-nothing.txt,add:/nothing.txt") if err := testCopy(t, apply, exp); err != nil { t.Fatalf("Copy test failed: %+v", err) @@ -84,49 +83,6 @@ func TestCopyToWorkDir(t *testing.T) { require.NoError(t, err) } -func TestCopyDevicesAndFifo(t *testing.T) { - requiresRoot(t) - - t1 := t.TempDir() - - err := mknod(filepath.Join(t1, "char"), unix.S_IFCHR|0444, int(unix.Mkdev(1, 9))) - require.NoError(t, err) - - err = mknod(filepath.Join(t1, "block"), unix.S_IFBLK|0441, int(unix.Mkdev(3, 2))) - require.NoError(t, err) - - err = mknod(filepath.Join(t1, "socket"), unix.S_IFSOCK|0555, 0) - require.NoError(t, err) - - err = unix.Mkfifo(filepath.Join(t1, "fifo"), 0555) - require.NoError(t, err) - - t2 := t.TempDir() - - err = Copy(context.TODO(), t1, ".", t2, ".") - require.NoError(t, err) - - fi, err := os.Lstat(filepath.Join(t2, "char")) - require.NoError(t, err) - assert.Equal(t, os.ModeCharDevice, fi.Mode()&os.ModeCharDevice) - assert.Equal(t, os.FileMode(0444), fi.Mode()&0777) - - fi, err = os.Lstat(filepath.Join(t2, "block")) - require.NoError(t, err) - assert.Equal(t, os.ModeDevice, fi.Mode()&os.ModeDevice) - assert.Equal(t, os.FileMode(0441), fi.Mode()&0777) - - fi, err = os.Lstat(filepath.Join(t2, "fifo")) - require.NoError(t, err) - assert.Equal(t, os.ModeNamedPipe, fi.Mode()&os.ModeNamedPipe) - assert.Equal(t, os.FileMode(0555), fi.Mode()&0777) - - fi, err = os.Lstat(filepath.Join(t2, "socket")) - require.NoError(t, err) - assert.NotEqual(t, os.ModeSocket, fi.Mode()&os.ModeSocket) // socket copied as stub - assert.Equal(t, os.FileMode(0555), fi.Mode()&0777) -} - func TestCopySingleFile(t *testing.T) { t1 := t.TempDir() @@ -168,7 +124,7 @@ func TestCopySingleFile(t *testing.T) { _, err = os.Stat(filepath.Join(t4, "a/b/c/foo2.txt")) require.NoError(t, err) - require.Equal(t, "add:/a/b/c/foo2.txt", ch.String()) + require.Equal(t, filepath.FromSlash("add:/a/b/c/foo2.txt"), ch.String()) } func TestCopyOverrideFile(t *testing.T) { @@ -221,7 +177,7 @@ func TestCopyDirectoryBasename(t *testing.T) { err = fstest.CheckDirectoryEqual(t1, t2) require.NoError(t, err) - require.Equal(t, "add:/foo,add:/foo/bar,add:/foo/bar/baz.txt", ch.String()) + require.Equal(t, filepath.FromSlash("add:/foo,add:/foo/bar,add:/foo/bar/baz.txt"), ch.String()) ch = &changeCollector{} err = Copy(context.TODO(), t1, "foo", t2, "foo", WithCopyInfo(CopyInfo{ @@ -230,7 +186,7 @@ func TestCopyDirectoryBasename(t *testing.T) { })) require.NoError(t, err) - require.Equal(t, "add:/foo/bar,add:/foo/bar/baz.txt", ch.String()) + require.Equal(t, filepath.FromSlash("add:/foo/bar,add:/foo/bar/baz.txt"), ch.String()) err = fstest.CheckDirectoryEqual(t1, t2) require.NoError(t, err) @@ -311,9 +267,12 @@ func TestCopyExistingDirDest(t *testing.T) { st, err := os.Lstat(filepath.Join(t2, "dir")) require.NoError(t, err) require.Equal(t, st.Mode()&os.ModePerm, os.FileMode(0700)) - uid, gid := getUIDGID(st) - require.Equal(t, 1, uid) - require.Equal(t, 1, gid) + var uid, gid int + if runtime.GOOS != "windows" { + uid, gid = getUIDGID(st) + require.Equal(t, 1, uid) + require.Equal(t, 1, gid) + } // verify that non-existing file was created _, err = os.Lstat(filepath.Join(t2, "dir/foo.txt")) @@ -323,9 +282,11 @@ func TestCopyExistingDirDest(t *testing.T) { st, err = os.Lstat(filepath.Join(t2, "dir/bar.txt")) require.NoError(t, err) require.Equal(t, os.FileMode(0644), st.Mode()&os.ModePerm) - uid, gid = getUIDGID(st) - require.Equal(t, 0, uid) - require.Equal(t, 0, gid) + if runtime.GOOS != "windows" { + uid, gid = getUIDGID(st) + require.Equal(t, 0, uid) + require.Equal(t, 0, gid) + } dt, err := os.ReadFile(filepath.Join(t2, "dir/bar.txt")) require.NoError(t, err) require.Equal(t, "bar-contents", string(dt)) @@ -376,7 +337,9 @@ func TestCopySymlinks(t *testing.T) { // verify that existing destination dir's metadata was not overwritten st, err := os.Lstat(filepath.Join(t2, "foo")) require.NoError(t, err) - require.Equal(t, os.FileMode(0644), st.Mode()&os.ModePerm) + if runtime.GOOS != "windows" { + require.Equal(t, os.FileMode(0644), st.Mode()&os.ModePerm) + } require.Equal(t, 0, int(st.Mode()&os.ModeSymlink)) dt, err := os.ReadFile(filepath.Join(t2, "foo")) require.NoError(t, err) @@ -439,37 +402,37 @@ func TestCopyIncludeExclude(t *testing.T) { name: "include bar", opts: []Opt{WithIncludePattern("bar")}, expectedResults: []string{"bar", "bar/foo", "bar/baz", "bar/baz/foo3"}, - expectedChanges: "add:/bar,add:/bar/baz,add:/bar/baz/foo3,add:/bar/foo", + expectedChanges: filepath.FromSlash("add:/bar,add:/bar/baz,add:/bar/baz/foo3,add:/bar/foo"), }, { name: "include *", opts: []Opt{WithIncludePattern("*")}, expectedResults: []string{"bar", "bar/foo", "bar/baz", "bar/baz/foo3", "foo2"}, - expectedChanges: "add:/bar,add:/bar/baz,add:/bar/baz/foo3,add:/bar/foo,add:/foo2", + expectedChanges: filepath.FromSlash("add:/bar,add:/bar/baz,add:/bar/baz/foo3,add:/bar/foo,add:/foo2"), }, { name: "include bar/foo", opts: []Opt{WithIncludePattern("bar/foo")}, expectedResults: []string{"bar", "bar/foo"}, - expectedChanges: "add:/bar/foo", + expectedChanges: filepath.FromSlash("add:/bar/foo"), }, { name: "include bar except bar/foo", opts: []Opt{WithIncludePattern("bar"), WithIncludePattern("!bar/foo")}, expectedResults: []string{"bar", "bar/baz", "bar/baz/foo3"}, - expectedChanges: "add:/bar,add:/bar/baz,add:/bar/baz/foo3", + expectedChanges: filepath.FromSlash("add:/bar,add:/bar/baz,add:/bar/baz/foo3"), }, { name: "include bar/foo and foo*", opts: []Opt{WithIncludePattern("bar/foo"), WithIncludePattern("foo*")}, expectedResults: []string{"bar", "bar/foo", "foo2"}, - expectedChanges: "add:/bar/foo,add:/foo2", + expectedChanges: filepath.FromSlash("add:/bar/foo,add:/foo2"), }, { name: "include b*", opts: []Opt{WithIncludePattern("b*")}, expectedResults: []string{"bar", "bar/foo", "bar/baz", "bar/baz/foo3"}, - expectedChanges: "add:/bar,add:/bar/baz,add:/bar/baz/foo3,add:/bar/foo", + expectedChanges: filepath.FromSlash("add:/bar,add:/bar/baz,add:/bar/baz/foo3,add:/bar/foo"), }, { name: "include bar/f*", @@ -530,25 +493,25 @@ func TestCopyIncludeExclude(t *testing.T) { name: "doublestar matching second item in path", opts: []Opt{WithIncludePattern("**/baz")}, expectedResults: []string{"bar", "bar/baz", "bar/baz/foo3"}, - expectedChanges: "add:/bar/baz,add:/bar/baz/foo3", + expectedChanges: filepath.FromSlash("add:/bar/baz,add:/bar/baz/foo3"), }, { name: "doublestar matching first item in path", opts: []Opt{WithIncludePattern("**/bar")}, expectedResults: []string{"bar", "bar/foo", "bar/baz", "bar/baz/foo3"}, - expectedChanges: "add:/bar,add:/bar/baz,add:/bar/baz/foo3,add:/bar/foo", + expectedChanges: filepath.FromSlash("add:/bar,add:/bar/baz,add:/bar/baz/foo3,add:/bar/foo"), }, { name: "doublestar exclude", opts: []Opt{WithIncludePattern("bar"), WithExcludePattern("**/foo3")}, expectedResults: []string{"bar", "bar/foo", "bar/baz"}, - expectedChanges: "add:/bar,add:/bar/baz,add:/bar/foo", + expectedChanges: filepath.FromSlash("add:/bar,add:/bar/baz,add:/bar/foo"), }, { name: "exclude bar/baz", opts: []Opt{WithExcludePattern("bar/baz")}, expectedResults: []string{"bar", "bar/foo", "foo2"}, - expectedChanges: "add:/bar,add:/bar/foo,add:/foo2", + expectedChanges: filepath.FromSlash("add:/bar,add:/bar/foo,add:/foo2"), }, } diff --git a/copy/copy_unix_test.go b/copy/copy_unix_test.go new file mode 100644 index 00000000..df710d4c --- /dev/null +++ b/copy/copy_unix_test.go @@ -0,0 +1,58 @@ +//go:build !windows +// +build !windows + +package fs + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" +) + +func TestCopyDevicesAndFifo(t *testing.T) { + requiresRoot(t) + + t1 := t.TempDir() + + err := mknod(filepath.Join(t1, "char"), unix.S_IFCHR|0444, int(unix.Mkdev(1, 9))) + require.NoError(t, err) + + err = mknod(filepath.Join(t1, "block"), unix.S_IFBLK|0441, int(unix.Mkdev(3, 2))) + require.NoError(t, err) + + err = mknod(filepath.Join(t1, "socket"), unix.S_IFSOCK|0555, 0) + require.NoError(t, err) + + err = unix.Mkfifo(filepath.Join(t1, "fifo"), 0555) + require.NoError(t, err) + + t2 := t.TempDir() + + err = Copy(context.TODO(), t1, ".", t2, ".") + require.NoError(t, err) + + fi, err := os.Lstat(filepath.Join(t2, "char")) + require.NoError(t, err) + assert.Equal(t, os.ModeCharDevice, fi.Mode()&os.ModeCharDevice) + assert.Equal(t, os.FileMode(0444), fi.Mode()&0777) + + fi, err = os.Lstat(filepath.Join(t2, "block")) + require.NoError(t, err) + assert.Equal(t, os.ModeDevice, fi.Mode()&os.ModeDevice) + assert.Equal(t, os.FileMode(0441), fi.Mode()&0777) + + fi, err = os.Lstat(filepath.Join(t2, "fifo")) + require.NoError(t, err) + assert.Equal(t, os.ModeNamedPipe, fi.Mode()&os.ModeNamedPipe) + assert.Equal(t, os.FileMode(0555), fi.Mode()&0777) + + fi, err = os.Lstat(filepath.Join(t2, "socket")) + require.NoError(t, err) + assert.NotEqual(t, os.ModeSocket, fi.Mode()&os.ModeSocket) // socket copied as stub + assert.Equal(t, os.FileMode(0555), fi.Mode()&0777) +} diff --git a/copy/copy_windows.go b/copy/copy_windows.go index 19a44a75..58f822d0 100644 --- a/copy/copy_windows.go +++ b/copy/copy_windows.go @@ -13,53 +13,83 @@ const ( seTakeOwnershipPrivilege = "SeTakeOwnershipPrivilege" ) -func (c *copier) copyFileInfo(fi os.FileInfo, src, name string) error { - if err := os.Chmod(name, fi.Mode()); err != nil { - return errors.Wrapf(err, "failed to chmod %s", name) - } - - // Copy file ownership and ACL - // We need SeRestorePrivilege and SeTakeOwnershipPrivilege in order - // to restore security info on a file, especially if we're trying to - // apply security info which includes SIDs not necessarily present on - // the host. - privileges := []string{winio.SeRestorePrivilege, seTakeOwnershipPrivilege} - if err := winio.EnableProcessPrivileges(privileges); err != nil { - return err - } - defer winio.DisableProcessPrivileges(privileges) +func getUIDGID(fi os.FileInfo) (uid, gid int) { + return 0, 0 +} +func getFileSecurityInfo(name string) (*windows.SID, *windows.ACL, error) { secInfo, err := windows.GetNamedSecurityInfo( - src, windows.SE_FILE_OBJECT, + name, windows.SE_FILE_OBJECT, windows.OWNER_SECURITY_INFORMATION|windows.DACL_SECURITY_INFORMATION) if err != nil { - return err + return nil, nil, errors.Wrap(err, "fetching security info") + } + sid, _, err := secInfo.Owner() + if err != nil { + return nil, nil, errors.Wrap(err, "fetching owner SID") } - dacl, _, err := secInfo.DACL() if err != nil { - return err + return nil, nil, errors.Wrap(err, "fetching dacl") } + return sid, dacl, nil +} - sid, _, err := secInfo.Owner() +func (c *copier) copyFileInfo(fi os.FileInfo, src, name string) error { + if err := os.Chmod(name, fi.Mode()); err != nil { + return errors.Wrapf(err, "failed to chmod %s", name) + } + + sid, dacl, err := getFileSecurityInfo(src) if err != nil { - return err + return errors.Wrap(err, "getting file info") } - if err := windows.SetNamedSecurityInfo( - name, windows.SE_FILE_OBJECT, - windows.OWNER_SECURITY_INFORMATION|windows.DACL_SECURITY_INFORMATION, - sid, nil, dacl, nil); err != nil { + if c.chown != nil { + // Use the defined chowner. + usr := &User{SID: sid.String()} + if err := Chown(name, usr, c.chown); err != nil { + return errors.Wrapf(err, "failed to chown %s", name) + } + return nil + } else { + // Copy file ownership and ACL from the source file. + // We need SeRestorePrivilege and SeTakeOwnershipPrivilege in order + // to restore security info on a file, especially if we're trying to + // apply security info which includes SIDs not necessarily present on + // the host. + privileges := []string{winio.SeRestorePrivilege, seTakeOwnershipPrivilege} + if err := winio.EnableProcessPrivileges(privileges); err != nil { + return err + } + defer winio.DisableProcessPrivileges(privileges) + + if err := windows.SetNamedSecurityInfo( + name, windows.SE_FILE_OBJECT, + windows.OWNER_SECURITY_INFORMATION|windows.DACL_SECURITY_INFORMATION, + sid, nil, dacl, nil); err != nil { + + return err + } + } + if err := c.copyFileTimestamp(fi, name); err != nil { return err } return nil } func (c *copier) copyFileTimestamp(fi os.FileInfo, name string) error { - // TODO: copy windows specific metadata + if c.utime != nil { + return Utimes(name, c.utime) + } + if fi.Mode()&os.ModeSymlink == 0 { + if err := os.Chtimes(name, fi.ModTime(), fi.ModTime()); err != nil { + return errors.Wrap(err, "changing mtime") + } + } return nil } diff --git a/copy/mkdir_windows.go b/copy/mkdir_windows.go index 6edb1f5f..988a9d50 100644 --- a/copy/mkdir_windows.go +++ b/copy/mkdir_windows.go @@ -28,6 +28,15 @@ func fixRootDirectory(p string) string { } func Utimes(p string, tm *time.Time) error { + info, err := os.Lstat(p) + if err != nil { + return errors.Wrap(err, "fetching file info") + } + if tm != nil && info.Mode()&os.ModeSymlink == 0 { + if err := os.Chtimes(p, *tm, *tm); err != nil { + return errors.Wrap(err, "changing times") + } + } return nil } diff --git a/diff_containerd.go b/diff_containerd.go index d8619abf..9b2050e4 100644 --- a/diff_containerd.go +++ b/diff_containerd.go @@ -5,6 +5,7 @@ import ( "context" "io" "os" + "path/filepath" "strings" "github.com/tonistiigi/fsutil/types" @@ -110,7 +111,7 @@ func doubleWalkDiff(ctx context.Context, changeFn ChangeFunc, a, b walkerFn, fil if filter != nil { filter(f2.path, &statCopy) } - f2copy = ¤tPath{path: f2.path, stat: &statCopy} + f2copy = ¤tPath{path: filepath.FromSlash(f2.path), stat: &statCopy} } k, p := pathChange(f1, f2copy) switch k { @@ -169,7 +170,6 @@ func pathChange(lower, upper *currentPath) (ChangeKind, string) { if upper == nil { return ChangeKindDelete, lower.path } - switch i := ComparePath(lower.path, upper.path); { case i < 0: // File in lower that is not in upper diff --git a/diskwriter.go b/diskwriter.go index b822644d..37c85f57 100644 --- a/diskwriter.go +++ b/diskwriter.go @@ -205,7 +205,8 @@ func (dw *DiskWriter) HandleChange(kind ChangeKind, p string, fi os.FileInfo, er return errors.Wrapf(err, "failed to remove %s", destPath) } } - if err := os.Rename(newPath, destPath); err != nil { + + if err := renameFile(newPath, destPath); err != nil { return errors.Wrapf(err, "failed to rename %s to %s", newPath, destPath) } } diff --git a/diskwriter_test.go b/diskwriter_test.go index 4b6edb3b..7e069628 100644 --- a/diskwriter_test.go +++ b/diskwriter_test.go @@ -1,3 +1,6 @@ +//go:build !windows +// +build !windows + package fsutil import ( @@ -6,27 +9,15 @@ import ( "io" "os" "path/filepath" - "sync" "syscall" "testing" "time" - "github.com/opencontainers/go-digest" - "github.com/pkg/errors" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "golang.org/x/sys/unix" ) -// requiresRoot skips tests that require root -func requiresRoot(t *testing.T) { - t.Helper() - if os.Getuid() != 0 { - t.Skip("skipping test that requires root") - return - } -} - func TestWriterSimple(t *testing.T) { requiresRoot(t) @@ -54,7 +45,7 @@ func TestWriterSimple(t *testing.T) { err = Walk(context.Background(), dest, nil, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, string(b.Bytes()), `dir bar + assert.Equal(t, b.String(), `dir bar file bar/foo symlink:../foo bar/foo2 file foo @@ -91,7 +82,7 @@ func TestWriterFileToDir(t *testing.T) { err = Walk(context.Background(), dest, nil, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, string(b.Bytes()), `dir foo + assert.Equal(t, b.String(), `dir foo file foo/bar `) } @@ -124,7 +115,7 @@ func TestWriterDirToFile(t *testing.T) { err = Walk(context.Background(), dest, nil, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, string(b.Bytes()), `file foo + assert.Equal(t, b.String(), `file foo `) } @@ -153,7 +144,7 @@ func TestWalkerWriterSimple(t *testing.T) { err = Walk(context.Background(), dest, nil, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, string(b.Bytes()), `dir bar + assert.Equal(t, b.String(), `dir bar file bar/foo symlink:../foo bar/foo2 file foo @@ -294,41 +285,3 @@ func newWriteToFunc(baseDir string, delay time.Duration) WriteToFunc { return nil } } - -type notificationBuffer struct { - items map[string]digest.Digest - sync.Mutex -} - -func newNotificationBuffer() *notificationBuffer { - nb := ¬ificationBuffer{ - items: map[string]digest.Digest{}, - } - return nb -} - -type hashed interface { - Digest() digest.Digest -} - -func (nb *notificationBuffer) HandleChange(kind ChangeKind, p string, fi os.FileInfo, err error) (retErr error) { - nb.Lock() - defer nb.Unlock() - if kind == ChangeKindDelete { - delete(nb.items, p) - } else { - h, ok := fi.(hashed) - if !ok { - return errors.Errorf("invalid FileInfo: %s", p) - } - nb.items[p] = h.Digest() - } - return nil -} - -func (nb *notificationBuffer) Hash(p string) (digest.Digest, bool) { - nb.Lock() - v, ok := nb.items[p] - nb.Unlock() - return v, ok -} diff --git a/diskwriter_unix.go b/diskwriter_unix.go index 1d97d6f9..ccd554ab 100644 --- a/diskwriter_unix.go +++ b/diskwriter_unix.go @@ -51,3 +51,10 @@ func handleTarTypeBlockCharFifo(path string, stat *types.Stat) error { } return nil } + +func renameFile(src, dst string) error { + if err := os.Rename(src, dst); err != nil { + return errors.Wrapf(err, "failed to rename %s to %s", src, dst) + } + return nil +} diff --git a/diskwriter_windows.go b/diskwriter_windows.go index 036544f0..13dbcf19 100644 --- a/diskwriter_windows.go +++ b/diskwriter_windows.go @@ -1,8 +1,15 @@ +//go:build windows // +build windows package fsutil import ( + "fmt" + ioFS "io/fs" + "os" + "syscall" + + "github.com/Microsoft/go-winio" "github.com/pkg/errors" "github.com/tonistiigi/fsutil/types" ) @@ -16,3 +23,75 @@ func rewriteMetadata(p string, stat *types.Stat) error { func handleTarTypeBlockCharFifo(path string, stat *types.Stat) error { return errors.New("Not implemented on windows") } + +func getFileHandle(path string, info ioFS.FileInfo) (syscall.Handle, error) { + p, err := syscall.UTF16PtrFromString(path) + if err != nil { + return 0, errors.Wrap(err, "converting string to UTF-16") + } + attrs := uint32(syscall.FILE_FLAG_BACKUP_SEMANTICS) + if info.Mode()&os.ModeSymlink != 0 { + // Use FILE_FLAG_OPEN_REPARSE_POINT, otherwise CreateFile will follow symlink. + // See https://docs.microsoft.com/en-us/windows/desktop/FileIO/symbolic-link-effects-on-file-systems-functions#createfile-and-createfiletransacted + attrs |= syscall.FILE_FLAG_OPEN_REPARSE_POINT + } + h, err := syscall.CreateFile(p, 0, 0, nil, syscall.OPEN_EXISTING, attrs, 0) + if err != nil { + return 0, errors.Wrap(err, "getting file handle") + } + return h, nil +} + +func readlink(path string, info ioFS.FileInfo) ([]byte, error) { + h, err := getFileHandle(path, info) + if err != nil { + return nil, errors.Wrap(err, "getting file handle") + } + defer syscall.CloseHandle(h) + + rdbbuf := make([]byte, syscall.MAXIMUM_REPARSE_DATA_BUFFER_SIZE) + var bytesReturned uint32 + err = syscall.DeviceIoControl(h, syscall.FSCTL_GET_REPARSE_POINT, nil, 0, &rdbbuf[0], uint32(len(rdbbuf)), &bytesReturned, nil) + if err != nil { + return nil, errors.Wrap(err, "sending I/O control command") + } + return rdbbuf[:bytesReturned], nil +} + +func getReparsePoint(path string, info ioFS.FileInfo) (*winio.ReparsePoint, error) { + target, err := readlink(path, info) + if err != nil { + return nil, errors.Wrap(err, "fetching link") + } + rp, err := winio.DecodeReparsePoint(target) + if err != nil { + return nil, errors.Wrap(err, "decoding reparse point") + } + return rp, nil +} + +func renameFile(src, dst string) error { + info, err := os.Lstat(dst) + if err != nil { + if !os.IsNotExist(err) { + return errors.Wrap(err, "getting file info") + } + } + + if info != nil && info.Mode()&os.ModeSymlink != 0 { + dstInfoRp, err := getReparsePoint(dst, info) + if err != nil { + return errors.Wrap(err, "getting reparse point") + } + if dstInfoRp.IsMountPoint { + return fmt.Errorf("%s is a mount point", dst) + } + if err := os.Remove(dst); err != nil { + return errors.Wrapf(err, "removing %s", dst) + } + } + if err := os.Rename(src, dst); err != nil { + return errors.Wrapf(err, "failed to rename %s to %s", src, dst) + } + return nil +} diff --git a/followlinks.go b/followlinks.go index 136a9082..f6809f62 100644 --- a/followlinks.go +++ b/followlinks.go @@ -31,7 +31,13 @@ type symlinkResolver struct { } func (r *symlinkResolver) append(p string) error { - p = filepath.Join(".", p) + p = filepath.FromSlash(p) + if runtime.GOOS == "windows" && filepath.IsAbs(p) { + absParts := strings.SplitN(p, ":", 2) + if len(absParts) == 2 { + p = absParts[1] + } + } current := "." for { parts := strings.SplitN(p, string(filepath.Separator), 2) @@ -41,7 +47,6 @@ func (r *symlinkResolver) append(p string) error { if err != nil { return err } - p = "" if len(parts) == 2 { p = parts[1] @@ -76,7 +81,7 @@ func (r *symlinkResolver) readSymlink(p string, allowWildcard bool) ([]string, e if allowWildcard && containsWildcards(base) { fis, err := os.ReadDir(filepath.Dir(realPath)) if err != nil { - if errors.Is(err, os.ErrNotExist) { + if isNotFound(err) { return nil, nil } return nil, errors.Wrap(err, "readdir") @@ -96,7 +101,7 @@ func (r *symlinkResolver) readSymlink(p string, allowWildcard bool) ([]string, e fi, err := os.Lstat(realPath) if err != nil { - if errors.Is(err, os.ErrNotExist) { + if isNotFound(err) { return nil, nil } return nil, errors.WithStack(err) diff --git a/followlinks_test.go b/followlinks_test.go index 522763f6..4d32f1a9 100644 --- a/followlinks_test.go +++ b/followlinks_test.go @@ -1,6 +1,8 @@ package fsutil import ( + "path/filepath" + "runtime" "testing" "github.com/containerd/continuity/fs/fstest" @@ -24,7 +26,7 @@ func TestFollowLinks(t *testing.T) { out, err := FollowLinks(tmpDir, []string{"l2", "bar"}) require.NoError(t, err) - require.Equal(t, out, []string{"bar", "dir/foo", "dir/l1", "l2"}) + require.Equal(t, out, []string{"bar", filepath.FromSlash("dir/foo"), filepath.FromSlash("dir/l1"), "l2"}) } func TestFollowLinksLoop(t *testing.T) { @@ -46,9 +48,14 @@ func TestFollowLinksLoop(t *testing.T) { func TestFollowLinksAbsolute(t *testing.T) { tmpDir := t.TempDir() + abslutePathForBaz := "/foo/bar/baz" + if runtime.GOOS == "windows" { + abslutePathForBaz = "C:/foo/bar/baz" + } + apply := fstest.Apply( fstest.CreateDir("dir", 0700), - fstest.Symlink("/foo/bar/baz", "dir/l1"), + fstest.Symlink(abslutePathForBaz, "dir/l1"), fstest.CreateDir("foo", 0700), fstest.Symlink("../", "foo/bar"), fstest.CreateFile("baz", nil, 0600), @@ -58,14 +65,14 @@ func TestFollowLinksAbsolute(t *testing.T) { out, err := FollowLinks(tmpDir, []string{"dir/l1"}) require.NoError(t, err) - require.Equal(t, out, []string{"baz", "dir/l1", "foo/bar"}) + require.Equal(t, []string{"baz", filepath.FromSlash("dir/l1"), filepath.FromSlash("foo/bar")}, out) // same but a link outside root tmpDir = t.TempDir() apply = fstest.Apply( fstest.CreateDir("dir", 0700), - fstest.Symlink("/foo/bar/baz", "dir/l1"), + fstest.Symlink(abslutePathForBaz, "dir/l1"), fstest.CreateDir("foo", 0700), fstest.Symlink("../../../", "foo/bar"), fstest.CreateFile("baz", nil, 0600), @@ -75,7 +82,7 @@ func TestFollowLinksAbsolute(t *testing.T) { out, err = FollowLinks(tmpDir, []string{"dir/l1"}) require.NoError(t, err) - require.Equal(t, out, []string{"baz", "dir/l1", "foo/bar"}) + require.Equal(t, []string{"baz", filepath.FromSlash("dir/l1"), filepath.FromSlash("foo/bar")}, out) } func TestFollowLinksNotExists(t *testing.T) { @@ -84,7 +91,7 @@ func TestFollowLinksNotExists(t *testing.T) { out, err := FollowLinks(tmpDir, []string{"foo/bar/baz", "bar/baz"}) require.NoError(t, err) - require.Equal(t, out, []string{"bar/baz", "foo/bar/baz"}) + require.Equal(t, out, []string{filepath.FromSlash("bar/baz"), filepath.FromSlash("foo/bar/baz")}) // root works fine with empty directory out, err = FollowLinks(tmpDir, []string{"."}) @@ -95,7 +102,7 @@ func TestFollowLinksNotExists(t *testing.T) { out, err = FollowLinks(tmpDir, []string{"f*/foo/t*"}) require.NoError(t, err) - require.Equal(t, out, []string{"f*/foo/t*"}) + require.Equal(t, []string{filepath.FromSlash("f*/foo/t*")}, out) } func TestFollowLinksNormalized(t *testing.T) { @@ -104,12 +111,17 @@ func TestFollowLinksNormalized(t *testing.T) { out, err := FollowLinks(tmpDir, []string{"foo/bar/baz", "foo/bar"}) require.NoError(t, err) - require.Equal(t, out, []string{"foo/bar"}) + require.Equal(t, out, []string{filepath.FromSlash("foo/bar")}) + + rootPath := "/" + if runtime.GOOS == "windows" { + rootPath = "C:/" + } apply := fstest.Apply( fstest.CreateDir("dir", 0700), - fstest.Symlink("/foo", "dir/l1"), - fstest.Symlink("/", "dir/l2"), + fstest.Symlink(filepath.Join(rootPath, "foo"), "dir/l1"), + fstest.Symlink(rootPath, "dir/l2"), fstest.CreateDir("foo", 0700), fstest.CreateFile("foo/bar", nil, 0600), ) @@ -118,7 +130,7 @@ func TestFollowLinksNormalized(t *testing.T) { out, err = FollowLinks(tmpDir, []string{"dir/l1", "foo/bar"}) require.NoError(t, err) - require.Equal(t, out, []string{"dir/l1", "foo"}) + require.Equal(t, out, []string{filepath.FromSlash("dir/l1"), "foo"}) out, err = FollowLinks(tmpDir, []string{"dir/l2", "foo", "foo/bar"}) require.NoError(t, err) @@ -129,12 +141,17 @@ func TestFollowLinksNormalized(t *testing.T) { func TestFollowLinksWildcard(t *testing.T) { tmpDir := t.TempDir() + absolutePathForFoo := "/foo" + if runtime.GOOS == "windows" { + absolutePathForFoo = "C:/foo" + } + apply := fstest.Apply( fstest.CreateDir("dir", 0700), fstest.CreateDir("foo", 0700), - fstest.Symlink("/foo/bar1", "dir/l1"), - fstest.Symlink("/foo/bar2", "dir/l2"), - fstest.Symlink("/foo/bar3", "dir/anotherlink"), + fstest.Symlink(filepath.Join(absolutePathForFoo, "bar1"), "dir/l1"), + fstest.Symlink(filepath.Join(absolutePathForFoo, "bar2"), "dir/l2"), + fstest.Symlink(filepath.Join(absolutePathForFoo, "bar3"), "dir/anotherlink"), fstest.Symlink("../baz", "foo/bar2"), fstest.CreateFile("foo/bar1", nil, 0600), fstest.CreateFile("foo/bar3", nil, 0600), @@ -145,7 +162,7 @@ func TestFollowLinksWildcard(t *testing.T) { out, err := FollowLinks(tmpDir, []string{"dir/l*"}) require.NoError(t, err) - require.Equal(t, out, []string{"baz", "dir/l*", "foo/bar1", "foo/bar2"}) + require.Equal(t, []string{"baz", filepath.FromSlash("dir/l*"), filepath.FromSlash("foo/bar1"), filepath.FromSlash("foo/bar2")}, out) out, err = FollowLinks(tmpDir, []string{"dir"}) require.NoError(t, err) @@ -155,5 +172,5 @@ func TestFollowLinksWildcard(t *testing.T) { out, err = FollowLinks(tmpDir, []string{"dir", "dir/*link"}) require.NoError(t, err) - require.Equal(t, out, []string{"dir", "foo/bar3"}) + require.Equal(t, out, []string{"dir", filepath.FromSlash("foo/bar3")}) } diff --git a/followlinks_unix.go b/followlinks_unix.go new file mode 100644 index 00000000..41ae5e42 --- /dev/null +++ b/followlinks_unix.go @@ -0,0 +1,14 @@ +//go:build !windows +// +build !windows + +package fsutil + +import ( + "os" + + "github.com/pkg/errors" +) + +func isNotFound(err error) bool { + return errors.Is(err, os.ErrNotExist) +} diff --git a/followlinks_windows.go b/followlinks_windows.go new file mode 100644 index 00000000..443ebd7f --- /dev/null +++ b/followlinks_windows.go @@ -0,0 +1,15 @@ +package fsutil + +import ( + "os" + + "github.com/pkg/errors" + "golang.org/x/sys/windows" +) + +func isNotFound(err error) bool { + if errors.Is(err, os.ErrNotExist) || errors.Is(err, windows.ERROR_INVALID_NAME) { + return true + } + return false +} diff --git a/go.mod b/go.mod index 80b0c6bf..09b5d035 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.18 require ( github.com/Microsoft/go-winio v0.5.2 - github.com/containerd/continuity v0.3.0 + github.com/containerd/continuity v0.3.1-0.20221025173834-5817935fcfbe github.com/gogo/protobuf v1.3.2 github.com/moby/patternmatcher v0.5.0 github.com/opencontainers/go-digest v1.0.0 diff --git a/go.sum b/go.sum index eb2b7c57..bccda3ad 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= -github.com/containerd/continuity v0.3.0 h1:nisirsYROK15TAMVukJOUyGJjz4BNQJBVsNvAXZJ/eg= -github.com/containerd/continuity v0.3.0/go.mod h1:wJEAIwKOm/pBZuBd0JmeTvnLquTB1Ag8espWhkykbPM= +github.com/containerd/continuity v0.3.1-0.20221025173834-5817935fcfbe h1:vrHfzd6wMSEZe3870709Tl6m6ZN3rhKmxxOTOWZFGIA= +github.com/containerd/continuity v0.3.1-0.20221025173834-5817935fcfbe/go.mod h1:wJEAIwKOm/pBZuBd0JmeTvnLquTB1Ag8espWhkykbPM= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= diff --git a/receive.go b/receive.go index 209d1d2f..c69fda9d 100644 --- a/receive.go +++ b/receive.go @@ -4,6 +4,7 @@ import ( "context" "io" "os" + "path/filepath" "sync" "github.com/pkg/errors" @@ -190,7 +191,7 @@ func (r *receiver) run(ctx context.Context) error { r.mu.Unlock() } i++ - cp := ¤tPath{path: p.Stat.Path, stat: p.Stat} + cp := ¤tPath{path: filepath.FromSlash(p.Stat.Path), stat: p.Stat} if err := r.orderValidator.HandleChange(ChangeKindAdd, cp.path, &StatInfo{cp.stat}, nil); err != nil { return err } diff --git a/receive_test.go b/receive_test.go index 0f9eb310..e38ef200 100644 --- a/receive_test.go +++ b/receive_test.go @@ -8,6 +8,7 @@ import ( "io" "os" "path/filepath" + "runtime" "sync" "testing" "time" @@ -143,10 +144,10 @@ func TestCopySwitchDirToFile(t *testing.T) { dest, err := tmpDir(changeStream([]string{ "ADD foo dir", - "ADD foo/bar dile data2", + "ADD foo/bar file data2", })) assert.NoError(t, err) - defer os.RemoveAll(d) + defer os.RemoveAll(dest) copy := func(src, dest string) (*changes, error) { ts := newNotificationBuffer() @@ -170,8 +171,13 @@ func TestCopySwitchDirToFile(t *testing.T) { NotifyHashed: chs.HandleChange, ContentHasher: simpleSHA256Hasher, Filter: func(_ string, s *types.Stat) bool { - s.Uid = uint32(os.Getuid()) - s.Gid = uint32(os.Getgid()) + if runtime.GOOS != "windows" { + // On Windows, Getuid() and Getgid() always return -1 + // See: https://pkg.go.dev/os#Getgid + // See: https://pkg.go.dev/os#Geteuid + s.Uid = uint32(os.Getuid()) + s.Gid = uint32(os.Getgid()) + } return true }, }) @@ -196,7 +202,7 @@ func TestCopySwitchDirToFile(t *testing.T) { err = Walk(context.Background(), dest, nil, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, string(b.Bytes()), `file foo + assert.Equal(t, b.String(), `file foo `) } @@ -239,8 +245,13 @@ func TestCopySimple(t *testing.T) { NotifyHashed: chs.HandleChange, ContentHasher: simpleSHA256Hasher, Filter: func(p string, s *types.Stat) bool { - s.Uid = uint32(os.Getuid()) - s.Gid = uint32(os.Getgid()) + if runtime.GOOS != "windows" { + // On Windows, Getuid() and Getgid() always return -1 + // See: https://pkg.go.dev/os#Getgid + // See: https://pkg.go.dev/os#Geteuid + s.Uid = uint32(os.Getuid()) + s.Gid = uint32(os.Getgid()) + } s.ModTime = tm.UnixNano() return true }, @@ -253,7 +264,18 @@ func TestCopySimple(t *testing.T) { err = Walk(context.Background(), dest, nil, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, string(b.Bytes()), `file foo + if runtime.GOOS == "windows" { + assert.Equal(t, b.String(), `file foo +file foo2 +dir zzz +file zzz\aa +dir zzz\bb +dir zzz\bb\cc +symlink:..\..\ zzz\bb\cc\dd +file zzz.aa +`) + } else { + assert.Equal(t, b.String(), `file foo file foo2 dir zzz file zzz/aa @@ -262,6 +284,7 @@ dir zzz/bb/cc symlink:../../ zzz/bb/cc/dd file zzz.aa `) + } dt, err := os.ReadFile(filepath.Join(dest, "zzz/aa")) assert.NoError(t, err) @@ -272,22 +295,33 @@ file zzz.aa assert.Equal(t, "dat2", string(dt)) fi, err := os.Stat(filepath.Join(dest, "foo2")) - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, tm, fi.ModTime()) - h, ok := ts.Hash("zzz/aa") + h, ok := ts.Hash(filepath.FromSlash("zzz/aa")) assert.True(t, ok) - assert.Equal(t, digest.Digest("sha256:99b6ef96ee0572b5b3a4eb28f00b715d820bfd73836e59cc1565e241f4d1bb2f"), h) + if runtime.GOOS == "windows" { + assert.Equal(t, digest.Digest("sha256:5da6b6a222dca8d9260384da15378d4389f7e16943e812e08c39759b8514b456"), h) + } else { + assert.Equal(t, digest.Digest("sha256:99b6ef96ee0572b5b3a4eb28f00b715d820bfd73836e59cc1565e241f4d1bb2f"), h) + } h, ok = ts.Hash("foo2") assert.True(t, ok) - assert.Equal(t, digest.Digest("sha256:dd2529f7749ba45ea55de3b2e10086d6494cc45a94e57650c2882a6a14b4ff32"), h) + if runtime.GOOS == "windows" { + assert.Equal(t, digest.Digest("sha256:cd83620e5308f6ddb9953a82b2c7450832eac78521dbf067d2882318cabc1311"), h) + } else { + assert.Equal(t, digest.Digest("sha256:dd2529f7749ba45ea55de3b2e10086d6494cc45a94e57650c2882a6a14b4ff32"), h) + } - h, ok = ts.Hash("zzz/bb/cc/dd") + h, ok = ts.Hash(filepath.FromSlash("zzz/bb/cc/dd")) assert.True(t, ok) - assert.Equal(t, digest.Digest("sha256:eca07e8f2d09bd574ea2496312e6de1685ef15b8e6a49a534ed9e722bcac8adc"), h) - - k, ok := chs.c["zzz/aa"] + if runtime.GOOS == "windows" { + assert.Equal(t, digest.Digest("sha256:47dc68d117ae85dc688103d6ba2cee54caabbbcf606e54ca62fda6a3d9deae19"), h) + } else { + assert.Equal(t, digest.Digest("sha256:eca07e8f2d09bd574ea2496312e6de1685ef15b8e6a49a534ed9e722bcac8adc"), h) + } + k, ok := chs.c[filepath.FromSlash("zzz/aa")] assert.Equal(t, ok, true) assert.Equal(t, k, ChangeKindAdd) @@ -317,8 +351,13 @@ file zzz.aa NotifyHashed: chs.HandleChange, ContentHasher: simpleSHA256Hasher, Filter: func(_ string, s *types.Stat) bool { - s.Uid = uint32(os.Getuid()) - s.Gid = uint32(os.Getgid()) + if runtime.GOOS != "windows" { + // On Windows, Getuid() and Getgid() always return -1 + // See: https://pkg.go.dev/os#Getgid + // See: https://pkg.go.dev/os#Geteuid + s.Uid = uint32(os.Getuid()) + s.Gid = uint32(os.Getgid()) + } s.ModTime = tm.UnixNano() return true }, @@ -330,8 +369,18 @@ file zzz.aa b = &bytes.Buffer{} err = Walk(context.Background(), dest, nil, bufWalk(b)) assert.NoError(t, err) - - assert.Equal(t, string(b.Bytes()), `file foo + if runtime.GOOS == "windows" { + assert.Equal(t, `file foo +dir zzz +file zzz\aa +dir zzz\bb +dir zzz\bb\cc +symlink:..\..\ zzz\bb\cc\dd +file zzz\bb\cc\foo +file zzz.aa +`, b.String()) + } else { + assert.Equal(t, `file foo dir zzz file zzz/aa dir zzz/bb @@ -339,20 +388,27 @@ dir zzz/bb/cc symlink:../../ zzz/bb/cc/dd file zzz/bb/cc/foo file zzz.aa -`) +`, b.String()) + } dt, err = os.ReadFile(filepath.Join(dest, "zzz/bb/cc/foo")) assert.NoError(t, err) assert.Equal(t, "data5", string(dt)) - h, ok = ts.Hash("zzz/bb/cc/dd") + h, ok = ts.Hash(filepath.FromSlash("zzz/bb/cc/dd")) assert.True(t, ok) - assert.Equal(t, digest.Digest("sha256:eca07e8f2d09bd574ea2496312e6de1685ef15b8e6a49a534ed9e722bcac8adc"), h) - - h, ok = ts.Hash("zzz/bb/cc/foo") + if runtime.GOOS == "windows" { + assert.Equal(t, digest.Digest("sha256:47dc68d117ae85dc688103d6ba2cee54caabbbcf606e54ca62fda6a3d9deae19"), h) + } else { + assert.Equal(t, digest.Digest("sha256:eca07e8f2d09bd574ea2496312e6de1685ef15b8e6a49a534ed9e722bcac8adc"), h) + } + h, ok = ts.Hash(filepath.FromSlash("zzz/bb/cc/foo")) assert.True(t, ok) - assert.Equal(t, digest.Digest("sha256:cd14a931fc2e123ded338093f2864b173eecdee578bba6ec24d0724272326c3a"), h) - + if runtime.GOOS == "windows" { + assert.Equal(t, digest.Digest("sha256:9184a7db8d056ee43838613279db9a7ab02272e50d5e20d253393521bb34aa46"), h) + } else { + assert.Equal(t, digest.Digest("sha256:cd14a931fc2e123ded338093f2864b173eecdee578bba6ec24d0724272326c3a"), h) + } _, ok = ts.Hash("foo2") assert.False(t, ok) @@ -360,15 +416,15 @@ file zzz.aa assert.Equal(t, ok, true) assert.Equal(t, k, ChangeKindDelete) - k, ok = chs.c["zzz/bb/cc/foo"] + k, ok = chs.c[filepath.FromSlash("zzz/bb/cc/foo")] assert.Equal(t, ok, true) assert.Equal(t, k, ChangeKindAdd) - _, ok = chs.c["zzz/aa"] + _, ok = chs.c[filepath.FromSlash("zzz/aa")] assert.Equal(t, ok, false) _, ok = chs.c["zzz.aa"] - assert.Equal(t, ok, false) + assert.Equal(t, false, ok) } func sockPairProto(ctx context.Context) (Stream, Stream) { @@ -490,8 +546,15 @@ func simpleSHA256Hasher(s *types.Stat) (hash.Hash, error) { // Unlike Linux, on FreeBSD's stat() call returns -1 in st_rdev for regular files ss.Devminor = 0 ss.Devmajor = 0 + if runtime.GOOS == "windows" { + // On Windows, Getuid() and Getgid() always return -1 + // See: https://pkg.go.dev/os#Getgid + // See: https://pkg.go.dev/os#Geteuid + ss.Uid = 0 + ss.Gid = 0 + } - if os.FileMode(ss.Mode)&os.ModeSymlink != 0 { + if os.FileMode(ss.Mode)&os.ModeSymlink != 0 && runtime.GOOS != "windows" { ss.Mode = ss.Mode | 0777 } @@ -502,3 +565,50 @@ func simpleSHA256Hasher(s *types.Stat) (hash.Hash, error) { h.Write(dt) return h, nil } + +type notificationBuffer struct { + items map[string]digest.Digest + sync.Mutex +} + +func newNotificationBuffer() *notificationBuffer { + nb := ¬ificationBuffer{ + items: map[string]digest.Digest{}, + } + return nb +} + +type hashed interface { + Digest() digest.Digest +} + +func (nb *notificationBuffer) HandleChange(kind ChangeKind, p string, fi os.FileInfo, err error) (retErr error) { + nb.Lock() + defer nb.Unlock() + if kind == ChangeKindDelete { + delete(nb.items, p) + } else { + h, ok := fi.(hashed) + if !ok { + return errors.Errorf("invalid FileInfo: %s", p) + } + nb.items[p] = h.Digest() + } + return nil +} + +func (nb *notificationBuffer) Hash(p string) (digest.Digest, bool) { + nb.Lock() + v, ok := nb.items[p] + nb.Unlock() + return v, ok +} + +// requiresRoot skips tests that require root +func requiresRoot(t *testing.T) { + t.Helper() + if os.Getuid() != 0 { + t.Skip("skipping test that requires root") + return + } +} diff --git a/stat.go b/stat.go index 2ab8da11..44441cb6 100644 --- a/stat.go +++ b/stat.go @@ -19,7 +19,7 @@ func mkstat(path, relpath string, fi os.FileInfo, inodemap map[uint64]string) (* relpath = filepath.ToSlash(relpath) stat := &types.Stat{ - Path: relpath, + Path: filepath.FromSlash(relpath), Mode: uint32(fi.Mode()), ModTime: fi.ModTime().UnixNano(), } diff --git a/validator.go b/validator.go index 9bd7d94d..ac3473f2 100644 --- a/validator.go +++ b/validator.go @@ -1,9 +1,9 @@ package fsutil import ( + "fmt" "os" - "path" - "runtime" + "path/filepath" "sort" "strings" "syscall" @@ -28,21 +28,20 @@ func (v *Validator) HandleChange(kind ChangeKind, p string, fi os.FileInfo, err if v.parentDirs == nil { v.parentDirs = make([]parent, 1, 10) } - if runtime.GOOS == "windows" { - p = strings.Replace(p, "\\", "", -1) - } - if p != path.Clean(p) { + + if p != filepath.Clean(p) { return errors.WithStack(&os.PathError{Path: p, Err: syscall.EINVAL, Op: "unclean path"}) } - if path.IsAbs(p) { + + if filepath.IsAbs(p) { return errors.WithStack(&os.PathError{Path: p, Err: syscall.EINVAL, Op: "absolute path"}) } - dir := path.Dir(p) - base := path.Base(p) + dir := filepath.Dir(p) + base := filepath.Base(p) if dir == "." { dir = "" } - if dir == ".." || strings.HasPrefix(p, "../") { + if dir == ".." || strings.HasPrefix(p, fmt.Sprintf("..%c", os.PathSeparator)) { return errors.WithStack(&os.PathError{Path: p, Err: syscall.EINVAL, Op: "escape check"}) } @@ -56,12 +55,12 @@ func (v *Validator) HandleChange(kind ChangeKind, p string, fi os.FileInfo, err } if dir != v.parentDirs[len(v.parentDirs)-1].dir || v.parentDirs[i].last >= base { - return errors.Errorf("changes out of order: %q %q", p, path.Join(v.parentDirs[i].dir, v.parentDirs[i].last)) + return errors.Errorf("changes out of order: %q %q", p, filepath.Join(v.parentDirs[i].dir, v.parentDirs[i].last)) } v.parentDirs[i].last = base if kind != ChangeKindDelete && fi.IsDir() { v.parentDirs = append(v.parentDirs, parent{ - dir: path.Join(dir, base), + dir: filepath.Join(dir, base), last: "", }) } @@ -76,7 +75,7 @@ func ComparePath(p1, p2 string) int { switch { case p1[i] == p2[i]: continue - case p2[i] != '/' && p1[i] < p2[i] || p1[i] == '/': + case p2[i] != os.PathSeparator && p1[i] < p2[i] || p1[i] == os.PathSeparator: return -1 default: return 1 diff --git a/validator_test.go b/validator_test.go index aabb0732..b3acf688 100644 --- a/validator_test.go +++ b/validator_test.go @@ -3,6 +3,7 @@ package fsutil import ( "fmt" "os" + "path/filepath" "strings" "testing" @@ -210,7 +211,7 @@ func parseChange(str string) *change { default: panic(errStr) } - c.path = f[1] + c.path = filepath.FromSlash(f[1]) st := &types.Stat{} switch f[2] { case "file": diff --git a/walker_test.go b/walker_test.go index bcc89bb7..419c0daf 100644 --- a/walker_test.go +++ b/walker_test.go @@ -28,7 +28,7 @@ func TestWalkerSimple(t *testing.T) { err = Walk(context.Background(), d, nil, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, string(b.Bytes()), `file foo + assert.Equal(t, b.String(), `file foo file foo2 `) @@ -48,9 +48,9 @@ func TestWalkerInclude(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, `dir bar + assert.Equal(t, filepath.FromSlash(`dir bar file bar/foo -`, string(b.Bytes())) +`), b.String()) b.Reset() err = Walk(context.Background(), d, &WalkOpt{ @@ -58,9 +58,9 @@ file bar/foo }, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, `dir bar + assert.Equal(t, filepath.FromSlash(`dir bar file bar/foo -`, string(b.Bytes())) +`), b.String()) b.Reset() err = Walk(context.Background(), d, &WalkOpt{ @@ -68,9 +68,9 @@ file bar/foo }, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, `dir bar + assert.Equal(t, filepath.FromSlash(`dir bar file bar/foo -`, string(b.Bytes())) +`), b.String()) b.Reset() err = Walk(context.Background(), d, &WalkOpt{ @@ -78,9 +78,9 @@ file bar/foo }, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, `dir bar + assert.Equal(t, filepath.FromSlash(`dir bar file bar/foo -`, string(b.Bytes())) +`), b.String()) b.Reset() err = Walk(context.Background(), d, &WalkOpt{ @@ -97,7 +97,7 @@ file bar/foo assert.NoError(t, err) assert.Equal(t, `file foo2 -`, string(b.Bytes())) +`, b.String()) b.Reset() err = Walk(context.Background(), d, &WalkOpt{ @@ -105,9 +105,9 @@ file bar/foo }, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, `dir bar + assert.Equal(t, filepath.FromSlash(`dir bar file bar/foo -`, string(b.Bytes())) +`), b.String()) b.Reset() err = Walk(context.Background(), d, &WalkOpt{ @@ -115,9 +115,9 @@ file bar/foo }, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, `dir bar + assert.Equal(t, filepath.FromSlash(`dir bar file bar/foo -`, string(b.Bytes())) +`), b.String()) b.Reset() err = Walk(context.Background(), d, &WalkOpt{ @@ -125,9 +125,9 @@ file bar/foo }, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, `dir bar + assert.Equal(t, filepath.FromSlash(`dir bar file bar/foo -`, string(b.Bytes())) +`), b.String()) } func TestWalkerExclude(t *testing.T) { @@ -145,24 +145,40 @@ func TestWalkerExclude(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, `file bar + assert.Equal(t, filepath.FromSlash(`file bar dir foo file foo/bar2 -`, string(b.Bytes())) +`), b.String()) } func TestWalkerFollowLinks(t *testing.T) { - d, err := tmpDir(changeStream([]string{ - "ADD bar file", - "ADD foo dir", - "ADD foo/l1 symlink /baz/one", - "ADD foo/l2 symlink /baz/two", - "ADD baz dir", - "ADD baz/one file", - "ADD baz/two symlink ../bax", - "ADD bax file", - "ADD bay file", // not included - })) + var d string + var err error + if runtime.GOOS == "windows" { + d, err = tmpDir(changeStream([]string{ + "ADD bar file", + "ADD foo dir", + "ADD foo/l1 symlink C:/baz/one", + "ADD foo/l2 symlink C:/baz/two", + "ADD baz dir", + "ADD baz/one file", + "ADD baz/two symlink ../bax", + "ADD bax file", + "ADD bay file", // not included + })) + } else { + d, err = tmpDir(changeStream([]string{ + "ADD bar file", + "ADD foo dir", + "ADD foo/l1 symlink /baz/one", + "ADD foo/l2 symlink /baz/two", + "ADD baz dir", + "ADD baz/one file", + "ADD baz/two symlink ../bax", + "ADD bax file", + "ADD bay file", // not included + })) + } assert.NoError(t, err) defer os.RemoveAll(d) b := &bytes.Buffer{} @@ -171,7 +187,18 @@ func TestWalkerFollowLinks(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, `file bar + if runtime.GOOS == "windows" { + assert.Equal(t, filepath.FromSlash(`file bar +file bax +dir baz +file baz/one +symlink:../bax baz/two +dir foo +symlink:C:/baz/one foo/l1 +symlink:C:/baz/two foo/l2 +`), b.String()) + } else { + assert.Equal(t, filepath.FromSlash(`file bar file bax dir baz file baz/one @@ -179,7 +206,8 @@ symlink:../bax baz/two dir foo symlink:/baz/one foo/l1 symlink:/baz/two foo/l2 -`, string(b.Bytes())) +`), b.String()) + } } func TestWalkerFollowLinksToRoot(t *testing.T) { @@ -198,12 +226,12 @@ func TestWalkerFollowLinksToRoot(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, `file bar + assert.Equal(t, filepath.FromSlash(`file bar file bax dir bay file bay/baz symlink:. foo -`, string(b.Bytes())) +`), b.String()) } func TestWalkerMap(t *testing.T) { @@ -227,10 +255,10 @@ func TestWalkerMap(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, `dir _foo + assert.Equal(t, filepath.FromSlash(`dir _foo file _foo/bar2 file _foo2 -`, string(b.Bytes())) +`), b.String()) } func TestWalkerMapSkipDir(t *testing.T) { @@ -261,10 +289,10 @@ func TestWalkerMapSkipDir(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, `dir includeDir + assert.Equal(t, filepath.FromSlash(`dir includeDir file includeDir/a.txt -`, string(b.Bytes())) - assert.Equal(t, []string{"excludeDir", "includeDir", "includeDir/a.txt"}, walked) +`), b.String()) + assert.Equal(t, []string{"excludeDir", "includeDir", filepath.FromSlash("includeDir/a.txt")}, walked) } func TestWalkerPermissionDenied(t *testing.T) { @@ -299,7 +327,7 @@ func TestWalkerPermissionDenied(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) assert.Equal(t, `dir foo -`, string(b.Bytes())) +`, b.String()) b.Reset() err = Walk(context.Background(), d, &WalkOpt{ @@ -307,7 +335,7 @@ func TestWalkerPermissionDenied(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) assert.Equal(t, `dir foo -`, string(b.Bytes())) +`, b.String()) b.Reset() err = Walk(context.Background(), d, &WalkOpt{ @@ -323,7 +351,7 @@ func TestWalkerPermissionDenied(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) assert.Equal(t, `dir foo -`, string(b.Bytes())) +`, b.String()) } func bufWalk(buf *bytes.Buffer) filepath.WalkFunc { @@ -521,7 +549,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - trimEqual(t, ` + trimEqual(t, filepath.FromSlash(` dir a dir a/b dir a/b/bar @@ -535,7 +563,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { dir foo/bar file foo/bar/bee file foo2 - `, string(b.Bytes())) + `), b.String()) b.Reset() err = Walk(context.Background(), d, &WalkOpt{ @@ -543,7 +571,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - trimEqual(t, ` + trimEqual(t, filepath.FromSlash(` dir a dir a/b dir a/b/bar @@ -554,7 +582,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { dir foo dir foo/bar file foo/bar/bee - `, string(b.Bytes())) + `), b.String()) b.Reset() err = Walk(context.Background(), d, &WalkOpt{ @@ -562,14 +590,14 @@ func TestWalkerDoublestarInclude(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - trimEqual(t, ` + trimEqual(t, filepath.FromSlash(` dir a dir a/b dir a/b/bar file a/b/bar/foo dir bar file bar/foo - `, string(b.Bytes())) + `), b.String()) b.Reset() err = Walk(context.Background(), d, &WalkOpt{ @@ -577,7 +605,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - trimEqual(t, ` + trimEqual(t, filepath.FromSlash(` dir a dir a/b dir a/b/bar @@ -590,7 +618,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { dir foo dir foo/bar file foo/bar/bee - `, string(b.Bytes())) + `), b.String()) b.Reset() err = Walk(context.Background(), d, &WalkOpt{ @@ -598,7 +626,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - trimEqual(t, ` + trimEqual(t, filepath.FromSlash(` dir a dir a/b dir a/b/bar @@ -606,7 +634,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { file a/b/bar/fop dir bar file bar/foo - `, string(b.Bytes())) + `), b.String()) b.Reset() err = Walk(context.Background(), d, &WalkOpt{ @@ -614,7 +642,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - trimEqual(t, ``, string(b.Bytes())) + trimEqual(t, ``, b.String()) b.Reset() err = Walk(context.Background(), d, &WalkOpt{ @@ -622,7 +650,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - trimEqual(t, ` + trimEqual(t, filepath.FromSlash(` dir a dir a/b dir a/b/bar @@ -634,7 +662,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { dir foo/bar file foo/bar/bee file foo2 - `, string(b.Bytes())) + `), b.String()) b.Reset() err = Walk(context.Background(), d, &WalkOpt{ @@ -642,7 +670,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - trimEqual(t, ` + trimEqual(t, filepath.FromSlash(` dir a dir a/b dir a/b/bar @@ -650,7 +678,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { file a/b/bar/fop dir bar file bar/foo - `, string(b.Bytes())) + `), b.String()) b.Reset() err = Walk(context.Background(), d, &WalkOpt{ @@ -658,14 +686,14 @@ func TestWalkerDoublestarInclude(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - trimEqual(t, ` + trimEqual(t, filepath.FromSlash(` dir a dir a/b dir a/b/bar file a/b/bar/foo dir bar file bar/foo - `, string(b.Bytes())) + `), b.String()) b.Reset() err = Walk(context.Background(), d, &WalkOpt{ @@ -673,11 +701,11 @@ func TestWalkerDoublestarInclude(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - trimEqual(t, ` + trimEqual(t, filepath.FromSlash(` dir foo dir foo/bar file foo/bar/bee - `, string(b.Bytes())) + `), b.String()) b.Reset() err = Walk(context.Background(), d, &WalkOpt{ @@ -685,12 +713,12 @@ func TestWalkerDoublestarInclude(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - trimEqual(t, ` + trimEqual(t, filepath.FromSlash(` dir a dir a/b dir a/b/baz dir baz - `, string(b.Bytes())) + `), b.String()) } func trimEqual(t assert.TestingT, expected, actual string, msgAndArgs ...interface{}) bool {