From 8af9ef85f4b5059c1568eb5b83793282431958e1 Mon Sep 17 00:00:00 2001 From: Aaron Lehmann Date: Sun, 3 Mar 2024 15:37:03 -0800 Subject: [PATCH] Support local features provided inside .devcontainer dir (#96) --- devcontainer/devcontainer.go | 21 ++++++---- devcontainer/features/features.go | 56 ++++++++++++++++++-------- devcontainer/features/features_test.go | 10 ++--- go.mod | 2 +- go.sum | 4 +- integration/integration_test.go | 22 +++++++++- 6 files changed, 83 insertions(+), 32 deletions(-) diff --git a/devcontainer/devcontainer.go b/devcontainer/devcontainer.go index 3519d351..c19c08a4 100644 --- a/devcontainer/devcontainer.go +++ b/devcontainer/devcontainer.go @@ -169,14 +169,14 @@ func (s *Spec) Compile(fs billy.Filesystem, devcontainerDir, scratchDir, fallbac if remoteUser == "" { remoteUser = params.User } - params.DockerfileContent, err = s.compileFeatures(fs, scratchDir, params.User, remoteUser, params.DockerfileContent) + params.DockerfileContent, err = s.compileFeatures(fs, devcontainerDir, scratchDir, params.User, remoteUser, params.DockerfileContent) if err != nil { return nil, err } return params, nil } -func (s *Spec) compileFeatures(fs billy.Filesystem, scratchDir, containerUser, remoteUser, dockerfileContent string) (string, error) { +func (s *Spec) compileFeatures(fs billy.Filesystem, devcontainerDir, scratchDir, containerUser, remoteUser, dockerfileContent string) (string, error) { // If there are no features, we don't need to do anything! if len(s.Features) == 0 { return dockerfileContent, nil @@ -200,9 +200,16 @@ func (s *Spec) compileFeatures(fs billy.Filesystem, scratchDir, containerUser, r sort.Strings(featureOrder) for _, featureRefRaw := range featureOrder { - featureRefParsed, err := name.NewTag(featureRefRaw) - if err != nil { - return "", fmt.Errorf("parse feature ref %s: %w", featureRefRaw, err) + var ( + featureRef string + ok bool + ) + if _, featureRef, ok = strings.Cut(featureRefRaw, "./"); !ok { + featureRefParsed, err := name.NewTag(featureRefRaw) + if err != nil { + return "", fmt.Errorf("parse feature ref %s: %w", featureRefRaw, err) + } + featureRef = featureRefParsed.Repository.Name() } featureOpts := map[string]any{} @@ -222,13 +229,13 @@ func (s *Spec) compileFeatures(fs billy.Filesystem, scratchDir, containerUser, r // devcontainers/cli has a very complex method of computing the feature // name from the feature reference. We're just going to hash it for simplicity. featureSha := md5.Sum([]byte(featureRefRaw)) - featureName := filepath.Base(featureRefParsed.Repository.Name()) + featureName := filepath.Base(featureRef) featureDir := filepath.Join(featuresDir, fmt.Sprintf("%s-%x", featureName, featureSha[:4])) err = fs.MkdirAll(featureDir, 0644) if err != nil { return "", err } - spec, err := features.Extract(fs, featureDir, featureRefRaw) + spec, err := features.Extract(fs, devcontainerDir, featureDir, featureRefRaw) if err != nil { return "", fmt.Errorf("extract feature %s: %w", featureRefRaw, err) } diff --git a/devcontainer/features/features.go b/devcontainer/features/features.go index 6c19550c..bc2d86d7 100644 --- a/devcontainer/features/features.go +++ b/devcontainer/features/features.go @@ -16,23 +16,22 @@ import ( "github.com/go-git/go-billy/v5" "github.com/google/go-containerregistry/pkg/name" "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/otiai10/copy" "github.com/tailscale/hujson" ) -// Extract unpacks the feature from the image and returns the -// parsed specification. -func Extract(fs billy.Filesystem, directory, reference string) (*Spec, error) { +func extractFromImage(fs billy.Filesystem, directory, reference string) error { ref, err := name.ParseReference(reference) if err != nil { - return nil, fmt.Errorf("parse feature ref %s: %w", reference, err) + return fmt.Errorf("parse feature ref %s: %w", reference, err) } image, err := remote.Image(ref) if err != nil { - return nil, fmt.Errorf("fetch feature image %s: %w", reference, err) + return fmt.Errorf("fetch feature image %s: %w", reference, err) } manifest, err := image.Manifest() if err != nil { - return nil, fmt.Errorf("fetch feature manifest %s: %w", reference, err) + return fmt.Errorf("fetch feature manifest %s: %w", reference, err) } var tarLayer *tar.Reader @@ -42,17 +41,17 @@ func Extract(fs billy.Filesystem, directory, reference string) (*Spec, error) { } layer, err := image.LayerByDigest(manifestLayer.Digest) if err != nil { - return nil, fmt.Errorf("fetch feature layer %s: %w", reference, err) + return fmt.Errorf("fetch feature layer %s: %w", reference, err) } layerReader, err := layer.Uncompressed() if err != nil { - return nil, fmt.Errorf("uncompress feature layer %s: %w", reference, err) + return fmt.Errorf("uncompress feature layer %s: %w", reference, err) } tarLayer = tar.NewReader(layerReader) break } if tarLayer == nil { - return nil, fmt.Errorf("no tar layer found with media type %q: are you sure this is a devcontainer feature?", TarLayerMediaType) + return fmt.Errorf("no tar layer found with media type %q: are you sure this is a devcontainer feature?", TarLayerMediaType) } for { @@ -61,35 +60,60 @@ func Extract(fs billy.Filesystem, directory, reference string) (*Spec, error) { break } if err != nil { - return nil, fmt.Errorf("read feature layer %s: %w", reference, err) + return fmt.Errorf("read feature layer %s: %w", reference, err) } path := filepath.Join(directory, header.Name) switch header.Typeflag { case tar.TypeDir: err = fs.MkdirAll(path, 0755) if err != nil { - return nil, fmt.Errorf("mkdir %s: %w", path, err) + return fmt.Errorf("mkdir %s: %w", path, err) } case tar.TypeReg: outFile, err := fs.Create(path) if err != nil { - return nil, fmt.Errorf("create %s: %w", path, err) + return fmt.Errorf("create %s: %w", path, err) } _, err = io.Copy(outFile, tarLayer) if err != nil { - return nil, fmt.Errorf("copy %s: %w", path, err) + return fmt.Errorf("copy %s: %w", path, err) } err = outFile.Close() if err != nil { - return nil, fmt.Errorf("close %s: %w", path, err) + return fmt.Errorf("close %s: %w", path, err) } default: - return nil, fmt.Errorf("unknown type %d in %s", header.Typeflag, path) + return fmt.Errorf("unknown type %d in %s", header.Typeflag, path) + } + } + return nil +} + +// Extract unpacks the feature from the image and returns the +// parsed specification. +func Extract(fs billy.Filesystem, devcontainerDir, directory, reference string) (*Spec, error) { + if strings.HasPrefix(reference, "./") { + if err := copy.Copy(filepath.Join(devcontainerDir, reference), directory, copy.Options{ + PreserveTimes: true, + PreserveOwner: true, + OnSymlink: func(src string) copy.SymlinkAction { + return copy.Shallow + }, + OnError: func(src, dest string, err error) error { + if err == nil { + return nil + } + return fmt.Errorf("copy error: %q -> %q: %w", reference, directory, err) + }, + }); err != nil { + return nil, err } + } else if err := extractFromImage(fs, directory, reference); err != nil { + return nil, err } installScriptPath := filepath.Join(directory, "install.sh") - _, err = fs.Stat(installScriptPath) + _, err := fs.Stat(installScriptPath) if err != nil { if errors.Is(err, os.ErrNotExist) { return nil, errors.New("install.sh must be in the root of the feature") diff --git a/devcontainer/features/features_test.go b/devcontainer/features/features_test.go index d17e9711..d6b9db01 100644 --- a/devcontainer/features/features_test.go +++ b/devcontainer/features/features_test.go @@ -17,7 +17,7 @@ func TestExtract(t *testing.T) { registry := registrytest.New(t) ref := registrytest.WriteContainer(t, registry, "coder/test:latest", "some/type", nil) fs := memfs.New() - _, err := features.Extract(fs, "/", ref) + _, err := features.Extract(fs, "", "/", ref) require.ErrorContains(t, err, "no tar layer found") }) t.Run("MissingInstallScript", func(t *testing.T) { @@ -27,7 +27,7 @@ func TestExtract(t *testing.T) { "devcontainer-feature.json": "{}", }) fs := memfs.New() - _, err := features.Extract(fs, "/", ref) + _, err := features.Extract(fs, "", "/", ref) require.ErrorContains(t, err, "install.sh") }) t.Run("MissingFeatureFile", func(t *testing.T) { @@ -37,7 +37,7 @@ func TestExtract(t *testing.T) { "install.sh": "hey", }) fs := memfs.New() - _, err := features.Extract(fs, "/", ref) + _, err := features.Extract(fs, "", "/", ref) require.ErrorContains(t, err, "devcontainer-feature.json") }) t.Run("MissingFeatureProperties", func(t *testing.T) { @@ -48,7 +48,7 @@ func TestExtract(t *testing.T) { "devcontainer-feature.json": features.Spec{}, }) fs := memfs.New() - _, err := features.Extract(fs, "/", ref) + _, err := features.Extract(fs, "", "/", ref) require.ErrorContains(t, err, "id is required") }) t.Run("Success", func(t *testing.T) { @@ -63,7 +63,7 @@ func TestExtract(t *testing.T) { }, }) fs := memfs.New() - _, err := features.Extract(fs, "/", ref) + _, err := features.Extract(fs, "", "/", ref) require.NoError(t, err) }) } diff --git a/go.mod b/go.mod index 1d99d770..074d1142 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/google/go-containerregistry v0.15.2 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 github.com/mattn/go-isatty v0.0.19 + github.com/otiai10/copy v1.14.0 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.7.0 github.com/stretchr/testify v1.8.4 @@ -186,7 +187,6 @@ require ( github.com/opencontainers/runc v1.1.5 // indirect github.com/opencontainers/runtime-spec v1.1.0-rc.1 // indirect github.com/opencontainers/selinux v1.11.0 // indirect - github.com/otiai10/copy v1.12.0 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect diff --git a/go.sum b/go.sum index 6b7190d5..fe1075f4 100644 --- a/go.sum +++ b/go.sum @@ -725,8 +725,8 @@ github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuh github.com/opencontainers/selinux v1.11.0 h1:+5Zbo97w3Lbmb3PeqQtpmTkMwsW5nRI3YaLpt7tQ7oU= github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec= github.com/ory/dockertest/v3 v3.10.0 h1:4K3z2VMe8Woe++invjaTB7VRyQXQy5UY+loujO4aNE4= -github.com/otiai10/copy v1.12.0 h1:cLMgSQnXBs1eehF0Wy/FAGsgDTDmAqFR7rQylBb1nDY= -github.com/otiai10/copy v1.12.0/go.mod h1:rSaLseMUsZFFbsFGc7wCJnnkTAvdc5L6VWxPE4308Ww= +github.com/otiai10/copy v1.14.0 h1:dCI/t1iTdYGtkvCuBG2BgR6KZa83PTclw4U5n2wAllU= +github.com/otiai10/copy v1.14.0/go.mod h1:ECfuL02W+/FkTWZWgQqXPWZgW9oeKCSQ5qVfSc4qc4w= github.com/otiai10/mint v1.5.1 h1:XaPLeE+9vGbuyEHem1JNk3bYc7KKqyI/na0/mLd/Kks= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= diff --git a/integration/integration_test.go b/integration/integration_test.go index 674f40d9..5e605675 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -110,6 +110,18 @@ func TestBuildFromDevcontainerWithFeatures(t *testing.T) { "install.sh": "echo $PINEAPPLE > /test2output", }) + feature3Spec, err := json.Marshal(features.Spec{ + ID: "test3", + Name: "test3", + Version: "1.0.0", + Options: map[string]features.Option{ + "grape": { + Type: "string", + }, + }, + }) + require.NoError(t, err) + // Ensures that a Git repository with a devcontainer.json is cloned and built. url := createGitServer(t, gitServerOptions{ files: map[string]string{ @@ -124,10 +136,15 @@ func TestBuildFromDevcontainerWithFeatures(t *testing.T) { }, "` + feature2Ref + `": { "pineapple": "hello from test 2!" + }, + "./feature3": { + "grape": "hello from test 3!" } } }`, - ".devcontainer/Dockerfile": "FROM ubuntu", + ".devcontainer/Dockerfile": "FROM ubuntu", + ".devcontainer/feature3/devcontainer-feature.json": string(feature3Spec), + ".devcontainer/feature3/install.sh": "echo $GRAPE > /test3output", }, }) ctr, err := runEnvbuilder(t, options{env: []string{ @@ -140,6 +157,9 @@ func TestBuildFromDevcontainerWithFeatures(t *testing.T) { test2Output := execContainer(t, ctr, "cat /test2output") require.Equal(t, "hello from test 2!", strings.TrimSpace(test2Output)) + + test3Output := execContainer(t, ctr, "cat /test3output") + require.Equal(t, "hello from test 3!", strings.TrimSpace(test3Output)) } func TestBuildFromDockerfile(t *testing.T) {