diff --git a/src/spec-configuration/containerCollectionsOCI.ts b/src/spec-configuration/containerCollectionsOCI.ts index 737789059..4ad6331eb 100644 --- a/src/spec-configuration/containerCollectionsOCI.ts +++ b/src/spec-configuration/containerCollectionsOCI.ts @@ -504,7 +504,7 @@ export async function getPublishedTags(params: CommonParams, ref: OCIRef): Promi } } -export async function getBlob(params: CommonParams, url: string, ociCacheDir: string, destCachePath: string, ociRef: OCIRef, expectedDigest: string, ignoredFilesDuringExtraction: string[] = [], metadataFile?: string): Promise<{ files: string[]; metadata: {} | undefined } | undefined> { +export async function getBlob(params: CommonParams, url: string, ociCacheDir: string, destCachePath: string, ociRef: OCIRef, expectedDigest: string, omitDuringExtraction: string[] = [], metadataFile?: string): Promise<{ files: string[]; metadata: {} | undefined } | undefined> { // TODO: Parallelize if multiple layers (not likely). // TODO: Seeking might be needed if the size is too large. @@ -543,24 +543,37 @@ export async function getBlob(params: CommonParams, url: string, ociCacheDir: st await mkdirpLocal(destCachePath); await writeLocalFile(tempTarballPath, resBody); + // https://github.com/devcontainers/spec/blob/main/docs/specs/devcontainer-templates.md#the-optionalpaths-property + const directoriesToOmit = omitDuringExtraction.filter(f => f.endsWith('/*')).map(f => f.slice(0, -1)); + const filesToOmit = omitDuringExtraction.filter(f => !f.endsWith('/*')); + + output.write(`omitDuringExtraction: '${omitDuringExtraction.join(', ')}`, LogLevel.Trace); + output.write(`Files to omit: '${filesToOmit.join(', ')}'`, LogLevel.Info); + if (directoriesToOmit.length) { + output.write(`Dirs to omit : '${directoriesToOmit.join(', ')}'`, LogLevel.Info); + } + const files: string[] = []; await tar.x( { file: tempTarballPath, cwd: destCachePath, - filter: (path: string, stat: tar.FileStat) => { - // Skip files that are in the ignore list - if (ignoredFilesDuringExtraction.some(f => path.indexOf(f) !== -1)) { - // Skip. - output.write(`Skipping file '${path}' during blob extraction`, LogLevel.Trace); - return false; + filter: (tPath: string, stat: tar.FileStat) => { + output.write(`Testing '${tPath}'(${stat.type})`, LogLevel.Trace); + const cleanedPath = tPath + .replace(/\\/g, '/') + .replace(/^\.\//, ''); + + if (filesToOmit.includes(cleanedPath) || directoriesToOmit.some(d => cleanedPath.startsWith(d))) { + output.write(` Omitting '${tPath}'`, LogLevel.Trace); + return false; // Skip } - // Keep track of all files extracted, in case the caller is interested. - output.write(`${path} : ${stat.type}`, LogLevel.Trace); - if ((stat.type.toString() === 'File')) { - files.push(path); + + if (stat.type.toString() === 'File') { + files.push(tPath); } - return true; + + return true; // Keep } } ); @@ -576,8 +589,8 @@ export async function getBlob(params: CommonParams, url: string, ociCacheDir: st { file: tempTarballPath, cwd: ociCacheDir, - filter: (path: string, _: tar.FileStat) => { - return path === `./${metadataFile}`; + filter: (tPath: string, _: tar.FileStat) => { + return tPath === `./${metadataFile}`; } }); const pathToMetadataFile = path.join(ociCacheDir, metadataFile); diff --git a/src/spec-configuration/containerTemplatesOCI.ts b/src/spec-configuration/containerTemplatesOCI.ts index 9df3d8d79..e0ee1ae0f 100644 --- a/src/spec-configuration/containerTemplatesOCI.ts +++ b/src/spec-configuration/containerTemplatesOCI.ts @@ -19,12 +19,13 @@ export interface SelectedTemplate { id: string; options: TemplateOptions; features: TemplateFeatureOption[]; + omitPaths: string[]; } export async function fetchTemplate(params: CommonParams, selectedTemplate: SelectedTemplate, templateDestPath: string, userProvidedTmpDir?: string): Promise { const { output } = params; - let { id: userSelectedId, options: userSelectedOptions } = selectedTemplate; + let { id: userSelectedId, options: userSelectedOptions, omitPaths } = selectedTemplate; const templateRef = getRef(output, userSelectedId); if (!templateRef) { output.write(`Failed to parse template ref for ${userSelectedId}`, LogLevel.Error); @@ -46,7 +47,7 @@ export async function fetchTemplate(params: CommonParams, selectedTemplate: Sele output.write(`blob url: ${blobUrl}`, LogLevel.Trace); const tmpDir = userProvidedTmpDir || path.join(os.tmpdir(), 'vsch-template-temp', `${Date.now()}`); - const blobResult = await getBlob(params, blobUrl, tmpDir, templateDestPath, templateRef, blobDigest, ['devcontainer-template.json', 'README.md', 'NOTES.md'], 'devcontainer-template.json'); + const blobResult = await getBlob(params, blobUrl, tmpDir, templateDestPath, templateRef, blobDigest, [...omitPaths, 'devcontainer-template.json', 'README.md', 'NOTES.md'], 'devcontainer-template.json'); if (!blobResult) { throw new Error(`Failed to download package for ${templateRef.resource}`); diff --git a/src/spec-node/templatesCLI/apply.ts b/src/spec-node/templatesCLI/apply.ts index ebefbf924..aabc18c5e 100644 --- a/src/spec-node/templatesCLI/apply.ts +++ b/src/spec-node/templatesCLI/apply.ts @@ -15,6 +15,7 @@ export function templateApplyOptions(y: Argv) { 'features': { type: 'string', alias: 'f', default: '[]', description: 'Features to add to the provided Template, provided as JSON.' }, 'log-level': { choices: ['info' as 'info', 'debug' as 'debug', 'trace' as 'trace'], default: 'info' as 'info', description: 'Log level.' }, 'tmp-dir': { type: 'string', description: 'Directory to use for temporary files. If not provided, the system default will be inferred.' }, + 'omit-paths': { type: 'string', default: '[]', description: 'List of paths within the Template to omit applying, provided as JSON. To ignore a directory append \'/*\'. Eg: \'[".github/*", "dir/a/*", "file.ts"]\'' }, }) .check(_argv => { return true; @@ -34,6 +35,7 @@ async function templateApply({ 'features': featuresArgs, 'log-level': inputLogLevel, 'tmp-dir': userProvidedTmpDir, + 'omit-paths': omitPathsArg, }: TemplateApplyArgs) { const disposables: (() => Promise | undefined)[] = []; const dispose = async () => { @@ -65,13 +67,23 @@ async function templateApply({ process.exit(1); } + let omitPaths: string[] = []; + if (omitPathsArg) { + let omitPathsErrors: jsonc.ParseError[] = []; + omitPaths = jsonc.parse(omitPathsArg, omitPathsErrors); + if (!Array.isArray(omitPaths)) { + output.write('Invalid \'--omitPaths\' argument provided. Provide as a JSON array, eg: \'[".github/*", "dir/a/*", "file.ts"]\'', LogLevel.Error); + process.exit(1); + } + } + const selectedTemplate: SelectedTemplate = { id: templateId, options, - features + features, + omitPaths, }; - const files = await fetchTemplate({ output, env: process.env }, selectedTemplate, workspaceFolder, userProvidedTmpDir); if (!files) { output.write(`Failed to fetch template '${id}'.`, LogLevel.Error); diff --git a/src/test/container-templates/containerTemplatesOCI.test.ts b/src/test/container-templates/containerTemplatesOCI.test.ts index d1de5f9e7..42e73b5b5 100644 --- a/src/test/container-templates/containerTemplatesOCI.test.ts +++ b/src/test/container-templates/containerTemplatesOCI.test.ts @@ -1,8 +1,9 @@ -import { createPlainLog, LogLevel, makeLog } from '../../spec-utils/log'; import * as assert from 'assert'; +import * as os from 'os'; +import * as path from 'path'; +import { createPlainLog, LogLevel, makeLog } from '../../spec-utils/log'; export const output = makeLog(createPlainLog(text => process.stdout.write(text), () => LogLevel.Trace)); import { fetchTemplate, SelectedTemplate } from '../../spec-configuration/containerTemplatesOCI'; -import * as path from 'path'; import { readLocalFile } from '../../spec-utils/pfs'; describe('fetchTemplate', async function () { @@ -14,7 +15,8 @@ describe('fetchTemplate', async function () { const selectedTemplate: SelectedTemplate = { id: 'ghcr.io/devcontainers/templates/docker-from-docker:latest', options: { 'installZsh': 'false', 'upgradePackages': 'true', 'dockerVersion': '20.10', 'moby': 'true' }, - features: [] + features: [], + omitPaths: [], }; const dest = path.relative(process.cwd(), path.join(__dirname, 'tmp1')); @@ -43,7 +45,8 @@ describe('fetchTemplate', async function () { const selectedTemplate: SelectedTemplate = { id: 'ghcr.io/devcontainers/templates/docker-from-docker:latest', options: {}, - features: [] + features: [], + omitPaths: [], }; const dest = path.relative(process.cwd(), path.join(__dirname, 'tmp2')); @@ -72,7 +75,8 @@ describe('fetchTemplate', async function () { const selectedTemplate: SelectedTemplate = { id: 'ghcr.io/devcontainers/templates/docker-from-docker:latest', options: { 'installZsh': 'false', 'upgradePackages': 'true', 'dockerVersion': '20.10', 'moby': 'true', 'enableNonRootDocker': 'true' }, - features: [{ id: 'ghcr.io/devcontainers/features/azure-cli:1', options: {} }] + features: [{ id: 'ghcr.io/devcontainers/features/azure-cli:1', options: {} }], + omitPaths: [], }; const dest = path.relative(process.cwd(), path.join(__dirname, 'tmp3')); @@ -104,7 +108,8 @@ describe('fetchTemplate', async function () { const selectedTemplate: SelectedTemplate = { id: 'ghcr.io/devcontainers/templates/anaconda-postgres:latest', options: { 'nodeVersion': 'lts/*' }, - features: [{ id: 'ghcr.io/devcontainers/features/azure-cli:1', options: {} }, { id: 'ghcr.io/devcontainers/features/git:1', options: { 'version': 'latest', ppa: true } }] + features: [{ id: 'ghcr.io/devcontainers/features/azure-cli:1', options: {} }, { id: 'ghcr.io/devcontainers/features/git:1', options: { 'version': 'latest', ppa: true } }], + omitPaths: [], }; const dest = path.relative(process.cwd(), path.join(__dirname, 'tmp4')); @@ -123,4 +128,109 @@ describe('fetchTemplate', async function () { assert.match(devcontainer, /"ghcr.io\/devcontainers\/features\/azure-cli:1": {}/); assert.match(devcontainer, /"ghcr.io\/devcontainers\/features\/git:1": {\n\t\t\t"version": "latest",\n\t\t\t"ppa": true/); }); -}); \ No newline at end of file + + describe('omit-path', async function () { + this.timeout('120s'); + + // https://github.com/codspace/templates/pkgs/container/templates%2Fmytemplate/255979159?tag=1.0.4 + const id = 'ghcr.io/codspace/templates/mytemplate@sha256:57cbf968907c74c106b7b2446063d114743ab3f63345f7c108c577915c535185'; + const templateFiles = [ + './c1.ts', + './c2.ts', + './c3.ts', + './.devcontainer/devcontainer.json', + './.github/dependabot.yml', + './assets/hello.md', + './assets/hi.md', + './example-projects/exampleA/a1.ts', + './example-projects/exampleA/.github/dependabot.yml', + './example-projects/exampleA/subFolderA/a2.ts', + './example-projects/exampleB/b1.ts', + './example-projects/exampleB/.github/dependabot.yml', + './example-projects/exampleB/subFolderB/b2.ts', + ]; + + // NOTE: Certain files, like the 'devcontainer-template.json', are always filtered + // out as they are not part of the Template. + it('Omit nothing', async () => { + const selectedTemplate: SelectedTemplate = { + id, + options: {}, + features: [], + omitPaths: [], + }; + + const files = await fetchTemplate( + { output, env: process.env }, + selectedTemplate, + path.join(os.tmpdir(), 'vsch-test-template-temp', `${Date.now()}`) + ); + + assert.ok(files); + assert.strictEqual(files.length, templateFiles.length); + for (const file of templateFiles) { + assert.ok(files.includes(file)); + } + }); + + it('Omit nested folder', async () => { + const selectedTemplate: SelectedTemplate = { + id, + options: {}, + features: [], + omitPaths: ['example-projects/exampleB/*'], + }; + + const files = await fetchTemplate( + { output, env: process.env }, + selectedTemplate, + path.join(os.tmpdir(), 'vsch-test-template-temp', `${Date.now()}`) + ); + + const expectedRemovedFiles = [ + './example-projects/exampleB/b1.ts', + './example-projects/example/.github/dependabot.yml', + './example-projects/exampleB/subFolderB/b2.ts', + ]; + + assert.ok(files); + assert.strictEqual(files.length, templateFiles.length - 3); + for (const file of expectedRemovedFiles) { + assert.ok(!files.includes(file)); + } + }); + + it('Omit single file, root folder, and nested folder', async () => { + const selectedTemplate: SelectedTemplate = { + id, + options: {}, + features: [], + omitPaths: ['.github/*', 'example-projects/exampleA/*', 'c1.ts'], + }; + + const files = await fetchTemplate( + { output, env: process.env }, + selectedTemplate, + path.join(os.tmpdir(), 'vsch-test-template-temp', `${Date.now()}`) + ); + + const expectedRemovedFiles = [ + './c1.ts', + './.github/dependabot.yml', + './example-projects/exampleA/a1.ts', + './example-projects/exampleA/.github/dependabot.yml', + './example-projects/exampleA/subFolderA/a2.ts', + ]; + + assert.ok(files); + assert.strictEqual(files.length, templateFiles.length - 5); + for (const file of expectedRemovedFiles) { + assert.ok(!files.includes(file)); + } + }); + }); + + +}); + +