diff --git a/.github/dependency_graph.svg b/.github/dependency_graph.svg index e17bc5298ce2..db3b31678ced 100644 --- a/.github/dependency_graph.svg +++ b/.github/dependency_graph.svg @@ -4,457 +4,463 @@ - + std_deps - - - -archive - -archive - - - -io - -io - - - -archive->io - - - - - -bytes - -bytes - - - -io->bytes - - - + - + assert - -assert + +assert - + internal - -internal + +internal - + assert->internal - - + + - + async - -async + +async + + + +bytes + +bytes - + cache - -cache + +cache + + + +cbor + +cbor + + + +cbor->bytes + + + + + +streams + +streams + + + +cbor->streams + + + + + +streams->bytes + + cli - -cli + +cli collections - -collections + +collections crypto - -crypto + +crypto csv - -csv - - - -streams - -streams + +csv - + csv->streams - - - - - -streams->bytes - - + + - + data-\nstructures - -data- -structures + +data- +structures - + datetime - -datetime + +datetime - + dotenv - -dotenv + +dotenv - + encoding - -encoding + +encoding - + expect - -expect + +expect - + expect->assert - - + + - + expect->internal - - + + - + fmt - -fmt + +fmt - + front-\nmatter - -front- -matter + +front- +matter - + toml - -toml + +toml - + front-\nmatter->toml - - + + - + yaml - -yaml + +yaml - + front-\nmatter->yaml - - + + - + toml->collections - - + + - + fs - -fs + +fs - + path - -path + +path - + fs->path - - + + - + html - -html + +html - + http - -http - - - -http->cli - - + +http - + http->streams - - + + + + + +http->cli + + - + http->encoding - - + + - + http->fmt - - + + - + http->path - - + + - + media-\ntypes - -media- -types + +media- +types - + http->media-\ntypes - - + + - + net - -net + +net - + http->net - - + + - + ini - -ini + +ini + + + +io + +io + + + +io->bytes + + json - -json + +json - + json->streams - - + + jsonc - -jsonc + +jsonc - + jsonc->json - - + + log - -log - - - -log->io - - + +log - + log->fmt - - + + - + log->fs - - + + + + + +log->io + + msgpack - -msgpack + +msgpack - + msgpack->bytes - - + + random - -random + +random regexp - -regexp + +regexp semver - -semver + +semver tar - -tar + +tar - + tar->streams - - + + testing - -testing + +testing - + testing->assert - - + + - + testing->internal - - + + - + testing->async - - + + - + testing->data-\nstructures - - + + - + testing->fs - - + + - + testing->path - - + + text - -text + +text ulid - -ulid + +ulid uuid - -uuid + +uuid - + uuid->bytes - - + + - + uuid->crypto - - + + webgpu - -webgpu + +webgpu diff --git a/.github/labeler.yml b/.github/labeler.yml index 98731a0e5ce6..ff2a3ce52234 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -1,6 +1,3 @@ -archive: - - changed-files: - - any-glob-to-any-file: archive/** assert: - changed-files: - any-glob-to-any-file: assert/** diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2e8d7826684d..aa5da45dd3c8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,7 +52,7 @@ jobs: run: deno task cov:gen - name: Upload coverage - uses: codecov/codecov-action@v4 + uses: codecov/codecov-action@v5 env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} with: diff --git a/.github/workflows/title.yml b/.github/workflows/title.yml index 71ad1432880c..fee6276b07fb 100644 --- a/.github/workflows/title.yml +++ b/.github/workflows/title.yml @@ -35,7 +35,6 @@ jobs: revert # This should be kept up-to-date with the current packages list scopes: | - archive(/unstable)? assert(/unstable)? async(/unstable)? bytes(/unstable)? diff --git a/.gitignore b/.gitignore index 420de2d43829..02be11f2662b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,14 +1,21 @@ -.DS_Store -.idea -.vim -**/cov/ -crypto/_wasm/target deno.lock -cli/testdata/unicode_width_crate/target -html_cov/ -cov.lcov http/testdata/%25A.txt http/testdata/file#2.txt http/testdata/test file.txt -coverage/ docs/ + +# rust build output +crypto/_wasm/target +cli/testdata/unicode_width_crate/target + +# coverage +coverage/ +cov.lcov +codecov +codecov.SHA256SUM +codecov.SHA256SUM.sig + +# misc files +.DS_Store +.idea +.vim diff --git a/README.md b/README.md index f5eb3c4eb2e8..92e51ab665eb 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,6 @@ documentation: | Package | Latest version | | ------------------------------------------------------ | ----------------------------------------------------------------------------------------- | -| [archive](https://jsr.io/@std/archive) | [![JSR](https://jsr.io/badges/@std/archive)](https://jsr.io/@std/archive) | | [assert](https://jsr.io/@std/assert) | [![JSR](https://jsr.io/badges/@std/assert)](https://jsr.io/@std/assert) | | [async](https://jsr.io/@std/async) | [![JSR](https://jsr.io/badges/@std/async)](https://jsr.io/@std/async) | | [bytes](https://jsr.io/@std/bytes) | [![JSR](https://jsr.io/badges/@std/bytes)](https://jsr.io/@std/bytes) | diff --git a/_tools/check_circular_package_dependencies.ts b/_tools/check_circular_package_dependencies.ts index 3a4f5e111804..ff6b3c621d31 100644 --- a/_tools/check_circular_package_dependencies.ts +++ b/_tools/check_circular_package_dependencies.ts @@ -35,7 +35,6 @@ type Dep = { state: DepState; }; type Mod = - | "archive" | "assert" | "async" | "bytes" @@ -79,7 +78,6 @@ type Mod = | "yaml"; const ENTRYPOINTS: Record = { - archive: ["mod.ts"], assert: ["mod.ts"], async: ["mod.ts"], bytes: ["mod.ts"], diff --git a/_tools/check_docs.ts b/_tools/check_docs.ts index 54c553f1147b..4ab9af04f334 100644 --- a/_tools/check_docs.ts +++ b/_tools/check_docs.ts @@ -30,7 +30,6 @@ type DocNodeWithJsDoc = T & { }; const ENTRY_POINTS = [ - "../archive/mod.ts", "../assert/mod.ts", "../assert/unstable_never.ts", "../async/mod.ts", diff --git a/archive/_common.ts b/archive/_common.ts deleted file mode 100644 index 8fdf9ceadf56..000000000000 --- a/archive/_common.ts +++ /dev/null @@ -1,206 +0,0 @@ -// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. - -import type { Reader } from "@std/io/types"; - -/** - * Base interface for {@linkcode TarMeta}. - * - * @deprecated Use {@linkcode https://jsr.io/@std/tar | @std/tar} instead. - * `@std/archive` will be removed in the future. - * - * @experimental **UNSTABLE**: New API, yet to be vetted. - */ -export interface TarInfo { - /** - * The underlying raw `st_mode` bits that contain the standard Unix - * permissions for this file/directory. - */ - fileMode?: number; - /** - * Data modification time of the file at the time it was archived. It - * represents the integer number of seconds since January 1, 1970, 00:00 UTC. - */ - mtime?: number; - /** - * Numeric user ID of the file owner. This is ignored if the operating system - * does not support numeric user IDs. - */ - uid?: number; - /** - * Numeric group ID of the file owner. This is ignored if the operating - * system does not support numeric group IDs. - */ - gid?: number; - /** The name of the file owner. */ - owner?: string; - /** The group that the file owner belongs to. */ - group?: string; - /** - * The type of file archived. - * - * @see {@linkcode FileTypes} - */ - type?: string; -} - -/** - * Base interface for {@linkcode TarMetaWithLinkName}. - * - * @deprecated Use {@linkcode https://jsr.io/@std/tar | @std/tar} instead. - * `@std/archive` will be removed in the future. - * - * @experimental **UNSTABLE**: New API, yet to be vetted. - */ -export interface TarMeta extends TarInfo { - /** - * The name of the file, with directory names (if any) preceding the file - * name, separated by slashes. - */ - fileName: string; - /** - * The size of the file in bytes; for archive members that are symbolic or - * hard links to another file, this field is specified as zero. - */ - fileSize?: number; -} - -/** The type of file archived. */ -export enum FileTypes { - "file" = 0, - "link" = 1, - "symlink" = 2, - "character-device" = 3, - "block-device" = 4, - "directory" = 5, - "fifo" = 6, - "contiguous-file" = 7, -} - -export const HEADER_LENGTH = 512; - -/* -struct posix_header { // byte offset - char name[100]; // 0 - char mode[8]; // 100 - char uid[8]; // 108 - char gid[8]; // 116 - char size[12]; // 124 - char mtime[12]; // 136 - char chksum[8]; // 148 - char typeflag; // 156 - char linkname[100]; // 157 - char magic[6]; // 257 - char version[2]; // 263 - char uname[32]; // 265 - char gname[32]; // 297 - char devmajor[8]; // 329 - char devminor[8]; // 337 - char prefix[155]; // 345 - // 500 -}; -*/ - -export const USTAR_STRUCTURE = [ - { - field: "fileName", - length: 100, - }, - { - field: "fileMode", - length: 8, - }, - { - field: "uid", - length: 8, - }, - { - field: "gid", - length: 8, - }, - { - field: "fileSize", - length: 12, - }, - { - field: "mtime", - length: 12, - }, - { - field: "checksum", - length: 8, - }, - { - field: "type", - length: 1, - }, - { - field: "linkName", - length: 100, - }, - { - field: "ustar", - length: 8, - }, - { - field: "owner", - length: 32, - }, - { - field: "group", - length: 32, - }, - { - field: "majorNumber", - length: 8, - }, - { - field: "minorNumber", - length: 8, - }, - { - field: "fileNamePrefix", - length: 155, - }, - { - field: "padding", - length: 12, - }, -] as const; - -/** - * @internal - */ -export type UstarFields = (typeof USTAR_STRUCTURE)[number]["field"]; - -/** - * Thrown when a read from a stream fails to read the - * requested number of bytes. - */ -class PartialReadError extends Error { - partial: Uint8Array; - - constructor(partial: Uint8Array) { - super("Encountered UnexpectedEof, data only partially read"); - this.name = this.constructor.name; - this.partial = partial; - } -} - -export async function readBlock( - reader: Reader, - p: Uint8Array, -): Promise { - let bytesRead = 0; - while (bytesRead < p.length) { - const rr = await reader.read(p.subarray(bytesRead)); - if (rr === null) { - if (bytesRead === 0) { - return null; - } else { - throw new PartialReadError(p.subarray(0, bytesRead)); - } - } - bytesRead += rr; - } - return bytesRead; -} diff --git a/archive/_multi_reader.ts b/archive/_multi_reader.ts deleted file mode 100644 index 54533149a2d1..000000000000 --- a/archive/_multi_reader.ts +++ /dev/null @@ -1,24 +0,0 @@ -// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -// This module is browser compatible. - -import type { Reader } from "@std/io/types"; - -export class MultiReader implements Reader { - readonly #readers: Reader[]; - #currentIndex = 0; - - constructor(readers: Reader[]) { - this.#readers = [...readers]; - } - - async read(p: Uint8Array): Promise { - const r = this.#readers[this.#currentIndex]; - if (!r) return null; - const result = await r.read(p); - if (result === null) { - this.#currentIndex++; - return 0; - } - return result; - } -} diff --git a/archive/_test_utils.ts b/archive/_test_utils.ts deleted file mode 100644 index 7520808d4fee..000000000000 --- a/archive/_test_utils.ts +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -import { dirname, fromFileUrl, resolve } from "@std/path"; - -const moduleDir = dirname(fromFileUrl(import.meta.url)); -export const testdataDir = resolve(moduleDir, "testdata"); -export const filePath = resolve(testdataDir, "example.txt"); diff --git a/archive/deno.json b/archive/deno.json deleted file mode 100644 index c47e282cbff3..000000000000 --- a/archive/deno.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "name": "@std/archive", - "version": "0.225.4", - "exports": { - ".": "./mod.ts", - "./tar": "./tar.ts", - "./untar": "./untar.ts" - } -} diff --git a/archive/mod.ts b/archive/mod.ts deleted file mode 100644 index 4d732f5fcfd3..000000000000 --- a/archive/mod.ts +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. - -/*! - * Ported and modified from: https://github.com/beatgammit/tar-js and - * licensed as: - * - * (The MIT License) - * - * Copyright (c) 2011 T. Jameson Little - * Copyright (c) 2019 Jun Kato - * Copyright (c) 2018-2024 the Deno authors - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -/** - * Tar is a utility for collecting multiple files (or any arbitrary data) into one - * archive file, while untar is the inverse utility to extract the files from an - * archive. Files are not compressed, only collected into the archive. - * - * ```ts ignore - * import { Tar } from "@std/archive/tar"; - * import { Buffer } from "@std/io/buffer"; - * import { copy } from "@std/io/copy"; - * - * const tar = new Tar(); - * - * // Now that we've created our tar, let's add some files to it: - * - * const content = new TextEncoder().encode("Some arbitrary content"); - * await tar.append("deno.txt", { - * reader: new Buffer(content), - * contentSize: content.byteLength, - * }); - * - * // This file is sourced from the filesystem (and renamed in the archive) - * await tar.append("filename_in_archive.txt", { - * filePath: "./filename_on_filesystem.txt", - * }); - * - * // Now let's write the tar (with its two files) to the filesystem - * // use tar.getReader() to read the contents. - * - * const writer = await Deno.open("./out.tar", { write: true, create: true }); - * await copy(tar.getReader(), writer); - * writer.close(); - * ``` - * - * @deprecated Use {@linkcode https://jsr.io/@std/tar | @std/tar} instead. - * `@std/archive` will be removed in the future. - * - * @experimental **UNSTABLE**: New API, yet to be vetted. - * - * @module - */ -export * from "./tar.ts"; -export * from "./untar.ts"; diff --git a/archive/tar.ts b/archive/tar.ts deleted file mode 100644 index d8466cf3cc85..000000000000 --- a/archive/tar.ts +++ /dev/null @@ -1,497 +0,0 @@ -// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -/*! - * Ported and modified from: https://github.com/beatgammit/tar-js and - * licensed as: - * - * (The MIT License) - * - * Copyright (c) 2011 T. Jameson Little - * Copyright (c) 2019 Jun Kato - * Copyright (c) 2018-2024 the Deno authors - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -import { - FileTypes, - type TarInfo, - type TarMeta, - USTAR_STRUCTURE, -} from "./_common.ts"; -import type { Reader } from "@std/io/types"; -import { MultiReader } from "./_multi_reader.ts"; -import { Buffer } from "@std/io/buffer"; -import { HEADER_LENGTH } from "./_common.ts"; - -export type { TarInfo, TarMeta }; - -/** - * Options for {@linkcode Tar.append}. - * - * @deprecated Use {@linkcode https://jsr.io/@std/tar | @std/tar} instead. - * `@std/archive` will be removed in the future. - * - * @experimental **UNSTABLE**: New API, yet to be vetted. - */ -export interface TarOptions extends TarInfo { - /** - * Filepath of the file to append to the archive - */ - filePath?: string; - - /** - * A Reader of any arbitrary content to append to the archive - */ - reader?: Reader; - - /** - * Size of the content to be appended. This is only required - * when passing a reader to the archive. - */ - contentSize?: number; -} - -const USTAR_MAGIC_HEADER = "ustar\u000000" as const; - -/** - * Simple file reader - */ -class FileReader implements Reader { - #file: Deno.FsFile | undefined; - #filePath: string; - - constructor(filePath: string) { - this.#filePath = filePath; - } - - async read(p: Uint8Array): Promise { - if (!this.#file) { - this.#file = await Deno.open(this.#filePath, { read: true }); - } - const res = await this.#file.read(p); - if (res === null) { - this.#file.close(); - this.#file = undefined; - } - return res; - } -} - -/** - * Pads a number with leading zeros to a specified number of bytes. - * - * @param num The number to pad. - * @param bytes The number of bytes to pad the number to. - * @returns The padded number as a string. - */ -function pad(num: number, bytes: number): string { - return num.toString(8).padStart(bytes, "0"); -} - -/** - * Formats the header data for a tar file entry. - * - * @param data The data object containing the values for the tar header fields. - * @returns The formatted header data as a Uint8Array. - */ -function formatHeader(data: TarData): Uint8Array { - const encoder = new TextEncoder(); - const buffer = new Uint8Array(HEADER_LENGTH); - let offset = 0; - for (const { field, length } of USTAR_STRUCTURE) { - const entry = encoder.encode(data[field as keyof TarData] ?? ""); - buffer.set(entry, offset); - offset += length; - } - return buffer; -} - -/** - * Base interface for {@linkcode TarDataWithSource}. - * - * @deprecated Use {@linkcode https://jsr.io/@std/tar | @std/tar} instead. - * `@std/archive` will be removed in the future. - * - * @experimental **UNSTABLE**: New API, yet to be vetted. - */ -export interface TarData { - /** Name of the file, excluding directory names (if any). */ - fileName?: string; - /** Directory names preceding the file name (if any). */ - fileNamePrefix?: string; - /** - * The underlying raw `st_mode` bits that contain the standard Unix - * permissions for this file/directory. - */ - fileMode?: string; - /** - * Numeric user ID of the file owner. This is ignored if the operating system - * does not support numeric user IDs. - */ - uid?: string; - /** - * Numeric group ID of the file owner. This is ignored if the operating - * system does not support numeric group IDs. - */ - gid?: string; - /** - * The size of the file in bytes; for archive members that are symbolic or - * hard links to another file, this field is specified as zero. - */ - fileSize?: string; - /** - * Data modification time of the file at the time it was archived. It - * represents the integer number of seconds since January 1, 1970, 00:00 UTC. - */ - mtime?: string; - /** The simple sum of all bytes in the header block */ - checksum?: string; - /** - * The type of file archived. - * - * @see {@linkcode FileTypes} - */ - type?: string; - /** Ustar magic header */ - ustar?: string; - /** The name of the file owner. */ - owner?: string; - /** The group that the file owner belongs to. */ - group?: string; -} - -/** - * Tar data interface for {@linkcode Tar.data}. - * - * @deprecated Use {@linkcode https://jsr.io/@std/tar | @std/tar} instead. - * `@std/archive` will be removed in the future. - * - * @experimental **UNSTABLE**: New API, yet to be vetted. - */ -export interface TarDataWithSource extends TarData { - /** - * Path of the file to read. - */ - filePath?: string; - /** - * Buffer reader. - */ - reader?: Reader; -} - -/** - * ### Overview - * A class to create a tar archive. Tar archives allow for storing multiple files in a - * single file (called an archive, or sometimes a tarball). These archives typically - * have the '.tar' extension. - * - * @deprecated Use {@linkcode https://jsr.io/@std/tar | @std/tar} instead. - * `@std/archive` will be removed in the future. - * - * ### Usage - * The workflow is to create a Tar instance, append files to it, and then write the - * tar archive to the filesystem (or other output stream). See the worked example - * below for details. - * - * ### Compression - * Tar archives are not compressed by default. If you want to compress the archive, - * you may compress the tar archive after creation, but this capability is not provided - * here. - * - * ### File format and limitations - * - * The ustar file format is used for creating the archive file. - * While this format is compatible with most tar readers, - * the format has several limitations, including: - * * Files must be smaller than 8GiB - * * Filenames (including path) must be shorter than 256 characters - * * Filenames (including path) cannot contain non-ASCII characters - * * Sparse files are not supported - * - * @example Usage - * ```ts ignore - * import { Tar } from "@std/archive/tar"; - * import { Buffer } from "@std/io/buffer"; - * import { copy } from "@std/io/copy"; - * - * const tar = new Tar(); - * - * // Now that we've created our tar, let's add some files to it: - * - * const content = new TextEncoder().encode("Some arbitrary content"); - * await tar.append("deno.txt", { - * reader: new Buffer(content), - * contentSize: content.byteLength, - * }); - * - * // This file is sourced from the filesystem (and renamed in the archive) - * await tar.append("filename_in_archive.txt", { - * filePath: "./filename_on_filesystem.txt", - * }); - * - * // Now let's write the tar (with its two files) to the filesystem - * // use tar.getReader() to read the contents. - * - * const writer = await Deno.open("./out.tar", { write: true, create: true }); - * await copy(tar.getReader(), writer); - * writer.close(); - * ``` - * - * @experimental **UNSTABLE**: New API, yet to be vetted. - */ -export class Tar { - /** Tar data. */ - #data: TarDataWithSource[]; - - /** Constructs a new instance. */ - constructor() { - this.#data = []; - } - - /** - * Append a file or reader of arbitrary content to this tar archive. Directories - * appended to the archive append only the directory itself to the archive, not - * its contents. To add a directory and its contents, recursively append the - * directory's contents. Directories and subdirectories will be created automatically - * in the archive as required. - * - * @param filenameInArchive File name of the content in the archive. E.g. - * `test.txt`. Use slash for directory separators. - * @param source Details of the source of the content including the - * reference to the content itself and potentially any related metadata. - * - * @example Usage - * ```ts ignore - * import { Tar } from "@std/archive/tar"; - * import { Buffer } from "@std/io/buffer"; - * import { copy } from "@std/io/copy"; - * - * const tar = new Tar(); - * - * // Now that we've created our tar, let's add some files to it: - * - * const content = new TextEncoder().encode("Some arbitrary content"); - * await tar.append("deno.txt", { - * reader: new Buffer(content), - * contentSize: content.byteLength, - * }); - * - * // This file is sourced from the filesystem (and renamed in the archive) - * await tar.append("filename_in_archive.txt", { - * filePath: "./filename_on_filesystem.txt", - * }); - * - * // Now let's write the tar (with its two files) to the filesystem - * // use tar.getReader() to read the contents. - * - * const writer = await Deno.open("./out.tar", { write: true, create: true }); - * await copy(tar.getReader(), writer); - * writer.close(); - * ``` - */ - async append(filenameInArchive: string, source: TarOptions) { - if (typeof filenameInArchive !== "string") { - throw new Error("Cannot append data: File name is not a string"); - } - let fileName = filenameInArchive; - - /** - * Ustar format has a limitation of file name length. Specifically: - * 1. File names can contain at most 255 bytes. - * 2. File names longer than 100 bytes must be split at a directory separator in two parts, - * the first being at most 155 bytes long. So, in most cases file names must be a bit shorter - * than 255 bytes. - */ - // separate file name into two parts if needed - let fileNamePrefix: string | undefined; - if (fileName.length > 100) { - let i = fileName.length; - while (i >= 0) { - i = fileName.lastIndexOf("/", i); - if (i <= 155) { - fileNamePrefix = fileName.slice(0, i); - fileName = fileName.slice(i + 1); - break; - } - i--; - } - const errMsg = - "Cannot append data: The 'ustar' format does not allow a long file name (length of [file name" + - "prefix] + / + [file name] must be shorter than 256 bytes)"; - if (i < 0 || fileName.length > 100) { - throw new Error(errMsg); - } else { - if (fileNamePrefix === undefined) { - throw new TypeError("File name prefix is undefined"); - } - if (fileNamePrefix.length > 155) { - throw new Error(errMsg); - } - } - } - - source = source ?? {}; - - // set meta data - let info: Deno.FileInfo | undefined; - if (source.filePath) { - info = await Deno.stat(source.filePath); - if (info.isDirectory) { - info.size = 0; - source.reader = new Buffer(); - } - } - - const mode = source.fileMode || (info && info.mode) || - parseInt("777", 8) & 0xfff /* 511 */; - const mtime = Math.floor( - source.mtime ?? (info?.mtime ?? new Date()).valueOf() / 1000, - ); - const uid = source.uid ?? 0; - const gid = source.gid ?? 0; - - if (typeof source.owner === "string" && source.owner.length >= 32) { - throw new Error( - "Cannot append data: The 'ustar' format does not allow owner name length >= 32 bytes", - ); - } - if (typeof source.group === "string" && source.group.length >= 32) { - throw new Error( - "Cannot append data: The 'ustar' format does not allow group name length >= 32 bytes", - ); - } - - const fileSize = info?.size ?? source.contentSize; - if (fileSize === undefined) { - throw new TypeError("Cannot append data: The file size is not defined"); - } - - const type = source.type - ? FileTypes[source.type as keyof typeof FileTypes] - : (info?.isDirectory ? FileTypes.directory : FileTypes.file); - const tarData: TarDataWithSource = { - fileName, - fileMode: pad(mode, 7), - uid: pad(uid, 7), - gid: pad(gid, 7), - fileSize: pad(fileSize, 11), - mtime: pad(mtime, 11), - checksum: " ", - type: type.toString(), - ustar: USTAR_MAGIC_HEADER, - owner: source.owner ?? "", - group: source.group ?? "", - }; - if (fileNamePrefix !== undefined) { - tarData.fileNamePrefix = fileNamePrefix; - } - if (source.filePath !== undefined) { - tarData.filePath = source.filePath; - } - if (source.reader !== undefined) { - tarData.reader = source.reader; - } - - // calculate the checksum - let checksum = 0; - const encoder = new TextEncoder(); - Object.keys(tarData) - .filter((key): boolean => ["filePath", "reader"].indexOf(key) < 0) - .forEach(function (key) { - checksum += encoder - .encode(tarData[key as keyof TarData]) - .reduce((p, c): number => p + c, 0); - }); - - tarData.checksum = pad(checksum, 6) + "\u0000 "; - this.#data.push(tarData); - } - - /** - * Get a {@linkcode Reader} instance for this tar archive. - * - * @returns A reader instance for the tar archive. - * - * @example Usage - * ```ts ignore - * import { Tar } from "@std/archive/tar"; - * import { Buffer } from "@std/io/buffer"; - * import { copy } from "@std/io/copy"; - * - * const tar = new Tar(); - * - * // Now that we've created our tar, let's add some files to it: - * - * const content = new TextEncoder().encode("Some arbitrary content"); - * await tar.append("deno.txt", { - * reader: new Buffer(content), - * contentSize: content.byteLength, - * }); - * - * // This file is sourced from the filesystem (and renamed in the archive) - * await tar.append("filename_in_archive.txt", { - * filePath: "./filename_on_filesystem.txt", - * }); - * - * // Now let's write the tar (with its two files) to the filesystem - * // use tar.getReader() to read the contents. - * - * const writer = await Deno.open("./out.tar", { write: true, create: true }); - * await copy(tar.getReader(), writer); - * writer.close(); - * ``` - */ - getReader(): Reader { - const readers: Reader[] = []; - this.#data.forEach((tarData) => { - let { reader } = tarData; - const { filePath } = tarData; - const headerArr = formatHeader(tarData); - readers.push(new Buffer(headerArr)); - if (!reader) { - if (filePath === undefined) { - throw new TypeError( - "Cannot get the reader for the tar archive: FilePath is not defined", - ); - } - reader = new FileReader(filePath); - } - readers.push(reader); - - // to the nearest multiple of recordSize - if (tarData.fileSize === undefined) { - throw new TypeError( - "Cannot get the reader for the tar archive: FileSize is not defined", - ); - } - readers.push( - new Buffer( - new Uint8Array( - HEADER_LENGTH - - (parseInt(tarData.fileSize, 8) % HEADER_LENGTH || HEADER_LENGTH), - ), - ), - ); - }); - - // append 2 empty records - readers.push(new Buffer(new Uint8Array(HEADER_LENGTH * 2))); - return new MultiReader(readers); - } -} diff --git a/archive/tar_test.ts b/archive/tar_test.ts deleted file mode 100644 index 2dff2c47339f..000000000000 --- a/archive/tar_test.ts +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -/** - * Tar test - * - * **test summary** - * - create a tar archive in memory containing output.txt and dir/tar.ts. - * - read and deflate a tar archive containing output.txt - * - * **to run this test** - * deno run --allow-read archive/tar_test.ts - */ -import { assert, assertEquals } from "@std/assert"; -import { resolve } from "@std/path"; -import { Tar } from "./tar.ts"; -import { Untar } from "./untar.ts"; -import { Buffer } from "@std/io/buffer"; -import { copy } from "@std/io/copy"; -import { readAll } from "@std/io/read-all"; -import { filePath, testdataDir } from "./_test_utils.ts"; - -Deno.test("createTarArchive", async function () { - // initialize - const tar = new Tar(); - - // put data on memory - const content = new TextEncoder().encode("hello tar world!"); - await tar.append("output.txt", { - reader: new Buffer(content), - contentSize: content.byteLength, - }); - - // put a file - await tar.append("dir/tar.ts", { filePath }); - - // write tar data to a buffer - const writer = new Buffer(); - const wrote = await copy(tar.getReader(), writer); - - /** - * 3072 = 512 (header) + 512 (content) + 512 (header) + 512 (content) - * + 1024 (footer) - */ - assertEquals(wrote, 3072); -}); - -Deno.test("Tar() deflates tar archive", async function () { - const fileName = "output.txt"; - const text = "hello tar world!"; - - // create a tar archive - const tar = new Tar(); - const content = new TextEncoder().encode(text); - await tar.append(fileName, { - reader: new Buffer(content), - contentSize: content.byteLength, - }); - - // read data from a tar archive - const untar = new Untar(tar.getReader()); - const result = await untar.extract(); - assert(result !== null); - const untarText = new TextDecoder("utf-8").decode(await readAll(result)); - - assertEquals(await untar.extract(), null); // EOF - // tests - assertEquals(result.fileName, fileName); - assertEquals(untarText, text); -}); - -Deno.test("Tar() appends file with long name to tar archive", async function (): Promise< - void -> { - // 10 * 15 + 13 = 163 bytes - const fileName = "long-file-name/".repeat(10) + "file-name.txt"; - const text = "hello tar world!"; - - // create a tar archive - const tar = new Tar(); - const content = new TextEncoder().encode(text); - await tar.append(fileName, { - reader: new Buffer(content), - contentSize: content.byteLength, - }); - - // read data from a tar archive - const untar = new Untar(tar.getReader()); - const result = await untar.extract(); - assert(result !== null); - assert(!result.consumed); - const untarText = new TextDecoder("utf-8").decode(await readAll(result)); - assert(result.consumed); - - // tests - assertEquals(result.fileName, fileName); - assertEquals(untarText, text); -}); - -Deno.test("Tar() checks directory entry type", async function () { - const tar = new Tar(); - - await tar.append("directory/", { - reader: new Buffer(), - contentSize: 0, - type: "directory", - }); - - const filePath = resolve(testdataDir); - await tar.append("archive/testdata/", { - filePath, - }); - - const outputFile = resolve(testdataDir, "directory_type_test.tar"); - using file = await Deno.open(outputFile, { create: true, write: true }); - await copy(tar.getReader(), file); - - using reader = await Deno.open(outputFile, { read: true }); - const untar = new Untar(reader); - await Array.fromAsync( - untar, - (entry) => assertEquals(entry.type, "directory"), - ); - - await Deno.remove(outputFile); -}); diff --git a/archive/testdata/deno.tar b/archive/testdata/deno.tar deleted file mode 100644 index 300ce003b5bc..000000000000 Binary files a/archive/testdata/deno.tar and /dev/null differ diff --git a/archive/testdata/example.txt b/archive/testdata/example.txt deleted file mode 100644 index a04238969736..000000000000 --- a/archive/testdata/example.txt +++ /dev/null @@ -1 +0,0 @@ -hello world! diff --git a/archive/testdata/with_link.tar b/archive/testdata/with_link.tar deleted file mode 100644 index c39fe31daf8e..000000000000 Binary files a/archive/testdata/with_link.tar and /dev/null differ diff --git a/archive/untar.ts b/archive/untar.ts deleted file mode 100644 index c3218a132b0b..000000000000 --- a/archive/untar.ts +++ /dev/null @@ -1,570 +0,0 @@ -// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. - -/*! - * Ported and modified from: https://github.com/beatgammit/tar-js and - * licensed as: - * - * (The MIT License) - * - * Copyright (c) 2011 T. Jameson Little - * Copyright (c) 2019 Jun Kato - * Copyright (c) 2018-2022 the Deno authors - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in - * all copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN - * THE SOFTWARE. - */ - -import { - FileTypes, - HEADER_LENGTH, - readBlock, - type TarMeta, - USTAR_STRUCTURE, - type UstarFields, -} from "./_common.ts"; -import { readAll } from "@std/io/read-all"; -import type { Reader, Seeker } from "@std/io/types"; - -export type { Reader, Seeker }; - -/** - * Extend TarMeta with the `linkName` property so that readers can access - * symbolic link values without polluting the world of archive writers. - * - * @deprecated Use {@linkcode https://jsr.io/@std/tar | @std/tar} instead. - * `@std/archive` will be removed in the future. - * - * @experimental **UNSTABLE**: New API, yet to be vetted. - */ -export interface TarMetaWithLinkName extends TarMeta { - /** File name of the symbolic link. */ - linkName?: string; -} - -/** - * Tar header with raw, unprocessed bytes as values. - * - * @deprecated Use {@linkcode https://jsr.io/@std/tar | @std/tar} instead. - * `@std/archive` will be removed in the future. - * - * @experimental **UNSTABLE**: New API, yet to be vetted. - */ -export type TarHeader = { - [key in UstarFields]: Uint8Array; -}; - -// https://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html#tag_20_92_13_06 -// eight checksum bytes taken to be ascii spaces (decimal value 32) -const initialChecksum = 8 * 32; - -/** - * Trims a Uint8Array by removing any trailing zero bytes. - * - * @param buffer The Uint8Array to trim. - * @returns A new Uint8Array with trailing zero bytes removed, or the original - * buffer if no trailing zero bytes are found. - */ -function trim(buffer: Uint8Array): Uint8Array { - const index = buffer.indexOf(0); - return index === -1 ? buffer : buffer.subarray(0, index); -} - -/** - * Parse file header in a tar archive - * @param length - */ -function parseHeader(buffer: Uint8Array): TarHeader { - const data = {} as TarHeader; - let offset = 0; - USTAR_STRUCTURE.forEach(function (value) { - const arr = buffer.subarray(offset, offset + value.length); - data[value.field] = arr; - offset += value.length; - }); - return data; -} - -/** - * Tar entry - * - * @deprecated Use {@linkcode https://jsr.io/@std/tar | @std/tar} instead. - * `@std/archive` will be removed in the future. - * - * @experimental **UNSTABLE**: New API, yet to be vetted. - * - * @example Usage - * ```ts ignore - * import { TarEntry } from "@std/archive/untar"; - * import { Buffer } from "@std/io/buffer"; - * - * const content = new TextEncoder().encode("hello tar world!"); - * const reader = new Buffer(content); - * const tarMeta = { - * fileName: "archive/", - * fileSize: 0, - * fileMode: 509, - * mtime: 1591800767, - * uid: 1001, - * gid: 1001, - * owner: "deno", - * group: "deno", - * type: "directory", - * }; - * const tarEntry: TarEntry = new TarEntry(tarMeta, reader); - * ``` - */ -export interface TarEntry extends TarMetaWithLinkName {} - -/** - * Contains tar header metadata and a reader to the entry's body. - * - * @deprecated Use {@linkcode https://jsr.io/@std/tar | @std/tar} instead. - * `@std/archive` will be removed in the future. - * - * @experimental **UNSTABLE**: New API, yet to be vetted. - * - * @example Usage - * ```ts ignore - * import { TarEntry } from "@std/archive/untar"; - * import { Buffer } from "@std/io/buffer"; - * - * const content = new TextEncoder().encode("hello tar world!"); - * const reader = new Buffer(content); - * const tarMeta = { - * fileName: "archive/", - * fileSize: 0, - * fileMode: 509, - * mtime: 1591800767, - * uid: 1001, - * gid: 1001, - * owner: "deno", - * group: "deno", - * type: "directory", - * }; - * const tarEntry: TarEntry = new TarEntry(tarMeta, reader); - * ``` - */ -export class TarEntry implements Reader { - #reader: Reader | (Reader & Seeker); - #size: number; - #read = 0; - #consumed = false; - #entrySize: number; - - /** - * Constructs a new instance. - * - * @param meta The metadata of the entry. - * @param reader The reader to read the entry from. - */ - constructor( - meta: TarMetaWithLinkName, - reader: Reader | (Reader & Seeker), - ) { - Object.assign(this, meta); - this.#reader = reader; - - // File Size - this.#size = this.fileSize ?? 0; - // Entry Size - const blocks = Math.ceil(this.#size / HEADER_LENGTH); - this.#entrySize = blocks * HEADER_LENGTH; - } - - /** - * Returns whether the entry has already been consumed. - * - * @returns Whether the entry has already been consumed. - * - * @example Usage - * ```ts ignore - * import { TarEntry } from "@std/archive/untar"; - * import { Buffer } from "@std/io/buffer"; - * import { assertEquals } from "@std/assert/equals"; - * - * const content = new TextEncoder().encode("hello tar world!"); - * const reader = new Buffer(content); - * const tarMeta = { - * fileName: "archive/", - * fileSize: 0, - * fileMode: 509, - * mtime: 1591800767, - * uid: 1001, - * gid: 1001, - * owner: "deno", - * group: "deno", - * type: "directory", - * }; - * const tarEntry: TarEntry = new TarEntry(tarMeta, reader); - * - * assertEquals(tarEntry.consumed, false); - * ``` - */ - get consumed(): boolean { - return this.#consumed; - } - - /** - * Reads up to `p.byteLength` bytes of the tar entry into `p`. It resolves to - * the number of bytes read (`0 < n <= p.byteLength`) and rejects if any - * error encountered. Even if read() resolves to n < p.byteLength, it may use - * all of `p` as scratch space during the call. If some data is available but - * not `p.byteLength bytes`, read() conventionally resolves to what is available - * instead of waiting for more. - * - * @param p The buffer to read the entry into. - * @returns The number of bytes read (`0 < n <= p.byteLength`) or `null` if - * there are no more bytes to read. - * - * @example Usage - * ```ts ignore - * import { Tar, Untar } from "@std/archive"; - * import { assertEquals } from "@std/assert/equals"; - * import { Buffer } from "@std/io/buffer"; - * - * const content = new TextEncoder().encode("hello tar world!"); - * - * const tar = new Tar(); - * tar.append("test.txt", { - * reader: new Buffer(content), - * contentSize: content.byteLength, - * }); - * - * const untar = new Untar(tar.getReader()); - * const entry = await untar.extract(); - * const buffer = new Uint8Array(1024); - * const n = await entry!.read(buffer); - * - * assertEquals(buffer.subarray(0, n!), content); - * ``` - */ - async read(p: Uint8Array): Promise { - // Bytes left for entry - const entryBytesLeft = this.#entrySize - this.#read; - const bufSize = Math.min( - // bufSize can't be greater than p.length nor bytes left in the entry - p.length, - entryBytesLeft, - ); - - if (entryBytesLeft <= 0) { - this.#consumed = true; - return null; - } - - const block = new Uint8Array(bufSize); - const n = await readBlock(this.#reader, block); - const bytesLeft = this.#size - this.#read; - - this.#read += n ?? 0; - if (n === null || bytesLeft <= 0) { - if (n === null) this.#consumed = true; - return null; - } - - // Remove zero filled - const offset = bytesLeft < n ? bytesLeft : n; - p.set(block.subarray(0, offset), 0); - - return offset < 0 ? n - Math.abs(offset) : offset; - } - - /** - * Discords the current entry. - * - * @example Usage - * ```ts ignore - * import { Buffer } from "@std/io/buffer"; - * import { TarEntry } from "@std/archive/untar"; - * import { assertEquals } from "@std/assert/equals"; - * - * const text = "Hello, world!"; - * - * const reader = new Buffer(new TextEncoder().encode(text)); - * const tarMeta = { - * fileName: "text", - * fileSize: 0, - * fileMode: 509, - * mtime: 1591800767, - * uid: 1001, - * gid: 1001, - * owner: "deno", - * group: "deno", - * type: "file", - * }; - * - * const tarEntry: TarEntry = new TarEntry(tarMeta, reader); - * await tarEntry.discard(); - * - * assertEquals(tarEntry.consumed, true); - * ``` - */ - async discard() { - // Discard current entry - if (this.#consumed) return; - this.#consumed = true; - - if (typeof (this.#reader as Seeker).seek === "function") { - await (this.#reader as Seeker).seek( - this.#entrySize - this.#read, - Deno.SeekMode.Current, - ); - this.#read = this.#entrySize; - } else { - await readAll(this); - } - } -} - -/** - * ### Overview - * A class to extract from a tar archive. Tar archives allow for storing multiple - * files in a single file (called an archive, or sometimes a tarball). These - * archives typically have the '.tar' extension. - * - * @deprecated Use {@linkcode https://jsr.io/@std/tar | @std/tar} instead. - * `@std/archive` will be removed in the future. - * - * ### Supported file formats - * Only the ustar file format is supported. This is the most common format. The - * pax file format may also be read, but additional features, such as longer - * filenames may be ignored. - * - * ### Usage - * The workflow is to create a Untar instance referencing the source of the tar file. - * You can then use the untar reference to extract files one at a time. See the worked - * example below for details. - * - * ### Understanding compression - * A tar archive may be compressed, often identified by the `.tar.gz` extension. - * This utility does not support decompression which must be done before extracting - * the files. - * - * @example Usage - * ```ts ignore - * import { Untar } from "@std/archive/untar"; - * import { ensureFile } from "@std/fs/ensure-file"; - * import { ensureDir } from "@std/fs/ensure-dir"; - * import { copy } from "@std/io/copy"; - * - * using reader = await Deno.open("./out.tar", { read: true }); - * const untar = new Untar(reader); - * - * for await (const entry of untar) { - * console.log(entry); // metadata - * - * if (entry.type === "directory") { - * await ensureDir(entry.fileName); - * continue; - * } - * - * await ensureFile(entry.fileName); - * using file = await Deno.open(entry.fileName, { write: true }); - * // is a reader. - * await copy(entry, file); - * } - * ``` - * - * @experimental **UNSTABLE**: New API, yet to be vetted. - */ -export class Untar { - /** Internal reader. */ - #reader: Reader; - /** Internal block. */ - #block: Uint8Array; - #entry: TarEntry | undefined; - - /** - * Constructs a new instance. - * - * @param reader The reader to extract from. - */ - constructor(reader: Reader) { - this.#reader = reader; - this.#block = new Uint8Array(HEADER_LENGTH); - } - - #checksum(header: Uint8Array): number { - let sum = initialChecksum; - for (let i = 0; i < HEADER_LENGTH; i++) { - if (i >= 148 && i < 156) { - // Ignore checksum header - continue; - } - sum += header[i]!; - } - return sum; - } - - async #getAndValidateHeader(): Promise { - await readBlock(this.#reader, this.#block); - const header = parseHeader(this.#block); - - // calculate the checksum - const decoder = new TextDecoder(); - const checksum = this.#checksum(this.#block); - - if (parseInt(decoder.decode(header.checksum), 8) !== checksum) { - if (checksum === initialChecksum) { - // EOF - return null; - } - throw new Error("Cannot validate checksum"); - } - - const magic = decoder.decode(header.ustar); - - if (magic.indexOf("ustar")) { - throw new Error( - `Cannot validate the header as it has unsupported archive format: ${magic}`, - ); - } - - return header; - } - - #getMetadata(header: TarHeader): TarMetaWithLinkName { - const decoder = new TextDecoder(); - // get meta data - const meta: TarMetaWithLinkName = { - fileName: decoder.decode(trim(header.fileName)), - }; - const fileNamePrefix = trim(header.fileNamePrefix); - if (fileNamePrefix.byteLength > 0) { - meta.fileName = decoder.decode(fileNamePrefix) + "/" + meta.fileName; - } - (["fileMode", "mtime", "uid", "gid"] as const) - .forEach((key) => { - const arr = trim(header[key]); - if (arr.byteLength > 0) { - meta[key] = parseInt(decoder.decode(arr), 8); - } - }); - (["owner", "group", "type"] as const) - .forEach((key) => { - const arr = trim(header[key]); - if (arr.byteLength > 0) { - meta[key] = decoder.decode(arr); - } - }); - - meta.fileSize = parseInt(decoder.decode(header.fileSize), 8); - if (meta.type !== undefined) { - meta.type = FileTypes[parseInt(meta.type!)] ?? meta.type; - } - - // Only create the `linkName` property for symbolic links to minimize - // the effect on existing code that only deals with non-links. - if (meta.type === "symlink") { - meta.linkName = decoder.decode(trim(header.linkName)); - } - - return meta; - } - - /** - * Extract the next entry of the tar archive. - * - * @returns A TarEntry with header metadata and a reader to the entry's body, - * or null if there are no more entries to extract. - * - * @example Usage - * ```ts ignore - * import { Tar, Untar } from "@std/archive"; - * import { Buffer } from "@std/io/buffer"; - * import { readAll } from "@std/io/read-all"; - * import { assertEquals, assertNotEquals } from "@std/assert"; - * - * const content = new TextEncoder().encode("hello tar world!"); - * - * // Create a tar archive - * const tar = new Tar(); - * await tar.append("output.txt", { - * reader: new Buffer(content), - * contentSize: content.byteLength, - * }); - * - * // Read data from a tar archive - * const untar = new Untar(tar.getReader()); - * const result = await untar.extract(); - * - * assertNotEquals(result, null); - * assertEquals(result!.fileName, "output.txt"); - * assertEquals(result!.fileSize, content.byteLength); - * assertEquals(result!.type, "file"); - * assertEquals(await readAll(result!), content); - * ``` - */ - async extract(): Promise { - if (this.#entry && !this.#entry.consumed) { - // If entry body was not read, discard the body - // so we can read the next entry. - await this.#entry.discard(); - } - - const header = await this.#getAndValidateHeader(); - if (header === null) return null; - - const meta = this.#getMetadata(header); - - this.#entry = new TarEntry(meta, this.#reader); - - return this.#entry; - } - - /** - * Iterate over all entries of the tar archive. - * - * @yields A TarEntry with tar header metadata and a reader to the entry's body. - * @returns An async iterator. - * - * @example Usage - * ```ts ignore - * import { Untar } from "@std/archive/untar"; - * import { ensureFile } from "@std/fs/ensure-file"; - * import { ensureDir } from "@std/fs/ensure-dir"; - * import { copy } from "@std/io/copy"; - * - * using reader = await Deno.open("./out.tar", { read: true }); - * const untar = new Untar(reader); - * - * for await (const entry of untar) { - * console.log(entry); // metadata - * - * if (entry.type === "directory") { - * await ensureDir(entry.fileName); - * continue; - * } - * - * await ensureFile(entry.fileName); - * using file = await Deno.open(entry.fileName, { write: true }); - * // is a reader. - * await copy(entry, file); - * } - * ``` - */ - async *[Symbol.asyncIterator](): AsyncIterableIterator { - while (true) { - const entry = await this.extract(); - - if (entry === null) return; - - yield entry; - } - } -} diff --git a/archive/untar_test.ts b/archive/untar_test.ts deleted file mode 100644 index 3c51a3f0a3b7..000000000000 --- a/archive/untar_test.ts +++ /dev/null @@ -1,386 +0,0 @@ -// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -import { assert, assertEquals, assertExists } from "@std/assert"; -import { resolve } from "@std/path"; -import { Tar, type TarMeta } from "./tar.ts"; -import { TarEntry, type TarMetaWithLinkName, Untar } from "./untar.ts"; -import { Buffer } from "@std/io/buffer"; -import { copy } from "@std/io/copy"; -import { readAll } from "@std/io/read-all"; -import { filePath, testdataDir } from "./_test_utils.ts"; - -interface TestEntry { - name: string; - content?: Uint8Array; - filePath?: string; -} - -async function createTar(entries: TestEntry[]): Promise { - const tar = new Tar(); - // put data on memory - for (const file of entries) { - let options; - - if (file.content) { - options = { - reader: new Buffer(file.content), - contentSize: file.content.byteLength, - }; - } else { - options = { filePath: file.filePath! }; - } - - await tar.append(file.name, options); - } - - return tar; -} - -Deno.test("Untar() works as an async iterator", async () => { - const entries: TestEntry[] = [ - { - name: "output.txt", - content: new TextEncoder().encode("hello tar world!"), - }, - { - name: "dir/tar.ts", - filePath, - }, - ]; - - const tar = await createTar(entries); - - // read data from a tar archive - const untar = new Untar(tar.getReader()); - - let lastEntry; - for await (const entry of untar) { - const expected = entries.shift(); - assert(expected); - - let content = expected.content; - if (expected.filePath) { - content = await Deno.readFile(expected.filePath); - } - assertEquals(content, await readAll(entry)); - assertEquals(expected.name, entry.fileName); - - if (lastEntry) assert(lastEntry.consumed); - lastEntry = entry; - } - assert(lastEntry); - assert(lastEntry.consumed); - assertEquals(entries.length, 0); -}); - -Deno.test("Untar() reads without body", async () => { - const entries: TestEntry[] = [ - { - name: "output.txt", - content: new TextEncoder().encode("hello tar world!"), - }, - { - name: "dir/tar.ts", - filePath, - }, - ]; - - const tar = await createTar(entries); - - // read data from a tar archive - const untar = new Untar(tar.getReader()); - - for await (const entry of untar) { - const expected = entries.shift(); - assert(expected); - assertEquals(expected.name, entry.fileName); - } - - assertEquals(entries.length, 0); -}); - -Deno.test( - "Untar() reads without body from FileReader", - async () => { - const entries: TestEntry[] = [ - { - name: "output.txt", - content: new TextEncoder().encode("hello tar world!"), - }, - { - name: "dir/tar.ts", - filePath, - }, - ]; - - const outputFile = resolve(testdataDir, "test.tar"); - - const tar = await createTar(entries); - using file = await Deno.open(outputFile, { create: true, write: true }); - await copy(tar.getReader(), file); - - using reader = await Deno.open(outputFile, { read: true }); - // read data from a tar archive - const untar = new Untar(reader); - - for await (const entry of untar) { - const expected = entries.shift(); - assert(expected); - assertEquals(expected.name, entry.fileName); - } - - await Deno.remove(outputFile); - assertEquals(entries.length, 0); - }, -); - -Deno.test("Untar() reads from FileReader", async () => { - const entries: TestEntry[] = [ - { - name: "output.txt", - content: new TextEncoder().encode("hello tar world!"), - }, - { - name: "dir/tar.ts", - filePath, - }, - ]; - - const outputFile = resolve(testdataDir, "test.tar"); - - const tar = await createTar(entries); - using file = await Deno.open(outputFile, { create: true, write: true }); - await copy(tar.getReader(), file); - - using reader = await Deno.open(outputFile, { read: true }); - // read data from a tar archive - const untar = new Untar(reader); - - for await (const entry of untar) { - const expected = entries.shift(); - assert(expected); - - let content = expected.content; - if (expected.filePath) { - content = await Deno.readFile(expected.filePath); - } - - assertEquals(content, await readAll(entry)); - assertEquals(expected.name, entry.fileName); - } - - await Deno.remove(outputFile); - assertEquals(entries.length, 0); -}); - -Deno.test( - "Untar() reads less than record size", - async () => { - // record size is 512 - const bufSizes = [1, 53, 256, 511]; - - for (const bufSize of bufSizes) { - const entries: TestEntry[] = [ - { - name: "output.txt", - content: new TextEncoder().encode("hello tar world!".repeat(100)), - }, - // Need to test at least two files, to make sure the first entry doesn't over-read - // Causing the next to fail with: checksum error - { - name: "deni.txt", - content: new TextEncoder().encode("deno!".repeat(250)), - }, - ]; - - const tar = await createTar(entries); - - // read data from a tar archive - const untar = new Untar(tar.getReader()); - - for await (const entry of untar) { - const expected = entries.shift(); - assert(expected); - assertEquals(expected.name, entry.fileName); - - const writer = new Buffer(); - while (true) { - const buf = new Uint8Array(bufSize); - const n = await entry.read(buf); - if (n === null) break; - - await writer.write(buf.subarray(0, n)); - } - assertEquals(writer.bytes(), expected!.content); - } - - assertEquals(entries.length, 0); - } - }, -); - -Deno.test("Untar() works with Linux generated tar", async () => { - const filePath = resolve(testdataDir, "deno.tar"); - using file = await Deno.open(filePath, { read: true }); - - type ExpectedEntry = TarMeta & { content?: Uint8Array }; - - const expectedEntries: ExpectedEntry[] = [ - { - fileName: "archive/", - fileSize: 0, - fileMode: 509, - mtime: 1591800767, - uid: 1001, - gid: 1001, - owner: "deno", - group: "deno", - type: "directory", - }, - { - fileName: "archive/deno/", - fileSize: 0, - fileMode: 509, - mtime: 1591799635, - uid: 1001, - gid: 1001, - owner: "deno", - group: "deno", - type: "directory", - }, - { - fileName: "archive/deno/land/", - fileSize: 0, - fileMode: 509, - mtime: 1591799660, - uid: 1001, - gid: 1001, - owner: "deno", - group: "deno", - type: "directory", - }, - { - fileName: "archive/deno/land/land.txt", - fileMode: 436, - fileSize: 5, - mtime: 1591799660, - uid: 1001, - gid: 1001, - owner: "deno", - group: "deno", - type: "file", - content: new TextEncoder().encode("land\n"), - }, - { - fileName: "archive/file.txt", - fileMode: 436, - fileSize: 5, - mtime: 1591799626, - uid: 1001, - gid: 1001, - owner: "deno", - group: "deno", - type: "file", - content: new TextEncoder().encode("file\n"), - }, - { - fileName: "archive/deno.txt", - fileMode: 436, - fileSize: 5, - mtime: 1591799642, - uid: 1001, - gid: 1001, - owner: "deno", - group: "deno", - type: "file", - content: new TextEncoder().encode("deno\n"), - }, - ]; - - const untar = new Untar(file); - - for await (const entry of untar) { - const expected = expectedEntries.shift(); - assert(expected); - const content = expected.content; - delete expected.content; - - assertEquals({ ...entry }, expected); - - if (content) { - assertEquals(content, await readAll(entry)); - } - } -}); - -Deno.test({ - name: "TarEntry() handles meta", - // only: true, - fn() { - // test TarEntry class - assertExists(TarEntry); - const content = new TextEncoder().encode("hello tar world!"); - const reader = new Buffer(content); - const tarMeta = { - fileName: "archive/", - fileSize: 0, - fileMode: 509, - mtime: 1591800767, - uid: 1001, - gid: 1001, - owner: "deno", - group: "deno", - type: "directory", - }; - const tarEntry: TarEntry = new TarEntry(tarMeta, reader); - assertExists(tarEntry); - }, -}); - -Deno.test("Untar() handles archive with link", async function () { - const filePath = resolve(testdataDir, "with_link.tar"); - using file = await Deno.open(filePath, { read: true }); - - type ExpectedEntry = TarMetaWithLinkName & { content?: Uint8Array }; - - const expectedEntries: ExpectedEntry[] = [ - { - fileName: "hello.txt", - fileMode: 436, - fileSize: 14, - mtime: 1696384910, - uid: 1000, - gid: 1000, - owner: "user", - group: "user", - type: "file", - content: new TextEncoder().encode("Hello World!\n\n"), - }, - { - fileName: "link_to_hello.txt", - linkName: "./hello.txt", - fileMode: 511, - fileSize: 0, - mtime: 1696384945, - uid: 1000, - gid: 1000, - owner: "user", - group: "user", - type: "symlink", - }, - ]; - - const untar = new Untar(file); - - for await (const entry of untar) { - const expected = expectedEntries.shift(); - assert(expected); - const content = expected.content; - delete expected.content; - - assertEquals({ ...entry }, expected); - - if (content) { - assertEquals(content, await readAll(entry)); - } - } -}); diff --git a/browser-compat.tsconfig.json b/browser-compat.tsconfig.json index 202cafb54986..4a48cf3dd4da 100644 --- a/browser-compat.tsconfig.json +++ b/browser-compat.tsconfig.json @@ -8,7 +8,6 @@ }, "importMap": "./import_map.json", "workspace": [ - "./archive", "./assert", "./async", "./bytes", diff --git a/deno.json b/deno.json index 0ca7dfb6e272..09a08bc0e375 100644 --- a/deno.json +++ b/deno.json @@ -50,7 +50,6 @@ } }, "workspace": [ - "./archive", "./assert", "./async", "./bytes", diff --git a/import_map.json b/import_map.json index 0e700a47fddf..8dcab0b177a6 100644 --- a/import_map.json +++ b/import_map.json @@ -6,7 +6,6 @@ "automation/": "https://raw.githubusercontent.com/denoland/automation/0.10.0/", "graphviz": "npm:node-graphviz@^0.1.1", - "@std/archive": "jsr:@std/archive@^0.225.4", "@std/assert": "jsr:@std/assert@^1.0.8", "@std/async": "jsr:@std/async@^1.0.8", "@std/bytes": "jsr:@std/bytes@^1.0.4",