From f3fa874cc31b15e2b9c1642ea63b354f3378b23f Mon Sep 17 00:00:00 2001 From: David Sherret Date: Sat, 23 Nov 2024 12:18:11 -0500 Subject: [PATCH 1/2] fix(compile): handle TypeScript file included as asset --- cli/standalone/binary.rs | 6 +- cli/standalone/file_system.rs | 4 +- cli/standalone/mod.rs | 17 ++- cli/standalone/virtual_fs.rs | 120 +++++++++++++++--- .../specs/compile/include/self/__test__.jsonc | 24 ++++ tests/specs/compile/include/self/main.ts | 6 + tests/specs/compile/include/self/output.out | 7 + 7 files changed, 157 insertions(+), 27 deletions(-) create mode 100644 tests/specs/compile/include/self/__test__.jsonc create mode 100644 tests/specs/compile/include/self/main.ts create mode 100644 tests/specs/compile/include/self/output.out diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs index e35119e0aa7dfa..791f5052c0ab80 100644 --- a/cli/standalone/binary.rs +++ b/cli/standalone/binary.rs @@ -87,6 +87,7 @@ use super::serialization::RemoteModulesStore; use super::serialization::RemoteModulesStoreBuilder; use super::virtual_fs::FileBackedVfs; use super::virtual_fs::VfsBuilder; +use super::virtual_fs::VfsFileSubDataKind; use super::virtual_fs::VfsRoot; use super::virtual_fs::VirtualDirectory; @@ -275,7 +276,9 @@ impl StandaloneModules { if specifier.scheme() == "file" { let path = deno_path_util::url_to_file_path(specifier)?; let bytes = match self.vfs.file_entry(&path) { - Ok(entry) => self.vfs.read_file_all(entry)?, + Ok(entry) => self + .vfs + .read_file_all(entry, VfsFileSubDataKind::ModuleGraph)?, Err(err) if err.kind() == ErrorKind::NotFound => { let bytes = match RealFs.read_file_sync(&path, None) { Ok(bytes) => bytes, @@ -691,6 +694,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { Some(source) => source, None => RealFs.read_file_sync(&file_path, None)?, }, + VfsFileSubDataKind::ModuleGraph, ) .with_context(|| { format!("Failed adding '{}'", file_path.display()) diff --git a/cli/standalone/file_system.rs b/cli/standalone/file_system.rs index 712c6ee91878e3..48dc907570babd 100644 --- a/cli/standalone/file_system.rs +++ b/cli/standalone/file_system.rs @@ -17,6 +17,7 @@ use deno_runtime::deno_io::fs::FsResult; use deno_runtime::deno_io::fs::FsStat; use super::virtual_fs::FileBackedVfs; +use super::virtual_fs::VfsFileSubDataKind; #[derive(Debug, Clone)] pub struct DenoCompileFileSystem(Arc); @@ -36,7 +37,8 @@ impl DenoCompileFileSystem { fn copy_to_real_path(&self, oldpath: &Path, newpath: &Path) -> FsResult<()> { let old_file = self.0.file_entry(oldpath)?; - let old_file_bytes = self.0.read_file_all(old_file)?; + let old_file_bytes = + self.0.read_file_all(old_file, VfsFileSubDataKind::Raw)?; RealFs.write_file_sync( newpath, OpenOptions { diff --git a/cli/standalone/mod.rs b/cli/standalone/mod.rs index b9f0b1d5be0825..27b03fec634383 100644 --- a/cli/standalone/mod.rs +++ b/cli/standalone/mod.rs @@ -56,6 +56,8 @@ use serialization::DenoCompileModuleSource; use std::borrow::Cow; use std::rc::Rc; use std::sync::Arc; +use virtual_fs::FileBackedVfs; +use virtual_fs::VfsFileSubDataKind; use crate::args::create_default_npmrc; use crate::args::get_root_cert_store; @@ -111,6 +113,7 @@ use self::file_system::DenoCompileFileSystem; struct SharedModuleLoaderState { cjs_tracker: Arc, + code_cache: Option>, fs: Arc, modules: StandaloneModules, node_code_translator: Arc, @@ -118,8 +121,8 @@ struct SharedModuleLoaderState { npm_module_loader: Arc, npm_req_resolver: Arc, npm_resolver: Arc, + vfs: Arc, workspace_resolver: WorkspaceResolver, - code_cache: Option>, } impl SharedModuleLoaderState { @@ -514,7 +517,12 @@ impl NodeRequireLoader for EmbeddedModuleLoader { &self, path: &std::path::Path, ) -> Result { - Ok(self.shared.fs.read_text_file_lossy_sync(path, None)?) + let file_entry = self.shared.vfs.file_entry(path)?; + let file_bytes = self + .shared + .vfs + .read_file_all(file_entry, VfsFileSubDataKind::ModuleGraph)?; + Ok(String::from_utf8(file_bytes.into_owned())?) } fn is_maybe_cjs( @@ -817,6 +825,7 @@ pub async fn run(data: StandaloneData) -> Result { let module_loader_factory = StandaloneModuleLoaderFactory { shared: Arc::new(SharedModuleLoaderState { cjs_tracker: cjs_tracker.clone(), + code_cache: code_cache.clone(), fs: fs.clone(), modules, node_code_translator: node_code_translator.clone(), @@ -826,10 +835,10 @@ pub async fn run(data: StandaloneData) -> Result { fs.clone(), node_code_translator, )), - code_cache: code_cache.clone(), npm_resolver: npm_resolver.clone(), - workspace_resolver, npm_req_resolver, + vfs, + workspace_resolver, }), }; diff --git a/cli/standalone/virtual_fs.rs b/cli/standalone/virtual_fs.rs index be7e937ee1e5b4..90e37218e4afc0 100644 --- a/cli/standalone/virtual_fs.rs +++ b/cli/standalone/virtual_fs.rs @@ -32,6 +32,14 @@ use thiserror::Error; use crate::util; use crate::util::fs::canonicalize_path; +#[derive(Debug, Copy, Clone)] +pub enum VfsFileSubDataKind { + /// Raw bytes of the file. + Raw, + /// Used for the module graph. + ModuleGraph, +} + #[derive(Error, Debug)] #[error( "Failed to strip prefix '{}' from '{}'", root_path.display(), target.display() @@ -141,7 +149,11 @@ impl VfsBuilder { // inline the symlink and make the target file let file_bytes = std::fs::read(&target) .with_context(|| format!("Reading {}", path.display()))?; - self.add_file_with_data_inner(&path, file_bytes)?; + self.add_file_with_data_inner( + &path, + file_bytes, + VfsFileSubDataKind::Raw, + )?; } else { log::warn!( "{} Symlink target is outside '{}'. Excluding symlink at '{}' with target '{}'.", @@ -219,25 +231,27 @@ impl VfsBuilder { ) -> Result<(), AnyError> { let file_bytes = std::fs::read(path) .with_context(|| format!("Reading {}", path.display()))?; - self.add_file_with_data_inner(path, file_bytes) + self.add_file_with_data_inner(path, file_bytes, VfsFileSubDataKind::Raw) } pub fn add_file_with_data( &mut self, path: &Path, data: Vec, + sub_data_kind: VfsFileSubDataKind, ) -> Result<(), AnyError> { let target_path = canonicalize_path(path)?; if target_path != path { self.add_symlink(path, &target_path)?; } - self.add_file_with_data_inner(&target_path, data) + self.add_file_with_data_inner(&target_path, data, sub_data_kind) } fn add_file_with_data_inner( &mut self, path: &Path, data: Vec, + sub_data_kind: VfsFileSubDataKind, ) -> Result<(), AnyError> { log::debug!("Adding file '{}'", path.display()); let checksum = util::checksum::gen(&[&data]); @@ -253,8 +267,19 @@ impl VfsBuilder { let name = path.file_name().unwrap().to_string_lossy(); let data_len = data.len(); match dir.entries.binary_search_by(|e| e.name().cmp(&name)) { - Ok(_) => { - // already added, just ignore + Ok(index) => { + let entry = &mut dir.entries[index]; + match entry { + VfsEntry::File(virtual_file) => match sub_data_kind { + VfsFileSubDataKind::Raw => { + virtual_file.offset = offset; + } + VfsFileSubDataKind::ModuleGraph => { + virtual_file.module_graph_offset = offset; + } + }, + VfsEntry::Dir(_) | VfsEntry::Symlink(_) => unreachable!(), + } } Err(insert_index) => { dir.entries.insert( @@ -262,6 +287,7 @@ impl VfsBuilder { VfsEntry::File(VirtualFile { name: name.to_string(), offset, + module_graph_offset: offset, len: data.len() as u64, }), ); @@ -454,6 +480,12 @@ pub struct VirtualDirectory { pub struct VirtualFile { pub name: String, pub offset: u64, + /// Offset file to use for module loading when it differs from the + /// raw file. Often this will be the same offset as above for data + /// such as JavaScript files, but for TypeScript files the `offset` + /// will be the original raw bytes when included as an asset and this + /// offset will be to the transpiled JavaScript source. + pub module_graph_offset: u64, pub len: u64, } @@ -647,7 +679,7 @@ impl FileBackedVfsFile { .map_err(|err| err.into()) } - fn read_to_end(&self) -> FsResult> { + fn read_to_end(&self) -> FsResult> { let read_pos = { let mut pos = self.pos.lock(); let read_pos = *pos; @@ -659,12 +691,20 @@ impl FileBackedVfsFile { read_pos }; if read_pos > self.file.len { - return Ok(Vec::new()); + return Ok(Cow::Borrowed(&[])); + } + if read_pos == 0 { + Ok( + self + .vfs + .read_file_all(&self.file, VfsFileSubDataKind::Raw)?, + ) + } else { + let size = (self.file.len - read_pos) as usize; + let mut buf = vec![0; size]; + self.vfs.read_file(&self.file, read_pos, &mut buf)?; + Ok(Cow::Owned(buf)) } - let size = (self.file.len - read_pos) as usize; - let mut buf = vec![0; size]; - self.vfs.read_file(&self.file, read_pos, &mut buf)?; - Ok(buf) } } @@ -703,11 +743,14 @@ impl deno_io::fs::File for FileBackedVfsFile { } fn read_all_sync(self: Rc) -> FsResult> { - self.read_to_end() + self.read_to_end().map(|bytes| bytes.into_owned()) } async fn read_all_async(self: Rc) -> FsResult> { let inner = (*self).clone(); - tokio::task::spawn_blocking(move || inner.read_to_end()).await? + tokio::task::spawn_blocking(move || { + inner.read_to_end().map(|bytes| bytes.into_owned()) + }) + .await? } fn chmod_sync(self: Rc, _pathmode: u32) -> FsResult<()> { @@ -878,8 +921,9 @@ impl FileBackedVfs { pub fn read_file_all( &self, file: &VirtualFile, + sub_data_kind: VfsFileSubDataKind, ) -> std::io::Result> { - let read_range = self.get_read_range(file, 0, file.len)?; + let read_range = self.get_read_range(file, sub_data_kind, 0, file.len)?; match &self.vfs_data { Cow::Borrowed(data) => Ok(Cow::Borrowed(&data[read_range])), Cow::Owned(data) => Ok(Cow::Owned(data[read_range].to_vec())), @@ -892,7 +936,12 @@ impl FileBackedVfs { pos: u64, buf: &mut [u8], ) -> std::io::Result { - let read_range = self.get_read_range(file, pos, buf.len() as u64)?; + let read_range = self.get_read_range( + file, + VfsFileSubDataKind::Raw, + pos, + buf.len() as u64, + )?; let read_len = read_range.len(); buf[..read_len].copy_from_slice(&self.vfs_data[read_range]); Ok(read_len) @@ -901,6 +950,7 @@ impl FileBackedVfs { fn get_read_range( &self, file: &VirtualFile, + sub_data_kind: VfsFileSubDataKind, pos: u64, len: u64, ) -> std::io::Result> { @@ -910,7 +960,11 @@ impl FileBackedVfs { "unexpected EOF", )); } - let file_offset = self.fs_root.start_file_offset + file.offset; + let offset = match sub_data_kind { + VfsFileSubDataKind::Raw => file.offset, + VfsFileSubDataKind::ModuleGraph => file.module_graph_offset, + }; + let file_offset = self.fs_root.start_file_offset + offset; let start = file_offset + pos; let end = file_offset + std::cmp::min(pos + len, file.len); Ok(start as usize..end as usize) @@ -951,7 +1005,13 @@ mod test { #[track_caller] fn read_file(vfs: &FileBackedVfs, path: &Path) -> String { let file = vfs.file_entry(path).unwrap(); - String::from_utf8(vfs.read_file_all(file).unwrap().into_owned()).unwrap() + String::from_utf8( + vfs + .read_file_all(file, VfsFileSubDataKind::Raw) + .unwrap() + .into_owned(), + ) + .unwrap() } #[test] @@ -964,23 +1024,40 @@ mod test { let src_path = src_path.to_path_buf(); let mut builder = VfsBuilder::new(src_path.clone()).unwrap(); builder - .add_file_with_data_inner(&src_path.join("a.txt"), "data".into()) + .add_file_with_data_inner( + &src_path.join("a.txt"), + "data".into(), + VfsFileSubDataKind::Raw, + ) .unwrap(); builder - .add_file_with_data_inner(&src_path.join("b.txt"), "data".into()) + .add_file_with_data_inner( + &src_path.join("b.txt"), + "data".into(), + VfsFileSubDataKind::Raw, + ) .unwrap(); assert_eq!(builder.files.len(), 1); // because duplicate data builder - .add_file_with_data_inner(&src_path.join("c.txt"), "c".into()) + .add_file_with_data_inner( + &src_path.join("c.txt"), + "c".into(), + VfsFileSubDataKind::Raw, + ) .unwrap(); builder .add_file_with_data_inner( &src_path.join("sub_dir").join("d.txt"), "d".into(), + VfsFileSubDataKind::Raw, ) .unwrap(); builder - .add_file_with_data_inner(&src_path.join("e.txt"), "e".into()) + .add_file_with_data_inner( + &src_path.join("e.txt"), + "e".into(), + VfsFileSubDataKind::Raw, + ) .unwrap(); builder .add_symlink( @@ -1151,6 +1228,7 @@ mod test { .add_file_with_data_inner( temp_path.join("a.txt").as_path(), "0123456789".to_string().into_bytes(), + VfsFileSubDataKind::Raw, ) .unwrap(); let (dest_path, virtual_fs) = into_virtual_fs(builder, &temp_dir); diff --git a/tests/specs/compile/include/self/__test__.jsonc b/tests/specs/compile/include/self/__test__.jsonc new file mode 100644 index 00000000000000..5fb74534aa6cc3 --- /dev/null +++ b/tests/specs/compile/include/self/__test__.jsonc @@ -0,0 +1,24 @@ +{ + "tempDir": true, + "steps": [{ + "if": "unix", + "args": "compile --allow-read=. --include . --output main main.ts", + "output": "[WILDCARD]" + }, { + "if": "unix", + "commandName": "./main", + "args": [], + "output": "output.out", + "exitCode": 0 + }, { + "if": "windows", + "args": "compile --allow-read=. --include . --output main main.ts", + "output": "[WILDCARD]" + }, { + "if": "windows", + "commandName": "./main.exe", + "args": [], + "output": "output.out", + "exitCode": 0 + }] +} diff --git a/tests/specs/compile/include/self/main.ts b/tests/specs/compile/include/self/main.ts new file mode 100644 index 00000000000000..d86580e3d825e3 --- /dev/null +++ b/tests/specs/compile/include/self/main.ts @@ -0,0 +1,6 @@ +function add(a: number, b: number) { + return a + b; +} + +console.log(add(1, 2)); +console.log(Deno.readTextFileSync(import.meta.filename!).trim()); diff --git a/tests/specs/compile/include/self/output.out b/tests/specs/compile/include/self/output.out new file mode 100644 index 00000000000000..10c297caba155a --- /dev/null +++ b/tests/specs/compile/include/self/output.out @@ -0,0 +1,7 @@ +3 +function add(a: number, b: number) { + return a + b; +} + +console.log(add(1, 2)); +console.log(Deno.readTextFileSync(import.meta.filename!).trim()); From 6700ca8cc1b007a9aeae688fd31c75a86cd717c7 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Sat, 23 Nov 2024 12:20:21 -0500 Subject: [PATCH 2/2] update --- cli/standalone/virtual_fs.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cli/standalone/virtual_fs.rs b/cli/standalone/virtual_fs.rs index 90e37218e4afc0..5b8076549d1700 100644 --- a/cli/standalone/virtual_fs.rs +++ b/cli/standalone/virtual_fs.rs @@ -36,7 +36,8 @@ use crate::util::fs::canonicalize_path; pub enum VfsFileSubDataKind { /// Raw bytes of the file. Raw, - /// Used for the module graph. + /// Bytes to use for module loading. For example, for TypeScript + /// files this will be the transpiled JavaScript source. ModuleGraph, }