From b1a796d1633270cb6bdd728a609b0ea9b6309669 Mon Sep 17 00:00:00 2001 From: Kouji Matsui Date: Thu, 23 May 2024 22:21:30 +0900 Subject: [PATCH] Added extraction feature on chibiar. --- chibiar/chibiar.core.Tests/ArchiverTests.cs | 9 +- chibiar/chibiar.core/Archiver.cs | 100 ++++++++++++++++-- chibiar/chibiar.core/Cli/CliOptions.cs | 19 +++- chibiar/chibiar/Program.cs | 2 +- .../Generating/ArchivedObjectInputFragment.cs | 14 ++- .../Archiving/ArchiverUtilities.cs | 32 +++++- 6 files changed, 152 insertions(+), 24 deletions(-) diff --git a/chibiar/chibiar.core.Tests/ArchiverTests.cs b/chibiar/chibiar.core.Tests/ArchiverTests.cs index 52d8bf9..09c5fde 100644 --- a/chibiar/chibiar.core.Tests/ArchiverTests.cs +++ b/chibiar/chibiar.core.Tests/ArchiverTests.cs @@ -7,6 +7,8 @@ // ///////////////////////////////////////////////////////////////////////////////////// +using chibicc.toolchain.Archiving; +using chibicc.toolchain.Logging; using NUnit.Framework; using System; using System.IO; @@ -14,8 +16,7 @@ using System.Linq; using System.Text; using System.Threading.Tasks; -using chibicc.toolchain.Archiving; -using chibicc.toolchain.Logging; + using static VerifyNUnit.Verifier; using static chibiar.ArchiverTestRunner; @@ -51,6 +52,7 @@ public Task ArchiveOne() { Path.Combine(ArtifactsBasePath, "parse.o"), }, + true, false); Assert.That(actual, Is.True); @@ -80,6 +82,7 @@ public Task ArchiveTwo() Path.Combine(ArtifactsBasePath, "parse.o"), Path.Combine(ArtifactsBasePath, "codegen.o"), }, + true, false); Assert.That(actual, Is.True); @@ -116,6 +119,7 @@ public Task Update() Path.Combine(ArtifactsBasePath, "parse.o"), Path.Combine(ArtifactsBasePath, "codegen.o"), }, + true, false); Assert.That(actual1, Is.True); @@ -126,6 +130,7 @@ public Task Update() { newCodegenPath, }, + true, false); Assert.That(actual2, Is.False); diff --git a/chibiar/chibiar.core/Archiver.cs b/chibiar/chibiar.core/Archiver.cs index cd39846..b971575 100644 --- a/chibiar/chibiar.core/Archiver.cs +++ b/chibiar/chibiar.core/Archiver.cs @@ -8,6 +8,7 @@ ///////////////////////////////////////////////////////////////////////////////////// using System; +using System.Collections.Generic; using System.IO; using System.IO.Compression; using System.Linq; @@ -146,14 +147,19 @@ private static SymbolList[] GetSymbolLists( Parallel.ForEach(objectNames, (objectName, _, index) => { - using var stream = ArchiverUtilities.OpenArchivedObject( + if (ArchiverUtilities.TryOpenArchivedObject( archiveFilePath, - objectName); - - var symbols = ArchiverUtilities.EnumerateSymbolsFromObjectFile(stream). - ToArray(); + objectName, + true, + out var stream)) + { + using var _s = stream; + + var symbols = ArchiverUtilities.EnumerateSymbolsFromObjectFile(stream). + ToArray(); - symbolLists[index] = new(objectName, symbols); + symbolLists[index] = new(objectName, symbols); + } }); } else @@ -196,6 +202,7 @@ private static void AddSymbolTable( internal bool AddOrUpdate( string archiveFilePath, string[] objectFilePaths, + bool isCreateSymbolTable, bool isDryrun) { var isCreatedArchive = !File.Exists(archiveFilePath); @@ -211,14 +218,73 @@ internal bool AddOrUpdate( objectNames, isDryrun); - AddSymbolTable( - archiveFilePath, - symbolLists, - isDryrun); + if (isCreateSymbolTable) + { + AddSymbolTable( + archiveFilePath, + symbolLists, + isDryrun); + } return isCreatedArchive; } + internal void Extract( + string archiveFilePath, + string[] objectNames, + bool isDryrun) + { + if (!isDryrun || File.Exists(archiveFilePath)) + { + Parallel.ForEach(objectNames, + objectName => + { + if (ArchiverUtilities.TryOpenArchivedObject( + archiveFilePath, + objectName, + false, + out var inputStream)) + { + using var outputStream = isDryrun ? + new MemoryStream() : + StreamUtilities.OpenStream(objectName, true); + + inputStream.CopyTo(outputStream); + outputStream.Flush(); + } + else + { + this.logger.Error($"Object is not found: {objectName}"); + } + }); + } + } + + internal void List( + string archiveFilePath, + string[] objectNames) + { + if (objectNames.Length >= 1) + { + var existObjectNames = new HashSet( + ArchiverUtilities.EnumerateArchivedObjectNames(archiveFilePath)); + + foreach (var objectName in objectNames. + Where(existObjectNames.Contains)) + { + Console.WriteLine(objectName); + } + } + else + { + foreach (var objectName in + ArchiverUtilities.EnumerateArchivedObjectNames(archiveFilePath)) + { + Console.WriteLine(objectName); + } + } + } + public void Archive(CliOptions options) { switch (options.Mode) @@ -226,13 +292,25 @@ public void Archive(CliOptions options) case ArchiveModes.AddOrUpdate: if (this.AddOrUpdate( options.ArchiveFilePath, - options.ObjectFilePaths.ToArray(), + options.ObjectNames.ToArray(), + options.IsCreateSymbolTable, options.IsDryRun) && !options.IsSilent) { this.logger.Information($"creating {Path.GetFileName(options.ArchiveFilePath)}"); } break; + case ArchiveModes.Extract: + this.Extract( + options.ArchiveFilePath, + options.ObjectNames.ToArray(), + options.IsDryRun); + break; + case ArchiveModes.List: + this.List( + options.ArchiveFilePath, + options.ObjectNames.ToArray()); + break; default: throw new NotImplementedException(); } diff --git a/chibiar/chibiar.core/Cli/CliOptions.cs b/chibiar/chibiar.core/Cli/CliOptions.cs index d720e7c..07353d0 100644 --- a/chibiar/chibiar.core/Cli/CliOptions.cs +++ b/chibiar/chibiar.core/Cli/CliOptions.cs @@ -18,6 +18,7 @@ public enum ArchiveModes { Nothing, AddOrUpdate, + Extract, Delete, List, } @@ -27,10 +28,11 @@ public sealed class CliOptions public string ArchiveFilePath = null!; public ArchiveModes Mode = ArchiveModes.Nothing; public bool IsSilent = false; + public bool IsCreateSymbolTable = true; public bool IsDryRun = false; public LogLevels LogLevel = LogLevels.Warning; public bool ShowHelp = false; - public readonly List ObjectFilePaths = new(); + public readonly List ObjectNames = new(); private CliOptions() { @@ -60,6 +62,9 @@ public static CliOptions Parse(string[] args) case 'u': options.Mode = ArchiveModes.AddOrUpdate; break; + case 'x': + options.Mode = ArchiveModes.Extract; + break; case 'd': options.Mode = ArchiveModes.Delete; break; @@ -70,6 +75,10 @@ public static CliOptions Parse(string[] args) options.IsSilent = true; break; case 's': + options.IsCreateSymbolTable = true; + break; + case 'S': + options.IsCreateSymbolTable = false; break; case 'h': options.ShowHelp = true; @@ -125,7 +134,7 @@ public static CliOptions Parse(string[] args) for (var index = 2; index < args.Length; index++) { - options.ObjectFilePaths.Add(Path.GetFullPath(args[index])); + options.ObjectNames.Add(args[index]); } return options; @@ -134,9 +143,11 @@ public static CliOptions Parse(string[] args) public static void WriteUsage(TextWriter tw) { tw.WriteLine(" -r, -u Add or update object files into the archive"); - tw.WriteLine(" -c Add object files into the archive silently"); - tw.WriteLine(" -s Add symbol table (Always enabled)"); + tw.WriteLine(" -c Create archive file silently"); + tw.WriteLine(" -x Extract object files from the archive"); tw.WriteLine(" -d Delete object files from the archive"); + tw.WriteLine(" -s Add symbol table (default)"); + tw.WriteLine(" -S Will not add symbol table"); tw.WriteLine(" -t List object files in the archive"); tw.WriteLine(" --log Log level [debug|trace|information|warning|error|silent] (defaulted: warning)"); tw.WriteLine(" --dryrun Need to dryrun"); diff --git a/chibiar/chibiar/Program.cs b/chibiar/chibiar/Program.cs index 5e7f248..3fca868 100644 --- a/chibiar/chibiar/Program.cs +++ b/chibiar/chibiar/Program.cs @@ -23,7 +23,7 @@ public static int Main(string[] args) { var options = CliOptions.Parse(args); - if (options.ShowHelp || options.ObjectFilePaths.Count == 0) + if (options.ShowHelp) { Console.WriteLine(); Console.WriteLine($"cil-ecma-chibiar [{ThisAssembly.AssemblyVersion},{ThisAssembly.AssemblyMetadata.TargetFrameworkMoniker}] [{ThisAssembly.AssemblyMetadata.CommitId}]"); diff --git a/chibild/chibild.core/Generating/ArchivedObjectInputFragment.cs b/chibild/chibild.core/Generating/ArchivedObjectInputFragment.cs index 1adebc5..dc3a9c2 100644 --- a/chibild/chibild.core/Generating/ArchivedObjectInputFragment.cs +++ b/chibild/chibild.core/Generating/ArchivedObjectInputFragment.cs @@ -7,6 +7,7 @@ // ///////////////////////////////////////////////////////////////////////////////////// +using System; using chibicc.toolchain.Archiving; using chibicc.toolchain.Parsing; using chibicc.toolchain.Logging; @@ -179,9 +180,18 @@ public LoadObjectResults LoadObjectIfRequired( { logger.Information($"Loading: {this.ObjectPath}"); - using var stream = ArchiverUtilities.OpenArchivedObject( + if (!ArchiverUtilities.TryOpenArchivedObject( Path.Combine(this.BaseInputPath, this.RelativePath), - this.archivedObjectName); + this.archivedObjectName, + true, + out var stream)) + { + logger.Error( + $"Unable find an object on archive: ObjectName={this.archivedObjectName}, ArchiveFile={this.RelativePath}"); + return LoadObjectResults.CaughtError; + } + + using var _s = stream; var tr = new StreamReader(stream, Encoding.UTF8, true); var parser = new CilParser(logger); diff --git a/toolchain.common/Archiving/ArchiverUtilities.cs b/toolchain.common/Archiving/ArchiverUtilities.cs index 9d4d813..e567c56 100644 --- a/toolchain.common/Archiving/ArchiverUtilities.cs +++ b/toolchain.common/Archiving/ArchiverUtilities.cs @@ -7,6 +7,7 @@ // ///////////////////////////////////////////////////////////////////////////////////// +using System; using chibicc.toolchain.Internal; using chibicc.toolchain.Parsing; using chibicc.toolchain.Tokenizing; @@ -235,16 +236,39 @@ public static IEnumerable EnumerateArchivedObjectNames( } } - public static Stream OpenArchivedObject(string archiveFilePath, string objectName) + public static bool TryOpenArchivedObject( + string archiveFilePath, + string objectName, + bool is_decoded_body, + out Stream stream) { var archive = ZipFile.Open( archiveFilePath, ZipArchiveMode.Read, Encoding.UTF8); - var zs = archive.GetEntry(objectName)!.Open(); - var ofs = new GZipStream(zs, CompressionMode.Decompress); + try + { + if (archive.GetEntry(objectName) is { } entry) + { + var ofs = is_decoded_body ? + new GZipStream(entry.Open(), CompressionMode.Decompress) : + entry.Open(); - return new ArchiveObjectStream(archive, ofs); + stream = new ArchiveObjectStream(archive, ofs); + return true; + } + else + { + archive.Dispose(); + stream = null!; + return false; + } + } + catch + { + archive.Dispose(); + throw; + } } }