From da6d99a68f80553b44f1b99ee0e4d3bbf6c93a99 Mon Sep 17 00:00:00 2001 From: Kouji Matsui Date: Tue, 4 Jun 2024 13:36:12 +0900 Subject: [PATCH] WIP --- chibiar/chibiar.core.Tests/ArchiverTests.cs | 2 +- chibiar/chibiar.core/Archiver.cs | 15 +- chibias/chibias.core/Assembler.cs | 2 +- .../chibild.core.Tests/LinkerTestRunner.cs | 6 +- chibild/chibild.core.Tests/LinkerTests.cs | 2 +- chibild/chibild.core/CilLinker.cs | 18 +- .../Generating/ArchivedObjectInputFragment.cs | 68 +-- .../Generating/AssemblyInputFragment.cs | 533 +++++++++++------- .../chibild.core/Generating/CodeGenerator.cs | 45 +- .../Generating/CodeGenerator_Consumer.cs | 49 +- .../Generating/CodeGenerator_Emit.cs | 67 +-- .../chibild.core/Generating/InputFragment.cs | 14 + .../Generating/ObjectFileInputFragment.cs | 7 + .../Generating/SymbolAggregator.cs | 109 ++++ .../chibild.core/Internal/CecilUtilities.cs | 125 +++- .../Internal/MultipleSymbolReaderProvider.cs | 8 +- chibild/chibild.core/Internal/Utilities.cs | 5 + chibild/chibild.core/LinkerOptions.cs | 10 + .../Archiving/ArchiverUtilities.cs | 82 +-- .../{Archiving => Generating}/Symbol.cs | 2 +- .../Generating/SymbolUtilities.cs | 107 ++++ ...ities.cs => CompressionStreamUtilities.cs} | 6 +- toolchain.common/IO/StreamUtilities.cs | 3 +- toolchain.common/Internal/CommonUtilities.cs | 33 ++ 24 files changed, 904 insertions(+), 414 deletions(-) create mode 100644 chibild/chibild.core/Generating/SymbolAggregator.cs rename toolchain.common/{Archiving => Generating}/Symbol.cs (98%) create mode 100644 toolchain.common/Generating/SymbolUtilities.cs rename toolchain.common/IO/{ObjectStreamUtilities.cs => CompressionStreamUtilities.cs} (92%) diff --git a/chibiar/chibiar.core.Tests/ArchiverTests.cs b/chibiar/chibiar.core.Tests/ArchiverTests.cs index 7f1b7ab..f8552f7 100644 --- a/chibiar/chibiar.core.Tests/ArchiverTests.cs +++ b/chibiar/chibiar.core.Tests/ArchiverTests.cs @@ -1,4 +1,4 @@ -///////////////////////////////////////////////////////////////////////////////////// +///////////////////////////////////////////////////////////////////////////////////// // // chibicc-toolchain - The specialized backend toolchain for chibicc-cil // Copyright (c) Kouji Matsui(@kozy_kekyo, @kekyo @mastodon.cloud) diff --git a/chibiar/chibiar.core/Archiver.cs b/chibiar/chibiar.core/Archiver.cs index ba4a0ab..7535e8e 100644 --- a/chibiar/chibiar.core/Archiver.cs +++ b/chibiar/chibiar.core/Archiver.cs @@ -7,6 +7,12 @@ // ///////////////////////////////////////////////////////////////////////////////////// +using chibiar.Cli; +using chibicc.toolchain.Archiving; +using chibicc.toolchain.Generating; +using chibicc.toolchain.Internal; +using chibicc.toolchain.IO; +using chibicc.toolchain.Logging; using System; using System.Collections.Generic; using System.IO; @@ -14,11 +20,6 @@ using System.Linq; using System.Text; using System.Threading.Tasks; -using chibiar.Cli; -using chibicc.toolchain.Archiving; -using chibicc.toolchain.Internal; -using chibicc.toolchain.IO; -using chibicc.toolchain.Logging; namespace chibiar; @@ -216,11 +217,11 @@ private static void AddSymbolTable( using var outputStream = symbolTableEntry.Open(); - ArchiverUtilities.WriteSymbolTable(outputStream, symbolLists); + SymbolUtilities.WriteSymbolTable(outputStream, symbolLists); } else { - ArchiverUtilities.WriteSymbolTable(new MemoryStream(), symbolLists); + SymbolUtilities.WriteSymbolTable(new MemoryStream(), symbolLists); } } diff --git a/chibias/chibias.core/Assembler.cs b/chibias/chibias.core/Assembler.cs index cae043e..89c0848 100644 --- a/chibias/chibias.core/Assembler.cs +++ b/chibias/chibias.core/Assembler.cs @@ -64,7 +64,7 @@ public bool Assemble( { // Convert source code to object file. using var outputStream = isDryrun ? - null : ObjectStreamUtilities.OpenObjectStream(outputTemporaryFilePath, true); + null : CompressionStreamUtilities.OpenStream(outputTemporaryFilePath, true); if (outputStream != null) { diff --git a/chibild/chibild.core.Tests/LinkerTestRunner.cs b/chibild/chibild.core.Tests/LinkerTestRunner.cs index 7fd64cc..c78cd41 100644 --- a/chibild/chibild.core.Tests/LinkerTestRunner.cs +++ b/chibild/chibild.core.Tests/LinkerTestRunner.cs @@ -119,6 +119,7 @@ public static string RunCore( ApplyOptimization = false, CreationOptions = creationOptions, PrependExecutionSearchPaths = prependExecutionSearchPaths ?? Array.Empty(), + //CacheBasePath = null, }, injectToAssemblyPath, basePath, @@ -175,7 +176,10 @@ public static string RunCore( { try { - logtw.Flush(); + if (logfs.CanWrite) + { + logtw.Flush(); + } } catch { diff --git a/chibild/chibild.core.Tests/LinkerTests.cs b/chibild/chibild.core.Tests/LinkerTests.cs index 59b2714..7707a27 100644 --- a/chibild/chibild.core.Tests/LinkerTests.cs +++ b/chibild/chibild.core.Tests/LinkerTests.cs @@ -1,4 +1,4 @@ -///////////////////////////////////////////////////////////////////////////////////// +///////////////////////////////////////////////////////////////////////////////////// // // chibicc-toolchain - The specialized backend toolchain for chibicc-cil // Copyright (c) Kouji Matsui(@kozy_kekyo, @kekyo @mastodon.cloud) diff --git a/chibild/chibild.core/CilLinker.cs b/chibild/chibild.core/CilLinker.cs index f160445..8d853a3 100644 --- a/chibild/chibild.core/CilLinker.cs +++ b/chibild/chibild.core/CilLinker.cs @@ -1,4 +1,4 @@ -///////////////////////////////////////////////////////////////////////////////////// +///////////////////////////////////////////////////////////////////////////////////// // // chibicc-toolchain - The specialized backend toolchain for chibicc-cil // Copyright (c) Kouji Matsui(@kozy_kekyo, @kekyo @mastodon.cloud) @@ -62,7 +62,9 @@ private bool TryLoadInputReferences( string[] libraryReferenceBasePaths, string[] assemblyReferenceBasePaths, InputReference[] inputReferences, + string? injectToAssemblyPath, bool isLocationOriginSource, + string? cacheBasePath, out InputFragment[] fragments) { // Load input files in parallelism. @@ -107,7 +109,7 @@ private bool TryLoadInputReferences( $"Unable to find the object file: {relativePath}"); break; } - using (var fs = ObjectStreamUtilities.OpenObjectStream( + using (var fs = CompressionStreamUtilities.OpenStream( Path.Combine(baseInputPath, relativePath), false)) { var tr = new StreamReader(fs, Encoding.UTF8, true); @@ -144,6 +146,7 @@ when Path.GetExtension(relativePath) == ".a": // Asssembly: case LibraryPathReference(var relativePath): + var isInjectAssembly = (index == 0) && (relativePath == injectToAssemblyPath); var libraryFilePath = Path.Combine(baseInputPath, relativePath); if (!File.Exists(libraryFilePath)) { @@ -157,7 +160,8 @@ when Path.GetExtension(relativePath) == ".a": this.logger, baseInputPath, relativePath, - this.CreateAssemblyResolver( + cacheBasePath, + () => this.CreateAssemblyResolver( ReadingMode.Deferred, assemblyReferenceBasePaths)), }; @@ -191,7 +195,8 @@ when Path.GetExtension(relativePath) == ".a": this.logger, foundEntry.basePath, foundEntry.fileName, - this.CreateAssemblyResolver( + cacheBasePath, + () => this.CreateAssemblyResolver( ReadingMode.Deferred, assemblyReferenceBasePaths)), }; @@ -320,7 +325,9 @@ injectToAssemblyPath is { } injectPath ? options.LibraryReferenceBasePaths, assemblyReferenceBasePaths, totalInputReferences, + injectToAssemblyPath, produceDebuggingInformation, + options.CacheBasePath, out var loadedFragments)) { return false; @@ -350,7 +357,8 @@ injectToAssemblyPath is { } injectPath ? // Will be injected exist (loaded) assembly. if (injectToAssemblyPath != null) { - primaryAssembly = ((AssemblyInputFragment)loadedFragments[0]).Assembly; + primaryAssembly = ((AssemblyInputFragment)loadedFragments[0]). + GetAssembly(); } // Will be created a new assembly. else diff --git a/chibild/chibild.core/Generating/ArchivedObjectInputFragment.cs b/chibild/chibild.core/Generating/ArchivedObjectInputFragment.cs index dc3a9c2..14269ee 100644 --- a/chibild/chibild.core/Generating/ArchivedObjectInputFragment.cs +++ b/chibild/chibild.core/Generating/ArchivedObjectInputFragment.cs @@ -7,7 +7,7 @@ // ///////////////////////////////////////////////////////////////////////////////////// -using System; +using chibicc.toolchain.Generating; using chibicc.toolchain.Archiving; using chibicc.toolchain.Parsing; using chibicc.toolchain.Logging; @@ -57,19 +57,16 @@ private enum RequiredStates private ArchivedObjectInputFragment( string baseInputPath, string relativePath, - string archivedObjectName, - Dictionary typeSymbols, - Dictionary variableSymbols, - Dictionary functionSymbols) : + AggregatedSymbols extraction) : base(baseInputPath, relativePath) { - this.archivedObjectName = archivedObjectName; + this.archivedObjectName = extraction.ObjectName; this.ObjectName = Path.GetFileNameWithoutExtension(this.archivedObjectName); this.ObjectPath = $"{this.archivedObjectName}@{base.ObjectPath}"; - this.typeSymbols = typeSymbols; - this.variableSymbols = variableSymbols; - this.functionSymbols = functionSymbols; + this.typeSymbols = extraction.TypeSymbols; + this.variableSymbols = extraction.VariableSymbols; + this.functionSymbols = extraction.FunctionSymbols; } public override string ObjectName { get; } @@ -162,14 +159,7 @@ public override bool ContainsFunctionAndSchedule( ////////////////////////////////////////////////////////////// - public enum LoadObjectResults - { - Ignored, - Loaded, - CaughtError, - } - - public LoadObjectResults LoadObjectIfRequired( + public override LoadObjectResults LoadObjectIfRequired( ILogger logger, bool isLocationOriginSource) { @@ -227,7 +217,9 @@ public LoadObjectResults LoadObjectIfRequired( LoadObjectResults.CaughtError : LoadObjectResults.Loaded; } - return LoadObjectResults.Ignored; + + return this.requiredState == (int)RequiredStates.Loaded ? + LoadObjectResults.Loaded : LoadObjectResults.Ignored; } public static ArchivedObjectInputFragment[] Load( @@ -239,42 +231,12 @@ public static ArchivedObjectInputFragment[] Load( var symbolLists = ArchiverUtilities.EnumerateSymbolTable( Path.Combine(baseInputPath, relativePath)); - - return symbolLists.Select(symbolList => - { - var symbols = symbolList.Symbols. - GroupBy(symbol => - { - switch (symbol.Directive) - { - case "enumeration": return "type"; - case "structure": return "type"; - case "global": return "variable"; - case "constant": return "variable"; - case "function": return "function"; - default: - logger.Warning($"Ignored invalid symbol table entry: {symbol.Directive}"); - return "unknown"; - } - }). - ToDictionary( - g => g.Key, - g => g. - // Takes largest member count. - OrderByDescending(symbol => symbol.MemberCount ?? 0). - DistinctBy(symbol => symbol.Name). - ToDictionary(symbol => symbol.Name)); - - var empty = new Dictionary(); - - return new ArchivedObjectInputFragment( + + return SymbolAggregator.AggregateSymbolsFromSymbolTable(logger, symbolLists). + Select(extraction => new ArchivedObjectInputFragment( baseInputPath, relativePath, - symbolList.ObjectName, - symbols.TryGetValue("type", out var types) ? types : empty, - symbols.TryGetValue("variable", out var variableNames) ? variableNames : empty, - symbols.TryGetValue("function", out var functionNames) ? functionNames : empty); - }). - ToArray(); + extraction)). + ToArray(); } } diff --git a/chibild/chibild.core/Generating/AssemblyInputFragment.cs b/chibild/chibild.core/Generating/AssemblyInputFragment.cs index bdd885e..d228cac 100644 --- a/chibild/chibild.core/Generating/AssemblyInputFragment.cs +++ b/chibild/chibild.core/Generating/AssemblyInputFragment.cs @@ -7,6 +7,9 @@ // ///////////////////////////////////////////////////////////////////////////////////// +using chibicc.toolchain.Generating; +using chibicc.toolchain.Internal; +using chibicc.toolchain.IO; using chibicc.toolchain.Parsing; using chibicc.toolchain.Logging; using chibild.Internal; @@ -16,34 +19,114 @@ using System.Diagnostics; using System.IO; using System.Linq; -using System.Threading.Tasks; +using System.Security.Cryptography; +using System.Text; +using System.Threading; namespace chibild.Generating; internal sealed class AssemblyInputFragment : InputFragment { - private readonly Dictionary types; - private readonly Dictionary fields; - private readonly Dictionary methods; - private readonly Dictionary resolvedModules = new(); - + private enum RequiredStates + { + Ignore, + Required, + Loaded, + } + + private readonly string assemblyPath; + + private readonly Dictionary typeSymbols; + private readonly Dictionary variableSymbols; + private readonly Dictionary functionSymbols; + + private Func assemblyResolverFactory; + private AssemblyDefinition? assembly; + private Dictionary types; + private Dictionary fields; + private Dictionary methods; + private Dictionary resolvedModules = new(); + + private int requiredState; + + private AssemblyInputFragment( + string baseInputPath, + string relativePath, + string assemblyPath, + Func assemblyResolverFactory, + AggregatedSymbols extraction) : + base(baseInputPath, relativePath) + { + this.assemblyPath = assemblyPath; + this.assemblyResolverFactory = assemblyResolverFactory; + + this.typeSymbols = extraction.TypeSymbols; + this.variableSymbols = extraction.VariableSymbols; + this.functionSymbols = extraction.FunctionSymbols; + + this.types = new(); + this.fields = new(); + this.methods = new(); + + this.requiredState = (int)RequiredStates.Ignore; + } + private AssemblyInputFragment( string baseInputPath, string relativePath, - AssemblyDefinition assembly, - Dictionary types, - Dictionary fields, - Dictionary methods) : + string assemblyPath, + AggregatedSymbols extraction, + LoadedCAbiMetadata cabiMetadata) : base(baseInputPath, relativePath) { - this.Assembly = assembly; - this.types = types; - this.fields = fields; - this.methods = methods; + this.assemblyPath = assemblyPath; + this.assemblyResolverFactory = null!; + + this.typeSymbols = extraction.TypeSymbols; + this.variableSymbols = extraction.VariableSymbols; + this.functionSymbols = extraction.FunctionSymbols; + + this.assembly = cabiMetadata.Assembly; + this.types = cabiMetadata.Types; + this.fields = cabiMetadata.Fields; + this.methods = cabiMetadata.Methods; + + this.requiredState = (int)RequiredStates.Loaded; + } + + ////////////////////////////////////////////////////////////// + + private void PrepareToLoadAssembly() + { + if (this.assembly == null) + { + var assemblyResolver = this.assemblyResolverFactory(); + + var cabiMetadata = CecilUtilities.LoadCAbiMetadataFromAssembly( + this.assemblyPath, + assemblyResolver); + + this.assemblyResolverFactory = null!; + + this.assembly = cabiMetadata.Assembly; + this.types = cabiMetadata.Types; + this.fields = cabiMetadata.Fields; + this.methods = cabiMetadata.Methods; + } } - public AssemblyDefinition Assembly { get; } + public AssemblyDefinition GetAssembly() + { + if (this.requiredState != (int)RequiredStates.Loaded) + { + this.PrepareToLoadAssembly(); + this.requiredState = (int)RequiredStates.Loaded; + } + Debug.Assert(this.assembly != null); + + return this.assembly!; + } public override string ToString() => $"Assembly: {this.ObjectPath}"; @@ -55,23 +138,14 @@ public override bool ContainsTypeAndSchedule( out Scopes scope, out int? memberCount) { - if (this.types.TryGetValue(type.TypeIdentity, out var td)) + if (this.typeSymbols.TryGetValue(type.TypeIdentity, out var ts)) { - scope = Scopes.Public; // Contains only public members. - if (td.IsEnum) - { - memberCount = td.Fields.Count(f => - f is { IsPublic: true, IsStatic: true, IsLiteral: true, }); - } - else if (td.IsValueType) - { - memberCount = td.Fields.Count(f => - f is { IsStatic: false, }); - } - else - { - memberCount = null; - } + Interlocked.CompareExchange( + ref this.requiredState, + (int)RequiredStates.Required, + (int)RequiredStates.Ignore); + CommonUtilities.TryParseEnum(ts.Scope, out scope); + memberCount = ts.MemberCount; return true; } scope = default; @@ -83,81 +157,32 @@ public override bool ContainsVariableAndSchedule( IdentityNode variable, out Scopes scope) { - if (this.fields.TryGetValue(variable.Identity, out _)) + if (this.variableSymbols.TryGetValue(variable.Identity, out var vs)) { - scope = Scopes.Public; // Contains only public members. + Interlocked.CompareExchange( + ref this.requiredState, + (int)RequiredStates.Required, + (int)RequiredStates.Ignore); + CommonUtilities.TryParseEnum(vs.Scope, out scope); return true; } scope = default; return false; } - private static bool TryGetMatchedMethodIndex( - FunctionSignatureNode signature, - MethodDefinition[] overloads, - out int index) - { - // If target signature is variadic, will ignore exact match. - if (signature.CallingConvention == - chibicc.toolchain.Parsing.MethodCallingConvention.Default) - { - for (index = 0; index < overloads.Length; index++) - { - var overload = overloads[index]; - - // Match exactly. - if (overload.Parameters. - Select(p => p.ParameterType.FullName). - SequenceEqual(signature.Parameters.Select(p => p.ParameterType.CilTypeName))) - { - return true; - } - } - } - - for (index = 0; index < overloads.Length; index++) - { - var overload = overloads[index]; - - // Match partially when overload is variadic. - if (overload.CallingConvention == Mono.Cecil.MethodCallingConvention.VarArg) - { - if (overload.Parameters. - Select(p => p.ParameterType.FullName). - SequenceEqual(signature.Parameters. - Take(overload.Parameters.Count). - Select(p => p.ParameterType.CilTypeName))) - { - return true; - } - } - if (signature.CallingConvention == - chibicc.toolchain.Parsing.MethodCallingConvention.VarArg) - { - if (overload.Parameters. - Take(signature.Parameters.Length). - Select(p => p.ParameterType.FullName). - SequenceEqual(signature.Parameters. - Select(p => p.ParameterType.CilTypeName))) - { - return true; - } - } - } - - index = -1; - return false; - } - public override bool ContainsFunctionAndSchedule( IdentityNode function, FunctionSignatureNode? signature, out Scopes scope) { - if (this.methods.TryGetValue(function.Identity, out var overloads) && - (signature == null || TryGetMatchedMethodIndex(signature, overloads, out _))) + // Ignored the signature, because contains only CABI functions. + if (this.functionSymbols.TryGetValue(function.Identity, out var fs)) { - scope = Scopes.Public; // Contains only public members. + Interlocked.CompareExchange( + ref this.requiredState, + (int)RequiredStates.Required, + (int)RequiredStates.Ignore); + CommonUtilities.TryParseEnum(fs.Scope, out scope); return true; } scope = default; @@ -166,7 +191,28 @@ public override bool ContainsFunctionAndSchedule( ////////////////////////////////////////////////////////////// - private ModuleDefinition ResovleOnFallbackModule( + public override LoadObjectResults LoadObjectIfRequired( + ILogger logger, + bool isLocationOriginSource) + { + if (Interlocked.CompareExchange( + ref this.requiredState, + (int)RequiredStates.Loaded, + (int)RequiredStates.Required) == (int)RequiredStates.Required) + { + logger.Information($"Loading: {this.assemblyPath}"); + + this.PrepareToLoadAssembly(); + return LoadObjectResults.Loaded; + } + + return this.requiredState == (int)RequiredStates.Loaded ? + LoadObjectResults.Loaded : LoadObjectResults.Ignored; + } + + ////////////////////////////////////////////////////////////// + + private ModuleDefinition ResolveOnFallbackModule( ModuleDefinition fallbackModule, MemberReference mr) { @@ -201,7 +247,7 @@ public override bool TryGetType( } // Resolve on fallback assembly resolver. - var exactModule = this.ResovleOnFallbackModule(fallbackModule, td); + var exactModule = this.ResolveOnFallbackModule(fallbackModule, td); if (exactModule.GetType(td.FullName) is { } ftd) { this.types[type.TypeIdentity] = ftd; @@ -233,7 +279,7 @@ public override bool TryGetField( } // Resolve on fallback assembly resolver. - var exactModule = this.ResovleOnFallbackModule(fallbackModule, fd); + var exactModule = this.ResolveOnFallbackModule(fallbackModule, fd); if (exactModule.GetType(fd.DeclaringType.FullName) is { } ftd && ftd.Fields.FirstOrDefault(f => f.Name == fd.Name) is { } ffd) { @@ -252,6 +298,63 @@ public override bool TryGetField( return false; } + private static bool TryGetMatchedMethodIndex( + FunctionSignatureNode signature, + MethodDefinition[] overloads, + out int index) + { + // If target signature is variadic, will ignore exact match. + if (signature.CallingConvention == + chibicc.toolchain.Parsing.MethodCallingConvention.Default) + { + for (index = 0; index < overloads.Length; index++) + { + var overload = overloads[index]; + + // Match exactly. + if (overload.Parameters. + Select(p => p.ParameterType.FullName). + SequenceEqual(signature.Parameters.Select(p => p.ParameterType.CilTypeName))) + { + return true; + } + } + } + + for (index = 0; index < overloads.Length; index++) + { + var overload = overloads[index]; + + // Match partially when overload is variadic. + if (overload.CallingConvention == Mono.Cecil.MethodCallingConvention.VarArg) + { + if (overload.Parameters. + Select(p => p.ParameterType.FullName). + SequenceEqual(signature.Parameters. + Take(overload.Parameters.Count). + Select(p => p.ParameterType.CilTypeName))) + { + return true; + } + } + if (signature.CallingConvention == + chibicc.toolchain.Parsing.MethodCallingConvention.VarArg) + { + if (overload.Parameters. + Take(signature.Parameters.Length). + Select(p => p.ParameterType.FullName). + SequenceEqual(signature.Parameters. + Select(p => p.ParameterType.CilTypeName))) + { + return true; + } + } + } + + index = -1; + return false; + } + public override bool TryGetMethod( IdentityNode function, FunctionSignatureNode? signature, @@ -267,7 +370,7 @@ public override bool TryGetMethod( // Resolve on fallback assembly resolver. MethodReference ResolveOnFallbackModule(MethodDefinition md) { - var exactModule = this.ResovleOnFallbackModule(fallbackModule, md); + var exactModule = this.ResolveOnFallbackModule(fallbackModule, md); if (exactModule.GetType(md.DeclaringType.FullName) is { } ftd && ftd.Methods.FirstOrDefault(m => CecilUtilities.Equals(m, md)) is { } fmd) { @@ -297,118 +400,170 @@ MethodReference ResolveOnFallbackModule(MethodDefinition md) ////////////////////////////////////////////////////////////// + private static string GetAssemblyPathHashedPath(string assemblyPath) + { + using var alg = MD5.Create(); + var path = BitConverter.ToString(alg.ComputeHash(Encoding.UTF8.GetBytes(assemblyPath))). + Replace("-", string.Empty). + ToLowerInvariant(); + + return Path.Combine( + path.Substring(0, 2), + path.Substring(2)); + } + + private static bool IsValidCache( + TextReader tr, + string assemblyPath) + { + var header1 = tr.ReadLine()?.Split(' ') ?? CommonUtilities.Empty(); + if (header1.FirstOrDefault() is ".path" && + header1.ElementAt(1) is { } path) + { + return assemblyPath == path; + } + + return false; + } + + ////////////////////////////////////////////////////////////// + public static AssemblyInputFragment Load( ILogger logger, string baseInputPath, string relativePath, - CachedAssemblyResolver assemblyResolver) + string? cacheBasePath, + Func assemblyResolverFactory) { + var assemblyPath = Path.Combine(baseInputPath, relativePath); + var assemblyHashedPath = GetAssemblyPathHashedPath(assemblyPath) + ".symtab"; + //var assemblyHash = GetAssemblyHash(assemblyPath); + + // Found cached file. + var cachePath = cacheBasePath != null ? + Path.Combine(cacheBasePath, assemblyHashedPath) : + null; + if (cachePath != null) + { + try + { + if (File.Exists(cachePath)) + { + var assemblyDate = File.GetLastWriteTime(assemblyPath); + var cacheDate = File.GetLastWriteTime(cachePath); + + if (cacheDate >= assemblyDate) + { + using var cacheStream = CompressionStreamUtilities.OpenStream( + cachePath, false); + var tr = new StreamReader(cacheStream, Encoding.UTF8, true); + + if (IsValidCache(tr, assemblyPath)) + { + var symbolLists = SymbolUtilities.EnumerateSymbolTable( + tr, + cachePath); + + var aggSymbols = SymbolAggregator.AggregateSymbolsFromSymbolTable( + logger, + symbolLists). + ToArray(); + + if (aggSymbols.SingleOrDefault() is { } aggSymbol && + aggSymbol.ObjectName == "symcache") + { + logger.Information($"Loading cached symbols: {relativePath}"); + logger.Trace($"Cached symbols: {cachePath}"); + + return new( + baseInputPath, + relativePath, + assemblyPath, + assemblyResolverFactory, + aggSymbol); + } + } + } + } + } + catch + { + } + } + // TODO: native dll logger.Information($"Loading assembly: {relativePath}"); - var assembly = assemblyResolver.ReadAssemblyFrom( - Path.Combine(baseInputPath, relativePath)); - - static IEnumerable IterateTypesDescendants(TypeDefinition type) + // Load from assembly immediate. + var assemblyResolver = assemblyResolverFactory(); + var cabiMetadata = CecilUtilities.LoadCAbiMetadataFromAssembly( + assemblyPath, + assemblyResolver); + + // Construct symbol aggregation and symbol table. + var aggSymbol2 = SymbolAggregator.CreateSymbolAggregationFromDefinitions( + "symcache", + cabiMetadata.Types, + cabiMetadata.Fields, + cabiMetadata.Methods); + + var symbolList = new SymbolList( + aggSymbol2.ObjectName, + aggSymbol2.TypeSymbols.Values. + Concat(aggSymbol2.VariableSymbols.Values). + Concat(aggSymbol2.FunctionSymbols.Values). + ToArray()); + + if (cachePath != null) { - yield return type; - - foreach (var childType in type.NestedTypes.Where(nestedType => - nestedType.IsNestedPublic && - (nestedType.IsClass || nestedType.IsInterface || nestedType.IsValueType || nestedType.IsEnum) && - // Excepts all generic types because CABI does not support it. - !nestedType.HasGenericParameters). - SelectMany(IterateTypesDescendants)) + var newCachePath = cachePath + $"_{Guid.NewGuid():N}"; + var newCacheBasePath = CommonUtilities.GetDirectoryPath(newCachePath); + if (!Directory.Exists(newCacheBasePath)) { - yield return childType; + try + { + Directory.CreateDirectory(newCacheBasePath); + } + catch + { + } } - } - var targetTypes = assembly.Modules. - SelectMany(module => module.Types). - Where(type => - type.IsPublic && - (type.IsClass || type.IsInterface || type.IsValueType || type.IsEnum) && - // Excepts all generic types because CABI does not support it. - !type.HasGenericParameters). - SelectMany(IterateTypesDescendants). - ToArray(); - - var types = targetTypes. - // Combine both CABI types and .NET types. - Where(type => type.Namespace is "C.type"). - Select(type => (name: type.Name, type)). - Concat(targetTypes. - Select(type => (name: type.FullName.Replace('/', '.'), type))). - ToDictionary(entry => entry.name, entry => entry.type); - - var targetFields = targetTypes. - Where(type => type is - { - IsPublic: true, IsClass: true, - } or - { - IsPublic: true, IsValueType: true, IsEnum: false, - }). - SelectMany(type => type.Fields). - Where(field => field is + try { - IsPublic: true, - }). - ToArray(); - - var fields = targetFields. - // Combine both CABI variables and .NET fields. - Where(field => field.DeclaringType.FullName is "C.data" or "C.rdata"). - Select(field => (name: field.Name, field)). - Concat(targetFields. - Select(field => (name: $"{field.DeclaringType.FullName}.{field.Name}", field))). - ToDictionary(entry => entry.name, entry => entry.field); - - var targetMethods = targetTypes. - Where(type => type is + using (var cacheStream = CompressionStreamUtilities.OpenStream( + newCachePath, true)) + { + var tw = new StreamWriter(cacheStream, Encoding.UTF8); + tw.WriteLine($".path {assemblyPath}"); + + SymbolUtilities.WriteSymbolTable(tw, new[] { symbolList }); + } + } + catch { - IsPublic: true, IsClass: true, - } or + File.Delete(newCachePath); + throw; + } + + try { - IsPublic: true, IsValueType: true, - }). - SelectMany(type => type.Methods). - Where(method => method is + // Swap new cache file. Totally ignored any failures. + File.Delete(cachePath); + File.Move(newCachePath, cachePath); + } + catch { - IsPublic: true, - // Excepts all generic methods because CABI does not support it. - HasGenericParameters: false - }). - ToArray(); - - var methods = targetMethods. - // Combine both CABI function and .NET methods. - Where(method => - method.IsStatic && - method.DeclaringType.FullName is "C.text"). - Select(method => (name: method.Name, method)). - Concat(targetMethods. - Select(method => - ( - name: $"{method.DeclaringType.FullName}.{method.Name}", - method - ))). - GroupBy( - entry => entry.name, - entry => entry.method). - ToDictionary( - g => g.Key, - // Sorted descending longer parameters. - g => g.OrderByDescending(method => method.Parameters.Count).ToArray()); + File.Delete(newCachePath); + } + } return new( baseInputPath, relativePath, - assembly, - types, - fields, - methods); + assemblyPath, + aggSymbol2, + cabiMetadata); } } diff --git a/chibild/chibild.core/Generating/CodeGenerator.cs b/chibild/chibild.core/Generating/CodeGenerator.cs index 4cefbfa..54e014f 100644 --- a/chibild/chibild.core/Generating/CodeGenerator.cs +++ b/chibild/chibild.core/Generating/CodeGenerator.cs @@ -169,18 +169,22 @@ private void ConsumeArchivedObject( { found = false; #if DEBUG - foreach (var currentFragment in inputFragments. - OfType()) + foreach (var currentFragment in inputFragments) { switch (currentFragment.LoadObjectIfRequired( this.logger, isLocationOriginSource)) { - case ArchivedObjectInputFragment.LoadObjectResults.Loaded: - found = true; - this.ConsumeFragment(currentFragment, inputFragments); + case LoadObjectResults.Loaded: + if (currentFragment is ArchivedObjectInputFragment afif) + { + // This consume may cause another `InputFragment` to become `Required` state. + // Therefore, it is a `do-while` loop to check again. + found = true; + this.ConsumeFragment(afif, inputFragments); + } break; - case ArchivedObjectInputFragment.LoadObjectResults.Ignored: + case LoadObjectResults.Ignored: break; default: this.caughtError = true; @@ -192,26 +196,29 @@ private void ConsumeArchivedObject( new() { MaxDegreeOfParallelism = Math.Max(1, Environment.ProcessorCount - 1) }, currentFragment => { - if (currentFragment is ArchivedObjectInputFragment afif) + switch (currentFragment.LoadObjectIfRequired( + this.logger, + isLocationOriginSource)) { - switch (afif.LoadObjectIfRequired( - this.logger, - isLocationOriginSource)) - { - case ArchivedObjectInputFragment.LoadObjectResults.Loaded: + case LoadObjectResults.Loaded: + if (currentFragment is ArchivedObjectInputFragment afif) + { + // This consume may cause another `InputFragment` to become `Required` state. + // Therefore, it is a `do-while` loop to check again. found = true; this.ConsumeFragment(afif, inputFragments); - break; - case ArchivedObjectInputFragment.LoadObjectResults.Ignored: - break; - default: - this.caughtError = true; - break; - } + } + break; + case LoadObjectResults.Ignored: + break; + default: + this.caughtError = true; + break; } }); #endif } + // Repeat if at least one `ArchivedObjectInputFragment` is processed and there are no errors. while (found && !this.caughtError); } diff --git a/chibild/chibild.core/Generating/CodeGenerator_Consumer.cs b/chibild/chibild.core/Generating/CodeGenerator_Consumer.cs index 26c8fd6..0e74b5a 100644 --- a/chibild/chibild.core/Generating/CodeGenerator_Consumer.cs +++ b/chibild/chibild.core/Generating/CodeGenerator_Consumer.cs @@ -10,6 +10,7 @@ using chibicc.toolchain.Internal; using chibicc.toolchain.Logging; using chibicc.toolchain.Parsing; +using chibicc.toolchain.Tokenizing; using chibild.Internal; using Mono.Cecil; using Mono.Cecil.Cil; @@ -866,13 +867,13 @@ private void ConsumeEnumeration( _ => TypeAttributes.NestedPublic | TypeAttributes.Sealed, }; - if (!context.UnsafeGetCoreType("System.Enum", out var setr)) - { - setr = this.CreatePlaceholderType(); - this.OutputError( - enumeration.Token, - $"Could not find System.Enum type."); - } + //if (!context.UnsafeGetCoreType("System.Enum", out var setr)) + //{ + // setr = this.CreatePlaceholderType(); + // this.OutputError( + // enumeration.Token, + // $"Could not find System.Enum type."); + //} var enumerationType = new TypeDefinition( enumeration.Scope.Scope switch @@ -884,7 +885,15 @@ private void ConsumeEnumeration( }, enumeration.Name.Identity, typeAttributes, - context.SafeImport(setr)); + // HACK: This is immitation TypeReference which is value type supplier. + // Will be replaced strict imported TypeReference. + new TypeReference("System", "Enum", context.FallbackModule, context.FallbackModule, true)); + + this.DelayLookingUpType( + context, + TypeParser.UnsafeParse(Token.Identity("System.Enum")), + false, + etr => enumerationType.BaseType = context.SafeImport(etr)); this.DelayLookingUpType( context, @@ -963,13 +972,13 @@ private void ConsumeStructure( typeAttributes |= (structure.IsExplicit?.Value ?? false) ? TypeAttributes.ExplicitLayout : TypeAttributes.SequentialLayout; - if (!context.UnsafeGetCoreType("System.ValueType", out var vttr)) - { - vttr = this.CreatePlaceholderType(); - this.OutputError( - structure.Token, - $"Could not find System.ValueType type."); - } + //if (!context.UnsafeGetCoreType("System.ValueType", out var vttr)) + //{ + // vttr = this.CreatePlaceholderType(); + // this.OutputError( + // structure.Token, + // $"Could not find System.ValueType type."); + //} var structureType = new TypeDefinition( structure.Scope.Scope switch @@ -981,7 +990,9 @@ private void ConsumeStructure( }, structure.Name.Identity, typeAttributes, - context.SafeImport(vttr)); + // HACK: This is immitation TypeReference which is value type supplier. + // Will be replaced strict imported TypeReference. + new TypeReference("System", "ValueType", context.FallbackModule, context.FallbackModule, true)); if (structure.PackSize?.Value is { } packSize) { @@ -989,6 +1000,12 @@ private void ConsumeStructure( structureType.ClassSize = 0; } + this.DelayLookingUpType( + context, + TypeParser.UnsafeParse(Token.Identity("System.ValueType")), + false, + vtr => structureType.BaseType = context.SafeImport(vtr)); + foreach (var structureField in structure.Fields) { var fieldAttribute = structureField.Scope.Scope switch diff --git a/chibild/chibild.core/Generating/CodeGenerator_Emit.cs b/chibild/chibild.core/Generating/CodeGenerator_Emit.cs index cc10310..d5de348 100644 --- a/chibild/chibild.core/Generating/CodeGenerator_Emit.cs +++ b/chibild/chibild.core/Generating/CodeGenerator_Emit.cs @@ -675,7 +675,7 @@ private bool TryLoadAndConsumeCAbiStartUpObjectIfRequired( return false; } - using var fs = ObjectStreamUtilities.OpenObjectStream( + using var fs = CompressionStreamUtilities.OpenStream( objectPath, false); @@ -704,27 +704,6 @@ private bool TryLoadAndConsumeCAbiStartUpObjectIfRequired( return false; } - private bool TryUnsafeGetMethod( - ModuleDefinition targetModule, - InputFragment[] inputFragments, - IdentityNode function, - out MethodReference method) - { - foreach (var fragment in inputFragments) - { - if (fragment.TryGetMethod( - function, - null, - targetModule, - out method)) - { - return true; - } - } - method = null!; - return false; - } - private void AssignEntryPoint( string entryPointSymbol) { @@ -755,28 +734,38 @@ private void InsertPrependExecutionPath( string[] prependExecutionSearchPaths, MethodDefinition targetMethod) { - if (this.TryUnsafeGetMethod( - targetModule, - inputFragments, - IdentityNode.Create("__prepend_path_env"), - out var m)) - { - var method = targetModule.SafeImport(m); - var instructions = targetMethod.Body.Instructions; + var function = IdentityNode.Create("__prepend_path_env"); - foreach (var prependPath in prependExecutionSearchPaths.Reverse()) + foreach (var fragment in inputFragments) + { + if (fragment.ContainsFunctionAndSchedule( + function, + null, + out var scope) && + fragment.LoadObjectIfRequired(this.logger, false) == LoadObjectResults.Loaded && + fragment.TryGetMethod( + function, + null, + this.targetModule, + out var m)) { - instructions.Insert(0, Instruction.Create(OpCodes.Ldstr, prependPath)); - instructions.Insert(1, Instruction.Create(OpCodes.Call, method)); + var method = targetModule.SafeImport(m); + var instructions = targetMethod.Body.Instructions; + + foreach (var prependPath in prependExecutionSearchPaths.Reverse()) + { + instructions.Insert(0, Instruction.Create(OpCodes.Ldstr, prependPath)); + instructions.Insert(1, Instruction.Create(OpCodes.Call, method)); - this.logger.Information($"Set prepend execution search path: {prependPath}"); + this.logger.Information($"Set prepend execution search path: {prependPath}"); + } + + return; } } - else - { - this.caughtError = true; - this.logger.Error($"Could not find prepender implementation."); - } + + this.caughtError = true; + this.logger.Error($"Could not find prepender implementation."); } /////////////////////////////////////////////////////////////////////////////// diff --git a/chibild/chibild.core/Generating/InputFragment.cs b/chibild/chibild.core/Generating/InputFragment.cs index 8776c9f..168c99f 100644 --- a/chibild/chibild.core/Generating/InputFragment.cs +++ b/chibild/chibild.core/Generating/InputFragment.cs @@ -7,12 +7,20 @@ // ///////////////////////////////////////////////////////////////////////////////////// +using chibicc.toolchain.Logging; using chibicc.toolchain.Parsing; using System.IO; using Mono.Cecil; namespace chibild.Generating; +internal enum LoadObjectResults +{ + Ignored, + Loaded, + CaughtError, +} + internal abstract class InputFragment { public readonly string BaseInputPath; @@ -50,6 +58,12 @@ public abstract bool ContainsFunctionAndSchedule( ////////////////////////////////////////////////////////////// + public abstract LoadObjectResults LoadObjectIfRequired( + ILogger logger, + bool isLocationOriginSource); + + ////////////////////////////////////////////////////////////// + public abstract bool TryGetType( TypeNode type, ModuleDefinition fallbackModule, diff --git a/chibild/chibild.core/Generating/ObjectFileInputFragment.cs b/chibild/chibild.core/Generating/ObjectFileInputFragment.cs index 7e1c753..d06ea76 100644 --- a/chibild/chibild.core/Generating/ObjectFileInputFragment.cs +++ b/chibild/chibild.core/Generating/ObjectFileInputFragment.cs @@ -134,6 +134,13 @@ public override bool ContainsFunctionAndSchedule( ////////////////////////////////////////////////////////////// + public override LoadObjectResults LoadObjectIfRequired( + ILogger logger, + bool isLocationOriginSource) => + LoadObjectResults.Ignored; + + ////////////////////////////////////////////////////////////// + public static bool TryLoad( ILogger logger, string baseInputPath, diff --git a/chibild/chibild.core/Generating/SymbolAggregator.cs b/chibild/chibild.core/Generating/SymbolAggregator.cs new file mode 100644 index 0000000..2c88c61 --- /dev/null +++ b/chibild/chibild.core/Generating/SymbolAggregator.cs @@ -0,0 +1,109 @@ +///////////////////////////////////////////////////////////////////////////////////// +// +// chibicc-toolchain - The specialized backend toolchain for chibicc-cil +// Copyright (c) Kouji Matsui(@kozy_kekyo, @kekyo @mastodon.cloud) +// +// Licensed under MIT: https://opensource.org/licenses/MIT +// +///////////////////////////////////////////////////////////////////////////////////// + +using chibicc.toolchain.Generating; +using chibicc.toolchain.Logging; +using chibild.Internal; +using Mono.Cecil; +using System.Collections.Generic; +using System.Linq; + +namespace chibild.Generating; + +internal readonly struct AggregatedSymbols +{ + public readonly string ObjectName; + public readonly Dictionary TypeSymbols; + public readonly Dictionary VariableSymbols; + public readonly Dictionary FunctionSymbols; + + public AggregatedSymbols( + string objectName, + Dictionary typeSymbols, + Dictionary variableSymbols, + Dictionary functionSymbols) + { + this.ObjectName = objectName; + this.TypeSymbols = typeSymbols; + this.VariableSymbols = variableSymbols; + this.FunctionSymbols = functionSymbols; + } +} + +internal static class SymbolAggregator +{ + public static IEnumerable AggregateSymbolsFromSymbolTable( + ILogger logger, + IEnumerable symbolLists) => + symbolLists.Select(symbolList => + { + var symbols = symbolList.Symbols. + GroupBy(symbol => + { + switch (symbol.Directive) + { + case "enumeration": return "type"; + case "structure": return "type"; + case "global": return "variable"; + case "constant": return "variable"; + case "function": return "function"; + default: + logger.Warning($"Ignored invalid symbol table entry: {symbol.Directive}"); + return "unknown"; + } + }). + ToDictionary( + g => g.Key, + g => g. + // Takes largest member count. + OrderByDescending(symbol => symbol.MemberCount ?? 0). + DistinctBy(symbol => symbol.Name). + ToDictionary(symbol => symbol.Name)); + + var empty = new Dictionary(); + + return new AggregatedSymbols( + symbolList.ObjectName, + symbols.TryGetValue("type", out var types) ? types : empty, + symbols.TryGetValue("variable", out var variableNames) ? variableNames : empty, + symbols.TryGetValue("function", out var functionNames) ? functionNames : empty); + }); + + public static AggregatedSymbols CreateSymbolAggregationFromDefinitions( + string objectName, + Dictionary types, + Dictionary fields, + Dictionary methods) => + new AggregatedSymbols( + objectName, + types.ToDictionary( + entry => entry.Key, + entry => new Symbol( + entry.Value.IsEnum ? "enumeration" : "structure", + entry.Value.IsPublic ? "public" : entry.Value.Namespace == "C.type" ? "internal" : "file", + entry.Key, + entry.Value.IsEnum ? entry.Value.Fields.Count(f => f is { IsStatic: true, IsInitOnly: true }) : entry.Value.Fields.Count)), + fields.ToDictionary( + entry => entry.Key, + entry => new Symbol( + entry.Value.IsInitOnly ? "constant" : "global", + entry.Value.IsPublic ? "public" : entry.Value.DeclaringType.FullName is "C.data" or "C.rdata" ? "internal" : "file", + entry.Key, + null)), + methods.SelectMany(entries => + entries.Value.Select(m => (entries.Key, symbol: new Symbol( + "function", + m.IsPublic ? "public" : m.DeclaringType.FullName is "C.text" ? "internal" : "file", + entries.Key, + null)))). + DistinctBy(entry => entry.Key). // Omitted overload definitions. + ToDictionary( + entry => entry.Key, + entry => entry.symbol)); +} diff --git a/chibild/chibild.core/Internal/CecilUtilities.cs b/chibild/chibild.core/Internal/CecilUtilities.cs index dd45b9d..15613f0 100644 --- a/chibild/chibild.core/Internal/CecilUtilities.cs +++ b/chibild/chibild.core/Internal/CecilUtilities.cs @@ -7,6 +7,7 @@ // ///////////////////////////////////////////////////////////////////////////////////// +using chibicc.toolchain.Parsing; using Mono.Cecil; using Mono.Cecil.Cil; using System; @@ -14,10 +15,29 @@ using System.Diagnostics; using System.Linq; using System.Text; -using chibicc.toolchain.Parsing; namespace chibild.Internal; +internal readonly struct LoadedCAbiMetadata +{ + public readonly AssemblyDefinition Assembly; + public readonly Dictionary Types; + public readonly Dictionary Fields; + public readonly Dictionary Methods; + + public LoadedCAbiMetadata( + AssemblyDefinition assemblyDefinition, + Dictionary types, + Dictionary fields, + Dictionary methods) + { + this.Assembly = assemblyDefinition; + this.Types = types; + this.Fields = fields; + this.Methods = methods; + } +} + internal static class CecilUtilities { private static readonly Dictionary opCodes = @@ -167,4 +187,107 @@ public static MemberReference SafeImport( MethodReference method => targetModule.SafeImport(method), _ => throw new InvalidOperationException(), }; + + public static LoadedCAbiMetadata LoadCAbiMetadataFromAssembly( + string assemblyPath, + CachedAssemblyResolver assemblyResolver) + { + var assembly = assemblyResolver.ReadAssemblyFrom( + assemblyPath); + + static IEnumerable IterateTypesDescendants(TypeDefinition type) + { + yield return type; + + foreach (var childType in type.NestedTypes.Where(nestedType => + nestedType.IsNestedPublic && + (nestedType.IsClass || nestedType.IsInterface || nestedType.IsValueType || nestedType.IsEnum) && + // Excepts all generic types because CABI does not support it. + !nestedType.HasGenericParameters). + SelectMany(IterateTypesDescendants)) + { + yield return childType; + } + } + + var targetTypes = assembly.Modules. + SelectMany(module => module.Types). + Where(type => + type.IsPublic && + (type.IsClass || type.IsInterface || type.IsValueType || type.IsEnum) && + // Excepts all generic types because CABI does not support it. + !type.HasGenericParameters). + SelectMany(IterateTypesDescendants). + ToArray(); + + var types = targetTypes. + // Combine both CABI types and .NET types. + Where(type => type.Namespace is "C.type"). + Select(type => (name: type.Name, type)). + Concat(targetTypes. + Select(type => (name: type.FullName.Replace('/', '.'), type))). + ToDictionary(entry => entry.name, entry => entry.type); + + var targetFields = targetTypes. + Where(type => type is + { + IsPublic: true, IsClass: true, + } or + { + IsPublic: true, IsValueType: true, IsEnum: false, + }). + SelectMany(type => type.Fields). + Where(field => field is + { + IsPublic: true, + }). + ToArray(); + + var fields = targetFields. + // Combine both CABI variables and .NET fields. + Where(field => field.DeclaringType.FullName is "C.data" or "C.rdata"). + Select(field => (name: field.Name, field)). + Concat(targetFields. + Select(field => (name: $"{field.DeclaringType.FullName}.{field.Name}", field))). + ToDictionary(entry => entry.name, entry => entry.field); + + var targetMethods = targetTypes. + Where(type => type is + { + IsPublic: true, IsClass: true, + } or + { + IsPublic: true, IsValueType: true, + }). + SelectMany(type => type.Methods). + Where(method => method is + { + IsPublic: true, + // Excepts all generic methods because CABI does not support it. + HasGenericParameters: false + }). + ToArray(); + + var methods = targetMethods. + // Combine both CABI function and .NET methods. + Where(method => + method.IsStatic && + method.DeclaringType.FullName is "C.text"). + Select(method => (name: method.Name, method)). + Concat(targetMethods. + Select(method => + ( + name: $"{method.DeclaringType.FullName}.{method.Name}", + method + ))). + GroupBy( + entry => entry.name, + entry => entry.method). + ToDictionary( + g => g.Key, + // Sorted descending longer parameters. + g => g.OrderByDescending(method => method.Parameters.Count).ToArray()); + + return new(assembly, types, fields, methods); + } } diff --git a/chibild/chibild.core/Internal/MultipleSymbolReaderProvider.cs b/chibild/chibild.core/Internal/MultipleSymbolReaderProvider.cs index 85b4474..31a8daf 100644 --- a/chibild/chibild.core/Internal/MultipleSymbolReaderProvider.cs +++ b/chibild/chibild.core/Internal/MultipleSymbolReaderProvider.cs @@ -1,4 +1,4 @@ -///////////////////////////////////////////////////////////////////////////////////// +///////////////////////////////////////////////////////////////////////////////////// // // chibicc-toolchain - The specialized backend toolchain for chibicc-cil // Copyright (c) Kouji Matsui(@kozy_kekyo, @kekyo @mastodon.cloud) @@ -65,7 +65,7 @@ public MultipleSymbolReaderProvider(ILogger logger) => var sr = provider.GetSymbolReader(module, ms); if (this.loaded.Add(path)) { - this.logger.Debug($"Symbol is loaded from: {path}"); + this.logger.Debug($"Debug symbol is loaded from: {path}"); } return sr; @@ -101,7 +101,7 @@ public MultipleSymbolReaderProvider(ILogger logger) => { if (this.loaded.Add(fullPath)) { - this.logger.Debug($"Embedded symbol is loaded from: {fullPath}"); + this.logger.Debug($"Embedded debug symbol is loaded from: {fullPath}"); } return sr2; } @@ -122,7 +122,7 @@ public MultipleSymbolReaderProvider(ILogger logger) => if (this.notFound.Add(fileName)) { - this.logger.Trace($"Symbol is not found: {fileName}"); + this.logger.Trace($"Debug symbol is not found: {fileName}"); } } } diff --git a/chibild/chibild.core/Internal/Utilities.cs b/chibild/chibild.core/Internal/Utilities.cs index f457e51..4394c9c 100644 --- a/chibild/chibild.core/Internal/Utilities.cs +++ b/chibild/chibild.core/Internal/Utilities.cs @@ -33,6 +33,11 @@ internal enum chmodFlags internal static class Utilities { + private static readonly HashSet invalidPathStrings = new() + { + ":", "//", @"\\", + }; + public const int EINTR = 4; [DllImport("libc", SetLastError = true)] diff --git a/chibild/chibild.core/LinkerOptions.cs b/chibild/chibild.core/LinkerOptions.cs index 691be63..b058ede 100644 --- a/chibild/chibild.core/LinkerOptions.cs +++ b/chibild/chibild.core/LinkerOptions.cs @@ -229,5 +229,15 @@ public sealed class LinkerOptions public string[] PrependExecutionSearchPaths = CommonUtilities.Empty(); + public string? CacheBasePath = + GetDefaultCacheBasePath(); + public bool IsDryRun = false; + + public static string GetDefaultCacheBasePath() => + Path.Combine( + CommonUtilities.GetHomePath() ?? Environment.CurrentDirectory, + ".cache", + "chibild-cil", + "symtab"); } diff --git a/toolchain.common/Archiving/ArchiverUtilities.cs b/toolchain.common/Archiving/ArchiverUtilities.cs index e567c56..97b750f 100644 --- a/toolchain.common/Archiving/ArchiverUtilities.cs +++ b/toolchain.common/Archiving/ArchiverUtilities.cs @@ -7,7 +7,7 @@ // ///////////////////////////////////////////////////////////////////////////////////// -using System; +using chibicc.toolchain.Generating; using chibicc.toolchain.Internal; using chibicc.toolchain.Parsing; using chibicc.toolchain.Tokenizing; @@ -32,11 +32,9 @@ private enum ObjectSymbolStates Structure, } - private static IEnumerable InternalEnumerateSymbolsFromObjectFile( - Stream objectFileStream) + private static IEnumerable InternalEnumerateSymbols( + TextReader tr) { - var tr = new StreamReader(objectFileStream, Encoding.UTF8, true); - var state = ObjectSymbolStates.Idle; string? currentDirective = null; string? currentScope = null; @@ -133,27 +131,11 @@ tokens[0] is (TokenTypes.Identity, _)) } public static IEnumerable EnumerateSymbolsFromObjectFile( - Stream objectFileStream) => - InternalEnumerateSymbolsFromObjectFile(objectFileStream). - Distinct(); - - public static void WriteSymbolTable( - Stream symbolTableStream, - SymbolList[] symbolLists) + Stream objectFileStream) { - var tw = new StreamWriter(symbolTableStream, Encoding.UTF8); - - foreach (var symbolList in symbolLists) - { - tw.WriteLine($".object {symbolList.ObjectName}"); - - foreach (var symbol in symbolList.Symbols.Distinct()) - { - tw.WriteLine($" {symbol.Directive} {symbol.Scope} {symbol.Name}{(symbol.MemberCount is { } mc ? $" {mc}" : "")}"); - } - } - - tw.Flush(); + var tr = new StreamReader(objectFileStream, Encoding.UTF8, true); + return InternalEnumerateSymbols(tr). + Distinct(); } public static IEnumerable EnumerateSymbolTable( @@ -169,54 +151,10 @@ public static IEnumerable EnumerateSymbolTable( using var stream = entry.Open(); var tr = new StreamReader(stream, Encoding.UTF8, true); - Token? currentObjectName = null; - var symbols = new List(); - - foreach (var tokens in CilTokenizer.TokenizeAll("", SymbolTableFileName, tr). - Where(tokens => tokens.Length >= 2)) - { - switch (tokens[0]) - { - case (TokenTypes.Directive, "object") - when tokens[1] is (TokenTypes.Identity, _): - if (currentObjectName is (_, var objectName)) - { - yield return new( - objectName, - symbols.Distinct().ToArray()); - } - currentObjectName = tokens[1]; - symbols.Clear(); - break; - // function public funcfoo - // global public varbar - // enumeration public enumbaz 3 - // structure public structhoge 5 - case (TokenTypes.Identity, var directive) - when tokens.Length >= 3 && - tokens[1] is (TokenTypes.Identity, var scope) && - CommonUtilities.TryParseEnum(scope, out _) && - tokens[2] is (TokenTypes.Identity, var name): - if (tokens.Length >= 4 && - tokens[3] is (TokenTypes.Identity, var mc) && - int.TryParse(mc, NumberStyles.Integer, CultureInfo.InvariantCulture, out var memberCount) && - memberCount >= 0) - { - symbols.Add(new(directive, scope, name, memberCount)); - } - else - { - symbols.Add(new(directive, scope, name, null)); - } - break; - } - } - - if (currentObjectName is var (_, con2)) + foreach (var symbolList in + SymbolUtilities.EnumerateSymbolTable(tr, SymbolTableFileName)) { - yield return new( - con2, - symbols.Distinct().ToArray()); + yield return symbolList; } } } diff --git a/toolchain.common/Archiving/Symbol.cs b/toolchain.common/Generating/Symbol.cs similarity index 98% rename from toolchain.common/Archiving/Symbol.cs rename to toolchain.common/Generating/Symbol.cs index 8d96301..7726c6e 100644 --- a/toolchain.common/Archiving/Symbol.cs +++ b/toolchain.common/Generating/Symbol.cs @@ -10,7 +10,7 @@ using System; using System.Linq; -namespace chibicc.toolchain.Archiving; +namespace chibicc.toolchain.Generating; public readonly struct Symbol : IEquatable { diff --git a/toolchain.common/Generating/SymbolUtilities.cs b/toolchain.common/Generating/SymbolUtilities.cs new file mode 100644 index 0000000..6ef7593 --- /dev/null +++ b/toolchain.common/Generating/SymbolUtilities.cs @@ -0,0 +1,107 @@ +///////////////////////////////////////////////////////////////////////////////////// +// +// chibicc-toolchain - The specialized backend toolchain for chibicc-cil +// Copyright (c) Kouji Matsui(@kozy_kekyo, @kekyo @mastodon.cloud) +// +// Licensed under MIT: https://opensource.org/licenses/MIT +// +///////////////////////////////////////////////////////////////////////////////////// + +using chibicc.toolchain.Internal; +using chibicc.toolchain.Parsing; +using chibicc.toolchain.Tokenizing; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; + +namespace chibicc.toolchain.Generating; + +public static class SymbolUtilities +{ + public static void WriteSymbolTable( + TextWriter tw, + SymbolList[] symbolLists) + { + foreach (var symbolList in symbolLists) + { + tw.WriteLine($".object {symbolList.ObjectName}"); + + foreach (var symbol in symbolList.Symbols.Distinct()) + { + tw.WriteLine($" {symbol.Directive} {symbol.Scope} {symbol.Name}{(symbol.MemberCount is { } mc ? $" {mc}" : "")}"); + } + } + + tw.Flush(); + } + + public static void WriteSymbolTable( + Stream symbolTableStream, + SymbolList[] symbolLists) + { + var tw = new StreamWriter(symbolTableStream, Encoding.UTF8); + WriteSymbolTable(tw, symbolLists); + } + + public static IEnumerable EnumerateSymbolTable( + TextReader tr, + string symbolTableName) + { + Token? currentObjectName = null; + var symbols = new List(); + + foreach (var tokens in CilTokenizer.TokenizeAll("", symbolTableName, tr)) + { + if (tokens.Length < 2) + { + continue; + } + + switch (tokens[0]) + { + // .object [objectName] + case (TokenTypes.Directive, "object") + when tokens[1] is (TokenTypes.Identity, _): + if (currentObjectName is (_, var objectName)) + { + yield return new( + objectName, + symbols.Distinct().ToArray()); + } + currentObjectName = tokens[1]; + symbols.Clear(); + break; + // function public funcfoo + // global public varbar + // enumeration public enumbaz 3 + // structure public structhoge 5 + case (TokenTypes.Identity, var directive) + when tokens.Length >= 3 && + tokens[1] is (TokenTypes.Identity, var scope) && + CommonUtilities.TryParseEnum(scope, out _) && + tokens[2] is (TokenTypes.Identity, var name): + if (tokens.Length >= 4 && + tokens[3] is (TokenTypes.Identity, var mc) && + int.TryParse(mc, NumberStyles.Integer, CultureInfo.InvariantCulture, out var memberCount) && + memberCount >= 0) + { + symbols.Add(new(directive, scope, name, memberCount)); + } + else + { + symbols.Add(new(directive, scope, name, null)); + } + break; + } + } + + if (currentObjectName is var (_, con2)) + { + yield return new( + con2, + symbols.Distinct().ToArray()); + } + } +} diff --git a/toolchain.common/IO/ObjectStreamUtilities.cs b/toolchain.common/IO/CompressionStreamUtilities.cs similarity index 92% rename from toolchain.common/IO/ObjectStreamUtilities.cs rename to toolchain.common/IO/CompressionStreamUtilities.cs index 4e3341c..00c25c9 100644 --- a/toolchain.common/IO/ObjectStreamUtilities.cs +++ b/toolchain.common/IO/CompressionStreamUtilities.cs @@ -15,9 +15,9 @@ namespace chibicc.toolchain.IO; -public static class ObjectStreamUtilities +public static class CompressionStreamUtilities { - public static Stream OpenObjectStream( + public static Stream OpenStream( string objectFilePath, bool writable) { var s = StreamUtilities.OpenStream(objectFilePath, writable); @@ -34,7 +34,7 @@ public static Stream OpenObjectStream( } [MethodImpl(MethodImplOptions.NoInlining | MethodImplOptions.NoOptimization)] - public static Stream OpenEmbeddedObjectStream( + public static Stream OpenEmbeddedStream( string objectResourcePath, Assembly? assembly = null) { diff --git a/toolchain.common/IO/StreamUtilities.cs b/toolchain.common/IO/StreamUtilities.cs index a6ed0e6..2d48faa 100644 --- a/toolchain.common/IO/StreamUtilities.cs +++ b/toolchain.common/IO/StreamUtilities.cs @@ -21,5 +21,6 @@ public static Stream OpenStream(string path, bool writable) => path, writable ? FileMode.Create : FileMode.Open, writable ? FileAccess.ReadWrite : FileAccess.Read, - FileShare.Read); + FileShare.Read, + 1024 * 1024); } diff --git a/toolchain.common/Internal/CommonUtilities.cs b/toolchain.common/Internal/CommonUtilities.cs index 13f238b..0dd4c51 100644 --- a/toolchain.common/Internal/CommonUtilities.cs +++ b/toolchain.common/Internal/CommonUtilities.cs @@ -55,6 +55,39 @@ internal static class CommonUtilities public static readonly bool IsInWindows = Environment.OSVersion.Platform == PlatformID.Win32NT; + public static string? GetHomePath() + { + if (IsInWindows) + { + var homeDrive = Environment.GetEnvironmentVariable("HOMEDRIVE"); + var homePath = Environment.GetEnvironmentVariable("HOMEPATH"); + if (!string.IsNullOrWhiteSpace(homeDrive) && + !string.IsNullOrWhiteSpace(homePath)) + { + var path1 = Path.GetFullPath( + $"{homeDrive}{homePath}". + TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar)); + if (Directory.Exists(path1)) + { + return path1; + } + } + } + + if (Environment.GetEnvironmentVariable("HOME") is { } path2 && + !string.IsNullOrWhiteSpace(path2)) + { + path2 = Path.GetFullPath( + path2.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar)); + if (Directory.Exists(path2)) + { + return path2; + } + } + + return null; + } + public static string GetDirectoryPath(string path) => Path.GetDirectoryName(path) is { } d ? Path.GetFullPath(string.IsNullOrWhiteSpace(d) ? "." : d) :