Skip to content

Commit

Permalink
Use cfg parser for Netkan
Browse files Browse the repository at this point in the history
  • Loading branch information
HebaruSan committed Apr 4, 2022
1 parent 8d68856 commit 65a4880
Show file tree
Hide file tree
Showing 12 changed files with 252 additions and 61 deletions.
4 changes: 4 additions & 0 deletions Netkan/CKAN-netkan.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
<PackageReference Include="Namotion.Reflection" Version="1.0.7" />
<PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
<PackageReference Include="YamlDotNet" Version="9.1.0" />
<PackageReference Include="ParsecSharp" Version="3.4.0" PrivateAssets="All" />
</ItemGroup>
<ItemGroup>
<Reference Include="System" />
Expand All @@ -71,7 +72,9 @@
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="QueueAppender.cs" />
<Compile Include="Services\CachingHttpService.cs" />
<Compile Include="Services\CachingConfigParser.cs" />
<Compile Include="Services\FileService.cs" />
<Compile Include="Services\IConfigParser.cs" />
<Compile Include="Services\IFileService.cs" />
<Compile Include="Services\IHttpService.cs" />
<Compile Include="Services\IModuleService.cs" />
Expand Down Expand Up @@ -129,6 +132,7 @@
<Compile Include="Validators\CkanValidator.cs" />
<Compile Include="Validators\CraftsInShipsValidator.cs" />
<Compile Include="Validators\DownloadVersionValidator.cs" />
<Compile Include="Validators\ForClauseValidator.cs" />
<Compile Include="Validators\HasIdentifierValidator.cs" />
<Compile Include="Validators\HarmonyValidator.cs" />
<Compile Include="Validators\InstallsFilesValidator.cs" />
Expand Down
6 changes: 4 additions & 2 deletions Netkan/Processors/Inflator.cs
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,11 @@ public Inflator(string cacheDir, bool overwriteCache, string githubToken, bool p

IModuleService moduleService = new ModuleService();
IFileService fileService = new FileService(cache);
IConfigParser configParser = new CachingConfigParser(moduleService);
http = new CachingHttpService(cache, overwriteCache);
ckanValidator = new CkanValidator(http, moduleService);
transformer = new NetkanTransformer(http, fileService, moduleService, githubToken, prerelease, netkanValidator);
ckanValidator = new CkanValidator(http, moduleService, configParser);
transformer = new NetkanTransformer(http, fileService, moduleService, configParser,
githubToken, prerelease, netkanValidator);
}

internal IEnumerable<Metadata> Inflate(string filename, Metadata netkan, TransformOptions opts)
Expand Down
85 changes: 85 additions & 0 deletions Netkan/Services/CachingConfigParser.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
using System;
using System.IO;
using System.Linq;
using System.Collections.Generic;

using log4net;
using ICSharpCode.SharpZipLib.Zip;
using ParsecSharp;

namespace CKAN.NetKAN.Services
{
using NodeCache = Dictionary<CkanModule, ConfigNodesCacheEntry>;

/// <summary>
/// Since parsing cfg files can be expensive, cache results for 15 minutes
/// </summary>
internal sealed class CachingConfigParser : IConfigParser
{
public CachingConfigParser(IModuleService modSvc)
{
moduleService = modSvc;
}

public Dictionary<InstallableFile, KSPConfigNode[]> GetConfigNodes(CkanModule module, ZipFile zip, GameInstance inst)
=> GetCachedNodes(module) ?? AddAndReturn(
module,
moduleService.GetConfigFiles(module, zip, inst).ToDictionary(
cfg => cfg,
cfg => KSPConfigParser.ConfigFile.ToArray().Parse(
new StreamReader(zip.GetInputStream(cfg.source)).ReadToEnd())
.CaseOf(failure =>
{
log.InfoFormat("{0}:{1}:{2}: {3}",
inst.ToRelativeGameDir(cfg.destination),
failure.State.Position.Line,
failure.State.Position.Column,
failure.Message);
return new KSPConfigNode[] { };
},
success => success.Value)));

private Dictionary<InstallableFile, KSPConfigNode[]> AddAndReturn(CkanModule module,
Dictionary<InstallableFile, KSPConfigNode[]> nodes)
{
log.DebugFormat("Caching config nodes for {0}", module);
cache.Add(module,
new ConfigNodesCacheEntry()
{
Value = nodes,
Timestamp = DateTime.Now,
});
return nodes;
}

private Dictionary<InstallableFile, KSPConfigNode[]> GetCachedNodes(CkanModule module)
{
if (cache.TryGetValue(module, out ConfigNodesCacheEntry entry))
{
if (DateTime.Now - entry.Timestamp < stringCacheLifetime)
{
log.DebugFormat("Using cached nodes for {0}", module);
return entry.Value;
}
else
{
log.DebugFormat("Purging stale nodes for {0}", module);
cache.Remove(module);
}
}
return null;
}

private readonly IModuleService moduleService;
private readonly NodeCache cache = new NodeCache();
// Re-use parse results within 15 minutes
private static readonly TimeSpan stringCacheLifetime = new TimeSpan(0, 15, 0);
private static readonly ILog log = LogManager.GetLogger(typeof(CachingConfigParser));
}

public class ConfigNodesCacheEntry
{
public Dictionary<InstallableFile, KSPConfigNode[]> Value;
public DateTime Timestamp;
}
}
12 changes: 12 additions & 0 deletions Netkan/Services/IConfigParser.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
using System;
using System.Collections.Generic;

using ICSharpCode.SharpZipLib.Zip;

namespace CKAN.NetKAN.Services
{
internal interface IConfigParser
{
Dictionary<InstallableFile, KSPConfigNode[]> GetConfigNodes(CkanModule module, ZipFile zip, GameInstance inst);
}
}
34 changes: 13 additions & 21 deletions Netkan/Transformers/LocalizationsTransformer.cs
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;

using ICSharpCode.SharpZipLib.Zip;
using log4net;
using Newtonsoft.Json.Linq;

using CKAN.Extensions;
using CKAN.NetKAN.Extensions;
using CKAN.NetKAN.Model;
Expand All @@ -21,10 +22,11 @@ internal sealed class LocalizationsTransformer : ITransformer
/// </summary>
/// <param name="http">HTTP service</param>
/// <param name="moduleService">Module service</param>
public LocalizationsTransformer(IHttpService http, IModuleService moduleService)
public LocalizationsTransformer(IHttpService http, IModuleService moduleService, IConfigParser parser)
{
_http = http;
_moduleService = moduleService;
_parser = parser;
}

/// <summary>
Expand Down Expand Up @@ -56,16 +58,13 @@ public IEnumerable<Metadata> Transform(Metadata metadata, TransformOptions opts)

log.Debug("Extracting locales");
// Extract the locale names from the ZIP's cfg files
var locales = _moduleService.GetConfigFiles(mod, zip, inst)
.Select(cfg => new StreamReader(zip.GetInputStream(cfg.source)).ReadToEnd())
.SelectMany(contents => localizationRegex.Matches(contents).Cast<Match>()
.Select(m => m.Groups["contents"].Value))
.SelectMany(contents => localeRegex.Matches(contents).Cast<Match>()
.Where(m => m.Groups["contents"].Value.Contains("="))
.Select(m => m.Groups["locale"].Value))
.Distinct()
.OrderBy(l => l)
.Memoize();
var locales = _parser.GetConfigNodes(mod, zip, inst)
.SelectMany(kvp => kvp.Value)
.Where(node => node.Name == localizationsNodeName)
.SelectMany(node => node.Children.Select(child => child.Name))
.Distinct()
.OrderBy(l => l)
.Memoize();
log.Debug("Locales extracted");

if (locales.Any())
Expand All @@ -82,20 +81,13 @@ public IEnumerable<Metadata> Transform(Metadata metadata, TransformOptions opts)
}
}

private const string localizationsNodeName = "Localization";
private const string localizationsProperty = "localizations";

private readonly IHttpService _http;
private readonly IModuleService _moduleService;
private readonly IConfigParser _parser;

private static readonly ILog log = LogManager.GetLogger(typeof(LocalizationsTransformer));

private static readonly Regex localizationRegex = new Regex(
@"^\s*Localization\b\s*{(?<contents>[^{}]+({[^{}]*}[^{}]*)+)}",
RegexOptions.Compiled | RegexOptions.Multiline | RegexOptions.Singleline
);
private static readonly Regex localeRegex = new Regex(
@"^\s*(?<locale>[-a-zA-Z]+).*?{(?<contents>.*?)}",
RegexOptions.Compiled | RegexOptions.Multiline | RegexOptions.Singleline
);
}
}
3 changes: 2 additions & 1 deletion Netkan/Transformers/NetkanTransformer.cs
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ public NetkanTransformer(
IHttpService http,
IFileService fileService,
IModuleService moduleService,
IConfigParser configParser,
string githubToken,
bool prerelease,
IValidator validator
Expand All @@ -43,7 +44,7 @@ IValidator validator
new AvcKrefTransformer(http, ghApi),
new InternalCkanTransformer(http, moduleService),
new AvcTransformer(http, moduleService, ghApi),
new LocalizationsTransformer(http, moduleService),
new LocalizationsTransformer(http, moduleService, configParser),
new VersionEditTransformer(),
new ForcedVTransformer(),
new EpochTransformer(),
Expand Down
12 changes: 4 additions & 8 deletions Netkan/Validators/CkanValidator.cs
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,8 @@ internal sealed class CkanValidator : IValidator
{
private readonly List<IValidator> _validators;

public CkanValidator(IHttpService downloader, IModuleService moduleService)
public CkanValidator(IHttpService downloader, IModuleService moduleService, IConfigParser configParser)
{
this.downloader = downloader;
this.moduleService = moduleService;
_validators = new List<IValidator>
{
new IsCkanModuleValidator(),
Expand All @@ -21,8 +19,9 @@ public CkanValidator(IHttpService downloader, IModuleService moduleService)
new ObeysCKANSchemaValidator(),
new KindValidator(),
new HarmonyValidator(downloader, moduleService),
new ModuleManagerDependsValidator(downloader, moduleService),
new PluginsValidator(downloader, moduleService),
new ModuleManagerDependsValidator(downloader, moduleService, configParser),
new PluginsValidator(downloader, moduleService, configParser),
new ForClauseValidator(downloader, moduleService, configParser),
new CraftsInShipsValidator(downloader, moduleService),
};
}
Expand All @@ -40,8 +39,5 @@ public void ValidateCkan(Metadata metadata, Metadata netkan)
Validate(metadata);
new MatchingIdentifiersValidator(netkan.Identifier).Validate(metadata);
}

private IHttpService downloader;
private IModuleService moduleService;
}
}
60 changes: 60 additions & 0 deletions Netkan/Validators/ForClauseValidator.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
using System.Linq;

using Newtonsoft.Json.Linq;
using ICSharpCode.SharpZipLib.Zip;
using log4net;

using CKAN.NetKAN.Services;
using CKAN.NetKAN.Model;
using CKAN.Games;

namespace CKAN.NetKAN.Validators
{
internal sealed class ForClauseValidator : IValidator
{
public ForClauseValidator(IHttpService http, IModuleService moduleService, IConfigParser parser)
{
_http = http;
_moduleService = moduleService;
_parser = parser;
}

public void Validate(Metadata metadata)
{
Log.Info("Validating that :FOR[] clauses specify the right mod");

JObject json = metadata.Json();
CkanModule mod = CkanModule.FromJson(json.ToString());
if (!mod.IsDLC)
{
var package = _http.DownloadModule(metadata);
if (!string.IsNullOrEmpty(package))
{
ZipFile zip = new ZipFile(package);
GameInstance inst = new GameInstance(new KerbalSpaceProgram(), "/", "dummy", new NullUser());

// Check for :FOR[identifier] in .cfg files
var mismatchedIdentifiers = KerbalSpaceProgram
.IdentifiersFromConfigNodes(
_parser.GetConfigNodes(mod, zip, inst)
.SelectMany(kvp => kvp.Value))
.Where(ident => ident != mod.identifier
&& Identifier.ValidIdentifierPattern.IsMatch(ident))
.OrderBy(s => s)
.ToArray();
if (mismatchedIdentifiers.Any())
{
Log.WarnFormat("Found :FOR[] clauses with the wrong identifiers: {0}",
string.Join(", ", mismatchedIdentifiers));
}
}
}
}

private readonly IHttpService _http;
private readonly IModuleService _moduleService;
private readonly IConfigParser _parser;

private static readonly ILog Log = LogManager.GetLogger(typeof(ForClauseValidator));
}
}
33 changes: 24 additions & 9 deletions Netkan/Validators/ModuleManagerDependsValidator.cs
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;

using Newtonsoft.Json.Linq;
using ICSharpCode.SharpZipLib.Zip;
using log4net;

using CKAN.NetKAN.Services;
using CKAN.NetKAN.Model;
using CKAN.Extensions;
Expand All @@ -13,10 +15,11 @@ namespace CKAN.NetKAN.Validators
{
internal sealed class ModuleManagerDependsValidator : IValidator
{
public ModuleManagerDependsValidator(IHttpService http, IModuleService moduleService)
public ModuleManagerDependsValidator(IHttpService http, IModuleService moduleService, IConfigParser parser)
{
_http = http;
_moduleService = moduleService;
_parser = parser;
}

public void Validate(Metadata metadata)
Expand All @@ -32,10 +35,10 @@ public void Validate(Metadata metadata)
{
ZipFile zip = new ZipFile(package);
GameInstance inst = new GameInstance(new KerbalSpaceProgram(), "/", "dummy", new NullUser());
var mmConfigs = _moduleService.GetConfigFiles(mod, zip, inst)
.Where(cfg => moduleManagerRegex.IsMatch(
new StreamReader(zip.GetInputStream(cfg.source)).ReadToEnd()))
.Memoize();
var mmConfigs = _parser.GetConfigNodes(mod, zip, inst)
.Where(kvp => kvp.Value.Any(node => HasAnyModuleManager(node)))
.Select(kvp => kvp.Key)
.ToArray();

bool dependsOnMM = mod?.depends?.Any(r => r.ContainsAny(identifiers)) ?? false;

Expand All @@ -56,13 +59,25 @@ public void Validate(Metadata metadata)

private string[] identifiers = new string[] { "ModuleManager" };

private static readonly Regex moduleManagerRegex = new Regex(
@"^\s*[@+$\-!%]|^\s*[a-zA-Z0-9_]+:",
RegexOptions.Compiled | RegexOptions.Multiline | RegexOptions.Singleline
);
private static bool HasAnyModuleManager(KSPConfigNode node)
=> node.Operator != MMOperator.Insert
|| node.Filters != null
|| node.Needs != null
|| node.Has != null
|| node.Index != null
|| node.Properties.Any(prop => HasAnyModuleManager(prop))
|| node.Children.Any( child => HasAnyModuleManager(child));

private static bool HasAnyModuleManager(KSPConfigProperty prop)
=> prop.Operator != MMOperator.Insert
|| prop.Needs != null
|| prop.Index != null
|| prop.ArrayIndex != null
|| prop.AssignmentOperator != null;

private readonly IHttpService _http;
private readonly IModuleService _moduleService;
private readonly IConfigParser _parser;

private static readonly ILog Log = LogManager.GetLogger(typeof(ModuleManagerDependsValidator));
}
Expand Down
Loading

0 comments on commit 65a4880

Please sign in to comment.