Skip to content

Commit

Permalink
Use cfg parser for Netkan
Browse files Browse the repository at this point in the history
  • Loading branch information
@HebaruSan
HebaruSan committed Mar 28, 2022
1 parent e9be017 commit cae4671
Show file tree
Hide file tree
Showing 11 changed files with 192 additions and 61 deletions.
3 changes: 3 additions & 0 deletions Netkan/CKAN-netkan.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@
<PackageReference Include="Namotion.Reflection" Version="1.0.7" />
<PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
<PackageReference Include="YamlDotNet" Version="9.1.0" />
<PackageReference Include="ParsecSharp" Version="3.4.0" PrivateAssets="All" />
</ItemGroup>
<ItemGroup>
<Reference Include="System" />
Expand All @@ -71,7 +72,9 @@
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="QueueAppender.cs" />
<Compile Include="Services\CachingHttpService.cs" />
<Compile Include="Services\CachingConfigParser.cs" />
<Compile Include="Services\FileService.cs" />
<Compile Include="Services\IConfigParser.cs" />
<Compile Include="Services\IFileService.cs" />
<Compile Include="Services\IHttpService.cs" />
<Compile Include="Services\IModuleService.cs" />
Expand Down
6 changes: 4 additions & 2 deletions Netkan/Processors/Inflator.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,9 +26,11 @@ public Inflator(string cacheDir, bool overwriteCache, string githubToken, bool p

IModuleService moduleService = new ModuleService();
IFileService fileService = new FileService(cache);
IConfigParser configParser = new CachingConfigParser(moduleService);
http = new CachingHttpService(cache, overwriteCache);
ckanValidator = new CkanValidator(http, moduleService);
transformer = new NetkanTransformer(http, fileService, moduleService, githubToken, prerelease, netkanValidator);
ckanValidator = new CkanValidator(http, moduleService, configParser);
transformer = new NetkanTransformer(http, fileService, moduleService, configParser,
githubToken, prerelease, netkanValidator);
}

internal IEnumerable<Metadata> Inflate(string filename, Metadata netkan, TransformOptions opts)
Expand Down
87 changes: 87 additions & 0 deletions Netkan/Services/CachingConfigParser.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
using System;
using System.IO;
using System.Linq;
using System.Collections.Generic;

using log4net;
using ICSharpCode.SharpZipLib.Zip;
using ParsecSharp;

using CKAN.NetKAN.Services;

namespace CKAN.NetKAN.Services
{
using NodeCache = Dictionary<CkanModule, ConfigNodesCacheEntry>;

/// <summary>
/// Since parsing cfg files can be expensive, cache results for 15 minutes
/// </summary>
internal sealed class CachingConfigParser : IConfigParser
{
public CachingConfigParser(IModuleService modSvc)
{
moduleService = modSvc;
}

public Dictionary<InstallableFile, KSPConfigNode[]> GetConfigNodes(CkanModule module, ZipFile zip, GameInstance inst)
=> GetCachedNodes(module) ?? AddAndReturn(
module,
moduleService.GetConfigFiles(module, zip, inst).ToDictionary(
cfg => cfg,
cfg => KSPConfigParser.ConfigFile.ToArray().Parse(
new StreamReader(zip.GetInputStream(cfg.source)).ReadToEnd())
.CaseOf(failure =>
{
log.InfoFormat("{0}:{1}:{2}: {3}",
inst.ToRelativeGameDir(cfg.destination),
failure.State.Position.Line,
failure.State.Position.Column,
failure.Message);
return new KSPConfigNode[] { };
},
success => success.Value)));

private Dictionary<InstallableFile, KSPConfigNode[]> AddAndReturn(CkanModule module,
Dictionary<InstallableFile, KSPConfigNode[]> nodes)
{
log.DebugFormat("Caching config nodes for {0}", module);
cache.Add(module,
new ConfigNodesCacheEntry()
{
Value = nodes,
Timestamp = DateTime.Now,
});
return nodes;
}

private Dictionary<InstallableFile, KSPConfigNode[]> GetCachedNodes(CkanModule module)
{
if (cache.TryGetValue(module, out ConfigNodesCacheEntry entry))
{
if (DateTime.Now - entry.Timestamp < stringCacheLifetime)
{
log.DebugFormat("Using cached nodes for {0}", module);
return entry.Value;
}
else
{
log.DebugFormat("Purging stale nodes for {0}", module);
cache.Remove(module);
}
}
return null;
}

private readonly IModuleService moduleService;
private readonly NodeCache cache = new NodeCache();
// Re-use parse results within 15 minutes
private static readonly TimeSpan stringCacheLifetime = new TimeSpan(0, 15, 0);
private static readonly ILog log = LogManager.GetLogger(typeof(CachingConfigParser));
}

public class ConfigNodesCacheEntry
{
public Dictionary<InstallableFile, KSPConfigNode[]> Value;
public DateTime Timestamp;
}
}
12 changes: 12 additions & 0 deletions Netkan/Services/IConfigParser.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
using System;
using System.Collections.Generic;

using ICSharpCode.SharpZipLib.Zip;

namespace CKAN.NetKAN.Services
{
internal interface IConfigParser
{
Dictionary<InstallableFile, KSPConfigNode[]> GetConfigNodes(CkanModule module, ZipFile zip, GameInstance inst);
}
}
34 changes: 13 additions & 21 deletions Netkan/Transformers/LocalizationsTransformer.cs
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;

using ICSharpCode.SharpZipLib.Zip;
using log4net;
using Newtonsoft.Json.Linq;

using CKAN.Extensions;
using CKAN.NetKAN.Extensions;
using CKAN.NetKAN.Model;
Expand All @@ -21,10 +22,11 @@ internal sealed class LocalizationsTransformer : ITransformer
/// </summary>
/// <param name="http">HTTP service</param>
/// <param name="moduleService">Module service</param>
public LocalizationsTransformer(IHttpService http, IModuleService moduleService)
public LocalizationsTransformer(IHttpService http, IModuleService moduleService, IConfigParser parser)
{
_http = http;
_moduleService = moduleService;
_parser = parser;
}

/// <summary>
Expand Down Expand Up @@ -56,16 +58,13 @@ public IEnumerable<Metadata> Transform(Metadata metadata, TransformOptions opts)

log.Debug("Extracting locales");
// Extract the locale names from the ZIP's cfg files
var locales = _moduleService.GetConfigFiles(mod, zip, inst)
.Select(cfg => new StreamReader(zip.GetInputStream(cfg.source)).ReadToEnd())
.SelectMany(contents => localizationRegex.Matches(contents).Cast<Match>()
.Select(m => m.Groups["contents"].Value))
.SelectMany(contents => localeRegex.Matches(contents).Cast<Match>()
.Where(m => m.Groups["contents"].Value.Contains("="))
.Select(m => m.Groups["locale"].Value))
.Distinct()
.OrderBy(l => l)
.Memoize();
var locales = _parser.GetConfigNodes(mod, zip, inst)
.SelectMany(kvp => kvp.Value)
.Where(node => node.Name == localizationsNodeName)
.SelectMany(node => node.Children.Select(child => child.Name))
.Distinct()
.OrderBy(l => l)
.Memoize();
log.Debug("Locales extracted");

if (locales.Any())
Expand All @@ -82,20 +81,13 @@ public IEnumerable<Metadata> Transform(Metadata metadata, TransformOptions opts)
}
}

private const string localizationsNodeName = "Localization";
private const string localizationsProperty = "localizations";

private readonly IHttpService _http;
private readonly IModuleService _moduleService;
private readonly IConfigParser _parser;

private static readonly ILog log = LogManager.GetLogger(typeof(LocalizationsTransformer));

private static readonly Regex localizationRegex = new Regex(
@"^\s*Localization\b\s*{(?<contents>[^{}]+({[^{}]*}[^{}]*)+)}",
RegexOptions.Compiled | RegexOptions.Multiline | RegexOptions.Singleline
);
private static readonly Regex localeRegex = new Regex(
@"^\s*(?<locale>[-a-zA-Z]+).*?{(?<contents>.*?)}",
RegexOptions.Compiled | RegexOptions.Multiline | RegexOptions.Singleline
);
}
}
3 changes: 2 additions & 1 deletion Netkan/Transformers/NetkanTransformer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ public NetkanTransformer(
IHttpService http,
IFileService fileService,
IModuleService moduleService,
IConfigParser configParser,
string githubToken,
bool prerelease,
IValidator validator
Expand All @@ -43,7 +44,7 @@ IValidator validator
new AvcKrefTransformer(http, ghApi),
new InternalCkanTransformer(http, moduleService),
new AvcTransformer(http, moduleService, ghApi),
new LocalizationsTransformer(http, moduleService),
new LocalizationsTransformer(http, moduleService, configParser),
new VersionEditTransformer(),
new ForcedVTransformer(),
new EpochTransformer(),
Expand Down
11 changes: 3 additions & 8 deletions Netkan/Validators/CkanValidator.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,8 @@ internal sealed class CkanValidator : IValidator
{
private readonly List<IValidator> _validators;

public CkanValidator(IHttpService downloader, IModuleService moduleService)
public CkanValidator(IHttpService downloader, IModuleService moduleService, IConfigParser configParser)
{
this.downloader = downloader;
this.moduleService = moduleService;
_validators = new List<IValidator>
{
new IsCkanModuleValidator(),
Expand All @@ -21,8 +19,8 @@ public CkanValidator(IHttpService downloader, IModuleService moduleService)
new ObeysCKANSchemaValidator(),
new KindValidator(),
new HarmonyValidator(downloader, moduleService),
new ModuleManagerDependsValidator(downloader, moduleService),
new PluginsValidator(downloader, moduleService),
new ModuleManagerDependsValidator(downloader, moduleService, configParser),
new PluginsValidator(downloader, moduleService, configParser),
new CraftsInShipsValidator(downloader, moduleService),
};
}
Expand All @@ -40,8 +38,5 @@ public void ValidateCkan(Metadata metadata, Metadata netkan)
Validate(metadata);
new MatchingIdentifiersValidator(netkan.Identifier).Validate(metadata);
}

private IHttpService downloader;
private IModuleService moduleService;
}
}
33 changes: 24 additions & 9 deletions Netkan/Validators/ModuleManagerDependsValidator.cs
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;

using Newtonsoft.Json.Linq;
using ICSharpCode.SharpZipLib.Zip;
using log4net;

using CKAN.NetKAN.Services;
using CKAN.NetKAN.Model;
using CKAN.Extensions;
Expand All @@ -13,10 +15,11 @@ namespace CKAN.NetKAN.Validators
{
internal sealed class ModuleManagerDependsValidator : IValidator
{
public ModuleManagerDependsValidator(IHttpService http, IModuleService moduleService)
public ModuleManagerDependsValidator(IHttpService http, IModuleService moduleService, IConfigParser parser)
{
_http = http;
_moduleService = moduleService;
_parser = parser;
}

public void Validate(Metadata metadata)
Expand All @@ -32,10 +35,10 @@ public void Validate(Metadata metadata)
{
ZipFile zip = new ZipFile(package);
GameInstance inst = new GameInstance(new KerbalSpaceProgram(), "/", "dummy", new NullUser());
var mmConfigs = _moduleService.GetConfigFiles(mod, zip, inst)
.Where(cfg => moduleManagerRegex.IsMatch(
new StreamReader(zip.GetInputStream(cfg.source)).ReadToEnd()))
.Memoize();
var mmConfigs = _parser.GetConfigNodes(mod, zip, inst)
.Where(kvp => kvp.Value.Any(node => HasAnyModuleManager(node)))
.Select(kvp => kvp.Key)
.ToArray();

bool dependsOnMM = mod?.depends?.Any(r => r.ContainsAny(identifiers)) ?? false;

Expand All @@ -56,13 +59,25 @@ public void Validate(Metadata metadata)

private string[] identifiers = new string[] { "ModuleManager" };

private static readonly Regex moduleManagerRegex = new Regex(
@"^\s*[@+$\-!%]|^\s*[a-zA-Z0-9_]+:",
RegexOptions.Compiled | RegexOptions.Multiline | RegexOptions.Singleline
);
private static bool HasAnyModuleManager(KSPConfigNode node)
=> node.Operator != MMOperator.Insert
|| node.Filters != null
|| node.Needs != null
|| node.Has != null
|| node.Index != null
|| node.Properties.Any(prop => HasAnyModuleManager(prop))
|| node.Children.Any( child => HasAnyModuleManager(child));

private static bool HasAnyModuleManager(KSPConfigProperty prop)
=> prop.Operator != MMOperator.Insert
|| prop.Needs != null
|| prop.Index != null
|| prop.ArrayIndex != null
|| prop.AssignmentOperator != null;

private readonly IHttpService _http;
private readonly IModuleService _moduleService;
private readonly IConfigParser _parser;

private static readonly ILog Log = LogManager.GetLogger(typeof(ModuleManagerDependsValidator));
}
Expand Down
22 changes: 16 additions & 6 deletions Netkan/Validators/PluginsValidator.cs
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;

using Newtonsoft.Json.Linq;
using ICSharpCode.SharpZipLib.Zip;
using log4net;

using CKAN.Extensions;
using CKAN.NetKAN.Services;
using CKAN.NetKAN.Model;
Expand All @@ -13,10 +14,11 @@ namespace CKAN.NetKAN.Validators
{
internal sealed class PluginsValidator : IValidator
{
public PluginsValidator(IHttpService http, IModuleService moduleService)
public PluginsValidator(IHttpService http, IModuleService moduleService, IConfigParser parser)
{
_http = http;
_moduleService = moduleService;
_parser = parser;
}

public void Validate(Metadata metadata)
Expand All @@ -42,15 +44,22 @@ public void Validate(Metadata metadata)
.OrderBy(f => f)
.ToList();
var dllIdentifiers = dllPaths
.Select(p => inst.DllPathToIdentifier(p))
.SelectMany(p => inst.game.IdentifiersFromFileName(inst, p))
.Where(ident => !string.IsNullOrEmpty(ident)
&& !identifiersToIgnore.Contains(ident))
.ToHashSet();
if (dllIdentifiers.Any() && !dllIdentifiers.Contains(metadata.Identifier))
{
Log.WarnFormat(
"No plugin matching the identifier, manual installations won't be detected: {0}",
string.Join(", ", dllPaths));
// Check for :FOR[identifier] in .cfg files
var cfgIdentifiers = KerbalSpaceProgram.IdentifiersFromConfigNodes(
_parser.GetConfigNodes(mod, zip, inst)
.SelectMany(kvp => kvp.Value));
if (!cfgIdentifiers.Contains(metadata.Identifier))
{
Log.WarnFormat(
"No plugin or :FOR[] clause matching the identifier, manual installations won't be detected: {0}",
string.Join(", ", dllPaths));
}
}

bool boundedCompatibility = json.ContainsKey("ksp_version") || json.ContainsKey("ksp_version_max");
Expand All @@ -76,6 +85,7 @@ public void Validate(Metadata metadata)

private readonly IHttpService _http;
private readonly IModuleService _moduleService;
private readonly IConfigParser _parser;

private static readonly ILog Log = LogManager.GetLogger(typeof(PluginsValidator));
}
Expand Down
Loading

0 comments on commit cae4671

Please sign in to comment.