Merge pull request #1432 from kubeshop/erdikose/feature/handle-plugin…

…-url

feat: Improve plugins install URL support

electron/extensions/downloadExtension.ts

@@ -1,9 +1,11 @@
+import {readFileSync} from 'fs';
+import path from 'path';
 import tar from 'tar';
 
 import {downloadFile} from '@utils/http';
 
 import downloadExtensionEntry from './downloadExtensionEntry';
-import {createOrRecreateFolder, deleteFile, doesPathExist} from './fileSystem';
+import {createOrRecreateFolder, deleteFile, deleteFolder, doesPathExist, getAllFiles} from './fileSystem';
 import {DownloadExtensionOptions} from './types';
 
 async function downloadExtension<ExtensionEntryType, ExtensionType>(
@@ -50,7 +52,25 @@ async function downloadExtension<ExtensionEntryType, ExtensionType>(
 
   await deleteFile(tarballFilePath);
 
+  checkAllJSONFilesOfExtensionAreValid(extensionFolderPath);
+
   return extension;
 }
 
 export default downloadExtension;
+
+export const checkAllJSONFilesOfExtensionAreValid = (folderPath: string) => {
+  const allFiles = getAllFiles(folderPath).filter(file => path.extname(file) === '.json');
+  let currentFile: string = '';
+  try {
+    allFiles.forEach(file => {
+      currentFile = file;
+      const fileData = readFileSync(file);
+      JSON.parse(fileData.toString());
+    });
+  } catch (error: any) {
+    deleteFolder(folderPath);
+    const paths: Array<string> = currentFile.split(path.sep);
+    throw new Error(`[${paths[paths.length - 1]}]: ${error.message}`);
+  }
+};

electron/extensions/downloadExtensionEntry.ts

@@ -47,7 +47,7 @@ async function downloadExtensionEntry<ExtensionEntryType>(
   if (doesEntryFileExist) {
     await deleteFile(entryFilePath);
   }
-  await writeFile(entryFilePath, entryFileContent);
+  await writeFile(entryFilePath, JSON.stringify(parsedEntryFileContent));
 
   return parsedEntryFileContent;
 }

electron/extensions/fileSystem.ts

@@ -1,4 +1,5 @@
 import fs, {promises} from 'fs';
+import { join, sep } from 'path';
 import util from 'util';
 
 export const doesPathExist = (path: string) => util.promisify(fs.exists)(path);
@@ -14,3 +15,18 @@ export const deleteFile = (filePath: string) => promises.unlink(filePath);
 export const deleteFolder = (folderPath: string) => promises.rm(folderPath, {recursive: true});
 export const readFile = (filePath: string) => promises.readFile(filePath, 'utf8');
 export const writeFile = (filePath: string, fileContent: string) => promises.writeFile(filePath, fileContent, 'utf8');
+
+export const getAllFiles = (dirPath: string, arrayOfFiles?: Array<any>) => {
+  const files = fs.readdirSync(dirPath);
+  arrayOfFiles = arrayOfFiles || [];
+
+  files.forEach((file: string) => {
+    if (fs.statSync(join(dirPath, file)).isDirectory()) {
+      arrayOfFiles = getAllFiles(join(dirPath, file), arrayOfFiles);
+    } else {
+      (<Array<any>>arrayOfFiles).push(join(dirPath, sep, file));
+    }
+  });
+
+  return arrayOfFiles;
+};

electron/extensions/loadExtension.ts

@@ -28,7 +28,7 @@ async function loadExtension<ExtensionEntryType, ExtensionType>(
     validateEntryFileContent(parsedEntryFileContent);
   } catch (e) {
     if (e instanceof Error) {
-      log.warn(`[LoadExtension]: Invalid ${entryFileName} in ${folderPath}: `, e.message);
+      log.warn(`[LoadExtension]: Invalid ${entryFileName} in ${folderPath}: `);
     }
     return undefined;
   }

electron/pluginService.ts

@@ -20,7 +20,9 @@ import {convertExtensionsToRecord, extractRepositoryOwnerAndNameFromUrl, makeExt
 const PLUGIN_ENTRY_FILE_NAME = 'package.json';
 
 function transformPackageJsonToAnyPlugin(packageJson: PluginPackageJson, folderPath: string): AnyPlugin {
-  const {repositoryOwner, repositoryName} = extractRepositoryOwnerAndNameFromUrl(packageJson.repository);
+  const {repositoryOwner, repositoryName, repositoryBranch} = extractRepositoryOwnerAndNameFromUrl(
+    packageJson.repository
+  );
   const plugin: AnyPlugin = {
     id: packageJson.monoklePlugin.id,
     name: packageJson.name,
@@ -31,7 +33,7 @@ function transformPackageJsonToAnyPlugin(packageJson: PluginPackageJson, folderP
     repository: {
       owner: repositoryOwner,
       name: repositoryName,
-      branch: 'main', // TODO: handle the branch name
+      branch: repositoryBranch,
     },
     modules: packageJson.monoklePlugin.modules.map(module => {
       if (isTemplatePluginModule(module)) {
@@ -60,7 +62,7 @@ export async function downloadPlugin(
     extensionTarballUrl: tarballUrl,
     entryFileName: 'package.json',
     entryFileUrl,
-    parseEntryFileContent: JSON.parse,
+    parseEntryFileContent: parseEntryFileContentHandler.bind(null, pluginUrl),
     validateEntryFileContent: validatePluginPackageJson,
     transformEntryFileContentToExtension: transformPackageJsonToAnyPlugin,
     makeExtensionFolderPath: () => {
@@ -97,7 +99,14 @@ export async function updatePlugin(
   pluginsDir: string,
   userTempDir: string
 ): Promise<AnyExtension<AnyPlugin> | undefined> {
-  const repositoryUrl = `https://github.com/${plugin.repository.owner}/${plugin.repository.name}`;
+  let repositoryUrl;
+
+  if (plugin.repository.branch) {
+    repositoryUrl = `https://github.com/${plugin.repository.owner}/${plugin.repository.name}/tree/${plugin.repository.branch}`;
+  } else {
+    repositoryUrl = `https://github.com/${plugin.repository.owner}/${plugin.repository.name}`;
+  }
+
   const {entryFileUrl, folderPath} = makeExtensionDownloadData(repositoryUrl, PLUGIN_ENTRY_FILE_NAME, userTempDir);
   let tempPluginEntry: PluginPackageJson;
   try {
@@ -114,6 +123,7 @@ export async function updatePlugin(
     }
     return;
   }
+
   if (semver.lt(plugin.version, tempPluginEntry.version)) {
     try {
       const pluginExtension = await downloadPlugin(repositoryUrl, pluginsDir);
@@ -128,3 +138,11 @@ export async function updatePlugin(
   }
   return undefined;
 }
+
+export const parseEntryFileContentHandler = (pluginUrl: string, entryFileContext: string) => {
+  const entryFile = JSON.parse(entryFileContext);
+  if (pluginUrl) {
+    entryFile.repository = pluginUrl;
+  }
+  return entryFile;
+};

electron/templateService.ts

@@ -141,7 +141,7 @@ const makeLoadTemplateOptions = (folderPath: string) => {
   return {
     folderPath,
     entryFileName: TEMPLATE_ENTRY_FILE_NAME,
-    parseEntryFileContent: JSON.parse,
+    parseEntryFileContent: parseEntryFileContentHandler.bind(null),
     validateEntryFileContent: validateAnyTemplate,
     transformEntryFileContentToExtension: parseTemplate,
   };
@@ -307,3 +307,11 @@ export async function updateTemplatePack(
   }
   return undefined;
 }
+
+export const parseEntryFileContentHandler = (entryFileContent: any) => {
+  try {
+    return JSON.parse(entryFileContent);
+  } catch (error: any) {
+    log.error(`[${entryFileContent.name}]: ${error.message}`);
+  }
+};

electron/utils.ts

@@ -3,6 +3,7 @@ import {dialog} from 'electron';
 import {AnyAction} from '@reduxjs/toolkit';
 
 import {existsSync, mkdirSync, writeFileSync} from 'fs';
+import gitUrlParse from 'git-url-parse';
 import _ from 'lodash';
 import {machineIdSync} from 'node-machine-id';
 import Nucleus from 'nucleus-nodejs';
@@ -18,7 +19,6 @@ import {loadResource} from '@redux/services';
 import electronStore from '@utils/electronStore';
 import {PROCESS_ENV} from '@utils/env';
 
-const GITHUB_URL = 'https://github.com';
 const GITHUB_REPOSITORY_REGEX = /^https:\/\/github.com\/[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+/i;
 
 export function isValidRepositoryUrl(repositoryUrl: string) {
@@ -27,14 +27,24 @@ export function isValidRepositoryUrl(repositoryUrl: string) {
 
 export function extractRepositoryOwnerAndNameFromUrl(pluginUrl: string) {
   if (!isValidRepositoryUrl(pluginUrl)) {
-    throw new Error('Invalid repository URL');
+    throw new Error('Currently we support only Github as provider');
+  }
+  const parsedURL = gitUrlParse(pluginUrl);
+  if (!parsedURL.owner || !parsedURL.name) {
+    throw new Error('Please enter a valid git URL!');
+  }
+  if (!parsedURL.protocols.includes('https')) {
+    throw new Error('Currently we support only HTTPS protocol!');
+  }
+  if (parsedURL.filepathtype && parsedURL.filepathtype !== 'tree') {
+    throw new Error('Please navigate main url of the branch!');
   }
-  const repositoryPath = pluginUrl.split(`${GITHUB_URL}/`)[1];
-  const [repositoryOwner, repositoryName] = repositoryPath.split('/');
-
   return {
-    repositoryOwner,
-    repositoryName,
+    repositoryOwner: parsedURL.owner,
+    repositoryName: parsedURL.name,
+    repositoryBranch: !parsedURL.filepathtype
+      ? 'main'
+      : `${parsedURL.ref}${parsedURL.filepath ? `/${parsedURL.filepath}` : ''}`,
   };
 }
 
@@ -43,10 +53,15 @@ export function makeExtensionDownloadData(
   extensionEntryFileName: string,
   downloadPath: string
 ) {
-  const {repositoryOwner, repositoryName} = extractRepositoryOwnerAndNameFromUrl(extensionRepositoryUrl);
-  const entryFileUrl = `https://raw.githubusercontent.com/${repositoryOwner}/${repositoryName}/main/${extensionEntryFileName}`;
-  const tarballUrl = `https://api.github.com/repos/${repositoryOwner}/${repositoryName}/tarball/main`;
-  const folderPath = path.join(downloadPath, `${repositoryOwner}-${repositoryName}`);
+  const {repositoryOwner, repositoryName, repositoryBranch} =
+    extractRepositoryOwnerAndNameFromUrl(extensionRepositoryUrl);
+  const entryFileUrl = `https://raw.githubusercontent.com/${repositoryOwner}/${repositoryName}/${repositoryBranch}/${extensionEntryFileName}`;
+  const tarballUrl = `https://api.github.com/repos/${repositoryOwner}/${repositoryName}/tarball/${repositoryBranch}`;
+  const folderPath = path.join(
+    downloadPath,
+    // @ts-ignore
+    `${repositoryOwner}-${repositoryName}-${repositoryBranch.replaceAll(path.sep, '-')}`
+  );
   return {entryFileUrl, tarballUrl, folderPath};
 }