Skip to content

Commit

Permalink
[jb-gha] improve nightly script and update file structure (#20220)
Browse files Browse the repository at this point in the history
* [jb-gha] draft nightly script and update file structure

* Update GHA with draft script

* Update summary

* Run leeway build

* Fix build and share slack notification

* Add force use dev latest images leeway script
  • Loading branch information
mustard-mh authored Sep 14, 2024
1 parent eb279a9 commit 4b05e71
Show file tree
Hide file tree
Showing 11 changed files with 452 additions and 217 deletions.
35 changes: 18 additions & 17 deletions .github/workflows/jetbrains-auto-update-template.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,39 +33,40 @@ jobs:
identity_provider: ${{ github.ref == 'refs/heads/main' && secrets.CORE_DEV_PROVIDER || secrets.DEV_PREVIEW_PROVIDER }}
service_account: ${{ github.ref == 'refs/heads/main' && secrets.CORE_DEV_SA || secrets.DEV_PREVIEW_SA }}
leeway_segment_key: ${{ secrets.LEEWAY_SEGMENT_KEY }}
- name: Find downloadUrl
id: download-url
- name: Install dependencies
run: |
downloadUrl=$(cat WORKSPACE.yaml | yq r - 'defaultArgs.${{ inputs.productId }}DownloadUrl')
echo "::set-output name=downloadUrl::$downloadUrl"
- name: Find IDE version to download
id: ide-version
cd ./components/ide/gha-update-image/
yarn
npm i -g bun
- name: Find Nightly Target
id: find-target
run: |
IDE_VERSIONS_JSON=$(bash ./components/ide/jetbrains/image/resolve-latest-ide-version.sh ${{ inputs.productCode }} ${{ steps.download-url.outputs.downloadUrl }})
IDE_BUILD_VERSION=$(echo "$IDE_VERSIONS_JSON" | jq -r .IDE_BUILD_VERSION)
IDE_VERSION=$(echo "$IDE_VERSIONS_JSON" | jq -r .IDE_VERSION)
echo "IDE_BUILD_VERSION: $IDE_BUILD_VERSION"
echo "IDE_VERSION: $IDE_VERSION"
echo "::set-output name=ideBuildVersion::$IDE_BUILD_VERSION"
echo "::set-output name=ideVersion::$IDE_VERSION"
cd ./components/ide/gha-update-image/
bun run index-jb-nightly.ts --task=1 --productCode=${{ inputs.productCode }}
if [ -f /tmp/__gh_output.txt ]
then
cat /tmp/__gh_output.txt >> $GITHUB_OUTPUT
fi
- name: Leeway build
if: ${{ steps.ide-version.outputs.ideBuildVersion }}
if: ${{ steps.find-target.outputs.buildNumber }}
env:
LEEWAY_MAX_PROVENANCE_BUNDLE_SIZE: "8388608"
LEEWAY_REMOTE_CACHE_BUCKET: ${{ github.ref == 'refs/heads/main' && 'leeway-cache-main-c514a01' || 'leeway-cache-dev-3ac8ef5' }}
run: |
imageRepoBase=${{ github.ref == 'refs/heads/main' && 'eu.gcr.io/gitpod-core-dev/build' || 'eu.gcr.io/gitpod-dev-artifact/build' }}
leeway build -Dversion=latest -DimageRepoBase=$imageRepoBase -DbuildNumber=${{ steps.ide-version.outputs.ideBuildVersion }} components/ide/jetbrains/image:${{ inputs.productId }}-latest -DjbBackendVersion=${{ steps.ide-version.outputs.ideVersion }}
echo "Upgrade latest ${{ inputs.productId }} image with ${{ steps.find-target.outputs.editorSummary }}" >> $GITHUB_STEP_SUMMARY
leeway build -Dversion=latest -DimageRepoBase=$imageRepoBase -DbuildNumber=${{ steps.find-target.outputs.buildNumber }} components/ide/jetbrains/image:${{ steps.find-target.outputs.image }}-latest -DjbBackendVersion=${{ steps.find-target.outputs.jbBackendVersion }}
- name: Get previous job's status
id: lastrun
uses: filiptronicek/get-last-job-status@main
- name: Slack Notification
if: ${{ (success() && steps.lastrun.outputs.status == 'failed') || failure() }}
if: ${{ (success() && steps.find-target.outputs.buildNumber) || failure() }}
uses: rtCamp/action-slack-notify@v2
env:
SLACK_WEBHOOK: ${{ secrets.IDE_SLACK_WEBHOOK }}
SLACK_COLOR: ${{ job.status }}
SLACK_TITLE: ${{ inputs.productId }}
SLACK_TITLE: Upgrade latest ${{ inputs.productId }} image with ${{ steps.find-target.outputs.editorSummary }}
SLACK_FOOTER: "<https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|Workflow logs>"

delete-runner:
Expand Down
19 changes: 19 additions & 0 deletions components/ide/gha-update-image/BUILD.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
scripts:
- name: jb-use-dev-latest
script: |
ide_list=("intellij" "goland" "pycharm" "phpstorm" "rubymine" "webstorm" "rider" "clion" "rustrover")
prop_list=("latestImage")
cf_patch=$(kubectl get cm ide-config -o=json | jq '.data."config.json"' |jq -r)
for ide in "${ide_list[@]}"; do
for prop in "${prop_list[@]}"; do
cf_patch=$(echo "$cf_patch" | jq ".ideOptions.options.$ide.$prop = \"eu.gcr.io/gitpod-dev-artifact/build/ide/$ide:latest\"")
done
done
cf_patch=$(echo "$cf_patch" |jq tostring)
cf_patch="{\"data\": {\"config.json\": $cf_patch}}"
# echo "$cf_patch"
kubectl patch cm ide-config --type=merge -p "$cf_patch"
kubectl rollout restart deployment ide-service
kubectl rollout restart deployment server
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
[
{
"image": "eu.gcr.io/gitpod-core-dev/build/ide/jb-backend-plugin:commit-d54058956db3274084249bdbc4f508ab111b8c51",
"detail": {
"pluginSinceBuild": "242.19533",
"pluginUntilBuild":"242.*"
}
},
{
"image": "eu.gcr.io/gitpod-core-dev/build/ide/jb-backend-plugin:commit-d54058956db3274084249bdbc4f508ab111b8c51-latest",
"detail": {
"pluginSinceBuild": "242.19533",
"pluginUntilBuild":"242.*"
}
}
]
77 changes: 77 additions & 0 deletions components/ide/gha-update-image/index-jb-nightly.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
// Copyright (c) 2024 Gitpod GmbH. All rights reserved.
// Licensed under the GNU Affero General Public License (AGPL).
// See License.AGPL.txt in the project root for license information.

// Update JetBrains latest editor images
//
// ```
// bun run index-jb-nightly.ts --task=<id> --productCode=<code>
// ```

import { $ } from "bun";
import { appendGitHubOutput, pathToBackendPluginGradleLatest, readWorkspaceYaml } from "./lib/common";
import { maybeCompatible, parseGradleProperties, parseGradlePropertiesFromTaskConfig } from "./lib/jb-helper/jb-helper";
import { fetchProductReleases, ReleaseItem, releaseItemStr } from "./lib/jb-helper/jb-releases";
import { getTaskFromArgs } from "./lib/jb-helper/jb-gradle-task-config";

$.nothrow(); // git likes to respond with non-zero codes, but it is alright for us

const task = getTaskFromArgs(true);

if (task.id !== 1) {
throw new Error(`Only task 1 is supported, got ${task.id}`);
}

console.log(`Updating nightly editor for ${task.productId} (${task.productType})`);

const { parsedObj: parsedWorkspaceYaml } = await readWorkspaceYaml();

const downloadUrl = parsedWorkspaceYaml.defaultArgs[task.productId + "DownloadUrl"] as string;

const latestGradle = parseGradleProperties(await Bun.file(pathToBackendPluginGradleLatest).text());

const platformVersionType = "build";

const releases = await fetchProductReleases({ productCode: task.productCode, productType: task.productType });

let maybeCompatibleRelease: ReleaseItem | undefined;
for (const release of releases) {
switch (platformVersionType) {
case "build": {
const ok = maybeCompatible(release, latestGradle);
if (ok) {
maybeCompatibleRelease = release;
break;
} else {
console.error(`${releaseItemStr(release)} incompatible`);
}
}
}
if (maybeCompatibleRelease) {
break;
}
}

if (maybeCompatibleRelease) {
console.log(`${releaseItemStr(maybeCompatibleRelease)} maybe compatible`);
}

const targetRelease = maybeCompatibleRelease || releases.find((e) => e.downloads.linux?.link === downloadUrl);

if (!targetRelease) {
throw new Error(`No compatible release found`);
}

console.log(`Preparing to use ${releaseItemStr(targetRelease)} as latest version for ${task.productId}`);

const targetConfig = parseGradlePropertiesFromTaskConfig(task, targetRelease);

// TODO: actually update nightly editor
console.log(
`Going to exec \`leeway build -Dversion=latest -DimageRepoBase=$imageRepoBase -DbuildNumber=${targetConfig.platformVersion} components/ide/jetbrains/image:${task.productId}-latest -DjbBackendVersion=${targetRelease.version}\``,
);

appendGitHubOutput(`buildNumber=${targetConfig.platformVersion}`);
appendGitHubOutput(`image=${task.productId}`);
appendGitHubOutput(`jbBackendVersion=${targetRelease.version}`);
appendGitHubOutput(`editorSummary=${releaseItemStr(targetRelease)}`);
132 changes: 28 additions & 104 deletions components/ide/gha-update-image/index-jb-platform-update.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,113 +2,37 @@
// Licensed under the GNU Affero General Public License (AGPL).
// See License.AGPL.txt in the project root for license information.

// Update JetBrains Platform Version
// Update JetBrains Plugins (gateway / backend) Platform Version
//
// ```
// bun run index-jb-platform-update.ts --task=<id>
// ```

import path from "path";
import { parseArgs } from "util";
import { pathToProjectRoot } from "./lib/common";
import { jbUpdatePlatformVersion } from "./lib/jb-update-platform-version";
import { getTaskFromArgs } from "./lib/jb-helper/jb-gradle-task-config";
import {
parseGradleProperties,
renderPropertiesTemplate,
parseGradlePropertiesFromTaskConfig,
} from "./lib/jb-helper/jb-helper";
import { fetchProductReleases } from "./lib/jb-helper/jb-releases";

const targetInfo = [
{
id: 1,
taskName: "Latest Backend Plugin",
productCode: "IIU",
productType: "eap,rc,release",
xmlName: "IntelliJ IDEA",
xmlChannels: ["IC-IU-EAP-licensing-RELEASE", "IC-IU-EAP-licensing-EAP", "IC-IU-RELEASE-licensing-RELEASE"],
useXml: true,
gradlePropertiesPath: path.resolve(
pathToProjectRoot,
"components/ide/jetbrains/backend-plugin/gradle-latest.properties",
),
gradlePropertiesTemplate: `# Code generated by gha-update-image/index-jb-platform-update.ts. DO NOT EDIT.
# See https://plugins.jetbrains.com/docs/intellij/build-number-ranges.html
# for insight into build numbers and IntelliJ Platform versions.
# revert pluginSinceBuild if it's unnecessary
pluginSinceBuild={{pluginSinceBuild}}
pluginUntilBuild={{pluginUntilBuild}}
# Plugin Verifier integration -> https://github.com/JetBrains/gradle-intellij-plugin#plugin-verifier-dsl
# See https://jb.gg/intellij-platform-builds-list for available build versions.
pluginVerifierIdeVersions={{pluginVerifierIdeVersions}}
# Version from "com.jetbrains.intellij.idea" which can be found at https://www.jetbrains.com/intellij-repository/snapshots
platformVersion={{platformVersion}}
`,
},
{
id: 2,
taskName: "Latest Frontend Plugin",
productCode: "GW",
productType: "eap,rc,release",
xmlName: "Gateway",
xmlChannels: ["GW-EAP-licensing-EAP", "GW-RELEASE-licensing-RELEASE", "GW-EAP-licensing-RELEASE"],
useXml: true,
useHumanreadableVersion: true,
gradlePropertiesPath: path.resolve(
pathToProjectRoot,
"components/ide/jetbrains/gateway-plugin/gradle-latest.properties",
),
gradlePropertiesTemplate: `# Code generated by gha-update-image/index-jb-platform-update.ts. DO NOT EDIT.
# See https://plugins.jetbrains.com/docs/intellij/build-number-ranges.html
# for insight into build numbers and IntelliJ Platform versions.
# revert pluginSinceBuild if it's unnecessary
pluginSinceBuild={{pluginSinceBuild}}
pluginUntilBuild={{pluginUntilBuild}}
# Plugin Verifier integration -> https://github.com/JetBrains/gradle-intellij-plugin#plugin-verifier-dsl
# See https://jb.gg/intellij-platform-builds-list for available build versions.
pluginVerifierIdeVersions={{pluginVerifierIdeVersions}}
# Version from "com.jetbrains.gateway" which can be found at https://www.jetbrains.com/intellij-repository/snapshots
platformVersion={{platformVersion}}
`,
},
{
id: 3,
taskName: "Stable Frontend Plugin",
productCode: "GW",
productType: "release",
xmlName: "Gateway",
xmlChannels: ["GW-RELEASE-licensing-RELEASE"],
useXml: true,
useHumanreadableVersion: true,
gradlePropertiesPath: path.resolve(
pathToProjectRoot,
"components/ide/jetbrains/gateway-plugin/gradle-stable.properties",
),
gradlePropertiesTemplate: `# Code generated by gha-update-image/index-jb-platform-update.ts. DO NOT EDIT.
# See https://plugins.jetbrains.com/docs/intellij/build-number-ranges.html
# for insight into build numbers and IntelliJ Platform versions.
# revert pluginSinceBuild if it's unnecessary
pluginSinceBuild={{pluginSinceBuild}}
pluginUntilBuild={{pluginUntilBuild}}
# Plugin Verifier integration -> https://github.com/JetBrains/gradle-intellij-plugin#plugin-verifier-dsl
# See https://jb.gg/intellij-platform-builds-list for available build versions.
pluginVerifierIdeVersions={{pluginVerifierIdeVersions}}
# Version from "com.jetbrains.gateway" which can be found at https://www.jetbrains.com/updates/updates.xml
platformVersion={{platformVersion}}
`,
},
];
const task = getTaskFromArgs(false);

const { values } = parseArgs({
args: Bun.argv,
options: {
task: {
type: "string",
},
},
strict: true,
allowPositionals: true,
});
console.log("Updating", task.taskName);

const taskID = Number.parseInt(values.task ?? "NaN");
const releases = await fetchProductReleases(task);
const newProps = parseGradlePropertiesFromTaskConfig(task, releases[0]);
console.log("New properties info:", newProps);
const oldProps = parseGradleProperties(await Bun.file(task.gradlePropertiesPath).text());
console.log("Old properties info:", oldProps);

const target = targetInfo.find((e) => e.id === taskID);
if (!target) {
throw new Error(
`Invalid task id: ${taskID}, update cmd with \`--task="<name>"\`, available tasks: \n\t- ${targetInfo
.map((e) => `(${e.id}) ${e.taskName}`)
.join("\n\t- ")}`,
);
if (newProps.platformVersion === oldProps.platformVersion) {
console.warn("PlatformVersion are the same, no need to update platform version");
process.exit(0);
}
console.log("Updating", target.taskName);
jbUpdatePlatformVersion(target);

const newGradleContent = renderPropertiesTemplate("gha-update-image/index-jb-platform-update.ts", task, newProps);

await Bun.write(task.gradlePropertiesPath, newGradleContent);

console.log("Updated platform version to:", newProps.platformVersion);
8 changes: 8 additions & 0 deletions components/ide/gha-update-image/lib/common.ts
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,14 @@ export const pathToBackendPluginGradleStable = path.resolve(
"components/ide/jetbrains/backend-plugin/gradle-stable.properties",
);

// gradle-latest.properties
export const pathToBackendPluginGradleLatest = path.resolve(
pathToProjectRoot,
"components/ide/jetbrains/backend-plugin/gradle-latest.properties",
);

export const pathToBackendPluginDir = path.resolve(pathToProjectRoot, "components/ide/jetbrains/backend-plugin");

// ide-configmap.json
export const pathToConfigmap = path.resolve(
pathToProjectRoot,
Expand Down
Loading

0 comments on commit 4b05e71

Please sign in to comment.