From 987bb3e3dd9198d354399acab2ab63001a323bb6 Mon Sep 17 00:00:00 2001 From: rhkp Date: Tue, 22 Oct 2024 19:07:06 +0000 Subject: [PATCH 01/18] Add kogito-db-migrator-tool and kogito-db-migrator-tool-image packages --- .gitignore | 3 + .../kogito-db-migrator-tool-image/README.md | 47 ++++ .../build-container-image.sh | 40 ++++ .../cekit/image.yaml | 45 ++++ .../kogito-postgres-db-migration-deps/install | 32 +++ .../migration.sh | 36 +++ .../module.yaml | 30 +++ .../env/index.js | 20 ++ .../kogito-db-migrator-tool-image/install.js | 20 ++ .../package.json | 26 +++ .../kogito-db-migrator-tool/.dockerignore | 5 + packages/kogito-db-migrator-tool/.gitignore | 44 ++++ packages/kogito-db-migrator-tool/README.md | 25 ++ .../build-db-migrator-jar.sh | 45 ++++ packages/kogito-db-migrator-tool/env/index.js | 31 +++ .../get-kogito-ddl-scripts.sh | 64 ++++++ packages/kogito-db-migrator-tool/install.js | 35 +++ packages/kogito-db-migrator-tool/package.json | 33 +++ packages/kogito-db-migrator-tool/pom.xml | 213 ++++++++++++++++++ .../src/main/cekit/image.yaml | 45 ++++ .../kogito-postgres-db-migration-deps/install | 32 +++ .../migration.sh | 36 +++ .../module.yaml | 30 +++ .../migrator/DBConnectionChecker.java | 66 ++++++ .../postgresql/migrator/DBMigrator.java | 73 ++++++ .../postgresql/migrator/MigrationService.java | 66 ++++++ .../src/main/resources/application.properties | 40 ++++ .../migrator/DBConnectionCheckerTest.java | 68 ++++++ .../postgresql/migrator/DBMigratorTest.java | 94 ++++++++ .../migrator/MigrationServiceTest.java | 69 ++++++ 30 files changed, 1413 insertions(+) create mode 100644 packages/kogito-db-migrator-tool-image/README.md create mode 100755 packages/kogito-db-migrator-tool-image/build-container-image.sh create mode 100644 packages/kogito-db-migrator-tool-image/cekit/image.yaml create mode 100644 packages/kogito-db-migrator-tool-image/cekit/modules/kogito-postgres-db-migration-deps/install create mode 100644 packages/kogito-db-migrator-tool-image/cekit/modules/kogito-postgres-db-migration-deps/migration.sh create mode 100644 packages/kogito-db-migrator-tool-image/cekit/modules/kogito-postgres-db-migration-deps/module.yaml create mode 100644 packages/kogito-db-migrator-tool-image/env/index.js create mode 100644 packages/kogito-db-migrator-tool-image/install.js create mode 100644 packages/kogito-db-migrator-tool-image/package.json create mode 100644 packages/kogito-db-migrator-tool/.dockerignore create mode 100644 packages/kogito-db-migrator-tool/.gitignore create mode 100644 packages/kogito-db-migrator-tool/README.md create mode 100755 packages/kogito-db-migrator-tool/build-db-migrator-jar.sh create mode 100644 packages/kogito-db-migrator-tool/env/index.js create mode 100755 packages/kogito-db-migrator-tool/get-kogito-ddl-scripts.sh create mode 100644 packages/kogito-db-migrator-tool/install.js create mode 100644 packages/kogito-db-migrator-tool/package.json create mode 100644 packages/kogito-db-migrator-tool/pom.xml create mode 100644 packages/kogito-db-migrator-tool/src/main/cekit/image.yaml create mode 100644 packages/kogito-db-migrator-tool/src/main/cekit/modules/kogito-postgres-db-migration-deps/install create mode 100644 packages/kogito-db-migrator-tool/src/main/cekit/modules/kogito-postgres-db-migration-deps/migration.sh create mode 100644 packages/kogito-db-migrator-tool/src/main/cekit/modules/kogito-postgres-db-migration-deps/module.yaml create mode 100644 packages/kogito-db-migrator-tool/src/main/java/org/kie/kogito/postgresql/migrator/DBConnectionChecker.java create mode 100644 packages/kogito-db-migrator-tool/src/main/java/org/kie/kogito/postgresql/migrator/DBMigrator.java create mode 100644 packages/kogito-db-migrator-tool/src/main/java/org/kie/kogito/postgresql/migrator/MigrationService.java create mode 100644 packages/kogito-db-migrator-tool/src/main/resources/application.properties create mode 100644 packages/kogito-db-migrator-tool/src/test/java/org/kie/kogito/postgresql/migrator/DBConnectionCheckerTest.java create mode 100644 packages/kogito-db-migrator-tool/src/test/java/org/kie/kogito/postgresql/migrator/DBMigratorTest.java create mode 100644 packages/kogito-db-migrator-tool/src/test/java/org/kie/kogito/postgresql/migrator/MigrationServiceTest.java diff --git a/.gitignore b/.gitignore index 0fa740642fb..750f639a452 100644 --- a/.gitignore +++ b/.gitignore @@ -346,3 +346,6 @@ turbo.json # IntelliJ projects should be on _intellij-projects folder, never on specific packages. packages/*/.idea examples/*/.idea + +# Jar files +**/*.jar \ No newline at end of file diff --git a/packages/kogito-db-migrator-tool-image/README.md b/packages/kogito-db-migrator-tool-image/README.md new file mode 100644 index 00000000000..db325462e39 --- /dev/null +++ b/packages/kogito-db-migrator-tool-image/README.md @@ -0,0 +1,47 @@ +# kogito-db-migrator-tool-image + +This is a cekit based image builder for the database migrator application. Details about the database migrator application are [here](../kogito-db-migrator-tool/README.md) + +## Build and Run container image locally + +You can build the cekit build of container image by using the provided image builder shell script + +```shell +./build-container-image.sh +``` + +_NOTE_: Before performing this step, be sure that the database migrator application jar has been built and available for inclusion in the image. + +Ensure the script completes without errors. + +Assuming you have a Postgres database running locally, e.g., a `di` database for data index and a `js` database for jobs service, you can run the image with the following command. Substitute appropriate values: + +```shell +podman run \ +--env MIGRATE_DB_DATAINDEX=true \ +--env QUARKUS_DATASOURCE_DATAINDEX_JDBC_URL= \ +--env QUARKUS_DATASOURCE_DATAINDEX_USERNAME= \ +--env QUARKUS_DATASOURCE_DATAINDEX_PASSWORD= \ +--env QUARKUS_FLYWAY_DATAINDEX_SCHEMAS=dataindex \ +--env MIGRATE_DB_JOBSSERVICE=true \ +--env QUARKUS_DATASOURCE_JOBSSERVICE_JDBC_URL= \ +--env QUARKUS_DATASOURCE_JOBSSERVICE_USERNAME= \ +--env QUARKUS_DATASOURCE_JOBSSERVICE_PASSWORD= \ +--env QUARKUS_FLYWAY_JOBSSERVICE_SCHEMAS=jobsservice \ +docker.io/apache/incubator-kie-kogito-service-db-migration-postgresql:999-SNAPSHOT +``` + +### Environment variables + +| NAME | DESCRIPTION | DEFAULT | +| --------------------------------------- | -------------------------------------------------------------------------------- | ----------------------------------------- | +| MIGRATE_DB_DATAINDEX | Set to true if you want to migrate data index database, set to false otherwise | false | +| QUARKUS_DATASOURCE_DATAINDEX_JDBC_URL | Data index database url e.g. jdbc:postgresql://host.docker.internal:5432/di | jdbc:postgresql://localhost:5432/postgres | +| QUARKUS_DATASOURCE_DATAINDEX_USERNAME | Data index database username | postgres | +| QUARKUS_DATASOURCE_DATAINDEX_PASSWORD | Data index database password | postgres | +| QUARKUS_FLYWAY_DATAINDEX_SCHEMAS | Data index database schema | dataindex | +| MIGRATE_DB_JOBSSERVICE | Set to true if you want to migrate jobs service database, set to false otherwise | false | +| QUARKUS_DATASOURCE_JOBSSERVICE_JDBC_URL | Jobs service database url e.g. jdbc:postgresql://host.docker.internal:5432/js | jdbc:postgresql://localhost:5432/postgres | +| QUARKUS_DATASOURCE_JOBSSERVICE_USERNAME | Jobs service database username | postgres | +| QUARKUS_DATASOURCE_JOBSSERVICE_PASSWORD | Jobs service database password | postgres | +| QUARKUS_FLYWAY_JOBSSERVICE_SCHEMAS | Jobs service database schema | jobsservice | diff --git a/packages/kogito-db-migrator-tool-image/build-container-image.sh b/packages/kogito-db-migrator-tool-image/build-container-image.sh new file mode 100755 index 00000000000..d4bac957570 --- /dev/null +++ b/packages/kogito-db-migrator-tool-image/build-container-image.sh @@ -0,0 +1,40 @@ +#!/bin/sh +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -x +set -e + +CEKIT_BUILDER=docker +SOURCE_FILE=/tmp/kogito-db-migrator-tool/sonataflow-db-migrator-0.0.0-runner.jar +TARGET_FILE=./cekit/modules/kogito-postgres-db-migration-deps/sonataflow-db-migrator-runner.jar + +if [ -f "$SOURCE_FILE" ]; then + echo "The file: $SOURCE_FILE found and will be used to build image." + rm -f $TARGET_FILE + cp $SOURCE_FILE $TARGET_FILE + + # Build the container image + cd ./cekit || exit + cekit -v build "$CEKIT_BUILDER" + rm -f $TARGET_FILE +else + echo "The file: $SOURCE_FILE not found. Please build kogito-db-migrator-tool package first before building the image." + exit 1 +fi \ No newline at end of file diff --git a/packages/kogito-db-migrator-tool-image/cekit/image.yaml b/packages/kogito-db-migrator-tool-image/cekit/image.yaml new file mode 100644 index 00000000000..a2517ee0154 --- /dev/null +++ b/packages/kogito-db-migrator-tool-image/cekit/image.yaml @@ -0,0 +1,45 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +name: "docker.io/apache/incubator-kie-kogito-service-db-migration-postgresql" +version: "999-SNAPSHOT" +from: registry.access.redhat.com/ubi8/openjdk-17-runtime:1.19 +description: Flyway image for DI/JS database migration + +labels: + - name: "org.kie.kogito.version" + value: "999-SNAPSHOT" + - name: "maintainer" + value: "Apache KIE " + - name: "io.k8s.description" + value: "Kogito DB Migration creates schemas and tables for Data Index and Jobs Service for PostgreSQL database" + - name: "io.k8s.display-name" + value: "Kogito DB Migration for Data Index and Jobs Service - PostgreSQL" + - name: "io.openshift.tags" + value: "kogito,db-migration" + +modules: + repositories: + - path: modules + install: + - name: kogito-postgres-db-migration-deps + +run: + workdir: "/home/kogito" + entrypoint: + - "/home/kogito/migration.sh" diff --git a/packages/kogito-db-migrator-tool-image/cekit/modules/kogito-postgres-db-migration-deps/install b/packages/kogito-db-migrator-tool-image/cekit/modules/kogito-postgres-db-migration-deps/install new file mode 100644 index 00000000000..5fa1a3e0e86 --- /dev/null +++ b/packages/kogito-db-migrator-tool-image/cekit/modules/kogito-postgres-db-migration-deps/install @@ -0,0 +1,32 @@ +#!/bin/sh +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -x +set -e + +useradd -m kogito + +cd /home/kogito || exit + +chgrp -R 0 /home/kogito +chown -R 0 /home/kogito +chmod -R g=u /home/kogito + +chmod +x /home/kogito/migration.sh \ No newline at end of file diff --git a/packages/kogito-db-migrator-tool-image/cekit/modules/kogito-postgres-db-migration-deps/migration.sh b/packages/kogito-db-migrator-tool-image/cekit/modules/kogito-postgres-db-migration-deps/migration.sh new file mode 100644 index 00000000000..fe41d336d5b --- /dev/null +++ b/packages/kogito-db-migrator-tool-image/cekit/modules/kogito-postgres-db-migration-deps/migration.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -x +set -e + +OPERATOR_VERSION=999.0.0 + +for arg in "$@"; do + case "$arg" in + OPERATOR_VERSION=*) OPERATOR_VERSION="${arg#*=}" ;; + esac +done + +cd /home/kogito/ + +mv sonataflow-db-migrator-runner.jar sonataflow-db-migrator-"$OPERATOR_VERSION"-runner.jar + +java -jar sonataflow-db-migrator-"$OPERATOR_VERSION"-runner.jar \ No newline at end of file diff --git a/packages/kogito-db-migrator-tool-image/cekit/modules/kogito-postgres-db-migration-deps/module.yaml b/packages/kogito-db-migrator-tool-image/cekit/modules/kogito-postgres-db-migration-deps/module.yaml new file mode 100644 index 00000000000..0605dacc689 --- /dev/null +++ b/packages/kogito-db-migrator-tool-image/cekit/modules/kogito-postgres-db-migration-deps/module.yaml @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +schema_version: 1 +name: kogito-postgres-db-migration-deps +version: "1.0" +artifacts: + - name: migration.sh + path: migration.sh + dest: /home/kogito + - name: sonataflow-db-migrator-runner.jar + path: sonataflow-db-migrator-runner.jar + dest: /home/kogito +execute: + - script: install diff --git a/packages/kogito-db-migrator-tool-image/env/index.js b/packages/kogito-db-migrator-tool-image/env/index.js new file mode 100644 index 00000000000..c3fe8a9b1e0 --- /dev/null +++ b/packages/kogito-db-migrator-tool-image/env/index.js @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +console.log("index.js"); diff --git a/packages/kogito-db-migrator-tool-image/install.js b/packages/kogito-db-migrator-tool-image/install.js new file mode 100644 index 00000000000..8529104bcd8 --- /dev/null +++ b/packages/kogito-db-migrator-tool-image/install.js @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +console.log("install.js"); diff --git a/packages/kogito-db-migrator-tool-image/package.json b/packages/kogito-db-migrator-tool-image/package.json new file mode 100644 index 00000000000..0580e059efb --- /dev/null +++ b/packages/kogito-db-migrator-tool-image/package.json @@ -0,0 +1,26 @@ +{ + "private": true, + "name": "@kie-tools/kogito-db-migrator-tool-image", + "version": "0.0.0", + "description": "", + "license": "Apache-2.0", + "homepage": "https://github.com/apache/incubator-kie-tools", + "repository": { + "type": "git", + "url": "https://github.com/apache/incubator-kie-tools.git" + }, + "bugs": { + "url": "https://github.com/apache/incubator-kie-tools/issues" + }, + "scripts": { + "build": "./build-container-image.sh", + "build:dev": "pnpm build", + "build:prod": "pnpm build", + "install": "node install.js" + }, + "dependencies": { + "@kie-tools/kogito-db-migrator-tool": "workspace:*", + "@kie-tools/python-venv": "workspace:*", + "@kie-tools/root-env": "workspace:*" + } +} diff --git a/packages/kogito-db-migrator-tool/.dockerignore b/packages/kogito-db-migrator-tool/.dockerignore new file mode 100644 index 00000000000..94810d006e7 --- /dev/null +++ b/packages/kogito-db-migrator-tool/.dockerignore @@ -0,0 +1,5 @@ +* +!target/*-runner +!target/*-runner.jar +!target/lib/* +!target/quarkus-app/* \ No newline at end of file diff --git a/packages/kogito-db-migrator-tool/.gitignore b/packages/kogito-db-migrator-tool/.gitignore new file mode 100644 index 00000000000..33b2771547b --- /dev/null +++ b/packages/kogito-db-migrator-tool/.gitignore @@ -0,0 +1,44 @@ +#Maven +target/ +pom.xml.tag +pom.xml.releaseBackup +pom.xml.versionsBackup +release.properties +.flattened-pom.xml + +# Eclipse +.project +.classpath +.settings/ +bin/ + +# IntelliJ +.idea +*.ipr +*.iml +*.iws + +# NetBeans +nb-configuration.xml + +# Visual Studio Code +.vscode +.factorypath + +# OSX +.DS_Store + +# Vim +*.swp +*.swo + +# patch +*.orig +*.rej + +# Local environment +.env + +# Plugin directory +/.quarkus/cli/plugins/ +*.iml \ No newline at end of file diff --git a/packages/kogito-db-migrator-tool/README.md b/packages/kogito-db-migrator-tool/README.md new file mode 100644 index 00000000000..229ce15d3be --- /dev/null +++ b/packages/kogito-db-migrator-tool/README.md @@ -0,0 +1,25 @@ +# kogito-db-migrator + +This is a java quarkus based postgres database migrator application for Sonataflow Data Index and Jobs Service applications for use by SonataFlow Operator. + +_NOTE_: This postgres database migrator application and its corresponding images are only envisaged to be made use of by SonataFlow Operator, Data Index and Jobs Service internally. Conversely this application is of no use outside the usecases involved with SonataFlow Operator, Data Index and Jobs Service. + +## Running the application in dev mode + +Though you can run the application locally in dev mode but it is advisable to run this application as a container image. +The primary reason not to run as standalone application in dev mode, is that by default there are no DDL migration files included in the source. The script `get-kogito-ddl-scripts.sh` can be used to download the needed postgres DDL files into the application. + +You can run your application in dev mode that enables live coding using: + +```shell script +./get-kogito-ddl-scripts.sh +./mvnw compile quarkus:dev +``` + +## Build jar file + +You can build the jar file with the script `build-db-migrator-jar.sh` which places the jar into /tmp/ kogito-db-migrator-tool directory for use by the corresponding image builder in package kogito-db-migrator-tool-image later, which can be found [here](../kogito-db-migrator-tool-image/README.md). + +```shell +./build-db-migrator-jar.sh +``` diff --git a/packages/kogito-db-migrator-tool/build-db-migrator-jar.sh b/packages/kogito-db-migrator-tool/build-db-migrator-jar.sh new file mode 100755 index 00000000000..6be70586900 --- /dev/null +++ b/packages/kogito-db-migrator-tool/build-db-migrator-jar.sh @@ -0,0 +1,45 @@ +#!/bin/sh +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -x +set -e + +# Directory where the DB Migrator Jar will be dropped +OUTPUT_DIR=/tmp/kogito-db-migrator-tool +TARGET_JAR_FILE=./target/sonataflow-db-migrator-0.0.0-runner.jar +DDL_DIR=./src/main/resources/postgresql + +rm -rf $OUTPUT_DIR +mkdir -p $OUTPUT_DIR + +./get-kogito-ddl-scripts.sh + +if [ "$1" = "true" ]; then + echo "Using dev version:" + mvn clean install -DskipTests -Dquarkus.package.jar.type=uber-jar +else + echo "Using prod version:" + mvn clean install -DskipTests=$(build-env tests.run --not) -Dmaven.test.failure.ignore=$(build-env tests.ignoreFailures) -Dquarkus.package.jar.type=uber-jar +fi + +cp $TARGET_JAR_FILE $OUTPUT_DIR + +# As the jar is built and ddl files are included, cleanup by deleting the ddl files +rm -rf $DDL_DIR diff --git a/packages/kogito-db-migrator-tool/env/index.js b/packages/kogito-db-migrator-tool/env/index.js new file mode 100644 index 00000000000..42ad5024d52 --- /dev/null +++ b/packages/kogito-db-migrator-tool/env/index.js @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +const { varsWithName, composeEnv } = require("@kie-tools-scripts/build-env"); + +module.exports = composeEnv([require("@kie-tools/root-env/env")], { + vars: varsWithName({}), + get env() { + return { + devDeploymentQuarkusApp: { + version: require("../package.json").version, + }, + }; + }, +}); diff --git a/packages/kogito-db-migrator-tool/get-kogito-ddl-scripts.sh b/packages/kogito-db-migrator-tool/get-kogito-ddl-scripts.sh new file mode 100755 index 00000000000..cc88b91eb0b --- /dev/null +++ b/packages/kogito-db-migrator-tool/get-kogito-ddl-scripts.sh @@ -0,0 +1,64 @@ +#!/bin/sh +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -x +set -e + +# Script variables with default values. These values will be updated by hack/bump-version.sh, don't change it. +DDL_VERSION=10.0.999-SNAPSHOT +DDL_FILE=kogito-ddl-10.0.999-20241018.012042-73-db-scripts.zip +DDL_BASE_URL=https://repository.apache.org/content/groups/snapshots/org/kie/kogito/kogito-ddl +TMP_DIR=./tmp +RESOURCES_DIR=src/main/resources +POSTGRESQL_DDL_DIR=postgresql + +# cleanup temporary files +cleanup () { + echo "Cleaning up" + rm -rf ${RESOURCES_DIR}"/"${POSTGRESQL_DDL_DIR} + rm -rf $TMP_DIR +} + +# Use specific variable values, if passed +for arg in "$@"; do + case "$arg" in + DDL_VERSION=*) DDL_VERSION="${arg#*=}" ;; + DDL_FILE=*) DDL_FILE="${arg#*=}" ;; + DDL_BASE_URL=*) DDL_BASE_URL="${arg#*=}" ;; + esac +done + +DDL_URL=$DDL_BASE_URL/$DDL_VERSION/$DDL_FILE + +printf "Variables being used for the DDL: \n DDL_VERSION: %s, \n DDL_FILE: %s, \n DDL_BASE_URL: %s \n\n" "$DDL_VERSION" "$DDL_FILE" "$DDL_BASE_URL" + +# Start with cleanup +cleanup + +# Get Data Index/ Jobs Service DDL Files +mkdir -p $TMP_DIR +# Change the variables below, as needed +wget "$DDL_URL" +mv "$DDL_FILE" $TMP_DIR +cd $TMP_DIR || exit +unzip "$DDL_FILE" +mv "./"${POSTGRESQL_DDL_DIR} "../"${RESOURCES_DIR} +cd .. || exit +rm -rf $TMP_DIR diff --git a/packages/kogito-db-migrator-tool/install.js b/packages/kogito-db-migrator-tool/install.js new file mode 100644 index 00000000000..61f0e221a7b --- /dev/null +++ b/packages/kogito-db-migrator-tool/install.js @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +const { env } = require("./env"); +const { setupMavenConfigFile, installMvnw, buildTailFromPackageJsonDependencies } = require("@kie-tools/maven-base"); + +setupMavenConfigFile( + ` + --batch-mode + -Dstyle.color=always + -Drevision=${env.devDeploymentQuarkusApp.version} + -Dversion.quarkus=${env.versions.quarkus} + -Dversion.org.kie.kogito=${env.versions.kogito} + -Dmaven.repo.local.tail=${buildTailFromPackageJsonDependencies()} +`, + { ignoreDefault: true } // Can't have special configuration that only works inside this repo. +); + +installMvnw(); diff --git a/packages/kogito-db-migrator-tool/package.json b/packages/kogito-db-migrator-tool/package.json new file mode 100644 index 00000000000..8dc4ca795f5 --- /dev/null +++ b/packages/kogito-db-migrator-tool/package.json @@ -0,0 +1,33 @@ +{ + "private": true, + "name": "@kie-tools/kogito-db-migrator-tool", + "version": "0.0.0", + "description": "", + "license": "Apache-2.0", + "homepage": "https://github.com/apache/incubator-kie-tools", + "repository": { + "type": "git", + "url": "https://github.com/apache/incubator-kie-tools.git" + }, + "bugs": { + "url": "https://github.com/apache/incubator-kie-tools/issues" + }, + "scripts": { + "build:dev": "./build-db-migrator-jar.sh true", + "build:prod": "./build-db-migrator-jar.sh false", + "install": "node install.js" + }, + "dependencies": { + "@kie-tools/maven-base": "workspace:*" + }, + "devDependencies": { + "@kie-tools/root-env": "workspace:*", + "run-script-os": "^1.1.6" + }, + "kieTools": { + "requiredPreinstalledCliCommands": [ + "java", + "mvn" + ] + } +} diff --git a/packages/kogito-db-migrator-tool/pom.xml b/packages/kogito-db-migrator-tool/pom.xml new file mode 100644 index 00000000000..25ae0882008 --- /dev/null +++ b/packages/kogito-db-migrator-tool/pom.xml @@ -0,0 +1,213 @@ + + + + + org.kie + kie-tools-maven-base + ${revision} + ./node_modules/@kie-tools/maven-base/pom.xml + + + 4.0.0 + org.kie.kogito + sonataflow-db-migrator + + 2024-11-01T00:00:00Z + 3.13.0 + UTF-8 + UTF-8 + quarkus-bom + io.quarkus.platform + 3.13.0 + true + 3.2.5 + 0.8.12 + 1.19.0 + + + + + + ${quarkus.platform.group-id} + ${quarkus.platform.artifact-id} + ${quarkus.platform.version} + pom + import + + + + + + + io.quarkus + quarkus-flyway + + + io.quarkus + quarkus-jdbc-postgresql + + + org.flywaydb + flyway-database-postgresql + + + io.quarkus + quarkus-arc + + + io.quarkus + quarkus-junit5 + test + + + org.mockito + mockito-junit-jupiter + test + + + org.mockito + mockito-core + test + + + com.github.stefanbirkner + system-rules + ${systemrules.version} + test + + + org.junit.jupiter + junit-jupiter + test + + + + + src/test/java + + + ${quarkus.platform.group-id} + quarkus-maven-plugin + ${quarkus.platform.version} + true + + + + build + generate-code + generate-code-tests + native-image-agent + + + + + + org.apache.maven.plugins + maven-compiler-plugin + ${compiler-plugin.version} + + + -parameters + + + + + maven-surefire-plugin + ${surefire-plugin.version} + + + org.jboss.logmanager.LogManager + ${maven.home} + + + + + maven-failsafe-plugin + ${surefire-plugin.version} + + + + integration-test + verify + + + + + + ${project.build.directory}/${project.build.finalName}-runner + + org.jboss.logmanager.LogManager + ${maven.home} + + + + + org.jacoco + jacoco-maven-plugin + ${jacoco.version} + + + + prepare-agent + + + + report + prepare-package + + report + + + + + + + + + + native + + + native + + + + false + true + + + + + + + apache + Apache Maven + default + https://repo.maven.apache.org/maven2/ + + false + + + + diff --git a/packages/kogito-db-migrator-tool/src/main/cekit/image.yaml b/packages/kogito-db-migrator-tool/src/main/cekit/image.yaml new file mode 100644 index 00000000000..a2517ee0154 --- /dev/null +++ b/packages/kogito-db-migrator-tool/src/main/cekit/image.yaml @@ -0,0 +1,45 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +name: "docker.io/apache/incubator-kie-kogito-service-db-migration-postgresql" +version: "999-SNAPSHOT" +from: registry.access.redhat.com/ubi8/openjdk-17-runtime:1.19 +description: Flyway image for DI/JS database migration + +labels: + - name: "org.kie.kogito.version" + value: "999-SNAPSHOT" + - name: "maintainer" + value: "Apache KIE " + - name: "io.k8s.description" + value: "Kogito DB Migration creates schemas and tables for Data Index and Jobs Service for PostgreSQL database" + - name: "io.k8s.display-name" + value: "Kogito DB Migration for Data Index and Jobs Service - PostgreSQL" + - name: "io.openshift.tags" + value: "kogito,db-migration" + +modules: + repositories: + - path: modules + install: + - name: kogito-postgres-db-migration-deps + +run: + workdir: "/home/kogito" + entrypoint: + - "/home/kogito/migration.sh" diff --git a/packages/kogito-db-migrator-tool/src/main/cekit/modules/kogito-postgres-db-migration-deps/install b/packages/kogito-db-migrator-tool/src/main/cekit/modules/kogito-postgres-db-migration-deps/install new file mode 100644 index 00000000000..5fa1a3e0e86 --- /dev/null +++ b/packages/kogito-db-migrator-tool/src/main/cekit/modules/kogito-postgres-db-migration-deps/install @@ -0,0 +1,32 @@ +#!/bin/sh +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -x +set -e + +useradd -m kogito + +cd /home/kogito || exit + +chgrp -R 0 /home/kogito +chown -R 0 /home/kogito +chmod -R g=u /home/kogito + +chmod +x /home/kogito/migration.sh \ No newline at end of file diff --git a/packages/kogito-db-migrator-tool/src/main/cekit/modules/kogito-postgres-db-migration-deps/migration.sh b/packages/kogito-db-migrator-tool/src/main/cekit/modules/kogito-postgres-db-migration-deps/migration.sh new file mode 100644 index 00000000000..fe41d336d5b --- /dev/null +++ b/packages/kogito-db-migrator-tool/src/main/cekit/modules/kogito-postgres-db-migration-deps/migration.sh @@ -0,0 +1,36 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +set -x +set -e + +OPERATOR_VERSION=999.0.0 + +for arg in "$@"; do + case "$arg" in + OPERATOR_VERSION=*) OPERATOR_VERSION="${arg#*=}" ;; + esac +done + +cd /home/kogito/ + +mv sonataflow-db-migrator-runner.jar sonataflow-db-migrator-"$OPERATOR_VERSION"-runner.jar + +java -jar sonataflow-db-migrator-"$OPERATOR_VERSION"-runner.jar \ No newline at end of file diff --git a/packages/kogito-db-migrator-tool/src/main/cekit/modules/kogito-postgres-db-migration-deps/module.yaml b/packages/kogito-db-migrator-tool/src/main/cekit/modules/kogito-postgres-db-migration-deps/module.yaml new file mode 100644 index 00000000000..0605dacc689 --- /dev/null +++ b/packages/kogito-db-migrator-tool/src/main/cekit/modules/kogito-postgres-db-migration-deps/module.yaml @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +schema_version: 1 +name: kogito-postgres-db-migration-deps +version: "1.0" +artifacts: + - name: migration.sh + path: migration.sh + dest: /home/kogito + - name: sonataflow-db-migrator-runner.jar + path: sonataflow-db-migrator-runner.jar + dest: /home/kogito +execute: + - script: install diff --git a/packages/kogito-db-migrator-tool/src/main/java/org/kie/kogito/postgresql/migrator/DBConnectionChecker.java b/packages/kogito-db-migrator-tool/src/main/java/org/kie/kogito/postgresql/migrator/DBConnectionChecker.java new file mode 100644 index 00000000000..c9143f850cc --- /dev/null +++ b/packages/kogito-db-migrator-tool/src/main/java/org/kie/kogito/postgresql/migrator/DBConnectionChecker.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.kie.kogito.postgresql.migrator; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import io.quarkus.logging.Log; + +import org.eclipse.microprofile.config.inject.ConfigProperty; + +import jakarta.enterprise.context.ApplicationScoped; + +@ApplicationScoped +public class DBConnectionChecker { + @ConfigProperty(name = "quarkus.datasource.dataindex.jdbc.url") + String dataIndexDBURL; + + @ConfigProperty(name = "quarkus.datasource.dataindex.username") + String dataIndexDBUserName; + + @ConfigProperty(name = "quarkus.datasource.dataindex.password") + String dataIndexDBPassword; + + @ConfigProperty(name = "quarkus.datasource.jobsservice.jdbc.url") + String jobsServiceDBURL; + + @ConfigProperty(name = "quarkus.datasource.jobsservice.username") + String jobsServiceDBUserName; + + @ConfigProperty(name = "quarkus.datasource.jobsservice.password") + String jobsServiceDBPassword; + + private void checkDBConnection(String dbURL, String dbUser, String dbPassword) throws SQLException { + try (Connection db = DriverManager.getConnection(dbURL, dbUser, dbPassword)) { + Log.infof("Checking DB connection: %s - success", dbURL); + } catch (SQLException sqe) { + Log.infof("Checking DB connection %s- failed", dbURL); + throw sqe; + } + } + + public void checkDataIndexDBConnection() throws SQLException { + checkDBConnection(dataIndexDBURL, dataIndexDBUserName, dataIndexDBPassword); + } + + public void checkJobsServiceDBConnection() throws SQLException { + checkDBConnection(jobsServiceDBURL, jobsServiceDBUserName, jobsServiceDBPassword); + } +} diff --git a/packages/kogito-db-migrator-tool/src/main/java/org/kie/kogito/postgresql/migrator/DBMigrator.java b/packages/kogito-db-migrator-tool/src/main/java/org/kie/kogito/postgresql/migrator/DBMigrator.java new file mode 100644 index 00000000000..bb69266f4e0 --- /dev/null +++ b/packages/kogito-db-migrator-tool/src/main/java/org/kie/kogito/postgresql/migrator/DBMigrator.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.kie.kogito.postgresql.migrator; + +import io.quarkus.runtime.Quarkus; +import io.quarkus.runtime.QuarkusApplication; +import io.quarkus.runtime.annotations.QuarkusMain; +import jakarta.inject.Inject; +import io.quarkus.logging.Log; + +import java.sql.SQLException; + +import org.eclipse.microprofile.config.inject.ConfigProperty; + +@QuarkusMain +public class DBMigrator implements QuarkusApplication { + + @Inject + MigrationService service; + + @Inject + DBConnectionChecker dbConnectionChecker; + + @ConfigProperty(name = "migrate.db.dataindex") + Boolean migrateDataIndex; + + @ConfigProperty(name = "migrate.db.jobsservice") + Boolean migrateJobsService; + + @Override + public int run(String... args) { + if (migrateDataIndex) { + try { + dbConnectionChecker.checkDataIndexDBConnection(); + } catch (SQLException e) { + Log.error( "Error obtaining data index database connection. Cannot proceed, exiting."); + Quarkus.asyncExit(-1); + return -1; + } + service.migrateDataIndex(); + } + + if (migrateJobsService) { + try { + dbConnectionChecker.checkJobsServiceDBConnection(); + } catch (SQLException e) { + Log.error( "Error obtaining jobs service database connection. Cannot proceed, exiting."); + Quarkus.asyncExit(-2); + return -2; + } + service.migrateJobsService(); + } + + Quarkus.asyncExit(0); + return 0; + } +} \ No newline at end of file diff --git a/packages/kogito-db-migrator-tool/src/main/java/org/kie/kogito/postgresql/migrator/MigrationService.java b/packages/kogito-db-migrator-tool/src/main/java/org/kie/kogito/postgresql/migrator/MigrationService.java new file mode 100644 index 00000000000..233539cc35a --- /dev/null +++ b/packages/kogito-db-migrator-tool/src/main/java/org/kie/kogito/postgresql/migrator/MigrationService.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.kie.kogito.postgresql.migrator; + +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.inject.Inject; +import io.quarkus.logging.Log; + +import org.eclipse.microprofile.config.inject.ConfigProperty; +import org.flywaydb.core.Flyway; + +import io.quarkus.flyway.FlywayDataSource; + +@ApplicationScoped +public class MigrationService { + @Inject + @FlywayDataSource("dataindex") + Flyway flywayDataIndex; + + @Inject + @FlywayDataSource("jobsservice") + Flyway flywayJobsService; + + @ConfigProperty(name = "quarkus.flyway.dataindex.clean-at-start") + Boolean cleanDataIndex; + + @ConfigProperty(name = "quarkus.flyway.jobsservice.clean-at-start") + Boolean cleanJobsService; + + private void migrateDB(Flyway flywayService, Boolean clean, String serviceName) { + Log.info("Migrating " + serviceName); + if (clean) { + Log.info("Cleaned the " + serviceName); + flywayService.clean(); + } + flywayService.migrate(); + if (flywayService.info() != null) { + Log.info(flywayService.info().current().getVersion().toString()); + } + } + + public void migrateDataIndex() { + migrateDB(flywayDataIndex, cleanDataIndex, "data-index"); + } + + public void migrateJobsService() { + migrateDB(flywayJobsService, cleanJobsService, "jobs-service"); + } +} \ No newline at end of file diff --git a/packages/kogito-db-migrator-tool/src/main/resources/application.properties b/packages/kogito-db-migrator-tool/src/main/resources/application.properties new file mode 100644 index 00000000000..59b8c2382a5 --- /dev/null +++ b/packages/kogito-db-migrator-tool/src/main/resources/application.properties @@ -0,0 +1,40 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +# Data Index data source +migrate.db.dataindex=false +quarkus.datasource.dataindex.db-kind=postgresql +quarkus.datasource.dataindex.username=postgres +quarkus.datasource.dataindex.password=postgres +quarkus.datasource.dataindex.jdbc.url=jdbc:postgresql://localhost:5432/postgres +quarkus.flyway.dataindex.locations=classpath:postgresql/data-index +quarkus.flyway.dataindex.schemas=dataindex +quarkus.flyway.dataindex.migrate-at-start=false +quarkus.flyway.dataindex.clean-at-start=false + +# Jobs Service data source +migrate.db.jobsservice=false +quarkus.datasource.jobsservice.db-kind=postgresql +quarkus.datasource.jobsservice.username=postgres +quarkus.datasource.jobsservice.password=postgres +quarkus.datasource.jobsservice.jdbc.url=jdbc:postgresql://localhost:5432/postgres +quarkus.flyway.jobsservice.locations=classpath:postgresql/jobs-service +quarkus.flyway.jobsservice.schemas=jobsservice +quarkus.flyway.jobsservice.migrate-at-start=false +quarkus.flyway.jobsservice.clean-at-start=false \ No newline at end of file diff --git a/packages/kogito-db-migrator-tool/src/test/java/org/kie/kogito/postgresql/migrator/DBConnectionCheckerTest.java b/packages/kogito-db-migrator-tool/src/test/java/org/kie/kogito/postgresql/migrator/DBConnectionCheckerTest.java new file mode 100644 index 00000000000..935eed5c382 --- /dev/null +++ b/packages/kogito-db-migrator-tool/src/test/java/org/kie/kogito/postgresql/migrator/DBConnectionCheckerTest.java @@ -0,0 +1,68 @@ +/* + * Copyright 2024 Apache Software Foundation (ASF) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.kie.kogito.postgresql.migrator; + +import io.quarkus.test.Mock; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mockStatic; + +public class DBConnectionCheckerTest { + DBConnectionChecker dbConnectionChecker = new DBConnectionChecker(); + + @Mock + static DriverManager driverManager; + + @BeforeAll + public static void init() { + mockStatic(DriverManager.class); + } + + @BeforeEach + public void setupEach() { + dbConnectionChecker.dataIndexDBURL = "jdbc:postgresql://db-service:5432/di"; + dbConnectionChecker.dataIndexDBUserName = "postgres"; + dbConnectionChecker.dataIndexDBPassword = "postgres"; + + dbConnectionChecker.jobsServiceDBURL = "jdbc:postgresql://db-service:5432/js"; + dbConnectionChecker.jobsServiceDBUserName = "postgres"; + dbConnectionChecker.jobsServiceDBPassword = "postgres"; + } + + @Test + public void testCheckDBConnections() throws SQLException { + Mockito.when(driverManager.getConnection(anyString(), anyString(), anyString())).thenReturn(Mockito.mock(Connection.class)); + assertDoesNotThrow(() -> dbConnectionChecker.checkDataIndexDBConnection()); + assertDoesNotThrow(() -> dbConnectionChecker.checkJobsServiceDBConnection()); + } + + @Test + public void testCheckDBConnectionsThrowSQLException() throws SQLException { + Mockito.when(driverManager.getConnection(anyString(), anyString(), anyString())).thenThrow(SQLException.class); + assertThrows(SQLException.class, () -> dbConnectionChecker.checkDataIndexDBConnection()); + assertThrows(SQLException.class, () -> dbConnectionChecker.checkJobsServiceDBConnection()); + } +} diff --git a/packages/kogito-db-migrator-tool/src/test/java/org/kie/kogito/postgresql/migrator/DBMigratorTest.java b/packages/kogito-db-migrator-tool/src/test/java/org/kie/kogito/postgresql/migrator/DBMigratorTest.java new file mode 100644 index 00000000000..3ee9c6e0555 --- /dev/null +++ b/packages/kogito-db-migrator-tool/src/test/java/org/kie/kogito/postgresql/migrator/DBMigratorTest.java @@ -0,0 +1,94 @@ +/* + * Copyright 2024 Apache Software Foundation (ASF) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.kie.kogito.postgresql.migrator; + +import io.quarkus.test.Mock; +import org.junit.Rule; +import org.junit.contrib.java.lang.system.ExpectedSystemExit; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.sql.SQLException; + +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.mock; + +public class DBMigratorTest { + @Rule + public final ExpectedSystemExit exitRule = ExpectedSystemExit.none(); + + @Mock + MigrationService migrationService; + + @Mock + DBConnectionChecker dbConnectionChecker; + + DBMigrator dbMigrator = new DBMigrator(); + + @BeforeEach + public void setupEach() { + migrationService = mock(MigrationService.class); + dbConnectionChecker = mock(DBConnectionChecker.class); + } + + @Test + public void testMigratorWithNoMigrations() throws Exception { + dbMigrator.migrateDataIndex = false; + dbMigrator.migrateJobsService = false; + + exitRule.expectSystemExitWithStatus(0); + dbMigrator.run(); + } + + @Test + public void testMigratorWithAllMigrations() throws Exception { + dbMigrator.migrateDataIndex = true; + dbMigrator.migrateJobsService = true; + dbMigrator.dbConnectionChecker = dbConnectionChecker; + dbMigrator.service = migrationService; + + exitRule.expectSystemExitWithStatus(0); + dbMigrator.run(); + } + + @Test + public void testDataIndexMigrationWithException() throws Exception { + dbMigrator.migrateDataIndex = true; + dbMigrator.migrateJobsService = false; + dbMigrator.dbConnectionChecker = dbConnectionChecker; + dbMigrator.service = migrationService; + + doThrow(new SQLException()).when(dbConnectionChecker).checkDataIndexDBConnection(); + + exitRule.expectSystemExitWithStatus(-1); + dbMigrator.run(); + } + + @Test + public void testJobsServiceWithException() throws Exception { + dbMigrator.migrateDataIndex = false; + dbMigrator.migrateJobsService = true; + dbMigrator.dbConnectionChecker = dbConnectionChecker; + dbMigrator.service = migrationService; + + doThrow(new SQLException()).when(dbConnectionChecker).checkJobsServiceDBConnection(); + + exitRule.expectSystemExitWithStatus(-2); + dbMigrator.run(); + } +} diff --git a/packages/kogito-db-migrator-tool/src/test/java/org/kie/kogito/postgresql/migrator/MigrationServiceTest.java b/packages/kogito-db-migrator-tool/src/test/java/org/kie/kogito/postgresql/migrator/MigrationServiceTest.java new file mode 100644 index 00000000000..0c4f497cc81 --- /dev/null +++ b/packages/kogito-db-migrator-tool/src/test/java/org/kie/kogito/postgresql/migrator/MigrationServiceTest.java @@ -0,0 +1,69 @@ +/* + * Copyright 2024 Apache Software Foundation (ASF) + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.kie.kogito.postgresql.migrator; + +import io.quarkus.test.Mock; +import org.flywaydb.core.Flyway; +import org.flywaydb.core.api.output.CleanResult; +import org.flywaydb.core.api.output.MigrateResult; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.mockito.Mockito.*; + +public class MigrationServiceTest { + @Mock + Flyway flyway; + + MigrationService migrationService = new MigrationService(); + + @BeforeEach + public void setupEach() { + flyway = mock(Flyway.class); + when(flyway.migrate()).thenReturn(new MigrateResult("flywayVersion", "db", "schema", "postgres")); + when(flyway.clean()).thenReturn(new CleanResult("flywayVersion", "db")); + } + + @Test + public void testMigrateDataIndexWithNoClean() { + migrationService.cleanDataIndex = false; + migrationService.flywayDataIndex = flyway; + migrationService.migrateDataIndex(); + } + + @Test + public void testMigrateDataIndexWithClean() { + migrationService.cleanDataIndex = true; + migrationService.flywayDataIndex = flyway; + migrationService.migrateDataIndex(); + } + + @Test + public void testMigrateJobsServiceWithNoClean() { + migrationService.cleanJobsService = false; + migrationService.flywayJobsService = flyway; + migrationService.migrateJobsService(); + } + + @Test + public void testMigrateJobsServiceWithClean() { + migrationService.cleanJobsService = true; + migrationService.flywayJobsService = flyway; + migrationService.migrateJobsService(); + } +} From b0c5d67575b18207e9f91495b977b84a2dcf5f30 Mon Sep 17 00:00:00 2001 From: rhkp Date: Wed, 23 Oct 2024 20:53:44 +0000 Subject: [PATCH 02/18] Address review comments --- .gitignore | 3 - .../kogito-db-migrator-tool-image/README.md | 123 ++++++++++++++---- .../build-container-image.sh | 1 + .../cekit/image.yaml | 15 ++- .../kogito-postgres-db-migration-deps/install | 32 ----- .../migration.sh | 36 ----- .../module.yaml | 5 - packages/kogito-db-migrator-tool/README.md | 2 +- packages/kogito-db-migrator-tool/install.js | 2 - packages/kogito-db-migrator-tool/pom.xml | 19 +-- .../src/main/cekit/image.yaml | 45 ------- .../kogito-postgres-db-migration-deps/install | 32 ----- .../migration.sh | 36 ----- .../module.yaml | 30 ----- 14 files changed, 112 insertions(+), 269 deletions(-) delete mode 100644 packages/kogito-db-migrator-tool-image/cekit/modules/kogito-postgres-db-migration-deps/install delete mode 100644 packages/kogito-db-migrator-tool-image/cekit/modules/kogito-postgres-db-migration-deps/migration.sh delete mode 100644 packages/kogito-db-migrator-tool/src/main/cekit/image.yaml delete mode 100644 packages/kogito-db-migrator-tool/src/main/cekit/modules/kogito-postgres-db-migration-deps/install delete mode 100644 packages/kogito-db-migrator-tool/src/main/cekit/modules/kogito-postgres-db-migration-deps/migration.sh delete mode 100644 packages/kogito-db-migrator-tool/src/main/cekit/modules/kogito-postgres-db-migration-deps/module.yaml diff --git a/.gitignore b/.gitignore index 750f639a452..0fa740642fb 100644 --- a/.gitignore +++ b/.gitignore @@ -346,6 +346,3 @@ turbo.json # IntelliJ projects should be on _intellij-projects folder, never on specific packages. packages/*/.idea examples/*/.idea - -# Jar files -**/*.jar \ No newline at end of file diff --git a/packages/kogito-db-migrator-tool-image/README.md b/packages/kogito-db-migrator-tool-image/README.md index db325462e39..dca5cd3bca1 100644 --- a/packages/kogito-db-migrator-tool-image/README.md +++ b/packages/kogito-db-migrator-tool-image/README.md @@ -1,37 +1,61 @@ -# kogito-db-migrator-tool-image + -This is a cekit based image builder for the database migrator application. Details about the database migrator application are [here](../kogito-db-migrator-tool/README.md) +# Kogito Postgres DB Migrator Tool Image -## Build and Run container image locally +This package contains the `Containerfile/Dockerfile` and scripts to build a container image for Kogito Postgres DB Migrator Tool. Details about the Kogito Postgres DB Migrator Tool can be found [here](../kogito-db-migrator-tool/README.md) -You can build the cekit build of container image by using the provided image builder shell script +## Additional requirements -```shell -./build-container-image.sh -``` +- docker + +## Build + +_NOTE_: Before performing this step, be sure that the Kogito Postgres DB Migrator Tool jar has been built and available for inclusion in the image. + +- Enable the image to be built: + + ```bash + export KIE_TOOLS_BUILD__buildContainerImages=true + ``` + +Run the following in the root folder of the repository to build the package: -_NOTE_: Before performing this step, be sure that the database migrator application jar has been built and available for inclusion in the image. - -Ensure the script completes without errors. - -Assuming you have a Postgres database running locally, e.g., a `di` database for data index and a `js` database for jobs service, you can run the image with the following command. Substitute appropriate values: - -```shell -podman run \ ---env MIGRATE_DB_DATAINDEX=true \ ---env QUARKUS_DATASOURCE_DATAINDEX_JDBC_URL= \ ---env QUARKUS_DATASOURCE_DATAINDEX_USERNAME= \ ---env QUARKUS_DATASOURCE_DATAINDEX_PASSWORD= \ ---env QUARKUS_FLYWAY_DATAINDEX_SCHEMAS=dataindex \ ---env MIGRATE_DB_JOBSSERVICE=true \ ---env QUARKUS_DATASOURCE_JOBSSERVICE_JDBC_URL= \ ---env QUARKUS_DATASOURCE_JOBSSERVICE_USERNAME= \ ---env QUARKUS_DATASOURCE_JOBSSERVICE_PASSWORD= \ ---env QUARKUS_FLYWAY_JOBSSERVICE_SCHEMAS=jobsservice \ -docker.io/apache/incubator-kie-kogito-service-db-migration-postgresql:999-SNAPSHOT +```bash +pnpm -F @kie-tools/kogito-db-migrator-tool-image... build:prod ``` -### Environment variables +- Then check if the image is correctly stored: + + ```bash + docker images + ``` + +## Run + +- Start up a clean container with: + + ```bash + docker run docker.io/apache/incubator-kie-kogito-service-db-migration-postgresql:latest + ``` + +## Customization + +1. Run a container with custom environment variables: | NAME | DESCRIPTION | DEFAULT | | --------------------------------------- | -------------------------------------------------------------------------------- | ----------------------------------------- | @@ -45,3 +69,48 @@ docker.io/apache/incubator-kie-kogito-service-db-migration-postgresql:999-SNAPSH | QUARKUS_DATASOURCE_JOBSSERVICE_USERNAME | Jobs service database username | postgres | | QUARKUS_DATASOURCE_JOBSSERVICE_PASSWORD | Jobs service database password | postgres | | QUARKUS_FLYWAY_JOBSSERVICE_SCHEMAS | Jobs service database schema | jobsservice | + +### Example + +An example to use diverse environment variables + +```bash + docker run \ + --env MIGRATE_DB_DATAINDEX=true \ + --env QUARKUS_DATASOURCE_DATAINDEX_JDBC_URL= \ + --env QUARKUS_DATASOURCE_DATAINDEX_USERNAME= \ + --env QUARKUS_DATASOURCE_DATAINDEX_PASSWORD= \ + --env QUARKUS_FLYWAY_DATAINDEX_SCHEMAS=dataindex \ + --env MIGRATE_DB_JOBSSERVICE=true \ + --env QUARKUS_DATASOURCE_JOBSSERVICE_JDBC_URL= \ + --env QUARKUS_DATASOURCE_JOBSSERVICE_USERNAME= \ + --env QUARKUS_DATASOURCE_JOBSSERVICE_PASSWORD= \ + --env QUARKUS_FLYWAY_JOBSSERVICE_SCHEMAS=jobsservice \ + docker.io/apache/incubator-kie-kogito-service-db-migration-postgresql:999-SNAPSHOT +``` + +--- + +Apache KIE (incubating) is an effort undergoing incubation at The Apache Software +Foundation (ASF), sponsored by the name of Apache Incubator. Incubation is +required of all newly accepted projects until a further review indicates that +the infrastructure, communications, and decision making process have stabilized +in a manner consistent with other successful ASF projects. While incubation +status is not necessarily a reflection of the completeness or stability of the +code, it does indicate that the project has yet to be fully endorsed by the ASF. + +Some of the incubating project’s releases may not be fully compliant with ASF +policy. For example, releases may have incomplete or un-reviewed licensing +conditions. What follows is a list of known issues the project is currently +aware of (note that this list, by definition, is likely to be incomplete): + +- Hibernate, an LGPL project, is being used. Hibernate is in the process of + relicensing to ASL v2 +- Some files, particularly test files, and those not supporting comments, may + be missing the ASF Licensing Header + +If you are planning to incorporate this work into your product/project, please +be aware that you will need to conduct a thorough licensing review to determine +the overall implications of including this work. For the current status of this +project through the Apache Incubator visit: +https://incubator.apache.org/projects/kie.html diff --git a/packages/kogito-db-migrator-tool-image/build-container-image.sh b/packages/kogito-db-migrator-tool-image/build-container-image.sh index d4bac957570..43952790e81 100755 --- a/packages/kogito-db-migrator-tool-image/build-container-image.sh +++ b/packages/kogito-db-migrator-tool-image/build-container-image.sh @@ -33,6 +33,7 @@ if [ -f "$SOURCE_FILE" ]; then # Build the container image cd ./cekit || exit cekit -v build "$CEKIT_BUILDER" + cd .. || exit rm -f $TARGET_FILE else echo "The file: $SOURCE_FILE not found. Please build kogito-db-migrator-tool package first before building the image." diff --git a/packages/kogito-db-migrator-tool-image/cekit/image.yaml b/packages/kogito-db-migrator-tool-image/cekit/image.yaml index a2517ee0154..833ae4931f2 100644 --- a/packages/kogito-db-migrator-tool-image/cekit/image.yaml +++ b/packages/kogito-db-migrator-tool-image/cekit/image.yaml @@ -18,12 +18,12 @@ # name: "docker.io/apache/incubator-kie-kogito-service-db-migration-postgresql" version: "999-SNAPSHOT" -from: registry.access.redhat.com/ubi8/openjdk-17-runtime:1.19 +from: registry.access.redhat.com/ubi8/openjdk-17-runtime:1.20 description: Flyway image for DI/JS database migration labels: - name: "org.kie.kogito.version" - value: "999-SNAPSHOT" + value: "### SET ME DURING BUILD PROCESS ###" - name: "maintainer" value: "Apache KIE " - name: "io.k8s.description" @@ -35,11 +35,18 @@ labels: modules: repositories: - - path: modules + - name: sonataflow-image-common + path: ../../sonataflow-image-common/resources/modules/kogito-system-user + - name: kogito-db-migrator-tool-image + path: modules install: + - name: org.kie.kogito.system.user - name: kogito-postgres-db-migration-deps run: workdir: "/home/kogito" entrypoint: - - "/home/kogito/migration.sh" + - "java" + cmd: + - "-jar" + - "sonataflow-db-migrator-runner.jar" diff --git a/packages/kogito-db-migrator-tool-image/cekit/modules/kogito-postgres-db-migration-deps/install b/packages/kogito-db-migrator-tool-image/cekit/modules/kogito-postgres-db-migration-deps/install deleted file mode 100644 index 5fa1a3e0e86..00000000000 --- a/packages/kogito-db-migrator-tool-image/cekit/modules/kogito-postgres-db-migration-deps/install +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/sh -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -set -x -set -e - -useradd -m kogito - -cd /home/kogito || exit - -chgrp -R 0 /home/kogito -chown -R 0 /home/kogito -chmod -R g=u /home/kogito - -chmod +x /home/kogito/migration.sh \ No newline at end of file diff --git a/packages/kogito-db-migrator-tool-image/cekit/modules/kogito-postgres-db-migration-deps/migration.sh b/packages/kogito-db-migrator-tool-image/cekit/modules/kogito-postgres-db-migration-deps/migration.sh deleted file mode 100644 index fe41d336d5b..00000000000 --- a/packages/kogito-db-migrator-tool-image/cekit/modules/kogito-postgres-db-migration-deps/migration.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -set -x -set -e - -OPERATOR_VERSION=999.0.0 - -for arg in "$@"; do - case "$arg" in - OPERATOR_VERSION=*) OPERATOR_VERSION="${arg#*=}" ;; - esac -done - -cd /home/kogito/ - -mv sonataflow-db-migrator-runner.jar sonataflow-db-migrator-"$OPERATOR_VERSION"-runner.jar - -java -jar sonataflow-db-migrator-"$OPERATOR_VERSION"-runner.jar \ No newline at end of file diff --git a/packages/kogito-db-migrator-tool-image/cekit/modules/kogito-postgres-db-migration-deps/module.yaml b/packages/kogito-db-migrator-tool-image/cekit/modules/kogito-postgres-db-migration-deps/module.yaml index 0605dacc689..7448604964f 100644 --- a/packages/kogito-db-migrator-tool-image/cekit/modules/kogito-postgres-db-migration-deps/module.yaml +++ b/packages/kogito-db-migrator-tool-image/cekit/modules/kogito-postgres-db-migration-deps/module.yaml @@ -20,11 +20,6 @@ schema_version: 1 name: kogito-postgres-db-migration-deps version: "1.0" artifacts: - - name: migration.sh - path: migration.sh - dest: /home/kogito - name: sonataflow-db-migrator-runner.jar path: sonataflow-db-migrator-runner.jar dest: /home/kogito -execute: - - script: install diff --git a/packages/kogito-db-migrator-tool/README.md b/packages/kogito-db-migrator-tool/README.md index 229ce15d3be..a22f4553051 100644 --- a/packages/kogito-db-migrator-tool/README.md +++ b/packages/kogito-db-migrator-tool/README.md @@ -1,4 +1,4 @@ -# kogito-db-migrator +# Kogito Postgres DB Migrator Tool This is a java quarkus based postgres database migrator application for Sonataflow Data Index and Jobs Service applications for use by SonataFlow Operator. diff --git a/packages/kogito-db-migrator-tool/install.js b/packages/kogito-db-migrator-tool/install.js index 61f0e221a7b..bc24d203a0d 100644 --- a/packages/kogito-db-migrator-tool/install.js +++ b/packages/kogito-db-migrator-tool/install.js @@ -31,5 +31,3 @@ setupMavenConfigFile( `, { ignoreDefault: true } // Can't have special configuration that only works inside this repo. ); - -installMvnw(); diff --git a/packages/kogito-db-migrator-tool/pom.xml b/packages/kogito-db-migrator-tool/pom.xml index 25ae0882008..52ab2825150 100644 --- a/packages/kogito-db-migrator-tool/pom.xml +++ b/packages/kogito-db-migrator-tool/pom.xml @@ -34,14 +34,11 @@ sonataflow-db-migrator 2024-11-01T00:00:00Z - 3.13.0 - UTF-8 - UTF-8 quarkus-bom io.quarkus.platform 3.13.0 true - 3.2.5 + 0.8.12 1.19.0 @@ -96,11 +93,6 @@ ${systemrules.version} test - - org.junit.jupiter - junit-jupiter - test - @@ -121,11 +113,6 @@ - - - org.apache.maven.plugins - maven-compiler-plugin - ${compiler-plugin.version} -parameters @@ -134,7 +121,7 @@ maven-surefire-plugin - ${surefire-plugin.version} + ${version.maven.surefire.plugin} org.jboss.logmanager.LogManager @@ -144,7 +131,7 @@ maven-failsafe-plugin - ${surefire-plugin.version} + ${version.maven.surefire.plugin} diff --git a/packages/kogito-db-migrator-tool/src/main/cekit/image.yaml b/packages/kogito-db-migrator-tool/src/main/cekit/image.yaml deleted file mode 100644 index a2517ee0154..00000000000 --- a/packages/kogito-db-migrator-tool/src/main/cekit/image.yaml +++ /dev/null @@ -1,45 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -name: "docker.io/apache/incubator-kie-kogito-service-db-migration-postgresql" -version: "999-SNAPSHOT" -from: registry.access.redhat.com/ubi8/openjdk-17-runtime:1.19 -description: Flyway image for DI/JS database migration - -labels: - - name: "org.kie.kogito.version" - value: "999-SNAPSHOT" - - name: "maintainer" - value: "Apache KIE " - - name: "io.k8s.description" - value: "Kogito DB Migration creates schemas and tables for Data Index and Jobs Service for PostgreSQL database" - - name: "io.k8s.display-name" - value: "Kogito DB Migration for Data Index and Jobs Service - PostgreSQL" - - name: "io.openshift.tags" - value: "kogito,db-migration" - -modules: - repositories: - - path: modules - install: - - name: kogito-postgres-db-migration-deps - -run: - workdir: "/home/kogito" - entrypoint: - - "/home/kogito/migration.sh" diff --git a/packages/kogito-db-migrator-tool/src/main/cekit/modules/kogito-postgres-db-migration-deps/install b/packages/kogito-db-migrator-tool/src/main/cekit/modules/kogito-postgres-db-migration-deps/install deleted file mode 100644 index 5fa1a3e0e86..00000000000 --- a/packages/kogito-db-migrator-tool/src/main/cekit/modules/kogito-postgres-db-migration-deps/install +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/sh -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -set -x -set -e - -useradd -m kogito - -cd /home/kogito || exit - -chgrp -R 0 /home/kogito -chown -R 0 /home/kogito -chmod -R g=u /home/kogito - -chmod +x /home/kogito/migration.sh \ No newline at end of file diff --git a/packages/kogito-db-migrator-tool/src/main/cekit/modules/kogito-postgres-db-migration-deps/migration.sh b/packages/kogito-db-migrator-tool/src/main/cekit/modules/kogito-postgres-db-migration-deps/migration.sh deleted file mode 100644 index fe41d336d5b..00000000000 --- a/packages/kogito-db-migrator-tool/src/main/cekit/modules/kogito-postgres-db-migration-deps/migration.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -set -x -set -e - -OPERATOR_VERSION=999.0.0 - -for arg in "$@"; do - case "$arg" in - OPERATOR_VERSION=*) OPERATOR_VERSION="${arg#*=}" ;; - esac -done - -cd /home/kogito/ - -mv sonataflow-db-migrator-runner.jar sonataflow-db-migrator-"$OPERATOR_VERSION"-runner.jar - -java -jar sonataflow-db-migrator-"$OPERATOR_VERSION"-runner.jar \ No newline at end of file diff --git a/packages/kogito-db-migrator-tool/src/main/cekit/modules/kogito-postgres-db-migration-deps/module.yaml b/packages/kogito-db-migrator-tool/src/main/cekit/modules/kogito-postgres-db-migration-deps/module.yaml deleted file mode 100644 index 0605dacc689..00000000000 --- a/packages/kogito-db-migrator-tool/src/main/cekit/modules/kogito-postgres-db-migration-deps/module.yaml +++ /dev/null @@ -1,30 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# -schema_version: 1 -name: kogito-postgres-db-migration-deps -version: "1.0" -artifacts: - - name: migration.sh - path: migration.sh - dest: /home/kogito - - name: sonataflow-db-migrator-runner.jar - path: sonataflow-db-migrator-runner.jar - dest: /home/kogito -execute: - - script: install From ff2f27e1566a66c3b876d98aedbffa86af66ff22 Mon Sep 17 00:00:00 2001 From: rhkp Date: Thu, 24 Oct 2024 19:52:28 +0000 Subject: [PATCH 03/18] Continuer addressing review comments --- .../env/index.js | 35 +++++++++++++- .../kogito-db-migrator-tool-image/install.js | 46 ++++++++++++++++++- .../package.json | 18 ++++++-- .../{ => resources}/build-container-image.sh | 23 ++++------ .../kogito-db-migrator-tool-image.yaml} | 11 ++--- .../module.yaml | 4 +- packages/kogito-db-migrator-tool/env/index.js | 2 +- packages/kogito-db-migrator-tool/install.js | 2 +- pnpm-lock.yaml | 40 ++++++++++++++++ 9 files changed, 151 insertions(+), 30 deletions(-) rename packages/kogito-db-migrator-tool-image/{ => resources}/build-container-image.sh (63%) rename packages/kogito-db-migrator-tool-image/{cekit/image.yaml => resources/kogito-db-migrator-tool-image.yaml} (82%) rename packages/kogito-db-migrator-tool-image/{cekit => resources}/modules/kogito-postgres-db-migration-deps/module.yaml (93%) diff --git a/packages/kogito-db-migrator-tool-image/env/index.js b/packages/kogito-db-migrator-tool-image/env/index.js index c3fe8a9b1e0..e0d521f19b4 100644 --- a/packages/kogito-db-migrator-tool-image/env/index.js +++ b/packages/kogito-db-migrator-tool-image/env/index.js @@ -17,4 +17,37 @@ * under the License. */ -console.log("index.js"); +const { varsWithName, composeEnv, getOrDefault } = require("@kie-tools-scripts/build-env"); + +const rootEnv = require("@kie-tools/root-env/env"); + +module.exports = composeEnv([rootEnv], { + vars: varsWithName({ + KOGITO_DB_MIGRATOR_TOOL_IMAGE__registry: { + default: "docker.io", + description: "The image registry.", + }, + KOGITO_DB_MIGRATOR_TOOL_IMAGE__account: { + default: "apache", + description: "The image registry account.", + }, + KOGITO_DB_MIGRATOR_TOOL_IMAGE__name: { + default: "kogito-db-migrator-tool", + description: "The image name.", + }, + KOGITO_DB_MIGRATOR_TOOL_IMAGE__buildTag: { + default: rootEnv.env.root.streamName, + description: "The image tag.", + }, + }), + get env() { + return { + kogitoDbMigratorToolImage: { + registry: getOrDefault(this.vars.KOGITO_DB_MIGRATOR_TOOL_IMAGE__registry), + account: getOrDefault(this.vars.KOGITO_DB_MIGRATOR_TOOL_IMAGE__account), + name: getOrDefault(this.vars.KOGITO_DB_MIGRATOR_TOOL_IMAGE__name), + buildTag: getOrDefault(this.vars.KOGITO_DB_MIGRATOR_TOOL_IMAGE__buildTag), + }, + }; + }, +}); diff --git a/packages/kogito-db-migrator-tool-image/install.js b/packages/kogito-db-migrator-tool-image/install.js index 8529104bcd8..e1eb165cea7 100644 --- a/packages/kogito-db-migrator-tool-image/install.js +++ b/packages/kogito-db-migrator-tool-image/install.js @@ -17,4 +17,48 @@ * under the License. */ -console.log("install.js"); +const { execSync } = require("child_process"); +const fs = require("fs"); + +const { env } = require("./env"); +const path = require("path"); +const pythonVenvDir = path.dirname(require.resolve("@kie-tools/python-venv/package.json")); +const sonataflowImageCommonDir = path.dirname(require.resolve("@kie-tools/sonataflow-image-common/package.json")); +const replaceInFile = require("replace-in-file"); + +const activateCmd = + process.platform === "win32" + ? `${pythonVenvDir}\\venv\\Scripts\\Activate.bat` + : `. ${pythonVenvDir}/venv/bin/activate`; + +execSync( + `${activateCmd} && \ + python3 ${sonataflowImageCommonDir}/resources/scripts/versions_manager.py --bump-to ${env.kogitoDbMigratorToolImage.buildTag} --source-folder ./resources`, + { stdio: "inherit" } +); + +// Find and read the -image.yaml file +const resourcesPath = path.resolve(__dirname, "./resources"); +const files = fs.readdirSync(resourcesPath); +const imageYamlFiles = files.filter((fileName) => fileName.endsWith("image.yaml")); +if (imageYamlFiles.length !== 1) { + throw new Error("There should only be one image.yaml file on ./resources!"); +} +const originalYamlPath = path.join(resourcesPath, imageYamlFiles[0]); +let imageYaml = fs.readFileSync(originalYamlPath, "utf8"); + +const imageUrl = `${env.kogitoDbMigratorToolImage.registry}/${env.kogitoDbMigratorToolImage.account}/${env.kogitoDbMigratorToolImage.name}`; + +// Replace the whole string between quotes ("") with the image name +imageYaml = imageYaml.replace(/(?<=")(.*incubator-kie-kogito-service-db-migration-postgresql.*)(?=")/gm, imageUrl); + +// Write file and then rename it to match the image name +fs.writeFileSync(originalYamlPath, imageYaml); +fs.renameSync(originalYamlPath, path.join(resourcesPath, `${env.kogitoDbMigratorToolImage.name}-image.yaml`)); + +// Replace image URL in .feature files +replaceInFile.sync({ + files: ["**/*.feature"], + from: /@docker.io\/apache\/.*/g, + to: `@${imageUrl}`, +}); diff --git a/packages/kogito-db-migrator-tool-image/package.json b/packages/kogito-db-migrator-tool-image/package.json index 0580e059efb..849cf631c14 100644 --- a/packages/kogito-db-migrator-tool-image/package.json +++ b/packages/kogito-db-migrator-tool-image/package.json @@ -13,14 +13,26 @@ "url": "https://github.com/apache/incubator-kie-tools/issues" }, "scripts": { - "build": "./build-container-image.sh", + "build": "pnpm copy-assets && pnpm image:build && rimraf build && rimraf target && rimraf dist-tests-e2e", "build:dev": "pnpm build", "build:prod": "pnpm build", - "install": "node install.js" + "copy-assets": "run-script-os", + "copy-assets:linux:darwin": "rimraf build && cp -R ./node_modules/@kie-tools/sonataflow-image-common/resources build && cp -R resources/* build", + "format": "prettier --write . --ignore-path=../../.prettierignore --ignore-path=../../.gitignore", + "image:build": "run-script-os", + "image:build:darwin:linux": "pnpm setup:env && ./resources/build-container-image.sh", + "image:build:win32": "echo \"Build skipped on Windows\"", + "install": "node install.js && pnpm format", + "setup:env": ". ./node_modules/@kie-tools/python-venv/venv/bin/activate && cross-env KOGITO_IMAGE_REGISTRY=$(build-env kogitoDbMigratorToolImage.registry) KOGITO_IMAGE_REGISTRY_ACCOUNT=$(build-env kogitoDbMigratorToolImage.account) KOGITO_IMAGE_NAME=$(build-env kogitoDbMigratorToolImage.name) KOGITO_IMAGE_TAG=$(build-env kogitoDbMigratorToolImage.buildTag) QUARKUS_PLATFORM_VERSION=$(build-env versions.quarkus) KOGITO_VERSION=$(build-env versions.kogito)" }, "dependencies": { "@kie-tools/kogito-db-migrator-tool": "workspace:*", "@kie-tools/python-venv": "workspace:*", - "@kie-tools/root-env": "workspace:*" + "@kie-tools/root-env": "workspace:*", + "@kie-tools/sonataflow-image-common": "workspace:*", + "cross-env": "^7.0.3", + "replace-in-file": "^7.1.0", + "rimraf": "^3.0.2", + "run-script-os": "^1.1.6" } } diff --git a/packages/kogito-db-migrator-tool-image/build-container-image.sh b/packages/kogito-db-migrator-tool-image/resources/build-container-image.sh similarity index 63% rename from packages/kogito-db-migrator-tool-image/build-container-image.sh rename to packages/kogito-db-migrator-tool-image/resources/build-container-image.sh index 43952790e81..9538db9e938 100755 --- a/packages/kogito-db-migrator-tool-image/build-container-image.sh +++ b/packages/kogito-db-migrator-tool-image/resources/build-container-image.sh @@ -23,19 +23,14 @@ set -e CEKIT_BUILDER=docker SOURCE_FILE=/tmp/kogito-db-migrator-tool/sonataflow-db-migrator-0.0.0-runner.jar -TARGET_FILE=./cekit/modules/kogito-postgres-db-migration-deps/sonataflow-db-migrator-runner.jar +TARGET_FILE=./resources/modules/kogito-postgres-db-migration-deps/sonataflow-db-migrator-runner.jar +CEKIT_DESCRIPTOR_FILE=build/kogito-db-migrator-tool-image.yaml -if [ -f "$SOURCE_FILE" ]; then - echo "The file: $SOURCE_FILE found and will be used to build image." - rm -f $TARGET_FILE - cp $SOURCE_FILE $TARGET_FILE +echo "The file: $SOURCE_FILE found and will be used to build image." +rm -f $TARGET_FILE +cp $SOURCE_FILE $TARGET_FILE - # Build the container image - cd ./cekit || exit - cekit -v build "$CEKIT_BUILDER" - cd .. || exit - rm -f $TARGET_FILE -else - echo "The file: $SOURCE_FILE not found. Please build kogito-db-migrator-tool package first before building the image." - exit 1 -fi \ No newline at end of file +# Build the container image +cekit --descriptor $CEKIT_DESCRIPTOR_FILE -v build "$CEKIT_BUILDER" +cd .. || exit +rm -f $TARGET_FILE \ No newline at end of file diff --git a/packages/kogito-db-migrator-tool-image/cekit/image.yaml b/packages/kogito-db-migrator-tool-image/resources/kogito-db-migrator-tool-image.yaml similarity index 82% rename from packages/kogito-db-migrator-tool-image/cekit/image.yaml rename to packages/kogito-db-migrator-tool-image/resources/kogito-db-migrator-tool-image.yaml index 833ae4931f2..2013ef82468 100644 --- a/packages/kogito-db-migrator-tool-image/cekit/image.yaml +++ b/packages/kogito-db-migrator-tool-image/resources/kogito-db-migrator-tool-image.yaml @@ -16,8 +16,8 @@ # specific language governing permissions and limitations # under the License. # -name: "docker.io/apache/incubator-kie-kogito-service-db-migration-postgresql" -version: "999-SNAPSHOT" +name: "docker.io/apache/kogito-db-migrator-tool" +version: "main" from: registry.access.redhat.com/ubi8/openjdk-17-runtime:1.20 description: Flyway image for DI/JS database migration @@ -35,13 +35,10 @@ labels: modules: repositories: - - name: sonataflow-image-common - path: ../../sonataflow-image-common/resources/modules/kogito-system-user - - name: kogito-db-migrator-tool-image - path: modules + - path: modules install: - name: org.kie.kogito.system.user - - name: kogito-postgres-db-migration-deps + - name: org.kie.kogito.postgresql.db-migration-deps run: workdir: "/home/kogito" diff --git a/packages/kogito-db-migrator-tool-image/cekit/modules/kogito-postgres-db-migration-deps/module.yaml b/packages/kogito-db-migrator-tool-image/resources/modules/kogito-postgres-db-migration-deps/module.yaml similarity index 93% rename from packages/kogito-db-migrator-tool-image/cekit/modules/kogito-postgres-db-migration-deps/module.yaml rename to packages/kogito-db-migrator-tool-image/resources/modules/kogito-postgres-db-migration-deps/module.yaml index 7448604964f..dd6c0d19b89 100644 --- a/packages/kogito-db-migrator-tool-image/cekit/modules/kogito-postgres-db-migration-deps/module.yaml +++ b/packages/kogito-db-migrator-tool-image/resources/modules/kogito-postgres-db-migration-deps/module.yaml @@ -17,8 +17,8 @@ # under the License. # schema_version: 1 -name: kogito-postgres-db-migration-deps -version: "1.0" +name: org.kie.kogito.postgresql.db-migration-deps +version: "main" artifacts: - name: sonataflow-db-migrator-runner.jar path: sonataflow-db-migrator-runner.jar diff --git a/packages/kogito-db-migrator-tool/env/index.js b/packages/kogito-db-migrator-tool/env/index.js index 42ad5024d52..8938b690b7f 100644 --- a/packages/kogito-db-migrator-tool/env/index.js +++ b/packages/kogito-db-migrator-tool/env/index.js @@ -23,7 +23,7 @@ module.exports = composeEnv([require("@kie-tools/root-env/env")], { vars: varsWithName({}), get env() { return { - devDeploymentQuarkusApp: { + kogitoDBMigratorTool: { version: require("../package.json").version, }, }; diff --git a/packages/kogito-db-migrator-tool/install.js b/packages/kogito-db-migrator-tool/install.js index bc24d203a0d..be3d97548ea 100644 --- a/packages/kogito-db-migrator-tool/install.js +++ b/packages/kogito-db-migrator-tool/install.js @@ -24,7 +24,7 @@ setupMavenConfigFile( ` --batch-mode -Dstyle.color=always - -Drevision=${env.devDeploymentQuarkusApp.version} + -Drevision=${env.kogitoDBMigratorTool.version} -Dversion.quarkus=${env.versions.quarkus} -Dversion.org.kie.kogito=${env.versions.kogito} -Dmaven.repo.local.tail=${buildTailFromPackageJsonDependencies()} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 217fdd48396..b19953b0b39 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -6385,6 +6385,46 @@ importers: specifier: ^1.1.6 version: 1.1.6 + packages/kogito-db-migrator-tool: + dependencies: + '@kie-tools/maven-base': + specifier: workspace:* + version: link:../maven-base + devDependencies: + '@kie-tools/root-env': + specifier: workspace:* + version: link:../root-env + run-script-os: + specifier: ^1.1.6 + version: 1.1.6 + + packages/kogito-db-migrator-tool-image: + dependencies: + '@kie-tools/kogito-db-migrator-tool': + specifier: workspace:* + version: link:../kogito-db-migrator-tool + '@kie-tools/python-venv': + specifier: workspace:* + version: link:../python-venv + '@kie-tools/root-env': + specifier: workspace:* + version: link:../root-env + '@kie-tools/sonataflow-image-common': + specifier: workspace:* + version: link:../sonataflow-image-common + cross-env: + specifier: ^7.0.3 + version: 7.0.3 + replace-in-file: + specifier: ^7.1.0 + version: 7.1.0 + rimraf: + specifier: ^3.0.2 + version: 3.0.2 + run-script-os: + specifier: ^1.1.6 + version: 1.1.6 + packages/kogito-jit-runner-image: devDependencies: '@kie-tools/maven-base': From ce1eaee7149bb6270b1d8b3404a45c8bde4331b5 Mon Sep 17 00:00:00 2001 From: rhkp Date: Fri, 25 Oct 2024 16:02:51 +0000 Subject: [PATCH 04/18] Continue addressing review comments --- packages/kogito-db-migrator-tool/README.md | 2 +- .../build-db-migrator-jar.sh | 5 +++-- packages/kogito-db-migrator-tool/env/index.js | 20 +++++++++++++++++-- .../get-kogito-ddl-scripts.sh | 16 ++++----------- packages/kogito-db-migrator-tool/install.js | 20 +++++++++++++++++++ 5 files changed, 46 insertions(+), 17 deletions(-) diff --git a/packages/kogito-db-migrator-tool/README.md b/packages/kogito-db-migrator-tool/README.md index a22f4553051..07260c4c5cc 100644 --- a/packages/kogito-db-migrator-tool/README.md +++ b/packages/kogito-db-migrator-tool/README.md @@ -1,6 +1,6 @@ # Kogito Postgres DB Migrator Tool -This is a java quarkus based postgres database migrator application for Sonataflow Data Index and Jobs Service applications for use by SonataFlow Operator. +This is a java quarkus based postgres database migrator application for SonataFlow Data Index and Jobs Service applications for use by SonataFlow Operator. _NOTE_: This postgres database migrator application and its corresponding images are only envisaged to be made use of by SonataFlow Operator, Data Index and Jobs Service internally. Conversely this application is of no use outside the usecases involved with SonataFlow Operator, Data Index and Jobs Service. diff --git a/packages/kogito-db-migrator-tool/build-db-migrator-jar.sh b/packages/kogito-db-migrator-tool/build-db-migrator-jar.sh index 6be70586900..71c3bae4848 100755 --- a/packages/kogito-db-migrator-tool/build-db-migrator-jar.sh +++ b/packages/kogito-db-migrator-tool/build-db-migrator-jar.sh @@ -29,7 +29,7 @@ DDL_DIR=./src/main/resources/postgresql rm -rf $OUTPUT_DIR mkdir -p $OUTPUT_DIR -./get-kogito-ddl-scripts.sh +./generated_get-kogito-ddl-scripts.sh if [ "$1" = "true" ]; then echo "Using dev version:" @@ -41,5 +41,6 @@ fi cp $TARGET_JAR_FILE $OUTPUT_DIR -# As the jar is built and ddl files are included, cleanup by deleting the ddl files +# As the jar is built and ddl files are included, cleanup by deleting the ddl files and generated files rm -rf $DDL_DIR +rm -f ./generated_get-kogito-ddl-scripts.sh diff --git a/packages/kogito-db-migrator-tool/env/index.js b/packages/kogito-db-migrator-tool/env/index.js index 8938b690b7f..a8d30b582e1 100644 --- a/packages/kogito-db-migrator-tool/env/index.js +++ b/packages/kogito-db-migrator-tool/env/index.js @@ -17,14 +17,30 @@ * under the License. */ -const { varsWithName, composeEnv } = require("@kie-tools-scripts/build-env"); +const { varsWithName, composeEnv, getOrDefault } = require("@kie-tools-scripts/build-env"); module.exports = composeEnv([require("@kie-tools/root-env/env")], { - vars: varsWithName({}), + vars: varsWithName({ + KOGITO_DB_MIGRATOR_TOOL__ddl_version: { + default: "10.0.999-SNAPSHOT", + description: "The version to use for ddl.", + }, + KOGITO_DB_MIGRATOR_TOOL__ddl_file_name: { + default: "kogito-ddl-10.0.999-20241018.012042-73-db-scripts.zip", + description: "The file name for ddl.", + }, + KOGITO_DB_MIGRATOR_TOOL__ddl_base_url: { + default: "https://repository.apache.org/content/groups/snapshots/org/kie/kogito/kogito-ddl", + description: "The url from where the ddl file will be downloaded.", + }, + }), get env() { return { kogitoDBMigratorTool: { version: require("../package.json").version, + ddlVersion: getOrDefault(this.vars.KOGITO_DB_MIGRATOR_TOOL__ddl_version), + ddlFileName: getOrDefault(this.vars.KOGITO_DB_MIGRATOR_TOOL__ddl_file_name), + ddlBaseUrl: getOrDefault(this.vars.KOGITO_DB_MIGRATOR_TOOL__ddl_base_url), }, }; }, diff --git a/packages/kogito-db-migrator-tool/get-kogito-ddl-scripts.sh b/packages/kogito-db-migrator-tool/get-kogito-ddl-scripts.sh index cc88b91eb0b..dc815b2e9d6 100755 --- a/packages/kogito-db-migrator-tool/get-kogito-ddl-scripts.sh +++ b/packages/kogito-db-migrator-tool/get-kogito-ddl-scripts.sh @@ -22,9 +22,10 @@ set -x set -e # Script variables with default values. These values will be updated by hack/bump-version.sh, don't change it. -DDL_VERSION=10.0.999-SNAPSHOT -DDL_FILE=kogito-ddl-10.0.999-20241018.012042-73-db-scripts.zip -DDL_BASE_URL=https://repository.apache.org/content/groups/snapshots/org/kie/kogito/kogito-ddl +DDL_VERSION= ### SET BY install.js ### +DDL_FILE= ### SET BY install.js ### +DDL_BASE_URL= ### SET BY install.js ### + TMP_DIR=./tmp RESOURCES_DIR=src/main/resources POSTGRESQL_DDL_DIR=postgresql @@ -36,15 +37,6 @@ cleanup () { rm -rf $TMP_DIR } -# Use specific variable values, if passed -for arg in "$@"; do - case "$arg" in - DDL_VERSION=*) DDL_VERSION="${arg#*=}" ;; - DDL_FILE=*) DDL_FILE="${arg#*=}" ;; - DDL_BASE_URL=*) DDL_BASE_URL="${arg#*=}" ;; - esac -done - DDL_URL=$DDL_BASE_URL/$DDL_VERSION/$DDL_FILE printf "Variables being used for the DDL: \n DDL_VERSION: %s, \n DDL_FILE: %s, \n DDL_BASE_URL: %s \n\n" "$DDL_VERSION" "$DDL_FILE" "$DDL_BASE_URL" diff --git a/packages/kogito-db-migrator-tool/install.js b/packages/kogito-db-migrator-tool/install.js index be3d97548ea..b05ad73710f 100644 --- a/packages/kogito-db-migrator-tool/install.js +++ b/packages/kogito-db-migrator-tool/install.js @@ -31,3 +31,23 @@ setupMavenConfigFile( `, { ignoreDefault: true } // Can't have special configuration that only works inside this repo. ); + +const fs = require("fs"); +const path = require("path"); +const dirName = path.resolve(__dirname); +const getKogitoDDLScriptFileName = "get-kogito-ddl-scripts.sh"; +const ddlScriptPath = path.join(dirName, getKogitoDDLScriptFileName); +let ddlScript = fs.readFileSync(ddlScriptPath, "utf8"); + +const ddlVersionLine = "DDL_VERSION=" + env.kogitoDBMigratorTool.ddlVersion; +const ddlFileNameLine = "DDL_FILE=" + env.kogitoDBMigratorTool.ddlFileName; +const ddlBaseUrlLine = "DDL_BASE_URL=" + env.kogitoDBMigratorTool.ddlBaseUrl; + +// Replace the whole string between quotes ("") with the image name +ddlScript = ddlScript.replace("DDL_VERSION= ### SET BY install.js ###", ddlVersionLine); +ddlScript = ddlScript.replace("DDL_FILE= ### SET BY install.js ###", ddlFileNameLine); +ddlScript = ddlScript.replace("DDL_BASE_URL= ### SET BY install.js ###", ddlBaseUrlLine); + +// Write file and then rename it to match the image name +const genDdlScriptPath = path.join(dirName, "generated_" + getKogitoDDLScriptFileName); +fs.writeFileSync(genDdlScriptPath, ddlScript); From 69ab2de83647e8aa9569c423d4a5c8fbb8fb0174 Mon Sep 17 00:00:00 2001 From: rhkp Date: Fri, 25 Oct 2024 17:22:24 +0000 Subject: [PATCH 05/18] Continue addressing review comments --- packages/kogito-db-migrator-tool/README.md | 40 +++++++++++++------ .../build-db-migrator-jar.sh | 1 + packages/kogito-db-migrator-tool/package.json | 4 +- repo/graph.dot | 5 +++ repo/graph.json | 19 +++++++++ 5 files changed, 54 insertions(+), 15 deletions(-) diff --git a/packages/kogito-db-migrator-tool/README.md b/packages/kogito-db-migrator-tool/README.md index 07260c4c5cc..454582c7364 100644 --- a/packages/kogito-db-migrator-tool/README.md +++ b/packages/kogito-db-migrator-tool/README.md @@ -4,22 +4,36 @@ This is a java quarkus based postgres database migrator application for SonataFl _NOTE_: This postgres database migrator application and its corresponding images are only envisaged to be made use of by SonataFlow Operator, Data Index and Jobs Service internally. Conversely this application is of no use outside the usecases involved with SonataFlow Operator, Data Index and Jobs Service. -## Running the application in dev mode - -Though you can run the application locally in dev mode but it is advisable to run this application as a container image. -The primary reason not to run as standalone application in dev mode, is that by default there are no DDL migration files included in the source. The script `get-kogito-ddl-scripts.sh` can be used to download the needed postgres DDL files into the application. - -You can run your application in dev mode that enables live coding using: - -```shell script -./get-kogito-ddl-scripts.sh -./mvnw compile quarkus:dev -``` - ## Build jar file You can build the jar file with the script `build-db-migrator-jar.sh` which places the jar into /tmp/ kogito-db-migrator-tool directory for use by the corresponding image builder in package kogito-db-migrator-tool-image later, which can be found [here](../kogito-db-migrator-tool-image/README.md). ```shell -./build-db-migrator-jar.sh +pnpm -F kogito-db-migrator-tool... build:dev ``` + +--- + +Apache KIE (incubating) is an effort undergoing incubation at The Apache Software +Foundation (ASF), sponsored by the name of Apache Incubator. Incubation is +required of all newly accepted projects until a further review indicates that +the infrastructure, communications, and decision making process have stabilized +in a manner consistent with other successful ASF projects. While incubation +status is not necessarily a reflection of the completeness or stability of the +code, it does indicate that the project has yet to be fully endorsed by the ASF. + +Some of the incubating project’s releases may not be fully compliant with ASF +policy. For example, releases may have incomplete or un-reviewed licensing +conditions. What follows is a list of known issues the project is currently +aware of (note that this list, by definition, is likely to be incomplete): + +- Hibernate, an LGPL project, is being used. Hibernate is in the process of + relicensing to ASL v2 +- Some files, particularly test files, and those not supporting comments, may + be missing the ASF Licensing Header + +If you are planning to incorporate this work into your product/project, please +be aware that you will need to conduct a thorough licensing review to determine +the overall implications of including this work. For the current status of this +project through the Apache Incubator visit: +https://incubator.apache.org/projects/kie.html diff --git a/packages/kogito-db-migrator-tool/build-db-migrator-jar.sh b/packages/kogito-db-migrator-tool/build-db-migrator-jar.sh index 71c3bae4848..d9e8c7cc41f 100755 --- a/packages/kogito-db-migrator-tool/build-db-migrator-jar.sh +++ b/packages/kogito-db-migrator-tool/build-db-migrator-jar.sh @@ -29,6 +29,7 @@ DDL_DIR=./src/main/resources/postgresql rm -rf $OUTPUT_DIR mkdir -p $OUTPUT_DIR +chmod a+x ./generated_get-kogito-ddl-scripts.sh ./generated_get-kogito-ddl-scripts.sh if [ "$1" = "true" ]; then diff --git a/packages/kogito-db-migrator-tool/package.json b/packages/kogito-db-migrator-tool/package.json index 8dc4ca795f5..691258f9a7c 100644 --- a/packages/kogito-db-migrator-tool/package.json +++ b/packages/kogito-db-migrator-tool/package.json @@ -13,8 +13,8 @@ "url": "https://github.com/apache/incubator-kie-tools/issues" }, "scripts": { - "build:dev": "./build-db-migrator-jar.sh true", - "build:prod": "./build-db-migrator-jar.sh false", + "build:dev": "pnpm install && ./build-db-migrator-jar.sh true", + "build:prod": "pnpm install && ./build-db-migrator-jar.sh false", "install": "node install.js" }, "dependencies": { diff --git a/repo/graph.dot b/repo/graph.dot index 3e22c2ef251..fbb8b1486bd 100644 --- a/repo/graph.dot +++ b/repo/graph.dot @@ -143,6 +143,8 @@ digraph G { "@kie-tools/sonataflow-image-common" [ color = "black", fontcolor = "black", style = "dashed, rounded" ]; "@kie/kogito-data-index-ephemeral-image" [ color = "black", fontcolor = "black", style = "dashed, rounded" ]; "@kie/kogito-data-index-postgresql-image" [ color = "black", fontcolor = "black", style = "dashed, rounded" ]; + "@kie-tools/kogito-db-migrator-tool" [ color = "black", fontcolor = "black", style = "dashed, rounded" ]; + "@kie-tools/kogito-db-migrator-tool-image" [ color = "black", fontcolor = "black", style = "dashed, rounded" ]; "@kie/kogito-jit-runner-image" [ color = "black", fontcolor = "black", style = "dashed, rounded" ]; "@kie/kogito-jobs-service-allinone-image" [ color = "black", fontcolor = "black", style = "dashed, rounded" ]; "@kie/kogito-jobs-service-ephemeral-image" [ color = "black", fontcolor = "black", style = "dashed, rounded" ]; @@ -449,6 +451,9 @@ digraph G { "@kie/kogito-data-index-ephemeral-image" -> "@kie-tools/sonataflow-image-common" [ style = "dashed", color = "black" ]; "@kie/kogito-data-index-postgresql-image" -> "@kie-tools/maven-base" [ style = "dashed", color = "black" ]; "@kie/kogito-data-index-postgresql-image" -> "@kie-tools/sonataflow-image-common" [ style = "dashed", color = "black" ]; + "@kie-tools/kogito-db-migrator-tool" -> "@kie-tools/maven-base" [ style = "solid", color = "black" ]; + "@kie-tools/kogito-db-migrator-tool-image" -> "@kie-tools/kogito-db-migrator-tool" [ style = "solid", color = "black" ]; + "@kie-tools/kogito-db-migrator-tool-image" -> "@kie-tools/sonataflow-image-common" [ style = "solid", color = "black" ]; "@kie/kogito-jit-runner-image" -> "@kie-tools/maven-base" [ style = "dashed", color = "black" ]; "@kie/kogito-jit-runner-image" -> "@kie-tools/sonataflow-image-common" [ style = "dashed", color = "black" ]; "@kie/kogito-jobs-service-allinone-image" -> "@kie-tools/maven-base" [ style = "dashed", color = "black" ]; diff --git a/repo/graph.json b/repo/graph.json index 0f3378f7b52..2eefc41b8e3 100644 --- a/repo/graph.json +++ b/repo/graph.json @@ -159,6 +159,8 @@ { "id": "@kie/kogito-base-builder-image" }, { "id": "@kie-tools/sonataflow-image-common" }, { "id": "@kie-tools/python-venv" }, + { "id": "@kie-tools/kogito-db-migrator-tool" }, + { "id": "@kie-tools/kogito-db-migrator-tool-image" }, { "id": "@kie/kogito-jit-runner-image" }, { "id": "@kie/kogito-jobs-service-allinone-image" }, { "id": "@kie-tools/unitables" }, @@ -1984,6 +1986,21 @@ "target": "@kie-tools/root-env", "weight": 1 }, + { + "source": "@kie-tools/kogito-db-migrator-tool", + "target": "@kie-tools/maven-base", + "weight": 1 + }, + { + "source": "@kie-tools/kogito-db-migrator-tool-image", + "target": "@kie-tools/kogito-db-migrator-tool", + "weight": 1 + }, + { + "source": "@kie-tools/kogito-db-migrator-tool-image", + "target": "@kie-tools/sonataflow-image-common", + "weight": 1 + }, { "source": "@kie/kogito-jit-runner-image", "target": "@kie-tools/maven-base", @@ -2670,6 +2687,8 @@ ["@kie/kogito-base-builder-image", "packages/kogito-base-builder-image"], ["@kie/kogito-data-index-ephemeral-image", "packages/kogito-data-index-ephemeral-image"], ["@kie/kogito-data-index-postgresql-image", "packages/kogito-data-index-postgresql-image"], + ["@kie-tools/kogito-db-migrator-tool", "packages/kogito-db-migrator-tool"], + ["@kie-tools/kogito-db-migrator-tool-image", "packages/kogito-db-migrator-tool-image"], ["@kie/kogito-jit-runner-image", "packages/kogito-jit-runner-image"], ["@kie/kogito-jobs-service-allinone-image", "packages/kogito-jobs-service-allinone-image"], ["@kie/kogito-jobs-service-ephemeral-image", "packages/kogito-jobs-service-ephemeral-image"], From 85d85cced1af99306603bd976140173037e3c909 Mon Sep 17 00:00:00 2001 From: rhkp Date: Fri, 25 Oct 2024 18:28:38 +0000 Subject: [PATCH 06/18] Continue addressing review comments --- .../kogito-db-migrator-tool/.dockerignore | 5 --- packages/kogito-db-migrator-tool/.gitignore | 44 ------------------- packages/kogito-db-migrator-tool/install.js | 13 ------ packages/kogito-db-migrator-tool/pom.xml | 1 - 4 files changed, 63 deletions(-) delete mode 100644 packages/kogito-db-migrator-tool/.dockerignore delete mode 100644 packages/kogito-db-migrator-tool/.gitignore diff --git a/packages/kogito-db-migrator-tool/.dockerignore b/packages/kogito-db-migrator-tool/.dockerignore deleted file mode 100644 index 94810d006e7..00000000000 --- a/packages/kogito-db-migrator-tool/.dockerignore +++ /dev/null @@ -1,5 +0,0 @@ -* -!target/*-runner -!target/*-runner.jar -!target/lib/* -!target/quarkus-app/* \ No newline at end of file diff --git a/packages/kogito-db-migrator-tool/.gitignore b/packages/kogito-db-migrator-tool/.gitignore deleted file mode 100644 index 33b2771547b..00000000000 --- a/packages/kogito-db-migrator-tool/.gitignore +++ /dev/null @@ -1,44 +0,0 @@ -#Maven -target/ -pom.xml.tag -pom.xml.releaseBackup -pom.xml.versionsBackup -release.properties -.flattened-pom.xml - -# Eclipse -.project -.classpath -.settings/ -bin/ - -# IntelliJ -.idea -*.ipr -*.iml -*.iws - -# NetBeans -nb-configuration.xml - -# Visual Studio Code -.vscode -.factorypath - -# OSX -.DS_Store - -# Vim -*.swp -*.swo - -# patch -*.orig -*.rej - -# Local environment -.env - -# Plugin directory -/.quarkus/cli/plugins/ -*.iml \ No newline at end of file diff --git a/packages/kogito-db-migrator-tool/install.js b/packages/kogito-db-migrator-tool/install.js index b05ad73710f..83482c86df3 100644 --- a/packages/kogito-db-migrator-tool/install.js +++ b/packages/kogito-db-migrator-tool/install.js @@ -18,19 +18,6 @@ */ const { env } = require("./env"); -const { setupMavenConfigFile, installMvnw, buildTailFromPackageJsonDependencies } = require("@kie-tools/maven-base"); - -setupMavenConfigFile( - ` - --batch-mode - -Dstyle.color=always - -Drevision=${env.kogitoDBMigratorTool.version} - -Dversion.quarkus=${env.versions.quarkus} - -Dversion.org.kie.kogito=${env.versions.kogito} - -Dmaven.repo.local.tail=${buildTailFromPackageJsonDependencies()} -`, - { ignoreDefault: true } // Can't have special configuration that only works inside this repo. -); const fs = require("fs"); const path = require("path"); diff --git a/packages/kogito-db-migrator-tool/pom.xml b/packages/kogito-db-migrator-tool/pom.xml index 52ab2825150..aeb0bd31dbe 100644 --- a/packages/kogito-db-migrator-tool/pom.xml +++ b/packages/kogito-db-migrator-tool/pom.xml @@ -33,7 +33,6 @@ org.kie.kogito sonataflow-db-migrator - 2024-11-01T00:00:00Z quarkus-bom io.quarkus.platform 3.13.0 From 597319b55ad12d1edfb41c9fdfcf6a629730882b Mon Sep 17 00:00:00 2001 From: rhkp Date: Mon, 28 Oct 2024 18:47:23 +0000 Subject: [PATCH 07/18] Continue addressing review comments --- .../resources/build-container-image.sh | 1 - .../kogito-db-migrator-tool-image.yaml | 2 +- .../module.yaml | 2 +- .../build-db-migrator-jar.sh | 4 +- packages/kogito-db-migrator-tool/pom.xml | 60 +------------------ .../migrator/MigrationServiceTest.java | 2 +- packages/maven-base/pom.xml | 2 + 7 files changed, 10 insertions(+), 63 deletions(-) diff --git a/packages/kogito-db-migrator-tool-image/resources/build-container-image.sh b/packages/kogito-db-migrator-tool-image/resources/build-container-image.sh index 9538db9e938..270bcba94aa 100755 --- a/packages/kogito-db-migrator-tool-image/resources/build-container-image.sh +++ b/packages/kogito-db-migrator-tool-image/resources/build-container-image.sh @@ -26,7 +26,6 @@ SOURCE_FILE=/tmp/kogito-db-migrator-tool/sonataflow-db-migrator-0.0.0-runner.jar TARGET_FILE=./resources/modules/kogito-postgres-db-migration-deps/sonataflow-db-migrator-runner.jar CEKIT_DESCRIPTOR_FILE=build/kogito-db-migrator-tool-image.yaml -echo "The file: $SOURCE_FILE found and will be used to build image." rm -f $TARGET_FILE cp $SOURCE_FILE $TARGET_FILE diff --git a/packages/kogito-db-migrator-tool-image/resources/kogito-db-migrator-tool-image.yaml b/packages/kogito-db-migrator-tool-image/resources/kogito-db-migrator-tool-image.yaml index 2013ef82468..d2a01560dc6 100644 --- a/packages/kogito-db-migrator-tool-image/resources/kogito-db-migrator-tool-image.yaml +++ b/packages/kogito-db-migrator-tool-image/resources/kogito-db-migrator-tool-image.yaml @@ -41,7 +41,7 @@ modules: - name: org.kie.kogito.postgresql.db-migration-deps run: - workdir: "/home/kogito" + workdir: "/home/kogito/bin" entrypoint: - "java" cmd: diff --git a/packages/kogito-db-migrator-tool-image/resources/modules/kogito-postgres-db-migration-deps/module.yaml b/packages/kogito-db-migrator-tool-image/resources/modules/kogito-postgres-db-migration-deps/module.yaml index dd6c0d19b89..31620117fc7 100644 --- a/packages/kogito-db-migrator-tool-image/resources/modules/kogito-postgres-db-migration-deps/module.yaml +++ b/packages/kogito-db-migrator-tool-image/resources/modules/kogito-postgres-db-migration-deps/module.yaml @@ -22,4 +22,4 @@ version: "main" artifacts: - name: sonataflow-db-migrator-runner.jar path: sonataflow-db-migrator-runner.jar - dest: /home/kogito + dest: /home/kogito/bin diff --git a/packages/kogito-db-migrator-tool/build-db-migrator-jar.sh b/packages/kogito-db-migrator-tool/build-db-migrator-jar.sh index d9e8c7cc41f..1933ca7eee9 100755 --- a/packages/kogito-db-migrator-tool/build-db-migrator-jar.sh +++ b/packages/kogito-db-migrator-tool/build-db-migrator-jar.sh @@ -34,10 +34,10 @@ chmod a+x ./generated_get-kogito-ddl-scripts.sh if [ "$1" = "true" ]; then echo "Using dev version:" - mvn clean install -DskipTests -Dquarkus.package.jar.type=uber-jar + mvn clean install -DskipTests -Dquarkus.package.type=uber-jar else echo "Using prod version:" - mvn clean install -DskipTests=$(build-env tests.run --not) -Dmaven.test.failure.ignore=$(build-env tests.ignoreFailures) -Dquarkus.package.jar.type=uber-jar + mvn clean install -DskipTests=$(build-env tests.run --not) -Dmaven.test.failure.ignore=$(build-env tests.ignoreFailures) -Dquarkus.package.type=uber-jar fi cp $TARGET_JAR_FILE $OUTPUT_DIR diff --git a/packages/kogito-db-migrator-tool/pom.xml b/packages/kogito-db-migrator-tool/pom.xml index aeb0bd31dbe..c525a48036f 100644 --- a/packages/kogito-db-migrator-tool/pom.xml +++ b/packages/kogito-db-migrator-tool/pom.xml @@ -32,27 +32,6 @@ 4.0.0 org.kie.kogito sonataflow-db-migrator - - quarkus-bom - io.quarkus.platform - 3.13.0 - true - - 0.8.12 - 1.19.0 - - - - - - ${quarkus.platform.group-id} - ${quarkus.platform.artifact-id} - ${quarkus.platform.version} - pom - import - - - @@ -63,10 +42,6 @@ io.quarkus quarkus-jdbc-postgresql - - org.flywaydb - flyway-database-postgresql - io.quarkus quarkus-arc @@ -89,7 +64,7 @@ com.github.stefanbirkner system-rules - ${systemrules.version} + ${version.com.github.stefanbirkner.systemrules} test @@ -98,9 +73,8 @@ src/test/java - ${quarkus.platform.group-id} + io.quarkus quarkus-maven-plugin - ${quarkus.platform.version} true @@ -108,7 +82,6 @@ build generate-code generate-code-tests - native-image-agent @@ -151,7 +124,7 @@ org.jacoco jacoco-maven-plugin - ${jacoco.version} + ${version.jacoco.maven.plugin} @@ -169,31 +142,4 @@ - - - - native - - - native - - - - false - true - - - - - - - apache - Apache Maven - default - https://repo.maven.apache.org/maven2/ - - false - - - diff --git a/packages/kogito-db-migrator-tool/src/test/java/org/kie/kogito/postgresql/migrator/MigrationServiceTest.java b/packages/kogito-db-migrator-tool/src/test/java/org/kie/kogito/postgresql/migrator/MigrationServiceTest.java index 0c4f497cc81..2506952c2b3 100644 --- a/packages/kogito-db-migrator-tool/src/test/java/org/kie/kogito/postgresql/migrator/MigrationServiceTest.java +++ b/packages/kogito-db-migrator-tool/src/test/java/org/kie/kogito/postgresql/migrator/MigrationServiceTest.java @@ -35,7 +35,7 @@ public class MigrationServiceTest { @BeforeEach public void setupEach() { flyway = mock(Flyway.class); - when(flyway.migrate()).thenReturn(new MigrateResult("flywayVersion", "db", "schema", "postgres")); + when(flyway.migrate()).thenReturn(new MigrateResult("flywayVersion", "db", "schema")); when(flyway.clean()).thenReturn(new CleanResult("flywayVersion", "db")); } diff --git a/packages/maven-base/pom.xml b/packages/maven-base/pom.xml index cb4ad84b499..01e88f9cf37 100644 --- a/packages/maven-base/pom.xml +++ b/packages/maven-base/pom.xml @@ -140,6 +140,8 @@ 4.11.0 0.4 0.23.0 + 0.8.12 + 1.19.0 From 4bab40671c0d87402dd0e39d1276d8d098a76eec Mon Sep 17 00:00:00 2001 From: rhkp Date: Mon, 28 Oct 2024 20:30:38 +0000 Subject: [PATCH 08/18] Move kogito db migration vars to root env --- packages/kogito-db-migrator-tool/env/index.js | 22 ++++--------------- packages/root-env/env/index.js | 19 ++++++++++++++++ 2 files changed, 23 insertions(+), 18 deletions(-) diff --git a/packages/kogito-db-migrator-tool/env/index.js b/packages/kogito-db-migrator-tool/env/index.js index a8d30b582e1..68ab781c9c7 100644 --- a/packages/kogito-db-migrator-tool/env/index.js +++ b/packages/kogito-db-migrator-tool/env/index.js @@ -17,30 +17,16 @@ * under the License. */ -const { varsWithName, composeEnv, getOrDefault } = require("@kie-tools-scripts/build-env"); +const rootEnv = require("@kie-tools/root-env/env"); +const { varsWithName, composeEnv } = require("@kie-tools-scripts/build-env"); module.exports = composeEnv([require("@kie-tools/root-env/env")], { - vars: varsWithName({ - KOGITO_DB_MIGRATOR_TOOL__ddl_version: { - default: "10.0.999-SNAPSHOT", - description: "The version to use for ddl.", - }, - KOGITO_DB_MIGRATOR_TOOL__ddl_file_name: { - default: "kogito-ddl-10.0.999-20241018.012042-73-db-scripts.zip", - description: "The file name for ddl.", - }, - KOGITO_DB_MIGRATOR_TOOL__ddl_base_url: { - default: "https://repository.apache.org/content/groups/snapshots/org/kie/kogito/kogito-ddl", - description: "The url from where the ddl file will be downloaded.", - }, - }), + vars: varsWithName({}), get env() { return { kogitoDBMigratorTool: { version: require("../package.json").version, - ddlVersion: getOrDefault(this.vars.KOGITO_DB_MIGRATOR_TOOL__ddl_version), - ddlFileName: getOrDefault(this.vars.KOGITO_DB_MIGRATOR_TOOL__ddl_file_name), - ddlBaseUrl: getOrDefault(this.vars.KOGITO_DB_MIGRATOR_TOOL__ddl_base_url), + ...rootEnv.env.kogitoDBMigratorToolVars, }, }; }, diff --git a/packages/root-env/env/index.js b/packages/root-env/env/index.js index 510050b39a3..89cf7103bb1 100644 --- a/packages/root-env/env/index.js +++ b/packages/root-env/env/index.js @@ -74,6 +74,20 @@ module.exports = composeEnv([], { description: "Kogito version to be used on dependency declaration.", }, /* (end) */ + /* (begin) Kogito DB Migrator tool build variables */ + KOGITO_DB_MIGRATOR_TOOL__ddl_version: { + default: "10.0.999-SNAPSHOT", + description: "The version to use for ddl.", + }, + KOGITO_DB_MIGRATOR_TOOL__ddl_file_name: { + default: "kogito-ddl-10.0.999-20241018.012042-73-db-scripts.zip", + description: "The file name for ddl.", + }, + KOGITO_DB_MIGRATOR_TOOL__ddl_base_url: { + default: "https://repository.apache.org/content/groups/snapshots/org/kie/kogito/kogito-ddl", + description: "The url from where the ddl file will be downloaded.", + }, + /* (end) */ }), get env() { return { @@ -106,6 +120,11 @@ module.exports = composeEnv([], { kogito: getOrDefault(this.vars.KOGITO_RUNTIME_version), quarkus: getOrDefault(this.vars.QUARKUS_PLATFORM_version), }, + kogitoDBMigratorToolVars: { + ddlVersion: getOrDefault(this.vars.KOGITO_DB_MIGRATOR_TOOL__ddl_version), + ddlFileName: getOrDefault(this.vars.KOGITO_DB_MIGRATOR_TOOL__ddl_file_name), + ddlBaseUrl: getOrDefault(this.vars.KOGITO_DB_MIGRATOR_TOOL__ddl_base_url), + }, }; }, }); From b8d48170f28542b77cc78010f5975f7b2593589d Mon Sep 17 00:00:00 2001 From: rhkp Date: Tue, 29 Oct 2024 16:11:44 +0000 Subject: [PATCH 09/18] Use maven dependency plugin and simplify --- .../build-db-migrator-jar.sh | 12 ++-- packages/kogito-db-migrator-tool/env/index.js | 2 - .../get-kogito-ddl-scripts.sh | 56 ------------------- packages/kogito-db-migrator-tool/install.js | 22 -------- packages/kogito-db-migrator-tool/package.json | 7 ++- packages/kogito-db-migrator-tool/pom.xml | 20 +++++++ packages/root-env/env/index.js | 19 ------- 7 files changed, 28 insertions(+), 110 deletions(-) delete mode 100755 packages/kogito-db-migrator-tool/get-kogito-ddl-scripts.sh diff --git a/packages/kogito-db-migrator-tool/build-db-migrator-jar.sh b/packages/kogito-db-migrator-tool/build-db-migrator-jar.sh index 1933ca7eee9..e2325d176e4 100755 --- a/packages/kogito-db-migrator-tool/build-db-migrator-jar.sh +++ b/packages/kogito-db-migrator-tool/build-db-migrator-jar.sh @@ -24,14 +24,12 @@ set -e # Directory where the DB Migrator Jar will be dropped OUTPUT_DIR=/tmp/kogito-db-migrator-tool TARGET_JAR_FILE=./target/sonataflow-db-migrator-0.0.0-runner.jar -DDL_DIR=./src/main/resources/postgresql +ANSI_DIR=src/main/resources/ansi +POSTGRES_DIR=src/main/resources/postgresql rm -rf $OUTPUT_DIR mkdir -p $OUTPUT_DIR -chmod a+x ./generated_get-kogito-ddl-scripts.sh -./generated_get-kogito-ddl-scripts.sh - if [ "$1" = "true" ]; then echo "Using dev version:" mvn clean install -DskipTests -Dquarkus.package.type=uber-jar @@ -41,7 +39,5 @@ else fi cp $TARGET_JAR_FILE $OUTPUT_DIR - -# As the jar is built and ddl files are included, cleanup by deleting the ddl files and generated files -rm -rf $DDL_DIR -rm -f ./generated_get-kogito-ddl-scripts.sh +rm -rf $ANSI_DIR +rm -rf $POSTGRES_DIR diff --git a/packages/kogito-db-migrator-tool/env/index.js b/packages/kogito-db-migrator-tool/env/index.js index 68ab781c9c7..8938b690b7f 100644 --- a/packages/kogito-db-migrator-tool/env/index.js +++ b/packages/kogito-db-migrator-tool/env/index.js @@ -17,7 +17,6 @@ * under the License. */ -const rootEnv = require("@kie-tools/root-env/env"); const { varsWithName, composeEnv } = require("@kie-tools-scripts/build-env"); module.exports = composeEnv([require("@kie-tools/root-env/env")], { @@ -26,7 +25,6 @@ module.exports = composeEnv([require("@kie-tools/root-env/env")], { return { kogitoDBMigratorTool: { version: require("../package.json").version, - ...rootEnv.env.kogitoDBMigratorToolVars, }, }; }, diff --git a/packages/kogito-db-migrator-tool/get-kogito-ddl-scripts.sh b/packages/kogito-db-migrator-tool/get-kogito-ddl-scripts.sh deleted file mode 100755 index dc815b2e9d6..00000000000 --- a/packages/kogito-db-migrator-tool/get-kogito-ddl-scripts.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/sh -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -set -x -set -e - -# Script variables with default values. These values will be updated by hack/bump-version.sh, don't change it. -DDL_VERSION= ### SET BY install.js ### -DDL_FILE= ### SET BY install.js ### -DDL_BASE_URL= ### SET BY install.js ### - -TMP_DIR=./tmp -RESOURCES_DIR=src/main/resources -POSTGRESQL_DDL_DIR=postgresql - -# cleanup temporary files -cleanup () { - echo "Cleaning up" - rm -rf ${RESOURCES_DIR}"/"${POSTGRESQL_DDL_DIR} - rm -rf $TMP_DIR -} - -DDL_URL=$DDL_BASE_URL/$DDL_VERSION/$DDL_FILE - -printf "Variables being used for the DDL: \n DDL_VERSION: %s, \n DDL_FILE: %s, \n DDL_BASE_URL: %s \n\n" "$DDL_VERSION" "$DDL_FILE" "$DDL_BASE_URL" - -# Start with cleanup -cleanup - -# Get Data Index/ Jobs Service DDL Files -mkdir -p $TMP_DIR -# Change the variables below, as needed -wget "$DDL_URL" -mv "$DDL_FILE" $TMP_DIR -cd $TMP_DIR || exit -unzip "$DDL_FILE" -mv "./"${POSTGRESQL_DDL_DIR} "../"${RESOURCES_DIR} -cd .. || exit -rm -rf $TMP_DIR diff --git a/packages/kogito-db-migrator-tool/install.js b/packages/kogito-db-migrator-tool/install.js index 83482c86df3..80b586513ce 100644 --- a/packages/kogito-db-migrator-tool/install.js +++ b/packages/kogito-db-migrator-tool/install.js @@ -16,25 +16,3 @@ * specific language governing permissions and limitations * under the License. */ - -const { env } = require("./env"); - -const fs = require("fs"); -const path = require("path"); -const dirName = path.resolve(__dirname); -const getKogitoDDLScriptFileName = "get-kogito-ddl-scripts.sh"; -const ddlScriptPath = path.join(dirName, getKogitoDDLScriptFileName); -let ddlScript = fs.readFileSync(ddlScriptPath, "utf8"); - -const ddlVersionLine = "DDL_VERSION=" + env.kogitoDBMigratorTool.ddlVersion; -const ddlFileNameLine = "DDL_FILE=" + env.kogitoDBMigratorTool.ddlFileName; -const ddlBaseUrlLine = "DDL_BASE_URL=" + env.kogitoDBMigratorTool.ddlBaseUrl; - -// Replace the whole string between quotes ("") with the image name -ddlScript = ddlScript.replace("DDL_VERSION= ### SET BY install.js ###", ddlVersionLine); -ddlScript = ddlScript.replace("DDL_FILE= ### SET BY install.js ###", ddlFileNameLine); -ddlScript = ddlScript.replace("DDL_BASE_URL= ### SET BY install.js ###", ddlBaseUrlLine); - -// Write file and then rename it to match the image name -const genDdlScriptPath = path.join(dirName, "generated_" + getKogitoDDLScriptFileName); -fs.writeFileSync(genDdlScriptPath, ddlScript); diff --git a/packages/kogito-db-migrator-tool/package.json b/packages/kogito-db-migrator-tool/package.json index 691258f9a7c..bde50331697 100644 --- a/packages/kogito-db-migrator-tool/package.json +++ b/packages/kogito-db-migrator-tool/package.json @@ -13,9 +13,10 @@ "url": "https://github.com/apache/incubator-kie-tools/issues" }, "scripts": { - "build:dev": "pnpm install && ./build-db-migrator-jar.sh true", - "build:prod": "pnpm install && ./build-db-migrator-jar.sh false", - "install": "node install.js" + "build:dev": "pnpm pre-build && ./build-db-migrator-jar.sh true", + "build:prod": "pnpm pre-build && ./build-db-migrator-jar.sh false", + "install": "node install.js", + "pre-build": "pnpm install && mvn dependency:unpack" }, "dependencies": { "@kie-tools/maven-base": "workspace:*" diff --git a/packages/kogito-db-migrator-tool/pom.xml b/packages/kogito-db-migrator-tool/pom.xml index c525a48036f..de8de0852cc 100644 --- a/packages/kogito-db-migrator-tool/pom.xml +++ b/packages/kogito-db-migrator-tool/pom.xml @@ -72,6 +72,26 @@ src/test/java + + + org.apache.maven.plugins + maven-dependency-plugin + 3.8.1 + + + + org.kie.kogito + kogito-ddl + 10.0.999-SNAPSHOT + db-scripts + zip + true + src/main/resources + kogito-ddl-db-scripts.zip + + + + io.quarkus quarkus-maven-plugin diff --git a/packages/root-env/env/index.js b/packages/root-env/env/index.js index 89cf7103bb1..510050b39a3 100644 --- a/packages/root-env/env/index.js +++ b/packages/root-env/env/index.js @@ -74,20 +74,6 @@ module.exports = composeEnv([], { description: "Kogito version to be used on dependency declaration.", }, /* (end) */ - /* (begin) Kogito DB Migrator tool build variables */ - KOGITO_DB_MIGRATOR_TOOL__ddl_version: { - default: "10.0.999-SNAPSHOT", - description: "The version to use for ddl.", - }, - KOGITO_DB_MIGRATOR_TOOL__ddl_file_name: { - default: "kogito-ddl-10.0.999-20241018.012042-73-db-scripts.zip", - description: "The file name for ddl.", - }, - KOGITO_DB_MIGRATOR_TOOL__ddl_base_url: { - default: "https://repository.apache.org/content/groups/snapshots/org/kie/kogito/kogito-ddl", - description: "The url from where the ddl file will be downloaded.", - }, - /* (end) */ }), get env() { return { @@ -120,11 +106,6 @@ module.exports = composeEnv([], { kogito: getOrDefault(this.vars.KOGITO_RUNTIME_version), quarkus: getOrDefault(this.vars.QUARKUS_PLATFORM_version), }, - kogitoDBMigratorToolVars: { - ddlVersion: getOrDefault(this.vars.KOGITO_DB_MIGRATOR_TOOL__ddl_version), - ddlFileName: getOrDefault(this.vars.KOGITO_DB_MIGRATOR_TOOL__ddl_file_name), - ddlBaseUrl: getOrDefault(this.vars.KOGITO_DB_MIGRATOR_TOOL__ddl_base_url), - }, }; }, }); From fa405b7e88385f4a15081e91fbc541cf52544db9 Mon Sep 17 00:00:00 2001 From: rhkp Date: Tue, 29 Oct 2024 17:48:36 +0000 Subject: [PATCH 10/18] Continue addressing review comments --- packages/kogito-db-migrator-tool/package.json | 14 +++++++++----- pnpm-lock.yaml | 3 +++ 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/packages/kogito-db-migrator-tool/package.json b/packages/kogito-db-migrator-tool/package.json index bde50331697..09f1d175162 100644 --- a/packages/kogito-db-migrator-tool/package.json +++ b/packages/kogito-db-migrator-tool/package.json @@ -13,13 +13,17 @@ "url": "https://github.com/apache/incubator-kie-tools/issues" }, "scripts": { - "build:dev": "pnpm pre-build && ./build-db-migrator-jar.sh true", - "build:prod": "pnpm pre-build && ./build-db-migrator-jar.sh false", - "install": "node install.js", - "pre-build": "pnpm install && mvn dependency:unpack" + "build:dev": "pnpm pre-build && mvn clean install -DskipTests -Dquarkus.package.type=uber-jar && pnpm drop-jar && pnpm cleanup-ddl-dirs", + "build:prod": "pnpm pre-build && mvn clean install -DskipTests=$(build-env tests.run --not) -Dmaven.test.failure.ignore=$(build-env tests.ignoreFailures) -Dquarkus.package.type=uber-jar && pnpm drop-jar && pnpm cleanup-ddl-dirs", + "cleanup": "pnpm cleanup-drop-jar && pnpm cleanup-ddl-dirs", + "cleanup-ddl-dirs": "rimraf src/main/resources/ansi && rimraf src/main/resources/postgresql", + "cleanup-drop-jar": "rimraf /tmp/kogito-db-migrator-tool/ && mkdir -p /tmp/kogito-db-migrator-tool", + "drop-jar": "cp target/sonataflow-db-migrator-0.0.0-runner.jar /tmp/kogito-db-migrator-tool", + "pre-build": " pnpm cleanup && mvn dependency:unpack" }, "dependencies": { - "@kie-tools/maven-base": "workspace:*" + "@kie-tools/maven-base": "workspace:*", + "rimraf": "^3.0.2" }, "devDependencies": { "@kie-tools/root-env": "workspace:*", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b19953b0b39..b51cd1c35f6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -6390,6 +6390,9 @@ importers: '@kie-tools/maven-base': specifier: workspace:* version: link:../maven-base + rimraf: + specifier: ^3.0.2 + version: 3.0.2 devDependencies: '@kie-tools/root-env': specifier: workspace:* From 219320da31fab169977242768fda84004a55c19e Mon Sep 17 00:00:00 2001 From: rhkp Date: Tue, 29 Oct 2024 19:11:09 +0000 Subject: [PATCH 11/18] Minor change to the pom file --- packages/kogito-db-migrator-tool/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/kogito-db-migrator-tool/pom.xml b/packages/kogito-db-migrator-tool/pom.xml index de8de0852cc..dc2f5128f8c 100644 --- a/packages/kogito-db-migrator-tool/pom.xml +++ b/packages/kogito-db-migrator-tool/pom.xml @@ -82,7 +82,7 @@ org.kie.kogito kogito-ddl - 10.0.999-SNAPSHOT + ${version.org.kie.kogito} db-scripts zip true From 4e17720869cb4bc68bc5f0bc31073c8a624e7822 Mon Sep 17 00:00:00 2001 From: rhkp Date: Tue, 29 Oct 2024 21:10:56 +0000 Subject: [PATCH 12/18] Continue addressing review comments --- .../env/index.js | 2 +- .../package.json | 4 +- .../kogito-db-migrator-tool-image.yaml | 4 +- packages/kogito-db-migrator-tool/README.md | 10 +--- .../build-db-migrator-jar.sh | 43 --------------- packages/kogito-db-migrator-tool/package.json | 6 +-- packages/kogito-db-migrator-tool/pom.xml | 54 +------------------ packages/maven-base/pom.xml | 25 +++++++++ 8 files changed, 37 insertions(+), 111 deletions(-) delete mode 100755 packages/kogito-db-migrator-tool/build-db-migrator-jar.sh diff --git a/packages/kogito-db-migrator-tool-image/env/index.js b/packages/kogito-db-migrator-tool-image/env/index.js index e0d521f19b4..9a584252b9e 100644 --- a/packages/kogito-db-migrator-tool-image/env/index.js +++ b/packages/kogito-db-migrator-tool-image/env/index.js @@ -32,7 +32,7 @@ module.exports = composeEnv([rootEnv], { description: "The image registry account.", }, KOGITO_DB_MIGRATOR_TOOL_IMAGE__name: { - default: "kogito-db-migrator-tool", + default: "incubator-kie-kogito-db-migrator-tool", description: "The image name.", }, KOGITO_DB_MIGRATOR_TOOL_IMAGE__buildTag: { diff --git a/packages/kogito-db-migrator-tool-image/package.json b/packages/kogito-db-migrator-tool-image/package.json index 849cf631c14..15d6153dccf 100644 --- a/packages/kogito-db-migrator-tool-image/package.json +++ b/packages/kogito-db-migrator-tool-image/package.json @@ -29,7 +29,9 @@ "@kie-tools/kogito-db-migrator-tool": "workspace:*", "@kie-tools/python-venv": "workspace:*", "@kie-tools/root-env": "workspace:*", - "@kie-tools/sonataflow-image-common": "workspace:*", + "@kie-tools/sonataflow-image-common": "workspace:*" + }, + "devDependencies": { "cross-env": "^7.0.3", "replace-in-file": "^7.1.0", "rimraf": "^3.0.2", diff --git a/packages/kogito-db-migrator-tool-image/resources/kogito-db-migrator-tool-image.yaml b/packages/kogito-db-migrator-tool-image/resources/kogito-db-migrator-tool-image.yaml index d2a01560dc6..84f553a6014 100644 --- a/packages/kogito-db-migrator-tool-image/resources/kogito-db-migrator-tool-image.yaml +++ b/packages/kogito-db-migrator-tool-image/resources/kogito-db-migrator-tool-image.yaml @@ -16,10 +16,10 @@ # specific language governing permissions and limitations # under the License. # -name: "docker.io/apache/kogito-db-migrator-tool" +name: "docker.io/apache/incubator-kie-kogito-db-migrator-tool" version: "main" from: registry.access.redhat.com/ubi8/openjdk-17-runtime:1.20 -description: Flyway image for DI/JS database migration +description: Flyway image for Data Index and Jobs Service database migration labels: - name: "org.kie.kogito.version" diff --git a/packages/kogito-db-migrator-tool/README.md b/packages/kogito-db-migrator-tool/README.md index 454582c7364..fef46b0f3e6 100644 --- a/packages/kogito-db-migrator-tool/README.md +++ b/packages/kogito-db-migrator-tool/README.md @@ -1,17 +1,9 @@ # Kogito Postgres DB Migrator Tool -This is a java quarkus based postgres database migrator application for SonataFlow Data Index and Jobs Service applications for use by SonataFlow Operator. +This is a Java, Quarkus-based PostgreSQL database migrator application for Data-Index and Jobs Service applications for use by SonataFlow Operator. _NOTE_: This postgres database migrator application and its corresponding images are only envisaged to be made use of by SonataFlow Operator, Data Index and Jobs Service internally. Conversely this application is of no use outside the usecases involved with SonataFlow Operator, Data Index and Jobs Service. -## Build jar file - -You can build the jar file with the script `build-db-migrator-jar.sh` which places the jar into /tmp/ kogito-db-migrator-tool directory for use by the corresponding image builder in package kogito-db-migrator-tool-image later, which can be found [here](../kogito-db-migrator-tool-image/README.md). - -```shell -pnpm -F kogito-db-migrator-tool... build:dev -``` - --- Apache KIE (incubating) is an effort undergoing incubation at The Apache Software diff --git a/packages/kogito-db-migrator-tool/build-db-migrator-jar.sh b/packages/kogito-db-migrator-tool/build-db-migrator-jar.sh deleted file mode 100755 index e2325d176e4..00000000000 --- a/packages/kogito-db-migrator-tool/build-db-migrator-jar.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/sh -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -set -x -set -e - -# Directory where the DB Migrator Jar will be dropped -OUTPUT_DIR=/tmp/kogito-db-migrator-tool -TARGET_JAR_FILE=./target/sonataflow-db-migrator-0.0.0-runner.jar -ANSI_DIR=src/main/resources/ansi -POSTGRES_DIR=src/main/resources/postgresql - -rm -rf $OUTPUT_DIR -mkdir -p $OUTPUT_DIR - -if [ "$1" = "true" ]; then - echo "Using dev version:" - mvn clean install -DskipTests -Dquarkus.package.type=uber-jar -else - echo "Using prod version:" - mvn clean install -DskipTests=$(build-env tests.run --not) -Dmaven.test.failure.ignore=$(build-env tests.ignoreFailures) -Dquarkus.package.type=uber-jar -fi - -cp $TARGET_JAR_FILE $OUTPUT_DIR -rm -rf $ANSI_DIR -rm -rf $POSTGRES_DIR diff --git a/packages/kogito-db-migrator-tool/package.json b/packages/kogito-db-migrator-tool/package.json index 09f1d175162..7bb73034eda 100644 --- a/packages/kogito-db-migrator-tool/package.json +++ b/packages/kogito-db-migrator-tool/package.json @@ -16,17 +16,17 @@ "build:dev": "pnpm pre-build && mvn clean install -DskipTests -Dquarkus.package.type=uber-jar && pnpm drop-jar && pnpm cleanup-ddl-dirs", "build:prod": "pnpm pre-build && mvn clean install -DskipTests=$(build-env tests.run --not) -Dmaven.test.failure.ignore=$(build-env tests.ignoreFailures) -Dquarkus.package.type=uber-jar && pnpm drop-jar && pnpm cleanup-ddl-dirs", "cleanup": "pnpm cleanup-drop-jar && pnpm cleanup-ddl-dirs", - "cleanup-ddl-dirs": "rimraf src/main/resources/ansi && rimraf src/main/resources/postgresql", + "cleanup-ddl-dirs": "rimraf src/main/resources/postgresql", "cleanup-drop-jar": "rimraf /tmp/kogito-db-migrator-tool/ && mkdir -p /tmp/kogito-db-migrator-tool", "drop-jar": "cp target/sonataflow-db-migrator-0.0.0-runner.jar /tmp/kogito-db-migrator-tool", "pre-build": " pnpm cleanup && mvn dependency:unpack" }, "dependencies": { - "@kie-tools/maven-base": "workspace:*", - "rimraf": "^3.0.2" + "@kie-tools/maven-base": "workspace:*" }, "devDependencies": { "@kie-tools/root-env": "workspace:*", + "rimraf": "^3.0.2", "run-script-os": "^1.1.6" }, "kieTools": { diff --git a/packages/kogito-db-migrator-tool/pom.xml b/packages/kogito-db-migrator-tool/pom.xml index dc2f5128f8c..4dad2c07a90 100644 --- a/packages/kogito-db-migrator-tool/pom.xml +++ b/packages/kogito-db-migrator-tool/pom.xml @@ -64,7 +64,6 @@ com.github.stefanbirkner system-rules - ${version.com.github.stefanbirkner.systemrules} test @@ -74,9 +73,7 @@ - org.apache.maven.plugins maven-dependency-plugin - 3.8.1 @@ -88,6 +85,8 @@ true src/main/resources kogito-ddl-db-scripts.zip + ansi + postgresql/* @@ -111,55 +110,6 @@ - - maven-surefire-plugin - ${version.maven.surefire.plugin} - - - org.jboss.logmanager.LogManager - ${maven.home} - - - - - maven-failsafe-plugin - ${version.maven.surefire.plugin} - - - - integration-test - verify - - - - - - ${project.build.directory}/${project.build.finalName}-runner - - org.jboss.logmanager.LogManager - ${maven.home} - - - - - org.jacoco - jacoco-maven-plugin - ${version.jacoco.maven.plugin} - - - - prepare-agent - - - - report - prepare-package - - report - - - - diff --git a/packages/maven-base/pom.xml b/packages/maven-base/pom.xml index 01e88f9cf37..42511516735 100644 --- a/packages/maven-base/pom.xml +++ b/packages/maven-base/pom.xml @@ -265,6 +265,12 @@ ${version.org.mockito} test + + com.github.stefanbirkner + system-rules + ${version.com.github.stefanbirkner.systemrules} + test + @@ -410,6 +416,25 @@ + + org.jacoco + jacoco-maven-plugin + ${version.jacoco.maven.plugin} + + + + prepare-agent + + + + report + prepare-package + + report + + + + From deb8e19ce46717222e20e8ed0669979517e353f4 Mon Sep 17 00:00:00 2001 From: rhkp Date: Wed, 30 Oct 2024 19:20:00 +0000 Subject: [PATCH 13/18] Continue addressing review comments --- .../package.json | 6 ++-- .../resources/build-container-image.sh | 35 ------------------- .../kogito-db-migrator-tool-image.yaml | 2 +- .../module.yaml | 4 +-- packages/kogito-db-migrator-tool/package.json | 9 ++--- packages/kogito-db-migrator-tool/pom.xml | 3 +- .../postgresql/migrator/MigrationService.java | 2 +- 7 files changed, 11 insertions(+), 50 deletions(-) delete mode 100755 packages/kogito-db-migrator-tool-image/resources/build-container-image.sh diff --git a/packages/kogito-db-migrator-tool-image/package.json b/packages/kogito-db-migrator-tool-image/package.json index 15d6153dccf..1a4e4425bac 100644 --- a/packages/kogito-db-migrator-tool-image/package.json +++ b/packages/kogito-db-migrator-tool-image/package.json @@ -13,14 +13,14 @@ "url": "https://github.com/apache/incubator-kie-tools/issues" }, "scripts": { - "build": "pnpm copy-assets && pnpm image:build && rimraf build && rimraf target && rimraf dist-tests-e2e", + "build": "pnpm copy-assets && pnpm image:build && rimraf build && rimraf target && rimraf dist-tests-e2e && rimraf resources/modules/kogito-postgres-db-migration-deps/quarkus-app", "build:dev": "pnpm build", "build:prod": "pnpm build", "copy-assets": "run-script-os", - "copy-assets:linux:darwin": "rimraf build && cp -R ./node_modules/@kie-tools/sonataflow-image-common/resources build && cp -R resources/* build", + "copy-assets:linux:darwin": "rimraf build && cp -R ./node_modules/@kie-tools/sonataflow-image-common/resources build && mkdir -p resources/modules/kogito-postgres-db-migration-deps/quarkus-app && cp -R ../kogito-db-migrator-tool/target/quarkus-app/* resources/modules/kogito-postgres-db-migration-deps/quarkus-app && cp -R resources/* build ", "format": "prettier --write . --ignore-path=../../.prettierignore --ignore-path=../../.gitignore", "image:build": "run-script-os", - "image:build:darwin:linux": "pnpm setup:env && ./resources/build-container-image.sh", + "image:build:darwin:linux": "pnpm setup:env && cekit --descriptor build/kogito-db-migrator-tool-image.yaml -v build docker", "image:build:win32": "echo \"Build skipped on Windows\"", "install": "node install.js && pnpm format", "setup:env": ". ./node_modules/@kie-tools/python-venv/venv/bin/activate && cross-env KOGITO_IMAGE_REGISTRY=$(build-env kogitoDbMigratorToolImage.registry) KOGITO_IMAGE_REGISTRY_ACCOUNT=$(build-env kogitoDbMigratorToolImage.account) KOGITO_IMAGE_NAME=$(build-env kogitoDbMigratorToolImage.name) KOGITO_IMAGE_TAG=$(build-env kogitoDbMigratorToolImage.buildTag) QUARKUS_PLATFORM_VERSION=$(build-env versions.quarkus) KOGITO_VERSION=$(build-env versions.kogito)" diff --git a/packages/kogito-db-migrator-tool-image/resources/build-container-image.sh b/packages/kogito-db-migrator-tool-image/resources/build-container-image.sh deleted file mode 100755 index 270bcba94aa..00000000000 --- a/packages/kogito-db-migrator-tool-image/resources/build-container-image.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/sh -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -# - -set -x -set -e - -CEKIT_BUILDER=docker -SOURCE_FILE=/tmp/kogito-db-migrator-tool/sonataflow-db-migrator-0.0.0-runner.jar -TARGET_FILE=./resources/modules/kogito-postgres-db-migration-deps/sonataflow-db-migrator-runner.jar -CEKIT_DESCRIPTOR_FILE=build/kogito-db-migrator-tool-image.yaml - -rm -f $TARGET_FILE -cp $SOURCE_FILE $TARGET_FILE - -# Build the container image -cekit --descriptor $CEKIT_DESCRIPTOR_FILE -v build "$CEKIT_BUILDER" -cd .. || exit -rm -f $TARGET_FILE \ No newline at end of file diff --git a/packages/kogito-db-migrator-tool-image/resources/kogito-db-migrator-tool-image.yaml b/packages/kogito-db-migrator-tool-image/resources/kogito-db-migrator-tool-image.yaml index 84f553a6014..cee53d87224 100644 --- a/packages/kogito-db-migrator-tool-image/resources/kogito-db-migrator-tool-image.yaml +++ b/packages/kogito-db-migrator-tool-image/resources/kogito-db-migrator-tool-image.yaml @@ -46,4 +46,4 @@ run: - "java" cmd: - "-jar" - - "sonataflow-db-migrator-runner.jar" + - "./quarkus-run.jar" diff --git a/packages/kogito-db-migrator-tool-image/resources/modules/kogito-postgres-db-migration-deps/module.yaml b/packages/kogito-db-migrator-tool-image/resources/modules/kogito-postgres-db-migration-deps/module.yaml index 31620117fc7..88986aea4ac 100644 --- a/packages/kogito-db-migrator-tool-image/resources/modules/kogito-postgres-db-migration-deps/module.yaml +++ b/packages/kogito-db-migrator-tool-image/resources/modules/kogito-postgres-db-migration-deps/module.yaml @@ -20,6 +20,6 @@ schema_version: 1 name: org.kie.kogito.postgresql.db-migration-deps version: "main" artifacts: - - name: sonataflow-db-migrator-runner.jar - path: sonataflow-db-migrator-runner.jar + - name: sonataflow-db-migrator-quarkus-app + path: ./quarkus-app dest: /home/kogito/bin diff --git a/packages/kogito-db-migrator-tool/package.json b/packages/kogito-db-migrator-tool/package.json index 7bb73034eda..503c14544bf 100644 --- a/packages/kogito-db-migrator-tool/package.json +++ b/packages/kogito-db-migrator-tool/package.json @@ -13,12 +13,9 @@ "url": "https://github.com/apache/incubator-kie-tools/issues" }, "scripts": { - "build:dev": "pnpm pre-build && mvn clean install -DskipTests -Dquarkus.package.type=uber-jar && pnpm drop-jar && pnpm cleanup-ddl-dirs", - "build:prod": "pnpm pre-build && mvn clean install -DskipTests=$(build-env tests.run --not) -Dmaven.test.failure.ignore=$(build-env tests.ignoreFailures) -Dquarkus.package.type=uber-jar && pnpm drop-jar && pnpm cleanup-ddl-dirs", - "cleanup": "pnpm cleanup-drop-jar && pnpm cleanup-ddl-dirs", - "cleanup-ddl-dirs": "rimraf src/main/resources/postgresql", - "cleanup-drop-jar": "rimraf /tmp/kogito-db-migrator-tool/ && mkdir -p /tmp/kogito-db-migrator-tool", - "drop-jar": "cp target/sonataflow-db-migrator-0.0.0-runner.jar /tmp/kogito-db-migrator-tool", + "build:dev": "pnpm pre-build && mvn clean install -DskipTests && pnpm cleanup", + "build:prod": "pnpm pre-build && mvn clean install -DskipTests=$(build-env tests.run --not) -Dmaven.test.failure.ignore=$(build-env tests.ignoreFailures) && pnpm cleanup", + "cleanup": "rimraf src/main/resources/postgresql && rimraf src/main/resources/ansi", "pre-build": " pnpm cleanup && mvn dependency:unpack" }, "dependencies": { diff --git a/packages/kogito-db-migrator-tool/pom.xml b/packages/kogito-db-migrator-tool/pom.xml index 4dad2c07a90..39694d66f39 100644 --- a/packages/kogito-db-migrator-tool/pom.xml +++ b/packages/kogito-db-migrator-tool/pom.xml @@ -85,8 +85,7 @@ true src/main/resources kogito-ddl-db-scripts.zip - ansi - postgresql/* + ansi/* diff --git a/packages/kogito-db-migrator-tool/src/main/java/org/kie/kogito/postgresql/migrator/MigrationService.java b/packages/kogito-db-migrator-tool/src/main/java/org/kie/kogito/postgresql/migrator/MigrationService.java index 233539cc35a..f6a86d704e0 100644 --- a/packages/kogito-db-migrator-tool/src/main/java/org/kie/kogito/postgresql/migrator/MigrationService.java +++ b/packages/kogito-db-migrator-tool/src/main/java/org/kie/kogito/postgresql/migrator/MigrationService.java @@ -52,7 +52,7 @@ private void migrateDB(Flyway flywayService, Boolean clean, String serviceName) } flywayService.migrate(); if (flywayService.info() != null) { - Log.info(flywayService.info().current().getVersion().toString()); + Log.info("Migrated to version " + flywayService.info().current().toString()); } } From a5309aa93aaf94143651f004e2348c1c99ee60bf Mon Sep 17 00:00:00 2001 From: rhkp Date: Thu, 31 Oct 2024 15:15:01 +0000 Subject: [PATCH 14/18] Continue addressing review comments --- .../kogito-db-migrator-tool-image/package.json | 2 +- packages/kogito-db-migrator-tool/install.js | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/packages/kogito-db-migrator-tool-image/package.json b/packages/kogito-db-migrator-tool-image/package.json index 1a4e4425bac..4ee0d0260f2 100644 --- a/packages/kogito-db-migrator-tool-image/package.json +++ b/packages/kogito-db-migrator-tool-image/package.json @@ -17,7 +17,7 @@ "build:dev": "pnpm build", "build:prod": "pnpm build", "copy-assets": "run-script-os", - "copy-assets:linux:darwin": "rimraf build && cp -R ./node_modules/@kie-tools/sonataflow-image-common/resources build && mkdir -p resources/modules/kogito-postgres-db-migration-deps/quarkus-app && cp -R ../kogito-db-migrator-tool/target/quarkus-app/* resources/modules/kogito-postgres-db-migration-deps/quarkus-app && cp -R resources/* build ", + "copy-assets:linux:darwin": "rimraf build && cp -R ./node_modules/@kie-tools/sonataflow-image-common/resources build && mkdir -p resources/modules/kogito-postgres-db-migration-deps/quarkus-app && cp -R ./node_modules/@kie-tools/kogito-db-migrator-tool/target/quarkus-app/* resources/modules/kogito-postgres-db-migration-deps/quarkus-app && cp -R resources/* build ", "format": "prettier --write . --ignore-path=../../.prettierignore --ignore-path=../../.gitignore", "image:build": "run-script-os", "image:build:darwin:linux": "pnpm setup:env && cekit --descriptor build/kogito-db-migrator-tool-image.yaml -v build docker", diff --git a/packages/kogito-db-migrator-tool/install.js b/packages/kogito-db-migrator-tool/install.js index 80b586513ce..26b71a097ba 100644 --- a/packages/kogito-db-migrator-tool/install.js +++ b/packages/kogito-db-migrator-tool/install.js @@ -16,3 +16,17 @@ * specific language governing permissions and limitations * under the License. */ +const { env } = require("./env"); +const { setupMavenConfigFile, buildTailFromPackageJsonDependencies } = require("@kie-tools/maven-base"); + +setupMavenConfigFile( + ` + --batch-mode + -Dstyle.color=always + -Drevision=${env.devDeploymentQuarkusApp.version} + -Dversion.quarkus=${env.versions.quarkus} + -Dversion.org.kie.kogito=${env.versions.kogito} + -Dmaven.repo.local.tail=${buildTailFromPackageJsonDependencies()} +`, + { ignoreDefault: true } // Can't have special configuration that only works inside this repo. +); From 443217e257d799e89f33bca39376719c710998e5 Mon Sep 17 00:00:00 2001 From: rhkp Date: Thu, 31 Oct 2024 16:03:37 +0000 Subject: [PATCH 15/18] Continue addressing review comments --- .gitignore | 4 ++++ packages/kogito-db-migrator-tool/package.json | 3 +-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 0fa740642fb..f7e7d52a8ea 100644 --- a/.gitignore +++ b/.gitignore @@ -346,3 +346,7 @@ turbo.json # IntelliJ projects should be on _intellij-projects folder, never on specific packages. packages/*/.idea examples/*/.idea + +# kogito-db-migrator-tool +packages/kogito-db-migrator-tool/src/main/resources/postgresql +packages/kogito-db-migrator-tool/src/main/resources/ansi \ No newline at end of file diff --git a/packages/kogito-db-migrator-tool/package.json b/packages/kogito-db-migrator-tool/package.json index 503c14544bf..42bb535d9ee 100644 --- a/packages/kogito-db-migrator-tool/package.json +++ b/packages/kogito-db-migrator-tool/package.json @@ -14,8 +14,7 @@ }, "scripts": { "build:dev": "pnpm pre-build && mvn clean install -DskipTests && pnpm cleanup", - "build:prod": "pnpm pre-build && mvn clean install -DskipTests=$(build-env tests.run --not) -Dmaven.test.failure.ignore=$(build-env tests.ignoreFailures) && pnpm cleanup", - "cleanup": "rimraf src/main/resources/postgresql && rimraf src/main/resources/ansi", + "build:prod": "pnpm pre-build && mvn clean install -DskipTests=$(build-env tests.run --not) -Dmaven.test.failure.ignore=$(build-env tests.ignoreFailures)", "pre-build": " pnpm cleanup && mvn dependency:unpack" }, "dependencies": { From 3c6fcf363675be481cc4193f47a99e6f5fdc21d1 Mon Sep 17 00:00:00 2001 From: rhkp Date: Thu, 31 Oct 2024 17:13:32 +0000 Subject: [PATCH 16/18] Update pnpm-lock file to reflect latest changes --- pnpm-lock.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b51cd1c35f6..6f6ec33a906 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -6390,13 +6390,13 @@ importers: '@kie-tools/maven-base': specifier: workspace:* version: link:../maven-base - rimraf: - specifier: ^3.0.2 - version: 3.0.2 devDependencies: '@kie-tools/root-env': specifier: workspace:* version: link:../root-env + rimraf: + specifier: ^3.0.2 + version: 3.0.2 run-script-os: specifier: ^1.1.6 version: 1.1.6 @@ -6415,6 +6415,7 @@ importers: '@kie-tools/sonataflow-image-common': specifier: workspace:* version: link:../sonataflow-image-common + devDependencies: cross-env: specifier: ^7.0.3 version: 7.0.3 From 5f346a0dddb775ab05cbf0b50c9f32ef3ca876bb Mon Sep 17 00:00:00 2001 From: rhkp Date: Fri, 1 Nov 2024 15:14:07 +0000 Subject: [PATCH 17/18] Attempt to fix CI errors --- packages/kogito-db-migrator-tool-image/package.json | 2 +- ....yaml => incubator-kie-kogito-db-migrator-tool-image.yaml} | 0 packages/kogito-db-migrator-tool/package.json | 4 ++-- 3 files changed, 3 insertions(+), 3 deletions(-) rename packages/kogito-db-migrator-tool-image/resources/{kogito-db-migrator-tool-image.yaml => incubator-kie-kogito-db-migrator-tool-image.yaml} (100%) diff --git a/packages/kogito-db-migrator-tool-image/package.json b/packages/kogito-db-migrator-tool-image/package.json index 4ee0d0260f2..d9395db372d 100644 --- a/packages/kogito-db-migrator-tool-image/package.json +++ b/packages/kogito-db-migrator-tool-image/package.json @@ -20,7 +20,7 @@ "copy-assets:linux:darwin": "rimraf build && cp -R ./node_modules/@kie-tools/sonataflow-image-common/resources build && mkdir -p resources/modules/kogito-postgres-db-migration-deps/quarkus-app && cp -R ./node_modules/@kie-tools/kogito-db-migrator-tool/target/quarkus-app/* resources/modules/kogito-postgres-db-migration-deps/quarkus-app && cp -R resources/* build ", "format": "prettier --write . --ignore-path=../../.prettierignore --ignore-path=../../.gitignore", "image:build": "run-script-os", - "image:build:darwin:linux": "pnpm setup:env && cekit --descriptor build/kogito-db-migrator-tool-image.yaml -v build docker", + "image:build:darwin:linux": "pnpm setup:env && cekit --descriptor build/incubator-kie-kogito-db-migrator-tool-image.yaml -v build docker", "image:build:win32": "echo \"Build skipped on Windows\"", "install": "node install.js && pnpm format", "setup:env": ". ./node_modules/@kie-tools/python-venv/venv/bin/activate && cross-env KOGITO_IMAGE_REGISTRY=$(build-env kogitoDbMigratorToolImage.registry) KOGITO_IMAGE_REGISTRY_ACCOUNT=$(build-env kogitoDbMigratorToolImage.account) KOGITO_IMAGE_NAME=$(build-env kogitoDbMigratorToolImage.name) KOGITO_IMAGE_TAG=$(build-env kogitoDbMigratorToolImage.buildTag) QUARKUS_PLATFORM_VERSION=$(build-env versions.quarkus) KOGITO_VERSION=$(build-env versions.kogito)" diff --git a/packages/kogito-db-migrator-tool-image/resources/kogito-db-migrator-tool-image.yaml b/packages/kogito-db-migrator-tool-image/resources/incubator-kie-kogito-db-migrator-tool-image.yaml similarity index 100% rename from packages/kogito-db-migrator-tool-image/resources/kogito-db-migrator-tool-image.yaml rename to packages/kogito-db-migrator-tool-image/resources/incubator-kie-kogito-db-migrator-tool-image.yaml diff --git a/packages/kogito-db-migrator-tool/package.json b/packages/kogito-db-migrator-tool/package.json index 42bb535d9ee..e094e6ac90d 100644 --- a/packages/kogito-db-migrator-tool/package.json +++ b/packages/kogito-db-migrator-tool/package.json @@ -13,9 +13,9 @@ "url": "https://github.com/apache/incubator-kie-tools/issues" }, "scripts": { - "build:dev": "pnpm pre-build && mvn clean install -DskipTests && pnpm cleanup", + "build:dev": "pnpm pre-build && mvn clean install -DskipTests", "build:prod": "pnpm pre-build && mvn clean install -DskipTests=$(build-env tests.run --not) -Dmaven.test.failure.ignore=$(build-env tests.ignoreFailures)", - "pre-build": " pnpm cleanup && mvn dependency:unpack" + "pre-build": "mvn dependency:unpack" }, "dependencies": { "@kie-tools/maven-base": "workspace:*" From a95e769a522f64be4f17a203b51b1fa40dd809ec Mon Sep 17 00:00:00 2001 From: rhkp Date: Mon, 25 Nov 2024 20:18:58 +0000 Subject: [PATCH 18/18] Minor change to pom file --- packages/kogito-db-migrator-tool/pom.xml | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/kogito-db-migrator-tool/pom.xml b/packages/kogito-db-migrator-tool/pom.xml index 39694d66f39..4948189277d 100644 --- a/packages/kogito-db-migrator-tool/pom.xml +++ b/packages/kogito-db-migrator-tool/pom.xml @@ -98,8 +98,6 @@ build - generate-code - generate-code-tests