Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

MIR-1282 improve expandability of configuration of docker images #967

Draft
wants to merge 3 commits into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .env
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ MIR_DATA=./docker/mir-data/
MIR_LOGS=./docker/mir-logs/
MIR_XMX=1024m
MIR_XMS=1024m
MIR_EXTRA_PROPERTIES=

SOLR_HTTP=8290
SOLR_DATA=./docker/solr-data/
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,7 @@ The docker container has its own install script which uses the environment varia
| XMS | 1g | The value of the -Xms parameter for Tomcat. |
| FIX_FILE_SYSTEM_RIGHTS | false | If true the file system rights of the mounted volumes get corrected to be owned by the right user. |
| MIR_OPTS | | Additional options which will be passed as JAVA_OPTS to the tomcat process |
| MIR_EXTRA_PROPERTIES | | A Path to a additional property file which will be included in to the default mycore.properties. |

### Mount Points

Expand Down
1 change: 1 addition & 0 deletions docker-compose.prod.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ services:
- XMX=${MIR_XMX}
- XMS=${MIR_XMS}
- FIX_FILE_SYSTEM_RIGHTS=false
- EXTRA_PROPERTIES=${MIR_EXTRA_PROPERTIES}
volumes:
- ${MIR_HOME}:/mcr/home/
- ${MIR_DATA}:/mcr/data/
Expand Down
111 changes: 47 additions & 64 deletions docker-entrypoint.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,21 +4,11 @@ set -e
MCR_SAVE_DIR="${MCR_CONFIG_DIR}save/"

MCR_CONFIG_DIR_ESCAPED=$(echo "$MCR_CONFIG_DIR" | sed 's/\//\\\//g')
MCR_DATA_DIR_ESCAPED=$(echo "$MCR_DATA_DIR" | sed 's/\//\\\//g')
MCR_SAVE_DIR_ESCAPED=$(echo "$MCR_SAVE_DIR" | sed 's/\//\\\//g')
MCR_LOG_DIR_ESCAPED=$(echo "$MCR_LOG_DIR" | sed 's/\//\\\//g')

SOLR_URL_ESCAPED=$(echo "$SOLR_URL" | sed 's/\//\\\//g')
SOLR_CORE_ESCAPED=$(echo "$SOLR_CORE" | sed 's/\//\\\//g')
SOLR_CLASSIFICATION_CORE_ESCAPED=$(echo "$SOLR_CLASSIFICATION_CORE" | sed 's/\//\\\//g')
MYCORE_PROPERTIES="${MCR_CONFIG_DIR}mycore.properties"

JDBC_NAME_ESCAPED=$(echo "$JDBC_NAME" | sed 's/\//\\\//g')
JDBC_PASSWORD_ESCAPED=$(echo "$JDBC_PASSWORD" | sed 's/\//\\\//g')
JDBC_DRIVER_ESCAPED=$(echo "$JDBC_DRIVER" | sed 's/\//\\\//g')
JDBC_URL_ESCAPED=$(echo "$JDBC_URL" | sed 's/\//\\\//g')
HIBERNATE_SCHEMA_ESCAPED=$(echo "$JDBC_URL" | sed 's/\//\\\//g')
DOCKER_PROPERTIES="${MCR_CONFIG_DIR}docker.properties"

MYCORE_PROPERTIES="${MCR_CONFIG_DIR}mycore.properties"
PERSISTENCE_XML="${MCR_CONFIG_DIR}resources/META-INF/persistence.xml"

function fixDirectoryRights() {
Expand Down Expand Up @@ -74,92 +64,79 @@ function migrateC3P0toHikari {
rm "${MCR_CONFIG_DIR}lib/h2-1.4.200.jar"
rm "${MCR_CONFIG_DIR}lib/mysql-connector-java-8.0.19.jar"

# delete old configuration and add new configuration
if grep -q "hibernate.c3p0" "${PERSISTENCE_XML}"; then
sed -ri "s/.*hibernate.c3p0.*//" "${PERSISTENCE_XML}"
sed -ri "s/(<property name=\"hibernate.connection.provider_class\" value=\")(.*)(\" \/>)/\1org.hibernate.hikaricp.internal.HikariCPConnectionProvider\3/" "${PERSISTENCE_XML}"
sed -ri "s/(<\/properties>)/<property name=\"hibernate.hikari.maximumPoolSize\" value=\"30\" \/>\n<property name=\"hibernate.hikari.minimumIdle\" value=\"2\" \/>\n<property name=\"hibernate.hikari.idleTimeout\" value=\"30000\" \/>\n<property name=\"hibernate.hikari.maxLifetime\" value=\"1800000\" \/>\n<property name=\"hibernate.hikari.leakDetectionThreshold\" value=\"9000\" \/>\n<property name=\"hibernate.hikari.registerMbeans\" value=\"true\" \/>\n\1/" "${PERSISTENCE_XML}"
fi

/opt/mir/mir/bin/mir.sh reload mappings in jpa configuration file
else
echo "No c3p0 driver found. Skip migration."
fi
}

function migrateJavaxPropertiesToJakarta() {
if grep -q "javax.persistence" "${PERSISTENCE_XML}"; then
echo "Migrate properties in persistence.xml from javax to jakarta"
sed -ri "s/(<property name=\")javax.persistence(.*\" value=\".*\" \/>)/\1jakarta.persistence\2/" "${PERSISTENCE_XML}"
fi
if grep -q "xmlns.jcp.org" "${PERSISTENCE_XML}"; then
echo "Migrate xmlns in persistence.xml from jcp.org to jakarta.ee"
sed -ri "s/xmlns=\".+persistence\"/xmlns=\"https:\/\/jakarta.ee\/xml\/ns\/persistence\"/" "${PERSISTENCE_XML}"
echo "Migrate schemaLocation in persistence.xml from jcp.org to jakarta.ee"
grep -ri "s/(xsi:schemaLocation=\").*jcp.org.*(\")/\1https:\/\/jakarta.ee\/xml\/ns\/persistence https:\/\/jakarta.ee\/xml\/ns\/persistence\/persistence_3_0.xsd\2/" "${PERSISTENCE_XML}"
fi
if grep -q "version=\"2" "${PERSISTENCE_XML}"; then
echo "Migrate version in persistence.xml from 2.* to 3.0"
sed -ri "s/version=\"2.*\"/version=\"3.0\"/" "${PERSISTENCE_XML}"
fi

function setOrAddProperty() {
KEY=$1
VALUE=$2
PROPERTIES_FILE=${3:-$DOCKER_PROPERTIES}

if grep -q "$KEY=" "${PROPERTIES_FILE}" ; then
ESCAPED_KEY=$(echo "${KEY}" | sed 's/\//\\\//g')
ESCAPED_VALUE=$(echo "${VALUE}" | sed 's/\//\\\//g')
sed -ri "s/($ESCAPED_KEY=).+/\1$ESCAPED_VALUE/" "${PROPERTIES_FILE}"
else
echo "$KEY=$VALUE">>"${PROPERTIES_FILE}"
fi
}

function setDockerValues() {
rm -f "${DOCKER_PROPERTIES}"
echo "Set Docker Values to Config!"

migrateJavaxPropertiesToJakarta

if [ -n "${SOLR_URL}" ]; then
sed -ri "s/#?(MCR\.Solr\.ServerURL=).+/\1${SOLR_URL_ESCAPED}/" "${MYCORE_PROPERTIES}";
setOrAddProperty "Solr.ServerURL" "${SOLR_URL}"
fi

if [ -n "${SOLR_CORE}" ]; then
sed -ri "s/#?(MCR\.Solr\.Core\.main\.Name=).+/\1${SOLR_CORE_ESCAPED}/" "${MYCORE_PROPERTIES}";
setOrAddProperty "Solr.Core.main.Name" "${SOLR_CORE}"
fi

if [ -n "${SOLR_CLASSIFICATION_CORE}" ]; then
sed -ri "s/#?(MCR\.Solr\.Core\.classification\.Name=).+/\1${SOLR_CLASSIFICATION_CORE_ESCAPED}/" "${MYCORE_PROPERTIES}"
setOrAddProperty "Solr.Core.classification.Name" "${SOLR_CLASSIFICATION_CORE}"
fi

if [ -n "${JDBC_NAME}" ]; then
sed -ri "s/(name=\"jakarta.persistence.jdbc.user\" value=\").*(\")/\1${JDBC_NAME_ESCAPED}\2/" "${PERSISTENCE_XML}"
setOrAddProperty "MCR.JPA.User" "${JDBC_NAME}"
fi

if [ -n "${JDBC_PASSWORD}" ]; then
sed -ri "s/(name=\"jakarta.persistence.jdbc.password\" value=\").*(\")/\1${JDBC_PASSWORD_ESCAPED}\2/" "${PERSISTENCE_XML}"
setOrAddProperty "MCR.JPA.Password" "${JDBC_PASSWORD}"
fi

if [ -n "${JDBC_DRIVER}" ]; then
sed -ri "s/(name=\"jakarta.persistence.jdbc.driver\" value=\").*(\")/\1${JDBC_DRIVER_ESCAPED}\2/" "${PERSISTENCE_XML}"
setOrAddProperty "MCR.JPA.Driver" "${JDBC_DRIVER}"
fi

if [ -n "${JDBC_URL}" ]; then
sed -ri "s/(name=\"jakarta.persistence.jdbc.url\" value=\").*(\")/\1${JDBC_URL_ESCAPED}\2/" "${PERSISTENCE_XML}"
setOrAddProperty "MCR.JPA.URL" "${JDBC_URL}"
fi

if [ -n "${SOLR_CLASSIFICATION_CORE}" ]; then
sed -ri "s/(name=\"hibernate.default_schema\" value=\").*(\")/\1${HIBERNATE_SCHEMA_ESCAPED}\2/" "${PERSISTENCE_XML}"
if [ -n "${HIBERNATE_SCHEMA}" ]; then
setOrAddProperty "MCR.JPA.DefaultSchema" "${HIBERNATE_SCHEMA}"
fi

sed -ri "s/(name=\"hibernate.hbm2ddl.auto\" value=\").*(\")/\1update\2/" "${PERSISTENCE_XML}"
setOrAddProperty "MCR.JPA.Hbm2ddlAuto" "update"
setOrAddProperty "MCR.JPA.PersistenceUnit.mir.Class" "org.mycore.backend.jpa.MCRSimpleConfigPersistenceUnitDescriptor"
setOrAddProperty "MCR.JPA.PersistenceUnitName" "mir"

if grep -q "MCR.datadir=" "${MYCORE_PROPERTIES}" ; then
sed -ri "s/#?(MCR\.datadir=).+/\1${MCR_DATA_DIR_ESCAPED}/" "${MYCORE_PROPERTIES}"
else
echo "MCR.datadir=${MCR_DATA_DIR}">>"${MYCORE_PROPERTIES}"
fi
setOrAddProperty "MCR.datadir" "${MCR_DATA_DIR}"
setOrAddProperty "MCR.Solr.NestedDocuments" "true"
setOrAddProperty "MCR.Save.FileSystem" "${MCR_SAVE_DIR}"

if grep -q "MCR.Solr.NestedDocuments=" "${MYCORE_PROPERTIES}" ; then
sed -ri "s/#?(MCR\.Solr\.NestedDocuments=).+/\1true/" "${MYCORE_PROPERTIES}"
else
echo "MCR.Solr.NestedDocuments=true">>"${MYCORE_PROPERTIES}";
fi
# s/(<\/properties>)/<property name=\"hibernate.hikari.maximumPoolSize\" value=\"30\" \/>\n<property name=\"hibernate.hikari.minimumIdle\" value=\"2\" \/>\n<property name=\"hibernate.hikari.idleTimeout\" value=\"30000\" \/>\n<property name=\"hibernate.hikari.maxLifetime\" value=\"1800000\" \/>\n<property name=\"hibernate.hikari.leakDetectionThreshold\" value=\"9000\" \/>\n<property name=\"\" value=\"true\" \/>
setOrAddProperty "MCR.JPA.Connection.ProviderClass" "org.hibernate.hikaricp.internal.HikariCPConnectionProvider"
setOrAddProperty "MCR.JPA.Connection.MaximumPoolSize" "30"
setOrAddProperty "MCR.JPA.Connection.MinimumIdle" "2"
setOrAddProperty "MCR.JPA.Connection.IdleTimeout" "30000"
setOrAddProperty "MCR.JPA.Connection.MaxLifetime" "180000"
setOrAddProperty "MCR.JPA.Connection.LeakDetectionThreshold" "9000"
setOrAddProperty "MCR.JPA.Connection.RegisterMbeans" "true"

if grep -q "MCR.Save.FileSystem=" "${MYCORE_PROPERTIES}" ; then
sed -ri "s/#?(MCR\.Save\.FileSystem=).+/\1${MCR_SAVE_DIR_ESCAPED}/" "${MYCORE_PROPERTIES}"
else
echo "MCR.Save.FileSystem=${MCR_SAVE_DIR}">>"${MYCORE_PROPERTIES}"
fi

migrateC3P0toHikari

Expand All @@ -173,15 +150,21 @@ function setDockerValues() {
mkdir -p "${MCR_CONFIG_DIR}lib"

downloadDriver https://repo1.maven.org/maven2/com/zaxxer/HikariCP/5.1.0/HikariCP-5.1.0.jar

if [[ -z "${EXTRA_PROPERTIES}" ]]; then
setOrAddProperty "MCR.Configuration.Include" "${DOCKER_PROPERTIES}" "${MYCORE_PROPERTIES}"
else
setOrAddProperty "MCR.Configuration.Include" "${DOCKER_PROPERTIES} ${EXTRA_PROPERTIES}" "${MYCORE_PROPERTIES}"
fi

rm -f "${PERSISTENCE_XML}"
}

function setUpMyCoRe {
echo "Set up MyCoRe!"
/opt/mir/mir/bin/mir.sh create configuration directory
setDockerValues
setupLog4jConfig
/opt/mir/mir/bin/mir.sh reload mappings in jpa configuration file
sed -ri "s/(<\/properties>)/<property name=\"hibernate.hikari.maximumPoolSize\" value=\"30\" \/>\n<property name=\"hibernate.hikari.minimumIdle\" value=\"2\" \/>\n<property name=\"hibernate.hikari.idleTimeout\" value=\"30000\" \/>\n<property name=\"hibernate.hikari.maxLifetime\" value=\"1800000\" \/>\n<property name=\"hibernate.hikari.leakDetectionThreshold\" value=\"9000\" \/>\n<property name=\"hibernate.hikari.registerMbeans\" value=\"true\" \/>\n\1/" "${MCR_CONFIG_DIR}resources/META-INF/persistence.xml"
/opt/mir/mir/bin/setup.sh
}

Expand Down
2 changes: 1 addition & 1 deletion mir-module/src/main/resources/config/mir/mycore.properties
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ MCR.ContentTransformer.mets-dfg.Stylesheet=xsl/mets/mets-dfg.xsl
# Configure Hibernate Mappings #
##############################################################################
MCR.Hibernate.Mappings=%MCR.Hibernate.Mappings%,org.mycore.mir.authorization.accesskeys.backend.MIRAccessKey,org.mycore.mir.authorization.accesskeys.backend.MIRAccessKeyPair

MCR.JPA.MappingFileNames=%MCR.JPA.MappingFileNames%,META-INF/mir-module-mappings.xml

#############################################################################
# HTMLEditor (for backwards compatibility) #
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,10 +43,9 @@ public class MIRWizard extends MCRServlet {
public static boolean isNecessary() {

File mcrProps = MCRConfigurationDir.getConfigFile("mycore.properties");
File jpaCfg = MCRConfigurationDir.getConfigFile("resources/META-INF/persistence.xml");

return (mcrProps == null || !mcrProps.canRead()) || (jpaCfg == null || !jpaCfg.canRead());

return (mcrProps == null || !mcrProps.canRead());
}

public Element doMagic(Element wizXML) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ private Element getResult(MCRServletJob job) throws IOException {

if (!MIRWizardRequestFilter.isAuthenticated(req)) {
final String loginToken = wizXML.getChildTextTrim("login");
String url = "wizard";
String url = "wizard/";

if (loginToken != null && MIRWizardRequestFilter.getLoginToken(req).equals(loginToken)) {
LOGGER.info("Authenticate with token \"" + loginToken + "\"...");
Expand All @@ -102,7 +102,7 @@ private Element getResult(MCRServletJob job) throws IOException {
req.getScheme() + "://" + req.getServerName() + ":" + req.getServerPort());
} else {
LOGGER.info("Redirect to login...");
url += "/?action=login"
url += "?action=login"
+ (!MIRWizardRequestFilter.getLoginToken(req).equals(loginToken) ? "&token=invalid" : "");

// output login token again
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,52 +23,132 @@
package org.mycore.mir.wizard.command;

import java.io.File;
import java.util.Optional;
import java.io.FileOutputStream;
import java.io.StringWriter;
import java.util.Objects;
import java.util.Properties;
import java.util.function.Predicate;

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.jdom2.Element;
import org.mycore.backend.jpa.MCRPersistenceProvider;
import org.mycore.backend.jpa.MCRSimpleConfigPersistenceUnitDescriptor;
import org.mycore.common.config.MCRConfiguration2;
import org.mycore.common.config.MCRConfigurationDir;
import org.mycore.common.config.MCRConfigurationDirSetup;
import org.mycore.common.content.MCRContent;
import org.mycore.common.content.MCRJDOMContent;
import org.mycore.common.content.transformer.MCRXSLTransformer;
import org.mycore.mir.wizard.MIRWizardCommand;

import jakarta.persistence.spi.PersistenceProviderResolverHolder;

public class MIRWizardGenerateJPAConfig extends MIRWizardCommand {

private static final Logger LOGGER = LogManager.getLogger();

public static final String PERSISTENCE_UNIT_NAME = MCRConfiguration2
.getStringOrThrow("MCR.JPA.PersistenceUnitName");

private static final String COMPLEX_DATABASE_PREFIX
= MCRPersistenceProvider.JPA_PERSISTENCE_UNIT_PROPERTY_NAME + PERSISTENCE_UNIT_NAME + ".";

public MIRWizardGenerateJPAConfig() {
this("persistence.xml");
this("persistence.properties");
}

private MIRWizardGenerateJPAConfig(String name) {
super(name);
}

private static Properties getDefaultProps() {
Properties props = new Properties();

props.setProperty(COMPLEX_DATABASE_PREFIX + "Class",
MCRSimpleConfigPersistenceUnitDescriptor.class.getName());
props.setProperty("MCR.JPA.Connection.MaximumPoolSize", "10");
props.setProperty(COMPLEX_DATABASE_PREFIX + "Properties.hibernate.auto_quote_keyword", "true");

return props;
}

private static Properties convertDatabaseElement(Element databaseElement) {
Properties props = getDefaultProps();
if (databaseElement != null) {
String driver = databaseElement.getChildText("driver");

if (driver != null && !driver.isBlank()) {
if (driver.contains("mariadb") || driver.contains("mysql")) {
props.setProperty(COMPLEX_DATABASE_PREFIX + "Properties.hibernate.dialect.storage_engine",
"innodb");
}
props.setProperty("MCR.JPA.Driver", driver);
}

String url = databaseElement.getChildText("url");
if (url != null && !url.isBlank()) {
props.setProperty("MCR.JPA.URL", url);
}

String username = databaseElement.getChildText("username");
if (username != null && !username.isBlank()) {
props.setProperty("MCR.JPA.User", username);
}

String password = databaseElement.getChildText("password");
if (password != null && !password.isBlank()) {
props.setProperty("MCR.JPA.Password", password);
}

Element extraPropertiesElement = databaseElement.getChild("extra_properties");
convertExtraPropertiesElement(extraPropertiesElement, props);
}
return props;
}

private static void convertExtraPropertiesElement(Element extraPropertiesElement, Properties props) {
if (extraPropertiesElement != null) {
extraPropertiesElement.getChildren().stream()
.filter(element -> Objects.equals(element.getName(), "property") &&
Objects.equals(element.getAttributeValue("name"), "schema"))
.map(Element::getText)
.filter(Predicate.not(String::isBlank))
.findFirst()
.ifPresent(schema -> props.setProperty("MCR.JPA.DefaultSchema", schema));

extraPropertiesElement.getChildren().stream()
.filter(element -> Objects.equals(element.getName(), "property") &&
Objects.equals(element.getAttributeValue("name"), "catalog"))
.map(Element::getText)
.filter(Predicate.not(String::isBlank))
.findFirst()
.ifPresent(catalog -> props
.setProperty(COMPLEX_DATABASE_PREFIX + "Properties.hibernate.default_catalog", catalog));
}
}

@Override
public void doExecute() {
File resDir = new File(MCRConfigurationDir.getConfigurationDirectory(), "resources/META-INF");
try {
resDir.mkdirs();

File file = new File(resDir, "persistence.xml");
this.result.setAttribute("file", file.getAbsolutePath());

// set extra engine property for MariaDB and MySQL
Optional.ofNullable(getInputXML().getChild("database"))
.map(c -> c.getChildText("driver")).filter(d -> d.contains("mariadb") || d.contains("mysql"))
.ifPresent(s -> System.setProperty("hibernate.dialect.storage_engine", "innodb"));
Element xml = getInputXML();
Element databaseElement = xml.getChild("database");

MCRContent source = new MCRJDOMContent(getInputXML().clone());
MCRXSLTransformer transformer = new MCRXSLTransformer("xsl/" + source.getDocType() + "-persistence.xsl");
MCRContent pXML = transformer.transform(source);
Properties props = convertDatabaseElement(databaseElement);
MCRConfigurationDirSetup.loadExternalLibs();

pXML.sendTo(file);
File file = MCRConfigurationDir.getConfigFile("mycore.properties");
try (FileOutputStream out = new FileOutputStream(file, true)) {
props.store(out, "Generated by MIR Wizard.");
}
props.forEach((key, value) -> MCRConfiguration2.set(key.toString(), value.toString()));

MCRConfigurationDirSetup.loadExternalLibs();
StringWriter stringWriter = new StringWriter();
props.store(stringWriter, "Generated by MIR Wizard.");
String propertiesAsString = stringWriter.toString();

PersistenceProviderResolverHolder.getPersistenceProviderResolver()
.clearCachedProviders();

this.result.setResult(pXML.asXML().getRootElement().clone());
this.result.setResult(propertiesAsString);
this.result.setSuccess(true);
} catch (Exception ex) {
LOGGER.error("Exception while generating JPA config.", ex);
Expand Down
Loading
Loading