Migrate Mysql TIMESTAMP to DATETIME data type #488

docker/linux/Dockerfile

@@ -46,7 +46,7 @@ VOLUME /jqm/hotdeploy/ \
 WORKDIR /jqm
 
 # Import initial config
-RUN java -jar jqm.jar -u ; java -jar jqm.jar -c selfConfig.single.xml ; java -jar jqm.jar -importjobdef ./jobs/jqm-demo ; rm -f .\logs\*
+RUN java -jar jqm.jar -u ; java -jar jqm.jar -c selfConfig.single.xml ; java -jar jqm.jar -importjobdef ./jobs/jqm-demo ; rm -f .\logs\* ; chmod 700 /jqm/bin/*.sh ;
 
 # Run node on startup
 ENTRYPOINT /jqm/bin/node.sh

docker/linux/demo/Dockerfile

@@ -8,5 +8,6 @@ FROM enioka/jqm:${JQM_VERSION}
 
 ARG JQM_VERSION
 RUN curl --silent https://jdbc.postgresql.org/download/postgresql-42.3.3.jar --output /jqm/ext/pgsql.jar
+RUN curl --silent https://repo1.maven.org/maven2/mysql/mysql-connector-java/5.1.9/mysql-connector-java-5.1.9.jar --output /jqm/ext/mysql.jar
 
 LABEL com.enioka.description="JQM with the pgsql driver included based on version ${JQM_VERSION}"

docker/linux/demo/docker-compose.mysql.yml

@@ -0,0 +1,91 @@
+version: "3.7"
+services:
+    db:
+        image: mysql:5.7
+        deploy:
+            mode: replicated
+            replicas: 1
+            endpoint_mode: vip
+        environment:
+            - MYSQL_ROOT_PASSWORD=superpassword
+            - MYSQL_DATABASE=jqm
+            - MYSQL_USER=jqm
+            - MYSQL_PASSWORD=jqm
+        volumes:
+            - jqmdbdata:/var/lib/mysql
+
+    jqmbootstrap:
+        # Create the database schema on startup and stops.
+        build:
+            context: .
+            args:
+                - JQM_VERSION=${JQM_VERSION:-latest}
+        image: enioka/jqm:pgsql-${JQM_VERSION:-latest}
+        environment:
+            - JQM_INIT_MODE=UPDATER
+            - JQM_POOL_CONNSTR=jdbc:mysql://db/jqm
+            - JQM_POOL_VALIDATION_QUERY=SELECT 1
+            - JQM_POOL_DRIVER=com.mysql.jdbc.Driver
+            - JQM_POOL_USER=jqm
+            - JQM_POOL_PASSWORD=jqm
+        deploy:
+            mode: replicated
+            replicas: 1
+            endpoint_mode: vip
+            restart_policy:
+                condition: none
+
+    jqmweb:
+        build:
+            context: .
+            args:
+                - JQM_VERSION=${JQM_VERSION:-latest}
+        image: enioka/jqm:pgsql-${JQM_VERSION:-latest}
+        environment:
+            - JQM_NODE_NAME=_localhost_
+            - JQM_NODE_WS_INTERFACE=_localhost_
+            - JQM_INIT_MODE=CLUSTER
+            - JQM_CREATE_NODE_TEMPLATE=TEMPLATE_WEB
+            - JQM_POOL_CONNSTR=jdbc:mysql://db/jqm
+            - JQM_POOL_VALIDATION_QUERY=SELECT 1
+            - JQM_POOL_DRIVER=com.mysql.jdbc.Driver
+            - JQM_POOL_USER=jqm
+            - JQM_POOL_PASSWORD=jqm
+        ports:
+            - target: 1789
+              published: 1789
+              protocol: tcp
+              mode: ingress
+        deploy:
+            mode: replicated
+            replicas: 1
+            endpoint_mode: vip
+
+    jqmworker:
+        build:
+            context: .
+            args:
+                - JQM_VERSION=${JQM_VERSION:-latest}
+        image: enioka/jqm:pgsql-${JQM_VERSION:-latest}
+        environment:
+            - JQM_NODE_NAME=_localhost_
+            - JQM_NODE_WS_INTERFACE=_localhost_
+            - JQM_INIT_MODE=CLUSTER
+            - JQM_CREATE_NODE_TEMPLATE=TEMPLATE_DRONE
+            - JQM_POOL_CONNSTR=jdbc:mysql://db/jqm
+            - JQM_POOL_VALIDATION_QUERY=SELECT 1
+            - JQM_POOL_DRIVER=com.mysql.jdbc.Driver
+            - JQM_POOL_USER=jqm
+            - JQM_POOL_PASSWORD=jqm
+        deploy:
+            mode: replicated
+            replicas: 2
+            endpoint_mode: vip
+
+    adminer:
+        image: adminer
+        ports:
+        - 8080:8080
+
+volumes:
+    jqmdbdata:

jqm-all/jqm-model/src/main/java/com/enioka/jqm/jdbc/Db.java

@@ -29,12 +29,12 @@ public class Db
     /**
      * The version of the schema as it described in the current Maven artifact
      */
-    private static final int SCHEMA_VERSION = 2;
+    private static final int SCHEMA_VERSION = 3;
 
     /**
      * The SCHEMA_VERSION version is backward compatible until this version
      */
-    private static final int SCHEMA_COMPATIBLE_VERSION = 2;
+    private static final int SCHEMA_COMPATIBLE_VERSION = 3;
 
     /**
      * The list of different database adapters. We are using reflection for loading them for future extensibility.
@@ -382,7 +382,7 @@ private void dbUpgrade()
             int loop_from = db_schema_version;
             int to = db_schema_version;
             List<String> toApply = new ArrayList<String>();
-            toApply.addAll(adapter.preSchemaCreationScripts());
+            toApply.addAll(adapter.preSchemaCreationScripts(cnx));
 
             while (to != SCHEMA_VERSION)
             {

jqm-all/jqm-model/src/main/java/com/enioka/jqm/jdbc/DbAdapter.java

@@ -59,6 +59,17 @@ public abstract class DbAdapter
      */
     public abstract String adaptSql(String sql);
 
+     /**
+     * Special version of adaptSql which only replaces the table prefix. This is used for queries which are already formatted for a specific database.
+     * 
+     * @param sql
+     * @return a ready to use query.
+     */
+    public String adaptSqlPrefixOnly(String sql)
+    {
+        return sql.replace("__T__", this.tablePrefix);
+    }
+
     /**
      * The name of the columns to retrieve for getGeneratedKeys calls. (some dbs want uppercase, other lowercase).
      *
@@ -72,7 +83,7 @@ public String[] keyRetrievalColumn()
     /**
      * A list of files to run (from the classpath) before running schema upgrades. Default is empty.
      */
-    public List<String> preSchemaCreationScripts()
+    public List<String> preSchemaCreationScripts(DbConn cnx)
     {
         return new ArrayList<String>();
     }

jqm-all/jqm-model/src/main/java/com/enioka/jqm/jdbc/DbConn.java

@@ -136,13 +136,18 @@ public QueryResult runUpdate(String query_key, Object... params)
     }
 
     void runRawUpdate(String query_sql)
+    {
+        runRawUpdate(query_sql, false);
+    }
+
+    void runRawUpdate(String query_sql, boolean runReallyRaw)
     {
         transac_open = true;
         Statement s = null;
         String sql = null;
         try
         {
-            sql = parent.getAdapter().adaptSql(query_sql);
+            sql = runReallyRaw ? parent.getAdapter().adaptSqlPrefixOnly(query_sql) : parent.getAdapter().adaptSql(query_sql);
             if (sql.trim().isEmpty())
             {
                 return;

jqm-all/jqm-model/src/main/java/com/enioka/jqm/jdbc/DbImplMySql.java

@@ -40,10 +40,10 @@ public String adaptSql(String sql)
         return sql.replace("MEMORY TABLE", "TABLE").replace("JQM_PK.nextval", "?").replace(" DOUBLE", " DOUBLE PRECISION")
                 .replace("UNIX_MILLIS()", "ROUND(UNIX_TIMESTAMP(NOW(4)) * 1000)").replace("IN(UNNEST(?))", "IN(?)")
                 .replace("CURRENT_TIMESTAMP - 1 MINUTE", "(UNIX_TIMESTAMP() - 60)")
-                .replace("CURRENT_TIMESTAMP - ? SECOND", "(NOW() - INTERVAL ? SECOND)").replace("FROM (VALUES(0))", "FROM DUAL")
-                .replace("DNS||':'||PORT", "CONCAT(DNS, ':', PORT)").replace(" TIMESTAMP ", " TIMESTAMP(3) ")
+                .replace("CURRENT_TIMESTAMP - ? SECOND", "(UTC_TIMESTAMP() - INTERVAL ? SECOND)").replace("FROM (VALUES(0))", "FROM DUAL")
+                .replace("DNS||':'||PORT", "CONCAT(DNS, ':', PORT)").replace(" TIMESTAMP ", " DATETIME(3) ")
                 .replace("CURRENT_TIMESTAMP", "FFFFFFFFFFFFFFFFF@@@@").replace("FFFFFFFFFFFFFFFFF@@@@", "UTC_TIMESTAMP(3)")
-                .replace("TIMESTAMP(3) NOT NULL", "TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3)").replace("__T__", this.tablePrefix);
+                .replace("DATETIME(3) NOT NULL", "DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3)").replace("__T__", this.tablePrefix);
     }
 
     @Override
@@ -56,10 +56,35 @@ public boolean compatibleWith(DatabaseMetaData product) throws SQLException
     }
 
     @Override
-    public List<String> preSchemaCreationScripts()
+    public List<String> preSchemaCreationScripts(DbConn cnx)
     {
+        // Sequence management
         List<String> res = new ArrayList<String>();
         res.add("/sql/mysql.sql");
+
+        // Deal with a change in version 2.3.0 where we changed timestamps into datetimes for better TZ support.
+        // This cannot be handled in normal migration scripts as the meaning of scripts version 1 & 2 has changed with the changes in
+        // adaptSql().
+        ResultSet rs = cnx.runRawSelect("select UPPER(DATA_TYPE) FROM __T__INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME=? AND COLUMN_NAME=?",
+                "VERSION", "INSTALL_DATE");
+        try
+        {
+            if (rs.next() && rs.getString(1).equals("TIMESTAMP"))
+            {
+                // Should be DATETIME not TIMESTAMP
+                res.add("/sql/mysql_00002_00003_raw.sql");
+            }
+            // If no result we do not care as it means DB schema is empty, hence the rs.next() condition.
+        }
+        catch (SQLException e)
+        {
+            throw new DatabaseException("Could not update MySQL existing schema", e);
+        }
+        finally
+        {
+            DbHelper.closeQuietly(rs);
+        }
+
         return res;
     }
 

jqm-all/jqm-model/src/main/java/com/enioka/jqm/jdbc/ScriptRunner.java

@@ -34,6 +34,7 @@ public static void run(DbConn cnx, String classpath)
         InputStreamReader isr = null;
         String line;
         StringBuilder sb = new StringBuilder();
+        jqmlogger.debug("Running SQL script {}", classpath);
         try
         {
             isr = new InputStreamReader(is, "UTF-8");
@@ -48,7 +49,8 @@ public static void run(DbConn cnx, String classpath)
                 if (line.contains(";"))
                 {
                     // End of order - run it.
-                    cnx.runRawUpdate(sb.substring(0, sb.length() - 1 - nl.length()));
+                    jqmlogger.trace("Running update SQL {}", sb.toString());
+                    cnx.runRawUpdate(sb.substring(0, sb.length() - 1 - nl.length()), classpath.contains("_raw.sql"));
                     sb = new StringBuilder();
                 }
 

jqm-all/jqm-model/src/main/resources/sql/00002_00003.sql

@@ -0,0 +1 @@
+/* Empty file. Version 3 is actually only MYSQL_specific */

jqm-all/jqm-model/src/main/resources/sql/mysql_00002_00003_raw.sql

@@ -0,0 +1,36 @@
+/* Data migration script for TIMESTAMP column to DATETIME column */
+
+/* This is very important to ensure the proper conversion from TIMESTAMP to DATETIME in UTC */
+SET time_zone = "UTC";
+
+ALTER TABLE __T__VERSION CHANGE INSTALL_DATE INSTALL_DATE DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3);
+
+ALTER TABLE __T__NODE CHANGE LAST_SEEN_ALIVE LAST_SEEN_ALIVE DATETIME(3) NULL;
+
+ALTER TABLE __T__QUEUE_NODE_MAPPING CHANGE LAST_MODIFIED LAST_MODIFIED DATETIME(3) NULL;
+
+ALTER TABLE __T__JOB_SCHEDULE CHANGE LAST_UPDATED LAST_UPDATED DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3);
+
+ALTER TABLE __T__JOB_INSTANCE CHANGE DATE_ENQUEUE DATE_ENQUEUE DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3);
+ALTER TABLE __T__JOB_INSTANCE CHANGE DATE_ATTRIBUTION DATE_ATTRIBUTION DATETIME(3) NULL;
+ALTER TABLE __T__JOB_INSTANCE CHANGE DATE_START DATE_START DATETIME(3) NULL;
+ALTER TABLE __T__JOB_INSTANCE CHANGE DATE_NOT_BEFORE DATE_NOT_BEFORE DATETIME(3) NULL;
+
+ALTER TABLE __T__HISTORY CHANGE DATE_ENQUEUE DATE_ENQUEUE DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3);
+ALTER TABLE __T__HISTORY CHANGE DATE_ATTRIBUTION DATE_ATTRIBUTION DATETIME(3) NULL;
+ALTER TABLE __T__HISTORY CHANGE DATE_START DATE_START DATETIME(3) NULL;
+ALTER TABLE __T__HISTORY CHANGE DATE_END DATE_END DATETIME(3) NULL;
+
+ALTER TABLE __T__JNDI_OBJECT_RESOURCE CHANGE LAST_MODIFIED LAST_MODIFIED DATETIME(3) NULL;
+
+ALTER TABLE __T__JNDI_OR_PARAMETER CHANGE LAST_MODIFIED LAST_MODIFIED DATETIME(3) NULL;
+
+ALTER TABLE __T__RUSER CHANGE CREATION_DATE CREATION_DATE DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3);
+
+ALTER TABLE __T__RUSER CHANGE EXPIRATION_DATE EXPIRATION_DATE DATETIME(3) NULL;
+
+ALTER TABLE __T__RUSER CHANGE LAST_MODIFIED LAST_MODIFIED DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3);
+
+ALTER TABLE __T__GLOBAL_PARAMETER CHANGE LAST_MODIFIED LAST_MODIFIED DATETIME(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3);
+
+ALTER TABLE __T__WITNESS CHANGE LATEST_CONTACT LATEST_CONTACT DATETIME(3) NULL;