remove_task_config_field (#2189)

DataLinkDC · Aug 9, 2023 · 48abeec · 48abeec
1 parent cbed39e
commit 48abeec
Show file tree

Hide file tree

Showing 11 changed files with 70 additions and 65 deletions.
diff --git a/dinky-admin/src/main/java/org/dinky/data/dto/StudioExecuteDTO.java b/dinky-admin/src/main/java/org/dinky/data/dto/StudioExecuteDTO.java
@@ -19,14 +19,11 @@
 
 package org.dinky.data.dto;
 
-import org.dinky.assertion.Asserts;
 import org.dinky.job.JobConfig;
 
 import java.util.HashMap;
 import java.util.Map;
 
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 
 import lombok.Getter;
@@ -65,24 +62,10 @@ public class StudioExecuteDTO extends AbstractStatementDTO {
     private Integer parallelism;
     private Integer savePointStrategy;
     private String savePointPath;
-    private String configJson;
+    private Map<String, String> config = new HashMap<>();
     private static final ObjectMapper mapper = new ObjectMapper();
 
     public JobConfig getJobConfig() {
-        Map<String, String> config = new HashMap<>();
-        if (Asserts.isNotNullString(configJson)) {
-            try {
-                JsonNode paras = mapper.readTree(configJson);
-                paras.forEach(
-                        (JsonNode node) -> {
-                            if (!node.isNull()) {
-                                config.put(node.get("key").asText(), node.get("value").asText());
-                            }
-                        });
-            } catch (JsonProcessingException e) {
-                log.error(e.getMessage());
-            }
-        }
         return new JobConfig(
                 type,
                 useResult,

diff --git a/dinky-admin/src/main/java/org/dinky/data/model/Task.java b/dinky-admin/src/main/java/org/dinky/data/model/Task.java
@@ -21,17 +21,16 @@
 
 import org.dinky.assertion.Asserts;
 import org.dinky.config.Dialect;
+import org.dinky.data.typehandler.ListTypeHandler;
 import org.dinky.job.JobConfig;
 import org.dinky.mybatis.model.SuperEntity;
 
-import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
 
 import com.baomidou.mybatisplus.annotation.TableField;
 import com.baomidou.mybatisplus.annotation.TableName;
-import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ObjectNode;
@@ -83,9 +82,6 @@ public class Task extends SuperEntity<Task> {
     private Integer envId;
 
     private Integer alertGroupId;
-
-    private String configJson;
-
     private String note;
 
     private Integer step;
@@ -103,8 +99,8 @@ public class Task extends SuperEntity<Task> {
     @TableField(exist = false)
     private List<Savepoints> savePoints;
 
-    @TableField(exist = false)
-    private List<Map<String, String>> config = new ArrayList<>();
+    @TableField(typeHandler = ListTypeHandler.class)
+    private List<Map<String, String>> configJson;
 
     @TableField(exist = false)
     private String path;
@@ -124,26 +120,10 @@ public class Task extends SuperEntity<Task> {
     @TableField(exist = false)
     private String alertGroupName;
 
-    public static final ObjectMapper objectMapper = new ObjectMapper();
-
-    @SuppressWarnings("unchecked")
-    public List<Map<String, String>> parseConfig() {
-        if (Asserts.isNullString(configJson)) {
-            return config;
-        }
-
-        try {
-            config = objectMapper.readValue(configJson, ArrayList.class);
-        } catch (JsonProcessingException e) {
-            log.error(e.getMessage());
-        }
-        return config;
-    }
-
     public JobConfig buildSubmitConfig() {
         boolean useRemote = clusterId != null && clusterId != 0;
         Map<String, String> map =
-                parseConfig().stream()
+                configJson.stream()
                         .filter(Asserts::isNotNull)
                         .collect(
                                 Collectors.toMap(
@@ -190,7 +170,6 @@ public JsonNode parseJsonNode(ObjectMapper mapper) {
         jsonNode.put("statementSet", this.statementSet);
         jsonNode.put("batchModel", this.batchModel);
         jsonNode.put("clusterName", this.clusterName);
-        jsonNode.put("configJson", this.configJson);
         jsonNode.put("note", this.note);
         jsonNode.put("step", this.step);
         jsonNode.put("enabled", this.getEnabled());

diff --git a/dinky-admin/src/main/java/org/dinky/data/typehandler/ListTypeHandler.java b/dinky-admin/src/main/java/org/dinky/data/typehandler/ListTypeHandler.java
@@ -0,0 +1,50 @@
+/*
+ *
+ *  Licensed to the Apache Software Foundation (ASF) under one or more
+ *  contributor license agreements.  See the NOTICE file distributed with
+ *  this work for additional information regarding copyright ownership.
+ *  The ASF licenses this file to You under the Apache License, Version 2.0
+ *  (the "License"); you may not use this file except in compliance with
+ *  the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package org.dinky.data.typehandler;
+
+import org.apache.ibatis.type.JdbcType;
+import org.apache.ibatis.type.MappedJdbcTypes;
+import org.apache.ibatis.type.MappedTypes;
+
+import java.util.List;
+import java.util.Map;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.baomidou.mybatisplus.extension.handlers.AbstractJsonTypeHandler;
+import com.baomidou.mybatisplus.extension.handlers.FastjsonTypeHandler;
+
+import cn.hutool.json.JSONArray;
+
+@MappedTypes({List.class})
+@MappedJdbcTypes({JdbcType.VARCHAR})
+public class ListTypeHandler extends AbstractJsonTypeHandler<List<Map>> {
+    private static final Logger log = LoggerFactory.getLogger(FastjsonTypeHandler.class);
+
+    protected List<Map> parse(String json) {
+        return new JSONArray(json).toList(Map.class);
+    }
+
+    @Override
+    protected String toJson(List<Map> obj) {
+        return new JSONArray(obj).toString();
+    }
+}
diff --git a/dinky-admin/src/main/java/org/dinky/service/impl/CatalogueServiceImpl.java b/dinky-admin/src/main/java/org/dinky/service/impl/CatalogueServiceImpl.java
@@ -104,7 +104,7 @@ public Catalogue saveOrUpdateCatalogueAndTask(CatalogueTaskDTO catalogueTaskDTO)
         }
         task.setName(catalogueTaskDTO.getName());
         task.setDialect(catalogueTaskDTO.getDialect());
-        task.setConfig(Collections.singletonList(catalogueTaskDTO.getConfig()));
+        task.setConfigJson(Collections.singletonList(catalogueTaskDTO.getConfig()));
         taskService.saveOrUpdateTask(task);
 
         catalogue.setTenantId(catalogueTaskDTO.getTenantId());

diff --git a/dinky-admin/src/main/java/org/dinky/service/impl/TaskServiceImpl.java b/dinky-admin/src/main/java/org/dinky/service/impl/TaskServiceImpl.java
@@ -411,8 +411,6 @@ public Task getTaskInfoById(Integer id) {
         if (task == null) {
             return null;
         }
-
-        task.parseConfig();
         if (task.getClusterId() != null) {
             Cluster cluster = clusterInstanceService.getById(task.getClusterId());
             if (cluster != null) {
@@ -444,10 +442,10 @@ public void initTenantByTaskId(Integer id) {
     @Override
     public boolean saveOrUpdateTask(Task task) {
         if (Dialect.isUDF(task.getDialect())) {
-            if (CollUtil.isNotEmpty(task.getConfig())
+            if (CollUtil.isNotEmpty(task.getConfigJson())
                     && Asserts.isNullString(task.getStatement())
-                    && Convert.toInt(task.getConfig().get(0).get("templateId"), 0) != 0) {
-                Map<String, String> config = task.getConfig().get(0);
+                    && Convert.toInt(task.getConfigJson().get(0).get("templateId"), 0) != 0) {
+                Map<String, String> config = task.getConfigJson().get(0);
                 UDFTemplate template = udfTemplateService.getById(config.get("templateId"));
                 if (template != null) {
                     String code =

diff --git a/dinky-admin/src/main/resources/application.yml b/dinky-admin/src/main/resources/application.yml
@@ -97,7 +97,7 @@ mybatis-plus:
     ##### mybatis-plus prints complete sql (only for development environment)
     #log-impl: org.apache.ibatis.logging.stdout.StdOutImpl
     log-impl: org.apache.ibatis.logging.nologging.NoLoggingImpl
-
+  type-handlers-package: org.dinky.data.typehandler
 #################################################################################################################
 ################################################# SMS Config ####################################################
 #################################################################################################################

diff --git a/dinky-web/src/hooks/useThemeValue.tsx b/dinky-web/src/hooks/useThemeValue.tsx
@@ -4,13 +4,14 @@ import {theme} from "antd";
 
 export type ThemeValue = {
   borderColor: string
+  footerColor: string
 }
 const { useToken } = theme;
 const getThemeValue = (isDark: boolean): ThemeValue => {
   if (isDark) {
-    return {borderColor: "#343434"}
+    return {borderColor: "#343434", footerColor: "rgba(255, 255, 255, 0.18)"}
   } else {
-    return {borderColor: "rgba(5, 5, 5, 0.06)"}
+    return {borderColor: "rgba(5, 5, 5, 0.06)", footerColor: "#f4f4f4"}
     // return {borderColor: "#E0E2E5"}
   }
 }

diff --git a/dinky-web/src/pages/DataStudio/FooterContainer/index.tsx b/dinky-web/src/pages/DataStudio/FooterContainer/index.tsx
@@ -6,6 +6,7 @@ import {getCurrentTab} from "@/pages/DataStudio/function";
 import {getSseData} from "@/services/api";
 import {l} from "@/utils/intl";
 import JobRunningModal from "@/pages/DataStudio/FooterContainer/JobRunningModal";
+import useThemeValue from "@/hooks/useThemeValue";
 
 export type FooterContainerProps = {
   token: GlobalToken
@@ -30,16 +31,16 @@ const FooterContainer: React.FC<FooterContainerProps & StateType> = (props) => {
       jobRunningMsg,
     }, token, tabs
   } = props;
+  const themeValue = useThemeValue();
 
   const [viewJobRunning, setViewJobRunning] = useState(false);
 
-
   const [memDetailInfo, setMemDetailInfo] = useState(memDetails);
   useEffect(() => {
     const eventSource = getSseData("/api/sse/getJvmInfo");
     eventSource.onmessage = (event) => {
       const data = JSON.parse(event.data);
-      setMemDetailInfo(Number(data["heapUsed"] / 1024 / 1024).toFixed(0) + "/" + Number(data["total"] / 1024 / 1024).toFixed(0) + "M")
+      setMemDetailInfo(Number(data["heapUsed"] / 1024 / 1024).toFixed(0) + "/" + Number(data["max"] / 1024 / 1024).toFixed(0) + "M")
     }
     return () => {
       eventSource.close()
@@ -112,7 +113,7 @@ const FooterContainer: React.FC<FooterContainerProps & StateType> = (props) => {
 
   return <>
     <div style={{
-      backgroundColor: token.colorFill,
+      backgroundColor: themeValue.footerColor,
       height: VIEW.footerHeight,
       width: "100%",
       display: "flex",

diff --git a/dinky-web/src/pages/DataStudio/HeaderContainer/index.tsx b/dinky-web/src/pages/DataStudio/HeaderContainer/index.tsx
@@ -97,7 +97,6 @@ const HeaderContainer = (props: any) => {
             ...current,
             jobName: current.name,
             taskId: current.id,
-            configJson: JSON.stringify(current.config),
         };
         const key = current.key;
         const taskKey = (Math.random() * 1000) + '';
@@ -148,7 +147,6 @@ const HeaderContainer = (props: any) => {
       title: l('button.save'),
       click: () => {
         const current = getCurrentData(panes, activeKey);
-        current.configJson = JSON.stringify(current.config)
         handlePutDataJson("/api/task", current).then(()=>saveTabs({...props.tabs}))
       },
       hotKey: (e: KeyboardEvent) => e.ctrlKey && e.key === 's',
@@ -176,11 +174,7 @@ const HeaderContainer = (props: any) => {
       title: l('button.graph'),
       click: () => {
         const currentData = getCurrentData(panes, activeKey);
-        const param = {
-          ...currentData,
-          configJson: JSON.stringify(currentData.config),
-        };
-        const res = getJobPlan(l("pages.datastudio.editor.explan.tip"), param);
+        const res = getJobPlan(l("pages.datastudio.editor.explan.tip"), currentData);
         res.then((result) => {
           if (result) {
             modal.confirm({

diff --git a/dinky-web/src/pages/DataStudio/LeftContainer/MetaData/index.tsx b/dinky-web/src/pages/DataStudio/LeftContainer/MetaData/index.tsx
@@ -13,11 +13,10 @@ import {TagAlignLeft} from "@/components/StyledComponents";
 import SchemaTree from "@/pages/RegCenter/DataSource/components/DataSourceDetail/SchemaTree";
 import {DataSources} from "@/types/RegCenter/data";
 import {BtnRoute} from "@/pages/DataStudio/route";
-import {transformTreeData} from "@/utils/function";
 
 const MetaData = (props: any) => {
 
-  const {dispatch ,toolContentHeight,leftContainer, database: { dbData , selectDatabaseId , expandKeys, selectKeys} } = props;
+  const {dispatch ,toolContentHeight, database: { dbData , selectDatabaseId , expandKeys, selectKeys} } = props;
   const [treeData, setTreeData] = useState<[]>([]);
   const [isLoadingDatabase, setIsLoadingDatabase] = useState(false);
   const selectDb = (dbData as DataSources.DataSource[]).filter(x=> x.id === selectDatabaseId)[0]

diff --git a/dinky-web/src/pages/DataStudio/RightContainer/JobConfig/index.tsx b/dinky-web/src/pages/DataStudio/RightContainer/JobConfig/index.tsx
@@ -226,7 +226,7 @@ const JobConfig = (props: any) => {
             <ProFormList
                 label={l('pages.datastudio.label.jobConfig.other')}
                 tooltip={l('pages.datastudio.label.jobConfig.other.tip')}
-                name={'config'}
+                name={'configJson'}
                 copyIconProps={false}
                 creatorButtonProps={{
                     style: {width: '100%'},