feat: add array element ignore, fix "arex_" prefix issue, and support…

… select list (#61)
arextest · Oct 30, 2024 · 0b0dba0 · 0b0dba0
1 parent 4aa38cc
commit 0b0dba0
Show file tree

Hide file tree

Showing 19 changed files with 354 additions and 53 deletions.
diff --git a/arex-compare-core/pom.xml b/arex-compare-core/pom.xml
@@ -5,7 +5,7 @@
     <parent>
         <artifactId>arex-compare-parent</artifactId>
         <groupId>com.arextest</groupId>
-        <version>0.2.16</version>
+        <version>0.2.17</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
 

diff --git a/arex-compare-core/src/main/java/com/arextest/diff/compare/ArrayCompare.java b/arex-compare-core/src/main/java/com/arextest/diff/compare/ArrayCompare.java
@@ -6,6 +6,7 @@
 import com.arextest.diff.handler.log.register.LogRegister;
 import com.arextest.diff.model.compare.CompareContext;
 import com.arextest.diff.model.compare.IndexPair;
+import com.arextest.diff.model.enumeration.ParentNodeType;
 import com.arextest.diff.model.log.NodeEntity;
 import com.arextest.diff.utils.ListUti;
 import com.fasterxml.jackson.databind.node.ArrayNode;
@@ -55,7 +56,8 @@ public static void arrayCompare(Object obj1, Object obj2, CompareContext compare
       boolean rightExist = false;
 
       if (correspondRightIndex == -1) {
-        LogRegister.register(element1, element2, LogMarker.RIGHT_ARRAY_MISSING_KEY, compareContext);
+        compareContext.parentNodeType = ParentNodeType.ARRAY;
+        GenericCompare.jsonCompare(element1, element2, compareContext);
       } else {
         element2 = obj2Array.get(correspondRightIndex);
         rightExist = true;
@@ -70,6 +72,7 @@ public static void arrayCompare(Object obj1, Object obj2, CompareContext compare
             compareContext.pkListIndexPair,
             compareContext.currentNodeLeft);
         if (needCompare && element1 != null && element2 != null) {
+          compareContext.parentNodeType = ParentNodeType.ARRAY;
           GenericCompare.jsonCompare(element1, element2, compareContext);
         }
       }
@@ -98,7 +101,8 @@ public static void arrayCompare(Object obj1, Object obj2, CompareContext compare
       Object element2 = obj2Array.get(i);
       boolean leftExist = false;
       if (correspondLeftIndex == -1) {
-        LogRegister.register(element1, element2, LogMarker.LEFT_ARRAY_MISSING_KEY, compareContext);
+        compareContext.parentNodeType = ParentNodeType.ARRAY;
+        GenericCompare.jsonCompare(element1, element2, compareContext);
       } else {
         element1 = obj1Array.get(correspondLeftIndex);
         leftExist = true;
@@ -111,6 +115,7 @@ public static void arrayCompare(Object obj1, Object obj2, CompareContext compare
             compareContext.pkListIndexPair,
             compareContext.currentNodeRight);
         if (needCompare && element1 != null && element2 != null) {
+          compareContext.parentNodeType = ParentNodeType.ARRAY;
           GenericCompare.jsonCompare(element1, element2, compareContext);
         }
       }

diff --git a/arex-compare-core/src/main/java/com/arextest/diff/compare/GenericCompare.java b/arex-compare-core/src/main/java/com/arextest/diff/compare/GenericCompare.java
@@ -3,13 +3,15 @@
 import com.arextest.diff.handler.log.LogMarker;
 import com.arextest.diff.handler.log.register.LogRegister;
 import com.arextest.diff.model.compare.CompareContext;
+import com.arextest.diff.model.enumeration.ParentNodeType;
 import com.arextest.diff.model.log.NodeEntity;
 import com.arextest.diff.utils.IgnoreUtil;
 import com.arextest.diff.utils.ListUti;
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.NullNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import java.util.List;
+import java.util.Objects;
 
 public class GenericCompare {
 
@@ -34,19 +36,26 @@ public static void jsonCompare(Object obj1, Object obj2, CompareContext compareC
     }
 
     // not compare by exclusions
-    if (IgnoreUtil.ignoreProcessor(fuzzyPath, currentNode, compareContext.exclusions,
-        compareContext.expressionExclusions, compareContext.ignoreNodeSet)) {
+    if (IgnoreUtil.ignoreProcessor(fuzzyPath, compareContext.currentNodeLeft,
+        compareContext.currentNodeRight, compareContext.exclusions,
+        compareContext.conditionExclusions, compareContext.ignoreNodeSet)) {
       return;
     }
 
     // field missing
     if (obj1 == null && obj2 == null) {
       return;
     } else if (obj1 == null) {
-      LogRegister.register(obj1, obj2, LogMarker.LEFT_OBJECT_MISSING, compareContext);
+      LogRegister.register(obj1, obj2,
+          Objects.equals(compareContext.parentNodeType, ParentNodeType.OBJECT)
+              ? LogMarker.LEFT_OBJECT_MISSING
+              : LogMarker.LEFT_ARRAY_MISSING, compareContext);
       return;
     } else if (obj2 == null) {
-      LogRegister.register(obj1, obj2, LogMarker.RIGHT_OBJECT_MISSING, compareContext);
+      LogRegister.register(obj1, obj2,
+          Objects.equals(compareContext.parentNodeType, ParentNodeType.OBJECT)
+              ? LogMarker.LEFT_OBJECT_MISSING
+              : LogMarker.RIGHT_ARRAY_MISSING, compareContext);
       return;
     }
 

diff --git a/arex-compare-core/src/main/java/com/arextest/diff/compare/ObjectCompare.java b/arex-compare-core/src/main/java/com/arextest/diff/compare/ObjectCompare.java
@@ -1,6 +1,7 @@
 package com.arextest.diff.compare;
 
 import com.arextest.diff.model.compare.CompareContext;
+import com.arextest.diff.model.enumeration.ParentNodeType;
 import com.arextest.diff.model.log.NodeEntity;
 import com.arextest.diff.utils.JacksonHelperUtil;
 import com.arextest.diff.utils.ListUti;
@@ -34,6 +35,7 @@ public static void objectCompare(Object obj1, Object obj2, CompareContext compar
         compareContext.currentNodeRight.add(new NodeEntity(fieldName, 0));
       }
 
+      compareContext.parentNodeType = ParentNodeType.OBJECT;
       GenericCompare.jsonCompare(obj1FieldValue, obj2FieldValue, compareContext);
 
       ListUti.removeLast(compareContext.currentNodeLeft);
@@ -57,6 +59,7 @@ public static void objectCompare(Object obj1, Object obj2, CompareContext compar
         compareContext.currentNodeLeft.add(new NodeEntity(fieldName, 0));
       }
 
+      compareContext.parentNodeType = ParentNodeType.OBJECT;
       GenericCompare.jsonCompare(obj1FieldValue, obj2FieldValue, compareContext);
 
       if (leftExist) {

diff --git a/arex-compare-core/src/main/java/com/arextest/diff/eigen/EigenHandler.java b/arex-compare-core/src/main/java/com/arextest/diff/eigen/EigenHandler.java
@@ -10,9 +10,10 @@
 import com.arextest.diff.model.pathparse.ExpressionNodeEntity;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
-import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 
 public class EigenHandler {
@@ -41,13 +42,13 @@ public EigenResult doHandler(RulesConfig rulesConfig) {
       obj = null;
     }
 
-    LinkedList<LinkedList<ExpressionNodeEntity>> expressionExclusions =
-        jsonPathExpressionHandler.doMultiExpressionParse(rulesConfig.getExpressionExclusions(),
+    List<List<ExpressionNodeEntity>> expressionExclusions = new LinkedList<>();
+    Map<List<ExpressionNodeEntity>, LinkedList<LinkedList<ExpressionNodeEntity>>> listLinkedListMap =
+        jsonPathExpressionHandler.doMultiExpressionParse(
+            rulesConfig.getExpressionExclusions(),
             obj);
-    if (expressionExclusions != null) {
-      rulesConfig.setExpressionExclusions(new ArrayList<>(expressionExclusions));
-    }
-
+    listLinkedListMap.values().forEach(expressionExclusions::addAll);
+    rulesConfig.setExpressionExclusions(expressionExclusions);
     return eigenMapCalculate.doCalculate(obj, rulesConfig, new HashMap<>());
   }
 

diff --git a/arex-compare-core/src/main/java/com/arextest/diff/eigen/EigenMapCalculate.java b/arex-compare-core/src/main/java/com/arextest/diff/eigen/EigenMapCalculate.java
@@ -12,7 +12,6 @@
 import java.time.Instant;
 import java.time.ZoneId;
 import java.time.format.DateTimeFormatter;
-import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.LinkedList;
@@ -54,7 +53,8 @@ private void doCalculateJsonNode(Object obj, CalculateContext calculateContext,
       Map<Integer, Long> eigenMap) {
 
     // ignore by node name and node path
-    if (IgnoreUtil.ignoreProcessor(calculateContext.nodePath, calculateContext.currentNodeEntity,
+    if (IgnoreUtil.ignoreProcessorEngine(calculateContext.nodePath,
+        calculateContext.currentNodeEntity,
         calculateContext.exclusions, calculateContext.expressionExclusions,
         calculateContext.ignoreNodeSet)) {
       return;

diff --git a/arex-compare-core/src/main/java/com/arextest/diff/handler/CompareHandler.java b/arex-compare-core/src/main/java/com/arextest/diff/handler/CompareHandler.java
@@ -29,7 +29,7 @@ public List<LogEntity> doHandler(RulesConfig rulesConfig, KeyComputeResponse key
     compareContext.setResponseReferences(keyComputeResponse.getAllReferenceEntities());
 
     compareContext.exclusions = rulesConfig.getExclusions();
-    compareContext.expressionExclusions = rulesConfig.getExpressionExclusions();
+    compareContext.conditionExclusions = rulesConfig.getConditionExclusions();
     compareContext.ignoreNodeSet = rulesConfig.getIgnoreNodeSet();
 
     compareContext.notDistinguishNullAndEmpty = rulesConfig.isNullEqualsEmpty();

diff --git a/arex-compare-core/src/main/java/com/arextest/diff/handler/parse/sqlparse/SqlParse.java b/arex-compare-core/src/main/java/com/arextest/diff/handler/parse/sqlparse/SqlParse.java
@@ -76,6 +76,9 @@ public ParsedResult sqlParse(ObjectNode jsonObj, boolean nameToLower) {
       return new ParsedResult(null, false);
     }
 
+    // additional: Handling multiple selections
+    databaseBody = processMultipleSelect(databaseBody);
+
     boolean successParse = true;
     ArrayNode parsedSql = JacksonHelperUtil.getArrayNode();
     List<Boolean> isSelect = new ArrayList<>();
@@ -245,6 +248,41 @@ private void fillOriginalSql(ObjectNode objectNode, JsonNode databaseBody) {
     objectNode.set(DbParseConstants.PARSED_SQL, parsedSql);
   }
 
+  private JsonNode processMultipleSelect(JsonNode databaseBody) {
+    JsonNode result = databaseBody;
+    if (databaseBody instanceof TextNode) {
+      String sql = databaseBody.asText();
+      if (sql.contains(";")) {
+        String[] sqls = sql.split(";");
+        ArrayNode arrayNode = JacksonHelperUtil.getArrayNode();
+        for (String s : sqls) {
+          if (!startsWithSelect(s)) {
+            break;
+          }
+          arrayNode.add(s);
+        }
+        result = arrayNode;
+      }
+
+    }
+    return result;
+  }
+
+  public static boolean startsWithSelect(String str) {
+    if (str == null) {
+      return false;
+    }
+    int len = str.length();
+    int i = 0;
+    // Skip leading whitespace
+    while (i < len && Character.isWhitespace(str.charAt(i))) {
+      i++;
+    }
+    // Check if the remaining string starts with "select" or "SELECT"
+    return str.regionMatches(true, i, "select", 0, 6);
+  }
+
+
   private static class ParsedResult {
 
     private List<Boolean> isSelect;

diff --git a/...are-core/src/main/java/com/arextest/diff/handler/pathparse/JsonPathExpressionHandler.java b/...are-core/src/main/java/com/arextest/diff/handler/pathparse/JsonPathExpressionHandler.java
@@ -11,9 +11,12 @@
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.fasterxml.jackson.databind.node.TextNode;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
 import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
@@ -29,36 +32,41 @@ public class JsonPathExpressionHandler {
   public void doExpressionParse(RulesConfig rulesConfig,
       Object baseObj, Object testObj) throws ExecutionException, InterruptedException {
 
-    Set<List<ExpressionNodeEntity>> result = new HashSet<>();
+    Map<LinkedList<LinkedList<ExpressionNodeEntity>>, LinkedList<LinkedList<ExpressionNodeEntity>>>
+        conditionExclusions = null;
+
     try {
       List<List<ExpressionNodeEntity>> expressionExclusions = rulesConfig.getExpressionExclusions();
 
-      CompletableFuture<LinkedList<LinkedList<ExpressionNodeEntity>>> future1 = CompletableFuture.supplyAsync(
+      CompletableFuture<Map<List<ExpressionNodeEntity>, LinkedList<LinkedList<ExpressionNodeEntity>>>>
+          future1 = CompletableFuture.supplyAsync(
           () -> doMultiExpressionParse(expressionExclusions, baseObj),
           TaskThreadFactory.jsonObjectThreadPool
       );
 
-      CompletableFuture<LinkedList<LinkedList<ExpressionNodeEntity>>> future2 = CompletableFuture.supplyAsync(
+      CompletableFuture<Map<List<ExpressionNodeEntity>, LinkedList<LinkedList<ExpressionNodeEntity>>>>
+          future2 = CompletableFuture.supplyAsync(
           () -> doMultiExpressionParse(expressionExclusions, testObj),
           TaskThreadFactory.jsonObjectThreadPool
       );
 
       CompletableFuture<Void> voidCompletableFuture = CompletableFuture.allOf(future1, future2);
       voidCompletableFuture.get(Constant.JSON_PATH_PARSE_MINUTES_TIME, TimeUnit.MINUTES);
 
-      result.addAll(future1.get());
-      result.addAll(future2.get());
+      Map<List<ExpressionNodeEntity>, LinkedList<LinkedList<ExpressionNodeEntity>>> baseExpression = future1.get();
+      Map<List<ExpressionNodeEntity>, LinkedList<LinkedList<ExpressionNodeEntity>>> testExpression = future2.get();
+      conditionExclusions = mergeExpression(baseExpression, testExpression);
 
     } catch (RuntimeException | TimeoutException e) {
       LOGGER.warning("doExpressionParse error: " + e.getMessage());
     }
-    rulesConfig.setExpressionExclusions(new LinkedList<>(result));
+    rulesConfig.setConditionExclusions(conditionExclusions);
   }
 
-  public LinkedList<LinkedList<ExpressionNodeEntity>> doMultiExpressionParse(
+  public Map<List<ExpressionNodeEntity>, LinkedList<LinkedList<ExpressionNodeEntity>>> doMultiExpressionParse(
       List<List<ExpressionNodeEntity>> expressionExclusions, Object object) {
-
-    LinkedList<LinkedList<ExpressionNodeEntity>> result = new LinkedList<>();
+    Map<List<ExpressionNodeEntity>, LinkedList<LinkedList<ExpressionNodeEntity>>> result =
+        new HashMap<>();
 
     try {
       if (ListUti.isEmpty(expressionExclusions)) {
@@ -69,7 +77,7 @@ public LinkedList<LinkedList<ExpressionNodeEntity>> doMultiExpressionParse(
         LinkedList<LinkedList<ExpressionNodeEntity>> linkedLists = doSinglePathExpressionParse(
             expressionNodeEntityList, 0, expressionNodeEntityList.size(), object, false);
         if (linkedLists != null) {
-          result.addAll(linkedLists);
+          result.put(expressionNodeEntityList, linkedLists);
         }
       }
     } catch (RuntimeException exception) {
@@ -279,5 +287,40 @@ private String getValueFormPath(Object objectNode, List<String> pathList) {
     return null;
   }
 
+
+  private Map<LinkedList<LinkedList<ExpressionNodeEntity>>, LinkedList<LinkedList<ExpressionNodeEntity>>> mergeExpression(
+      Map<List<ExpressionNodeEntity>, LinkedList<LinkedList<ExpressionNodeEntity>>> baseExpression,
+      Map<List<ExpressionNodeEntity>, LinkedList<LinkedList<ExpressionNodeEntity>>> testExpression) {
+
+    Map<LinkedList<LinkedList<ExpressionNodeEntity>>, LinkedList<LinkedList<ExpressionNodeEntity>>> result = new HashMap<>();
+
+    Set<List<ExpressionNodeEntity>> baseExpressionKeySet = baseExpression.keySet();
+    Set<List<ExpressionNodeEntity>> testExpresssionKeyset = testExpression.keySet();
+
+    // get the intersection of the two sets
+    Set<List<ExpressionNodeEntity>> temp = new HashSet<>(baseExpressionKeySet);
+    temp.retainAll(testExpresssionKeyset);
+
+    for (Entry<List<ExpressionNodeEntity>, LinkedList<LinkedList<ExpressionNodeEntity>>> entry : baseExpression.entrySet()) {
+      List<ExpressionNodeEntity> key = entry.getKey();
+      LinkedList<LinkedList<ExpressionNodeEntity>> baseValue = entry.getValue();
+      if (temp.contains(key)) {
+        LinkedList<LinkedList<ExpressionNodeEntity>> testValue = testExpression.get(key);
+        result.put(baseValue, testValue);
+      } else {
+        result.put(baseValue, null);
+      }
+    }
+
+    for (Entry<List<ExpressionNodeEntity>, LinkedList<LinkedList<ExpressionNodeEntity>>> entry : testExpression.entrySet()) {
+      List<ExpressionNodeEntity> key = entry.getKey();
+      LinkedList<LinkedList<ExpressionNodeEntity>> testValue = entry.getValue();
+      if (!temp.contains(key)) {
+        result.put(null, testValue);
+      }
+    }
+    return result;
+  }
+
 }
 
diff --git a/arex-compare-core/src/main/java/com/arextest/diff/model/RulesConfig.java b/arex-compare-core/src/main/java/com/arextest/diff/model/RulesConfig.java
@@ -6,9 +6,11 @@
 import com.arextest.diff.model.key.ReferenceEntity;
 import com.arextest.diff.model.pathparse.ExpressionNodeEntity;
 import java.util.Collections;
+import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import org.apache.commons.lang3.tuple.Pair;
 
 public class RulesConfig {
 
@@ -60,6 +62,8 @@ public class RulesConfig {
   //region: inner processed class
   private List<List<ExpressionNodeEntity>> expressionExclusions;
 
+  private Map<LinkedList<LinkedList<ExpressionNodeEntity>>, LinkedList<LinkedList<ExpressionNodeEntity>>> conditionExclusions;
+
   //endregion
 
   public RulesConfig() {
@@ -231,8 +235,16 @@ public List<List<ExpressionNodeEntity>> getExpressionExclusions() {
     return expressionExclusions;
   }
 
-  public void setExpressionExclusions(
-      List<List<ExpressionNodeEntity>> expressionExclusions) {
+  public void setExpressionExclusions(List<List<ExpressionNodeEntity>> expressionExclusions) {
     this.expressionExclusions = expressionExclusions;
   }
+
+  public Map<LinkedList<LinkedList<ExpressionNodeEntity>>, LinkedList<LinkedList<ExpressionNodeEntity>>> getConditionExclusions() {
+    return conditionExclusions;
+  }
+
+  public void setConditionExclusions(
+      Map<LinkedList<LinkedList<ExpressionNodeEntity>>, LinkedList<LinkedList<ExpressionNodeEntity>>> conditionExclusions) {
+    this.conditionExclusions = conditionExclusions;
+  }
 }