exception handling Meta Tests

andy/src/main/java/nl/tudelft/cse1110/andy/config/SecureExamRunConfiguration.java

@@ -39,6 +39,10 @@ public Map<String, Float> weights() {
         ));
     }
 
+    public List<MetaTest> metaTests() {
+        return Collections.emptyList();
+    }
+
     @Override
     public List<String> classesUnderTest() {
         return Collections.unmodifiableList(classesUnderTest);

andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/library/LibraryMetaTest.java

@@ -30,7 +30,7 @@ public String evaluate(String oldLibraryCode) {
     }
 
     @Override
-    public boolean execute(Context ctx, DirectoryConfiguration dirCfg, RunConfiguration runCfg) throws Exception {
+    public boolean execute(Context ctx, DirectoryConfiguration dirCfg, RunConfiguration runCfg) throws Exception{
         /* Get the student solution, which we will run for each meta test */
         String solutionFile = findSolution(dirCfg.getWorkingDir());
 

andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunMetaTestsStep.java

@@ -7,6 +7,7 @@
 import nl.tudelft.cse1110.andy.execution.ExecutionStep;
 import nl.tudelft.cse1110.andy.result.MetaTestResult;
 import nl.tudelft.cse1110.andy.result.ResultBuilder;
+import nl.tudelft.cse1110.andy.execution.mode.Mode;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -43,7 +44,14 @@ public void execute(Context ctx, ResultBuilder result) {
 
             result.logMetaTests(score, totalWeight, metaTestResults);
         } catch (Exception ex) {
-            result.genericFailure(this, ex);
+            if(runCfg.mode().equals(Mode.EXAM)){
+                result.genericFailure(this, ex, "Compilation Error occured while running meta tests.");
+            }
+            else {
+                System.err.println("Meta test compilation error occurred:");
+                result.genericFailure(this, ex);
+                ex.printStackTrace(System.err);
+            }
         } finally {
             /* restore the class loader to the one before meta tests */
             Thread.currentThread().setContextClassLoader(currentClassLoader);

andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java

@@ -262,6 +262,17 @@ public void genericFailure(ExecutionStep step, Throwable e) {
         this.genericFailure(step.getClass().getSimpleName(), exceptionMessage(e));
     }
 
+    public void genericFailure(String step, String genericFailureExceptionMessage, String msg) {
+        this.genericFailureStepName = step;
+        this.genericFailureExceptionMessage = genericFailureExceptionMessage;
+        this.genericFailureMessage = msg;
+        this.buildGenericFailure();
+    }
+
+    public void genericFailure(ExecutionStep step, Throwable e, String msg) {
+        this.genericFailure(step.getClass().getSimpleName(), exceptionMessage(e), msg);
+    }
+
     /*
      * Build the final result
      */

andy/src/test/java/integration/LibraryMetaTestsTest.java

@@ -88,4 +88,15 @@ void metaTestInternalFailure() {
                 .isEqualTo(RunMetaTestsStep.class.getSimpleName());
     }
 
+    @Test
+    void metaTestInternalFailureExamMode() {
+        Result result = run("NumberUtilsAddLibrary", "NumberUtilsAddAllTestsPass", "NumberUtilsAddConfigurationWithMetaTestInternalFailureExamMode");
+
+        assertThat(result.hasGenericFailure()).isTrue();
+        assertThat(result.getGenericFailure().getGenericFailureMessage())
+                .isPresent()
+                .get()
+                .isEqualTo("Compilation Error occured while running meta tests.");
+    }
+
 }

andy/src/test/java/unit/execution/metatest/LibraryMetaTestTest.java

@@ -2,15 +2,26 @@
 
 import nl.tudelft.cse1110.andy.config.MetaTest;
 import nl.tudelft.cse1110.andy.execution.metatest.library.LibraryMetaTest;
+import nl.tudelft.cse1110.andy.execution.mode.Mode;
+import nl.tudelft.cse1110.andy.execution.Context.Context;
+import nl.tudelft.cse1110.andy.config.RunConfiguration;
+import nl.tudelft.cse1110.andy.config.DirectoryConfiguration;
+
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.Arguments;
 import org.junit.jupiter.params.provider.MethodSource;
 
 import java.util.stream.Stream;
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.junit.jupiter.api.Assertions.assertThrows;
+import org.mockito.Mockito;
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.Mockito.*;
+
 
 public class LibraryMetaTestTest {
 
@@ -162,4 +173,5 @@ void withStringReplacementNotFound() {
         assertThrows(RuntimeException.class,
                 () -> metaTest.evaluate("line 1\nline 2\nline 3\nline 4\nline 5"));
     }
+
 }

andy/src/test/resources/grader/fixtures/Config/NumberUtilsAddConfigurationWithMetaTestInternalFailureExamMode.java

@@ -0,0 +1,84 @@
+package delft;
+
+import nl.tudelft.cse1110.andy.config.RunConfiguration;
+import nl.tudelft.cse1110.andy.config.SecureExamRunConfiguration;
+import nl.tudelft.cse1110.andy.config.MetaTest;
+import nl.tudelft.cse1110.andy.execution.mode.Mode;
+
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class Configuration extends SecureExamRunConfiguration {
+
+    @Override
+    public Map<String, Float> weights() {
+        return new HashMap<>() {{
+            put("coverage", 0.1f);
+            put("mutation", 0.3f);
+            put("meta", 0.4f);
+            put("codechecks", 0.2f);
+        }};
+    }
+
+    @Override
+    public List<String> classesUnderTest() {
+        return List.of("delft.NumberUtils");
+    }
+
+    @Override
+    public List<MetaTest> metaTests() {
+        return List.of(
+            MetaTest.withStringReplacement(3, "AppliesMultipleCarriesWrongly",
+                """
+                int sum = leftDigit + rightDigit + carry;
+
+                result.addFirst(sum % 10);
+                carry = sum / 10;
+                """,
+                """
+                int sum;
+
+                if (leftDigit + rightDigit >= 10) {
+                    sum = leftDigit + rightDigit;
+                    carry = 1;
+                }
+                else {
+                    sum = leftDigit + rightDigit + carry;
+                    carry = 0;
+                }
+                result.addFirst(sum % 10);
+                """),
+            MetaTest.withLineReplacement("DoesNotApplyCarryAtAll", 47, 68,
+                """
+                for (int i = 0; i < Math.max(reversedLeft.size(), reversedRight.size()); i++) {
+
+                    int leftDigit = reversedLeft.size() > i ? reversedLeft.get(i) : 0;
+                    int rightDigit = reversedRight.size() > i ? reversedRight.get(i) : 0;
+
+                    if (leftDigit < 0 || leftDigit > 9 || rightDigit < 0 || rightDigit > 9)
+                        throw new IllegalArgumentException();
+
+                    int sum = leftDigit + rightDigit;
+
+                    result.addFirst(sum % 10);
+                }
+
+                // remove leading zeroes from the result
+                while (result.size() > 1 && result.get(0) == 0)
+                    result.remove(0);
+
+                if (result.isEmpty()) {
+                    result.addFirst(0);
+                }
+
+                return result;
+                """),
+            MetaTest.withStringReplacement("BadMetaTest",
+                "something that doesn't exist",
+                    ""),
+            MetaTest.withLineReplacement(2, "DoesNotCheckNumbersOutOfRange", 52, 53, "")
+        );
+    }
+}