diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/config/SecureExamRunConfiguration.java b/andy/src/main/java/nl/tudelft/cse1110/andy/config/SecureExamRunConfiguration.java index 75af9acc6..8f2c93d47 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/config/SecureExamRunConfiguration.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/config/SecureExamRunConfiguration.java @@ -39,6 +39,10 @@ public Map weights() { )); } + public List metaTests() { + return Collections.emptyList(); + } + @Override public List classesUnderTest() { return Collections.unmodifiableList(classesUnderTest); diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/library/LibraryMetaTest.java b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/library/LibraryMetaTest.java index 4905983cf..3a3fe7d68 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/library/LibraryMetaTest.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/library/LibraryMetaTest.java @@ -30,7 +30,7 @@ public String evaluate(String oldLibraryCode) { } @Override - public boolean execute(Context ctx, DirectoryConfiguration dirCfg, RunConfiguration runCfg) throws Exception { + public boolean execute(Context ctx, DirectoryConfiguration dirCfg, RunConfiguration runCfg) throws Exception{ /* Get the student solution, which we will run for each meta test */ String solutionFile = findSolution(dirCfg.getWorkingDir()); diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunMetaTestsStep.java b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunMetaTestsStep.java index 654abff8e..654a28809 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunMetaTestsStep.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunMetaTestsStep.java @@ -7,6 +7,7 @@ import nl.tudelft.cse1110.andy.execution.ExecutionStep; import nl.tudelft.cse1110.andy.result.MetaTestResult; import nl.tudelft.cse1110.andy.result.ResultBuilder; +import nl.tudelft.cse1110.andy.execution.mode.Mode; import java.util.ArrayList; import java.util.List; @@ -43,7 +44,14 @@ public void execute(Context ctx, ResultBuilder result) { result.logMetaTests(score, totalWeight, metaTestResults); } catch (Exception ex) { - result.genericFailure(this, ex); + if(runCfg.mode().equals(Mode.EXAM)){ + result.genericFailure(this, ex, "Compilation Error occured while running meta tests."); + } + else { + System.err.println("Meta test compilation error occurred:"); + result.genericFailure(this, ex); + ex.printStackTrace(System.err); + } } finally { /* restore the class loader to the one before meta tests */ Thread.currentThread().setContextClassLoader(currentClassLoader); diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java b/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java index 86ebe84cf..a9a5b0ba0 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java @@ -262,6 +262,17 @@ public void genericFailure(ExecutionStep step, Throwable e) { this.genericFailure(step.getClass().getSimpleName(), exceptionMessage(e)); } + public void genericFailure(String step, String genericFailureExceptionMessage, String msg) { + this.genericFailureStepName = step; + this.genericFailureExceptionMessage = genericFailureExceptionMessage; + this.genericFailureMessage = msg; + this.buildGenericFailure(); + } + + public void genericFailure(ExecutionStep step, Throwable e, String msg) { + this.genericFailure(step.getClass().getSimpleName(), exceptionMessage(e), msg); + } + /* * Build the final result */ diff --git a/andy/src/test/java/integration/LibraryMetaTestsTest.java b/andy/src/test/java/integration/LibraryMetaTestsTest.java index 3199ca88f..899babedc 100644 --- a/andy/src/test/java/integration/LibraryMetaTestsTest.java +++ b/andy/src/test/java/integration/LibraryMetaTestsTest.java @@ -88,4 +88,15 @@ void metaTestInternalFailure() { .isEqualTo(RunMetaTestsStep.class.getSimpleName()); } + @Test + void metaTestInternalFailureExamMode() { + Result result = run("NumberUtilsAddLibrary", "NumberUtilsAddAllTestsPass", "NumberUtilsAddConfigurationWithMetaTestInternalFailureExamMode"); + + assertThat(result.hasGenericFailure()).isTrue(); + assertThat(result.getGenericFailure().getGenericFailureMessage()) + .isPresent() + .get() + .isEqualTo("Compilation Error occured while running meta tests."); + } + } \ No newline at end of file diff --git a/andy/src/test/java/unit/execution/metatest/LibraryMetaTestTest.java b/andy/src/test/java/unit/execution/metatest/LibraryMetaTestTest.java index 215c77409..4e82b7e2e 100644 --- a/andy/src/test/java/unit/execution/metatest/LibraryMetaTestTest.java +++ b/andy/src/test/java/unit/execution/metatest/LibraryMetaTestTest.java @@ -2,15 +2,26 @@ import nl.tudelft.cse1110.andy.config.MetaTest; import nl.tudelft.cse1110.andy.execution.metatest.library.LibraryMetaTest; +import nl.tudelft.cse1110.andy.execution.mode.Mode; +import nl.tudelft.cse1110.andy.execution.Context.Context; +import nl.tudelft.cse1110.andy.config.RunConfiguration; +import nl.tudelft.cse1110.andy.config.DirectoryConfiguration; + import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import java.util.stream.Stream; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertThrows; +import org.mockito.Mockito; +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + public class LibraryMetaTestTest { @@ -162,4 +173,5 @@ void withStringReplacementNotFound() { assertThrows(RuntimeException.class, () -> metaTest.evaluate("line 1\nline 2\nline 3\nline 4\nline 5")); } + } diff --git a/andy/src/test/resources/grader/fixtures/Config/NumberUtilsAddConfigurationWithMetaTestInternalFailureExamMode.java b/andy/src/test/resources/grader/fixtures/Config/NumberUtilsAddConfigurationWithMetaTestInternalFailureExamMode.java new file mode 100644 index 000000000..ff881a835 --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Config/NumberUtilsAddConfigurationWithMetaTestInternalFailureExamMode.java @@ -0,0 +1,84 @@ +package delft; + +import nl.tudelft.cse1110.andy.config.RunConfiguration; +import nl.tudelft.cse1110.andy.config.SecureExamRunConfiguration; +import nl.tudelft.cse1110.andy.config.MetaTest; +import nl.tudelft.cse1110.andy.execution.mode.Mode; + + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class Configuration extends SecureExamRunConfiguration { + + @Override + public Map weights() { + return new HashMap<>() {{ + put("coverage", 0.1f); + put("mutation", 0.3f); + put("meta", 0.4f); + put("codechecks", 0.2f); + }}; + } + + @Override + public List classesUnderTest() { + return List.of("delft.NumberUtils"); + } + + @Override + public List metaTests() { + return List.of( + MetaTest.withStringReplacement(3, "AppliesMultipleCarriesWrongly", + """ + int sum = leftDigit + rightDigit + carry; + + result.addFirst(sum % 10); + carry = sum / 10; + """, + """ + int sum; + + if (leftDigit + rightDigit >= 10) { + sum = leftDigit + rightDigit; + carry = 1; + } + else { + sum = leftDigit + rightDigit + carry; + carry = 0; + } + result.addFirst(sum % 10); + """), + MetaTest.withLineReplacement("DoesNotApplyCarryAtAll", 47, 68, + """ + for (int i = 0; i < Math.max(reversedLeft.size(), reversedRight.size()); i++) { + + int leftDigit = reversedLeft.size() > i ? reversedLeft.get(i) : 0; + int rightDigit = reversedRight.size() > i ? reversedRight.get(i) : 0; + + if (leftDigit < 0 || leftDigit > 9 || rightDigit < 0 || rightDigit > 9) + throw new IllegalArgumentException(); + + int sum = leftDigit + rightDigit; + + result.addFirst(sum % 10); + } + + // remove leading zeroes from the result + while (result.size() > 1 && result.get(0) == 0) + result.remove(0); + + if (result.isEmpty()) { + result.addFirst(0); + } + + return result; + """), + MetaTest.withStringReplacement("BadMetaTest", + "something that doesn't exist", + ""), + MetaTest.withLineReplacement(2, "DoesNotCheckNumbersOutOfRange", 52, 53, "") + ); + } +} \ No newline at end of file