From cec3e5a4bec904a2bc96c836d12f2db29d08a450 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Atilla=20=C3=87olak?= Date: Tue, 1 Aug 2023 17:39:42 +0200 Subject: [PATCH 1/3] exception handling Meta Tests --- .../metatest/library/LibraryMetaTest.java | 127 +++++++++++++----- .../andy/execution/step/RunMetaTestsStep.java | 11 +- .../metatest/LibraryMetaTestTest.java | 49 +++++++ 3 files changed, 149 insertions(+), 38 deletions(-) diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/library/LibraryMetaTest.java b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/library/LibraryMetaTest.java index 4905983cf..3e26907e2 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/library/LibraryMetaTest.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/library/LibraryMetaTest.java @@ -30,44 +30,97 @@ public String evaluate(String oldLibraryCode) { } @Override - public boolean execute(Context ctx, DirectoryConfiguration dirCfg, RunConfiguration runCfg) throws Exception { - /* Get the student solution, which we will run for each meta test */ - String solutionFile = findSolution(dirCfg.getWorkingDir()); - - /* Get the original library content that we will keep changing according to the meta test */ - File libraryFile = new File(findLibrary(dirCfg.getWorkingDir())); - String originalLibraryContent = readFile(libraryFile); - - /* - * For each meta test, we basically perform a string replace in the - * original library code, recompile it, and run the tests. - * If there's a failing test, the meta test is killed. - * - * We reuse our execution framework to run the code with the meta test. - */ - - /* Copy the library and replace the library by the meta test */ - File metaWorkingDir = createTemporaryDirectory("metaWorkplace").toFile(); - copyFile(solutionFile, metaWorkingDir.getAbsolutePath()); - String metaFileContent = generateMetaFileContent(originalLibraryContent); - createMetaTestFile(metaWorkingDir, metaFileContent); - - /* We then run the meta test, using our infrastructure */ - Result metaResult = runMetaTest(ctx, dirCfg, metaWorkingDir); - - /* And check the result. If there's a failing test, the test suite is good! */ - int testsRan = metaResult.getTests().getTestsRan(); - int testsSucceeded = metaResult.getTests().getTestsSucceeded(); - boolean passesTheMetaTest = testsSucceeded < testsRan; - - /* Clean up the directory */ - deleteDirectory(metaWorkingDir); - - /* Set the classloader back to the one with the student's original code */ - Thread.currentThread().setContextClassLoader(ctx.getClassloaderWithStudentsCode()); - - return passesTheMetaTest; + public boolean execute(Context ctx, DirectoryConfiguration dirCfg, RunConfiguration runCfg)throws Exception{ + {/* Get the student solution, which we will run for each meta test */ + String solutionFile = findSolution(dirCfg.getWorkingDir()); + + /* Get the original library content that we will keep changing according to the meta test */ + File libraryFile = new File(findLibrary(dirCfg.getWorkingDir())); + String originalLibraryContent = readFile(libraryFile); + + /* + * For each meta test, we basically perform a string replace in the + * original library code, recompile it, and run the tests. + * If there's a failing test, the meta test is killed. + * + * We reuse our execution framework to run the code with the meta test. + */ + + /* Copy the library and replace the library by the meta test */ + File metaWorkingDir = createTemporaryDirectory("metaWorkplace").toFile(); + copyFile(solutionFile, metaWorkingDir.getAbsolutePath()); + String metaFileContent = generateMetaFileContent(originalLibraryContent); + createMetaTestFile(metaWorkingDir, metaFileContent); + + /* We then run the meta test, using our infrastructure */ + Result metaResult = runMetaTest(ctx, dirCfg, metaWorkingDir); + + /* And check the result. If there's a failing test, the test suite is good! */ + int testsRan = metaResult.getTests().getTestsRan(); + int testsSucceeded = metaResult.getTests().getTestsSucceeded(); + boolean passesTheMetaTest = testsSucceeded < testsRan; + + /* Clean up the directory */ + deleteDirectory(metaWorkingDir); + + /* Set the classloader back to the one with the student's original code */ + Thread.currentThread().setContextClassLoader(ctx.getClassloaderWithStudentsCode()); + + return passesTheMetaTest; + } } +// This was the execute method I updated, but this was causing a failing test in LibraryMetaTestsTest. +// So I decided to work on the RunMetaTestsStep class for error handling. +// @Override +// public boolean execute(Context ctx, DirectoryConfiguration dirCfg, RunConfiguration runCfg){ +// try {/* Get the student solution, which we will run for each meta test */ +// String solutionFile = findSolution(dirCfg.getWorkingDir()); +// +// /* Get the original library content that we will keep changing according to the meta test */ +// File libraryFile = new File(findLibrary(dirCfg.getWorkingDir())); +// String originalLibraryContent = readFile(libraryFile); +// +// /* +// * For each meta test, we basically perform a string replace in the +// * original library code, recompile it, and run the tests. +// * If there's a failing test, the meta test is killed. +// * +// * We reuse our execution framework to run the code with the meta test. +// */ +// +// /* Copy the library and replace the library by the meta test */ +// File metaWorkingDir = createTemporaryDirectory("metaWorkplace").toFile(); +// copyFile(solutionFile, metaWorkingDir.getAbsolutePath()); +// String metaFileContent = generateMetaFileContent(originalLibraryContent); +// createMetaTestFile(metaWorkingDir, metaFileContent); +// +// /* We then run the meta test, using our infrastructure */ +// Result metaResult = runMetaTest(ctx, dirCfg, metaWorkingDir); +// +// /* And check the result. If there's a failing test, the test suite is good! */ +// int testsRan = metaResult.getTests().getTestsRan(); +// int testsSucceeded = metaResult.getTests().getTestsSucceeded(); +// boolean passesTheMetaTest = testsSucceeded < testsRan; +// +// /* Clean up the directory */ +// deleteDirectory(metaWorkingDir); +// +// /* Set the classloader back to the one with the student's original code */ +// Thread.currentThread().setContextClassLoader(ctx.getClassloaderWithStudentsCode()); +// +// return passesTheMetaTest; +// } catch (Exception e){ +// if(runCfg.mode().equals(Mode.EXAM)){ +// System.out.println("Meta test compilation error occurred."); +// } +// else{ +// System.err.println("Meta test compilation error occurred:"); +// e.printStackTrace(System.err); +// } +// // if test fails, return false to keep it organized. +// return false; +// } +// } private String generateMetaFileContent(String originalLibraryContent) { String metaFileContent = evaluate(originalLibraryContent); diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunMetaTestsStep.java b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunMetaTestsStep.java index 654abff8e..d2192d10c 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunMetaTestsStep.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunMetaTestsStep.java @@ -7,6 +7,7 @@ import nl.tudelft.cse1110.andy.execution.ExecutionStep; import nl.tudelft.cse1110.andy.result.MetaTestResult; import nl.tudelft.cse1110.andy.result.ResultBuilder; +import nl.tudelft.cse1110.andy.execution.mode.Mode; import java.util.ArrayList; import java.util.List; @@ -43,7 +44,15 @@ public void execute(Context ctx, ResultBuilder result) { result.logMetaTests(score, totalWeight, metaTestResults); } catch (Exception ex) { - result.genericFailure(this, ex); + if(runCfg.mode().equals(Mode.EXAM)){ + System.out.println("Meta test compilation error occurred."); + result.genericFailure(this, ex); + } + else { + System.err.println("Meta test compilation error occurred:"); + result.genericFailure(this, ex); + ex.printStackTrace(System.err); + } } finally { /* restore the class loader to the one before meta tests */ Thread.currentThread().setContextClassLoader(currentClassLoader); diff --git a/andy/src/test/java/unit/execution/metatest/LibraryMetaTestTest.java b/andy/src/test/java/unit/execution/metatest/LibraryMetaTestTest.java index 215c77409..0952dcc94 100644 --- a/andy/src/test/java/unit/execution/metatest/LibraryMetaTestTest.java +++ b/andy/src/test/java/unit/execution/metatest/LibraryMetaTestTest.java @@ -2,15 +2,26 @@ import nl.tudelft.cse1110.andy.config.MetaTest; import nl.tudelft.cse1110.andy.execution.metatest.library.LibraryMetaTest; +import nl.tudelft.cse1110.andy.execution.mode.Mode; +import nl.tudelft.cse1110.andy.execution.Context.Context; +import nl.tudelft.cse1110.andy.config.RunConfiguration; +import nl.tudelft.cse1110.andy.config.DirectoryConfiguration; + import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import java.util.stream.Stream; +import java.io.ByteArrayOutputStream; +import java.io.PrintStream; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertThrows; +import org.mockito.Mockito; +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + public class LibraryMetaTestTest { @@ -162,4 +173,42 @@ void withStringReplacementNotFound() { assertThrows(RuntimeException.class, () -> metaTest.evaluate("line 1\nline 2\nline 3\nline 4\nline 5")); } +// I initially wrote this test because try-catch was inside LibraryMetaTest. Will remove it when decision is final. +// @Test +// void testExecuteWithCompilationErrorExamMode() throws Exception { +// // Create mock objects +// Context ctxMock = mock(Context.class); +// DirectoryConfiguration dirCfgMock = mock(DirectoryConfiguration.class); +// RunConfiguration runCfgMock = mock(RunConfiguration.class); +// +// // Set up the behavior of the mock objects +// when(runCfgMock.mode()).thenReturn(Mode.EXAM); +// when(dirCfgMock.getWorkingDir()).thenThrow(new RuntimeException("Working directory exception.")); +// +// // Create the LibraryMetaTest object +// LibraryMetaTest metaTest = (LibraryMetaTest) MetaTest.withStringReplacement("some meta test", +// "line 5\nline 6", +// "extra line 1\nextra line 2"); +// +// // Create a stream to hold the output +// // if tests run in parallel, this could cause an issue as we capture logging and change system out settings. +// ByteArrayOutputStream outContent = new ByteArrayOutputStream(); +// PrintStream originalOut = System.out; +// System.setOut(new PrintStream(outContent)); +// +// // Test the execute method with the mocked objects +// boolean result = metaTest.execute(ctxMock, dirCfgMock, runCfgMock); +// +// // Assertions +// assertFalse(result); // The test should fail due to the compilation error +// +// // Verify that the proper methods were called on the mock objects +// verify(runCfgMock, times(1)).mode(); +// +// // Assert that the correct message was logged +// assertEquals("Meta test compilation error occurred.\n", outContent.toString()); +// +// // Restore System.out to its original +// System.setOut(originalOut); +// } } From e1e257a020ea86087e81b58e3808914cee7ae8d7 Mon Sep 17 00:00:00 2001 From: AtillaColak Date: Thu, 3 Aug 2023 23:39:17 +0200 Subject: [PATCH 2/3] Removed Unnecessary comments. --- .../metatest/library/LibraryMetaTest.java | 52 ------------------- .../metatest/LibraryMetaTestTest.java | 38 -------------- 2 files changed, 90 deletions(-) diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/library/LibraryMetaTest.java b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/library/LibraryMetaTest.java index 3e26907e2..c8326ae95 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/library/LibraryMetaTest.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/library/LibraryMetaTest.java @@ -69,58 +69,6 @@ public boolean execute(Context ctx, DirectoryConfiguration dirCfg, RunConfigurat return passesTheMetaTest; } } -// This was the execute method I updated, but this was causing a failing test in LibraryMetaTestsTest. -// So I decided to work on the RunMetaTestsStep class for error handling. -// @Override -// public boolean execute(Context ctx, DirectoryConfiguration dirCfg, RunConfiguration runCfg){ -// try {/* Get the student solution, which we will run for each meta test */ -// String solutionFile = findSolution(dirCfg.getWorkingDir()); -// -// /* Get the original library content that we will keep changing according to the meta test */ -// File libraryFile = new File(findLibrary(dirCfg.getWorkingDir())); -// String originalLibraryContent = readFile(libraryFile); -// -// /* -// * For each meta test, we basically perform a string replace in the -// * original library code, recompile it, and run the tests. -// * If there's a failing test, the meta test is killed. -// * -// * We reuse our execution framework to run the code with the meta test. -// */ -// -// /* Copy the library and replace the library by the meta test */ -// File metaWorkingDir = createTemporaryDirectory("metaWorkplace").toFile(); -// copyFile(solutionFile, metaWorkingDir.getAbsolutePath()); -// String metaFileContent = generateMetaFileContent(originalLibraryContent); -// createMetaTestFile(metaWorkingDir, metaFileContent); -// -// /* We then run the meta test, using our infrastructure */ -// Result metaResult = runMetaTest(ctx, dirCfg, metaWorkingDir); -// -// /* And check the result. If there's a failing test, the test suite is good! */ -// int testsRan = metaResult.getTests().getTestsRan(); -// int testsSucceeded = metaResult.getTests().getTestsSucceeded(); -// boolean passesTheMetaTest = testsSucceeded < testsRan; -// -// /* Clean up the directory */ -// deleteDirectory(metaWorkingDir); -// -// /* Set the classloader back to the one with the student's original code */ -// Thread.currentThread().setContextClassLoader(ctx.getClassloaderWithStudentsCode()); -// -// return passesTheMetaTest; -// } catch (Exception e){ -// if(runCfg.mode().equals(Mode.EXAM)){ -// System.out.println("Meta test compilation error occurred."); -// } -// else{ -// System.err.println("Meta test compilation error occurred:"); -// e.printStackTrace(System.err); -// } -// // if test fails, return false to keep it organized. -// return false; -// } -// } private String generateMetaFileContent(String originalLibraryContent) { String metaFileContent = evaluate(originalLibraryContent); diff --git a/andy/src/test/java/unit/execution/metatest/LibraryMetaTestTest.java b/andy/src/test/java/unit/execution/metatest/LibraryMetaTestTest.java index 0952dcc94..c4c7e6451 100644 --- a/andy/src/test/java/unit/execution/metatest/LibraryMetaTestTest.java +++ b/andy/src/test/java/unit/execution/metatest/LibraryMetaTestTest.java @@ -173,42 +173,4 @@ void withStringReplacementNotFound() { assertThrows(RuntimeException.class, () -> metaTest.evaluate("line 1\nline 2\nline 3\nline 4\nline 5")); } -// I initially wrote this test because try-catch was inside LibraryMetaTest. Will remove it when decision is final. -// @Test -// void testExecuteWithCompilationErrorExamMode() throws Exception { -// // Create mock objects -// Context ctxMock = mock(Context.class); -// DirectoryConfiguration dirCfgMock = mock(DirectoryConfiguration.class); -// RunConfiguration runCfgMock = mock(RunConfiguration.class); -// -// // Set up the behavior of the mock objects -// when(runCfgMock.mode()).thenReturn(Mode.EXAM); -// when(dirCfgMock.getWorkingDir()).thenThrow(new RuntimeException("Working directory exception.")); -// -// // Create the LibraryMetaTest object -// LibraryMetaTest metaTest = (LibraryMetaTest) MetaTest.withStringReplacement("some meta test", -// "line 5\nline 6", -// "extra line 1\nextra line 2"); -// -// // Create a stream to hold the output -// // if tests run in parallel, this could cause an issue as we capture logging and change system out settings. -// ByteArrayOutputStream outContent = new ByteArrayOutputStream(); -// PrintStream originalOut = System.out; -// System.setOut(new PrintStream(outContent)); -// -// // Test the execute method with the mocked objects -// boolean result = metaTest.execute(ctxMock, dirCfgMock, runCfgMock); -// -// // Assertions -// assertFalse(result); // The test should fail due to the compilation error -// -// // Verify that the proper methods were called on the mock objects -// verify(runCfgMock, times(1)).mode(); -// -// // Assert that the correct message was logged -// assertEquals("Meta test compilation error occurred.\n", outContent.toString()); -// -// // Restore System.out to its original -// System.setOut(originalOut); -// } } From 79432da903b56e31c8549ad44f3ad8b7920d6797 Mon Sep 17 00:00:00 2001 From: AtillaColak Date: Fri, 18 Aug 2023 17:55:55 +0200 Subject: [PATCH 3/3] trying to fix notify the teacher problem --- .../config/SecureExamRunConfiguration.java | 4 + .../metatest/library/LibraryMetaTest.java | 75 ++++++++--------- .../andy/execution/step/RunMetaTestsStep.java | 3 +- .../cse1110/andy/result/ResultBuilder.java | 11 +++ .../integration/LibraryMetaTestsTest.java | 11 +++ .../metatest/LibraryMetaTestTest.java | 1 + ...onWithMetaTestInternalFailureExamMode.java | 84 +++++++++++++++++++ 7 files changed, 149 insertions(+), 40 deletions(-) create mode 100644 andy/src/test/resources/grader/fixtures/Config/NumberUtilsAddConfigurationWithMetaTestInternalFailureExamMode.java diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/config/SecureExamRunConfiguration.java b/andy/src/main/java/nl/tudelft/cse1110/andy/config/SecureExamRunConfiguration.java index 75af9acc6..8f2c93d47 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/config/SecureExamRunConfiguration.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/config/SecureExamRunConfiguration.java @@ -39,6 +39,10 @@ public Map weights() { )); } + public List metaTests() { + return Collections.emptyList(); + } + @Override public List classesUnderTest() { return Collections.unmodifiableList(classesUnderTest); diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/library/LibraryMetaTest.java b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/library/LibraryMetaTest.java index c8326ae95..3a3fe7d68 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/library/LibraryMetaTest.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/metatest/library/LibraryMetaTest.java @@ -30,44 +30,43 @@ public String evaluate(String oldLibraryCode) { } @Override - public boolean execute(Context ctx, DirectoryConfiguration dirCfg, RunConfiguration runCfg)throws Exception{ - {/* Get the student solution, which we will run for each meta test */ - String solutionFile = findSolution(dirCfg.getWorkingDir()); - - /* Get the original library content that we will keep changing according to the meta test */ - File libraryFile = new File(findLibrary(dirCfg.getWorkingDir())); - String originalLibraryContent = readFile(libraryFile); - - /* - * For each meta test, we basically perform a string replace in the - * original library code, recompile it, and run the tests. - * If there's a failing test, the meta test is killed. - * - * We reuse our execution framework to run the code with the meta test. - */ - - /* Copy the library and replace the library by the meta test */ - File metaWorkingDir = createTemporaryDirectory("metaWorkplace").toFile(); - copyFile(solutionFile, metaWorkingDir.getAbsolutePath()); - String metaFileContent = generateMetaFileContent(originalLibraryContent); - createMetaTestFile(metaWorkingDir, metaFileContent); - - /* We then run the meta test, using our infrastructure */ - Result metaResult = runMetaTest(ctx, dirCfg, metaWorkingDir); - - /* And check the result. If there's a failing test, the test suite is good! */ - int testsRan = metaResult.getTests().getTestsRan(); - int testsSucceeded = metaResult.getTests().getTestsSucceeded(); - boolean passesTheMetaTest = testsSucceeded < testsRan; - - /* Clean up the directory */ - deleteDirectory(metaWorkingDir); - - /* Set the classloader back to the one with the student's original code */ - Thread.currentThread().setContextClassLoader(ctx.getClassloaderWithStudentsCode()); - - return passesTheMetaTest; - } + public boolean execute(Context ctx, DirectoryConfiguration dirCfg, RunConfiguration runCfg) throws Exception{ + /* Get the student solution, which we will run for each meta test */ + String solutionFile = findSolution(dirCfg.getWorkingDir()); + + /* Get the original library content that we will keep changing according to the meta test */ + File libraryFile = new File(findLibrary(dirCfg.getWorkingDir())); + String originalLibraryContent = readFile(libraryFile); + + /* + * For each meta test, we basically perform a string replace in the + * original library code, recompile it, and run the tests. + * If there's a failing test, the meta test is killed. + * + * We reuse our execution framework to run the code with the meta test. + */ + + /* Copy the library and replace the library by the meta test */ + File metaWorkingDir = createTemporaryDirectory("metaWorkplace").toFile(); + copyFile(solutionFile, metaWorkingDir.getAbsolutePath()); + String metaFileContent = generateMetaFileContent(originalLibraryContent); + createMetaTestFile(metaWorkingDir, metaFileContent); + + /* We then run the meta test, using our infrastructure */ + Result metaResult = runMetaTest(ctx, dirCfg, metaWorkingDir); + + /* And check the result. If there's a failing test, the test suite is good! */ + int testsRan = metaResult.getTests().getTestsRan(); + int testsSucceeded = metaResult.getTests().getTestsSucceeded(); + boolean passesTheMetaTest = testsSucceeded < testsRan; + + /* Clean up the directory */ + deleteDirectory(metaWorkingDir); + + /* Set the classloader back to the one with the student's original code */ + Thread.currentThread().setContextClassLoader(ctx.getClassloaderWithStudentsCode()); + + return passesTheMetaTest; } private String generateMetaFileContent(String originalLibraryContent) { diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunMetaTestsStep.java b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunMetaTestsStep.java index d2192d10c..654a28809 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunMetaTestsStep.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/execution/step/RunMetaTestsStep.java @@ -45,8 +45,7 @@ public void execute(Context ctx, ResultBuilder result) { result.logMetaTests(score, totalWeight, metaTestResults); } catch (Exception ex) { if(runCfg.mode().equals(Mode.EXAM)){ - System.out.println("Meta test compilation error occurred."); - result.genericFailure(this, ex); + result.genericFailure(this, ex, "Compilation Error occured while running meta tests."); } else { System.err.println("Meta test compilation error occurred:"); diff --git a/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java b/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java index 86ebe84cf..a9a5b0ba0 100644 --- a/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java +++ b/andy/src/main/java/nl/tudelft/cse1110/andy/result/ResultBuilder.java @@ -262,6 +262,17 @@ public void genericFailure(ExecutionStep step, Throwable e) { this.genericFailure(step.getClass().getSimpleName(), exceptionMessage(e)); } + public void genericFailure(String step, String genericFailureExceptionMessage, String msg) { + this.genericFailureStepName = step; + this.genericFailureExceptionMessage = genericFailureExceptionMessage; + this.genericFailureMessage = msg; + this.buildGenericFailure(); + } + + public void genericFailure(ExecutionStep step, Throwable e, String msg) { + this.genericFailure(step.getClass().getSimpleName(), exceptionMessage(e), msg); + } + /* * Build the final result */ diff --git a/andy/src/test/java/integration/LibraryMetaTestsTest.java b/andy/src/test/java/integration/LibraryMetaTestsTest.java index 3199ca88f..899babedc 100644 --- a/andy/src/test/java/integration/LibraryMetaTestsTest.java +++ b/andy/src/test/java/integration/LibraryMetaTestsTest.java @@ -88,4 +88,15 @@ void metaTestInternalFailure() { .isEqualTo(RunMetaTestsStep.class.getSimpleName()); } + @Test + void metaTestInternalFailureExamMode() { + Result result = run("NumberUtilsAddLibrary", "NumberUtilsAddAllTestsPass", "NumberUtilsAddConfigurationWithMetaTestInternalFailureExamMode"); + + assertThat(result.hasGenericFailure()).isTrue(); + assertThat(result.getGenericFailure().getGenericFailureMessage()) + .isPresent() + .get() + .isEqualTo("Compilation Error occured while running meta tests."); + } + } \ No newline at end of file diff --git a/andy/src/test/java/unit/execution/metatest/LibraryMetaTestTest.java b/andy/src/test/java/unit/execution/metatest/LibraryMetaTestTest.java index c4c7e6451..4e82b7e2e 100644 --- a/andy/src/test/java/unit/execution/metatest/LibraryMetaTestTest.java +++ b/andy/src/test/java/unit/execution/metatest/LibraryMetaTestTest.java @@ -173,4 +173,5 @@ void withStringReplacementNotFound() { assertThrows(RuntimeException.class, () -> metaTest.evaluate("line 1\nline 2\nline 3\nline 4\nline 5")); } + } diff --git a/andy/src/test/resources/grader/fixtures/Config/NumberUtilsAddConfigurationWithMetaTestInternalFailureExamMode.java b/andy/src/test/resources/grader/fixtures/Config/NumberUtilsAddConfigurationWithMetaTestInternalFailureExamMode.java new file mode 100644 index 000000000..ff881a835 --- /dev/null +++ b/andy/src/test/resources/grader/fixtures/Config/NumberUtilsAddConfigurationWithMetaTestInternalFailureExamMode.java @@ -0,0 +1,84 @@ +package delft; + +import nl.tudelft.cse1110.andy.config.RunConfiguration; +import nl.tudelft.cse1110.andy.config.SecureExamRunConfiguration; +import nl.tudelft.cse1110.andy.config.MetaTest; +import nl.tudelft.cse1110.andy.execution.mode.Mode; + + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class Configuration extends SecureExamRunConfiguration { + + @Override + public Map weights() { + return new HashMap<>() {{ + put("coverage", 0.1f); + put("mutation", 0.3f); + put("meta", 0.4f); + put("codechecks", 0.2f); + }}; + } + + @Override + public List classesUnderTest() { + return List.of("delft.NumberUtils"); + } + + @Override + public List metaTests() { + return List.of( + MetaTest.withStringReplacement(3, "AppliesMultipleCarriesWrongly", + """ + int sum = leftDigit + rightDigit + carry; + + result.addFirst(sum % 10); + carry = sum / 10; + """, + """ + int sum; + + if (leftDigit + rightDigit >= 10) { + sum = leftDigit + rightDigit; + carry = 1; + } + else { + sum = leftDigit + rightDigit + carry; + carry = 0; + } + result.addFirst(sum % 10); + """), + MetaTest.withLineReplacement("DoesNotApplyCarryAtAll", 47, 68, + """ + for (int i = 0; i < Math.max(reversedLeft.size(), reversedRight.size()); i++) { + + int leftDigit = reversedLeft.size() > i ? reversedLeft.get(i) : 0; + int rightDigit = reversedRight.size() > i ? reversedRight.get(i) : 0; + + if (leftDigit < 0 || leftDigit > 9 || rightDigit < 0 || rightDigit > 9) + throw new IllegalArgumentException(); + + int sum = leftDigit + rightDigit; + + result.addFirst(sum % 10); + } + + // remove leading zeroes from the result + while (result.size() > 1 && result.get(0) == 0) + result.remove(0); + + if (result.isEmpty()) { + result.addFirst(0); + } + + return result; + """), + MetaTest.withStringReplacement("BadMetaTest", + "something that doesn't exist", + ""), + MetaTest.withLineReplacement(2, "DoesNotCheckNumbersOutOfRange", 52, 53, "") + ); + } +} \ No newline at end of file