From 881454dfc765d2c9abc9c600de8e92f365a0d0cd Mon Sep 17 00:00:00 2001 From: Tom Brunet Date: Thu, 5 Sep 2024 21:40:16 -0500 Subject: [PATCH] Add Playwright support --- .github/workflows/test.yml | 38 ++- java-accessibility-checker/pom.xml | 45 ++- .../enginecontext/EngineContextManager.java | 25 ++ .../playwright/EngineContextPlaywright.java | 308 ++++++++++++++++++ .../AccessibilityCheckerPlaywrightTest.java | 223 +++++++++++++ ... => AccessibilityCheckerSeleniumTest.java} | 30 +- 6 files changed, 634 insertions(+), 35 deletions(-) create mode 100644 java-accessibility-checker/src/main/java/com/ibm/able/equalaccess/enginecontext/playwright/EngineContextPlaywright.java create mode 100644 java-accessibility-checker/src/test/java/com/ibm/able/equalaccess/AccessibilityCheckerPlaywrightTest.java rename java-accessibility-checker/src/test/java/com/ibm/able/equalaccess/{AccessibilityCheckerTest.java => AccessibilityCheckerSeleniumTest.java} (87%) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 319df1214..b22b0cad2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -364,7 +364,7 @@ jobs: ############################################################################### # Java test #### - java-accessibility-checker-test: + java-accessibility-checker-selenium-test: runs-on: ubuntu-22.04 strategy: @@ -392,7 +392,41 @@ jobs: - run: sleep 10 working-directory: rule-server/dist - name: Test package - run: mvn --batch-mode test + run: mvn --batch-mode test -Dtest="AccessibilityCheckerSeleniumTest" + working-directory: java-accessibility-checker + env: + chromedriverpath: ${{ steps.setup-chrome.outputs.chromedriver-path }} + chromebinpath: ${{ steps.setup-chrome.outputs.chrome-path }} + + java-accessibility-checker-playwright-test: + runs-on: ubuntu-22.04 + + strategy: + matrix: + node-version: [18.x] + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-java@v4 + with: + distribution: 'semeru' # See 'Supported distributions' for available options + java-version: '17' + - name: Latest Chrome + uses: browser-actions/setup-chrome@v1 + with: + chrome-version: latest + install-chromedriver: true + id: setup-chrome + - run: npm install + working-directory: rule-server + - run: npm run build + working-directory: rule-server + - run: node main.js & + working-directory: rule-server/dist + - run: sleep 10 + working-directory: rule-server/dist + - name: Test package + run: mvn --batch-mode test -Dtest="AccessibilityCheckerPlaywrightTest" working-directory: java-accessibility-checker env: chromedriverpath: ${{ steps.setup-chrome.outputs.chromedriver-path }} diff --git a/java-accessibility-checker/pom.xml b/java-accessibility-checker/pom.xml index 97db7c3c7..529001e71 100644 --- a/java-accessibility-checker/pom.xml +++ b/java-accessibility-checker/pom.xml @@ -39,7 +39,7 @@ org.junit junit-bom - xxxx + 5.11.0 pom import @@ -51,14 +51,14 @@ junit junit - xxxx + 4.13.2 test @@ -70,26 +70,42 @@ com.google.code.gson gson - xxxx + 2.11.0 org.mozilla rhino - xxxx + 1.7.14 org.seleniumhq.selenium selenium-java - xxxx + 4.23.0 + provided + + + com.microsoft.playwright + playwright + 1.46.0 + provided + org.apache.maven.plugins maven-source-plugin - xxxx + 2.2.1 attach-sources @@ -102,7 +118,7 @@ org.apache.maven.plugins maven-gpg-plugin - xxxx + 1.5 sign-artifacts @@ -123,7 +139,7 @@ org.apache.maven.plugins maven-javadoc-plugin - xxxx + 2.9.1 attach-javadocs @@ -136,7 +152,7 @@ org.sonatype.central central-publishing-maven-plugin - xxxx + 0.5.0 true central @@ -145,14 +161,7 @@ - + Tom Brunet diff --git a/java-accessibility-checker/src/main/java/com/ibm/able/equalaccess/enginecontext/EngineContextManager.java b/java-accessibility-checker/src/main/java/com/ibm/able/equalaccess/enginecontext/EngineContextManager.java index 6c157a6ee..4731e4e82 100644 --- a/java-accessibility-checker/src/main/java/com/ibm/able/equalaccess/enginecontext/EngineContextManager.java +++ b/java-accessibility-checker/src/main/java/com/ibm/able/equalaccess/enginecontext/EngineContextManager.java @@ -33,6 +33,7 @@ public static IEngineContext getEngineContext(Object contentContext) { if (contentContext == null) { engineContext = new EngineContextLocal(); } + // See if this is Selenium and we can load it if (engineContext == null && Misc.classIsAvailable("org.openqa.selenium.WebDriver")) { @@ -56,6 +57,30 @@ public static IEngineContext getEngineContext(Object contentContext) { } catch (InvocationTargetException e) { } } + // See if this is Playwright and we can load it + if (engineContext == null + && Misc.classIsAvailable("com.microsoft.playwright.Page")) + { + if (!Misc.classIsAvailable("com.ibm.able.equalaccess.enginecontext.playwright.EngineContextPlaywright")) { + System.err.println("Attempted scan with Page, but com.ibm.able.equalaccess.enginecontext.playwright.EngineContextPlaywright could not be loaded"); + throw new ACError("Attempted scan with Page, but com.ibm.able.equalaccess.enginecontext.playwright.EngineContextPlaywright could not be loaded"); + } + try { + Class playwrightPageClass = Class.forName("com.microsoft.playwright.Page"); + if (playwrightPageClass.isAssignableFrom(contentContext.getClass())) { + // We have a webdriver, use EngineContextSelenium to instantiate it + Class ecClass = Class.forName("com.ibm.able.equalaccess.enginecontext.playwright.EngineContextPlaywright"); + engineContext = (IEngineContext) ecClass.getConstructor(playwrightPageClass).newInstance(contentContext); + } + } catch (ClassNotFoundException e) { + } catch (NoSuchMethodException e) { + } catch (SecurityException e) { + } catch (InstantiationException e) { + } catch (IllegalAccessException e) { + } catch (IllegalArgumentException e) { + } catch (InvocationTargetException e) { + } + } if (engineContext != null) { engineContext.loadEngine(); } else { diff --git a/java-accessibility-checker/src/main/java/com/ibm/able/equalaccess/enginecontext/playwright/EngineContextPlaywright.java b/java-accessibility-checker/src/main/java/com/ibm/able/equalaccess/enginecontext/playwright/EngineContextPlaywright.java new file mode 100644 index 000000000..d770e3a98 --- /dev/null +++ b/java-accessibility-checker/src/main/java/com/ibm/able/equalaccess/enginecontext/playwright/EngineContextPlaywright.java @@ -0,0 +1,308 @@ +/****************************************************************************** + Copyright:: 2024- IBM, Inc + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + *****************************************************************************/ +package com.ibm.able.equalaccess.enginecontext.playwright; + +import java.io.IOException; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Arrays; + +import com.google.gson.Gson; +import com.ibm.able.equalaccess.config.ACConfigManager; +import com.ibm.able.equalaccess.config.Config; +import com.ibm.able.equalaccess.config.ConfigInternal; +import com.ibm.able.equalaccess.engine.ACError; +import com.ibm.able.equalaccess.engine.Guideline; +import com.ibm.able.equalaccess.engine.Rule; +import com.ibm.able.equalaccess.enginecontext.IEngineContext; +import com.ibm.able.equalaccess.engine.ACEReport; +import com.ibm.able.equalaccess.util.Fetch; +import com.microsoft.playwright.*; + +public class EngineContextPlaywright implements IEngineContext { + private static Gson gson = new Gson(); + private Page driver = null; + private String engineContent = null; + + public EngineContextPlaywright(Page driver) { + this.driver = driver; + } + + @Override + public void loadEngine() { + ConfigInternal config = ACConfigManager.getConfigUnsupported(); + String engineUrl = config.rulePack+"/ace.js"; + String engineLoadMode = config.engineMode; + if ("DEFAULT".equals(engineLoadMode)) { + engineLoadMode = "INJECT"; + } + try { + if ("INJECT".equals(engineLoadMode) && engineContent == null) { + engineContent = Fetch.get(engineUrl, config.ignoreHTTPSErrors); + } + + if (config.DEBUG) System.out.println("[INFO] aChecker.loadEngine detected Selenium"); + String scriptStr; + if ("REMOTE".equals(engineLoadMode)) { + scriptStr = """ +(scriptUrl) => { + try { + var ace_backup_in_ibma; + if ('undefined' !== typeof(ace)) { + if (!ace || !ace.Checker) + ace_backup_in_ibma = ace; + ace = null; + } + if ('undefined' === typeof (ace) || ace === null) { + return new Promise((resolve, reject) => { + let script = document.createElement('script'); + script.setAttribute('type', 'text/javascript'); + script.setAttribute('aChecker', 'ACE'); + script.setAttribute('src', scriptUrl); + script.addEventListener('load', function () { + globalThis.ace_ibma = ace; + if ('undefined' !== typeof(ace)) { + ace = ace_backup_in_ibma; + } + resolve(); + }); + script.addEventListener('error', function (evt) { + reject(new Error(`Unable to load engine into ${document.location.href}. This can happen if the page server sets a Content-Security-Policy that prevents ${scriptUrl} from loading.`)) + }); + let heads = document.getElementsByTagName('head'); + if (heads.length > 0) { heads[0].appendChild(script); } + else if (document.body) { document.body.appendChild(script); } + else { Promise.reject("Invalid document"); } + }) + } + } catch (e) { + return Promise.reject(e); + } +} +"""; + this.driver.evaluate(scriptStr, config.rulePack); + } else if ("INJECT".equals(engineLoadMode)) { + // Selenium + scriptStr = """ +(engineContent) => { + try { + var ace_backup_in_ibma; + if ('undefined' !== typeof(ace)) { + if (!ace || !ace.Checker) + ace_backup_in_ibma = ace; + ace = null; + } + if ('undefined' === typeof (ace) || ace === null) { + return new Promise((resolve, reject) => { + eval(engineContent); + globalThis.ace_ibma = ace; + if ('undefined' !== typeof(ace)) { + ace = ace_backup_in_ibma; + } + resolve(); + }) + } + } catch (e) { + return Promise.reject(e); + } +} +"""; + this.driver.evaluate(scriptStr, engineContent); + } else { + scriptStr = ""; + } + + // this.driver.manage().timeouts().scriptTimeout(Duration.ofSeconds(60)); + + } catch (Error e) { + System.err.println(e); + } catch (IOException e) { + System.err.println("aChecker: Unable to load engine from "+engineUrl+" due to IOException: "+e.toString()); + e.printStackTrace(); + } + } + + @Override + public ACEReport getCompliance(String label) { + Config config = ACConfigManager.getConfig(); + try { + List resultList = new ArrayList<>(config.reportLevels.length + config.failLevels.length); + Collections.addAll(resultList, config.reportLevels); + Collections.addAll(resultList, config.failLevels); + String scriptStr = """ +([policies, reportLevels]) => { + try { + const valueToLevel = (reportValue) => { + let reportLevel; + if (reportValue[1] === "PASS") { + reportLevel = "pass"; + } + else if ((reportValue[0] === "VIOLATION" || reportValue[0] === "RECOMMENDATION") && reportValue[1] === "MANUAL") { + reportLevel = "manual"; + } + else if (reportValue[0] === "VIOLATION") { + if (reportValue[1] === "FAIL") { + reportLevel = "violation"; + } + else if (reportValue[1] === "POTENTIAL") { + reportLevel = "potentialviolation"; + } + } + else if (reportValue[0] === "RECOMMENDATION") { + if (reportValue[1] === "FAIL") { + reportLevel = "recommendation"; + } + else if (reportValue[1] === "POTENTIAL") { + reportLevel = "potentialrecommendation"; + } + } + return reportLevel; + } + const getCounts = (engineReport) => { + let counts = { + violation: 0, + potentialviolation: 0, + recommendation: 0, + potentialrecommendation: 0, + manual: 0, + pass: 0 + } + for (const issue of engineReport.results) { + ++counts[issue.level]; + } + return counts; + } + + let checker = new window.ace_ibma.Checker(); + let customRulesets = []; + customRulesets.forEach((rs) => checker.addRuleset(rs)); + return new Promise((resolve, reject) => { + checker.check(document, policies).then(async function(report) { + for (const result of report.results) { + delete result.node; + result.level = valueToLevel(result.value) + } + report.summary ||= {}; + report.summary.counts ||= getCounts(report); + // Filter out pass results unless they asked for them in reports + // We don't want to mess with baseline functions, but pass results can break the response object + report.results = report.results.filter(result => reportLevels.includes(result.level) || result.level !== "pass"); + resolve(JSON.stringify(report)); + }) + }); + } catch (e) { + return Promise.reject(e); + } +} + """; + ACEReport report; + String jsonReport = this.driver.evaluate(scriptStr, Arrays.asList( + config.policies, + resultList.toArray() + /* TODO: ${JSON.stringify(ACEngineManager.customRulesets)}; */ + )).toString(); + if (!jsonReport.startsWith("{\"results\":[")) { + throw new ACError(jsonReport); + } else { + report = gson.fromJson(jsonReport, ACEReport.class); + } + + // TODO: + // String getPolicies = "return new window.ace_ibma.Checker().rulesetIds;"; + // if (curPol != null && !checkPolicy) { + // checkPolicy = true; + // const valPolicies = ACEngineManager.customRulesets.map(rs => rs.id).concat(await browser.executeScript(getPolicies)); + // areValidPolicy(valPolicies, curPol); + // } + + // If there is something to report... + if (report.results != null) { + if (config.captureScreenshots) { + // TODO: Screenshot? + // String image = ((TakesScreenshot)this.driver).getScreenshotAs(OutputType.BASE64); + // report.screenshot = image; + } + } + return report; + } catch (Error err) { + System.err.println(err); + throw err; + } + } + + @Override + public String getUrl() { + return this.driver.url(); + } + + @Override + public String getTitle() { + return this.driver.title(); + } + + @Override + public Guideline[] getGuidelines() { + String scriptStr = String.format(""" +() => { + try { + let checker = new window.ace_ibma.Checker(); + let customRulesets = []; + customRulesets.forEach((rs) => checker.addRuleset(rs)); + return Promise.resolve(JSON.stringify(checker.getGuidelines())); + } catch (e) { + return Promise.reject(e); + } +} +"""); + String jsonGuidelines = this.driver.evaluate(scriptStr).toString(); + return gson.fromJson(jsonGuidelines, Guideline[].class); + } + + @Override + public Rule[] getRules() { + String scriptStr = String.format(""" +() => { + try { + let checker = new window.ace_ibma.Checker(); + return Promise.resolve(JSON.stringify(checker.getRules())); + } catch (e) { + return Promise.reject(e); + } +} +"""); + String jsonGuidelines = this.driver.evaluate(scriptStr).toString(); + return gson.fromJson(jsonGuidelines, Rule[].class); + } + + @Override + public String getProfile() { + return "Selenium"; + } + + @Override + public String getHelp(String ruleId, String reasonId, String helpRoot) { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'getHelp'"); + } + + @Override + public String encodeURIComponent(String s) { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'encodeURIComponent'"); + } +} diff --git a/java-accessibility-checker/src/test/java/com/ibm/able/equalaccess/AccessibilityCheckerPlaywrightTest.java b/java-accessibility-checker/src/test/java/com/ibm/able/equalaccess/AccessibilityCheckerPlaywrightTest.java new file mode 100644 index 000000000..4c462436d --- /dev/null +++ b/java-accessibility-checker/src/test/java/com/ibm/able/equalaccess/AccessibilityCheckerPlaywrightTest.java @@ -0,0 +1,223 @@ +/****************************************************************************** + Copyright:: 2024- IBM, Inc + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + *****************************************************************************/ +package com.ibm.able.equalaccess; + +import org.junit.Test; +import static org.junit.Assert.*; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Paths; +import java.time.Duration; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.Arrays; +import java.util.Map; +import java.util.List; +import java.util.HashSet; +import java.util.Set; + +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; + +import com.google.common.io.Files; +import com.google.gson.Gson; +import com.ibm.able.equalaccess.config.ACConfigManager; +import com.ibm.able.equalaccess.engine.ACReport; +import com.ibm.able.equalaccess.engine.ACReport.Result; +import com.ibm.able.equalaccess.report.BaselineManager.eAssertResult; +import com.microsoft.playwright.*; + +public class AccessibilityCheckerPlaywrightTest { + public static class UnitTestInfoResult { + public String ruleId; + public String reasonId; + public String category; + public String message; + public String[] messageArgs; + public String[] value; + public Map path; + + public boolean matches(Result result) { + return ruleId.equals(result.ruleId) + && reasonId.equals(result.reasonId) + && category.equals(result.category) + && message.equals(result.message) + && value[1].equals(result.value[1]) + && path.get("dom").equals(result.path.get("dom")) + && path.get("aria").equals(result.path.get("aria")); + } + } + public static class UnitTestInfo { + public String[] ruleIds; + public UnitTestInfoResult[] results; + } + private static Page driver; + + /** + * Setup a Selenium Chrome environment before tests + */ + @BeforeClass public static void setup() { + try { + Playwright playwright = Playwright.create(); + Browser browser = playwright.chromium().launch(); + AccessibilityCheckerPlaywrightTest.driver = browser.newPage(); + } catch (Throwable err) { + System.err.println(err.toString()); + err.printStackTrace(); + } + } + + /** + * Close Selenium Chrome environment after tests + */ + @AfterClass public static void teardown() { + AccessibilityCheckerPlaywrightTest.driver.close(); + AccessibilityChecker.close(); + } + + @Test public void getCompliance() { + ACConfigManager.getConfig().label = new String[] { "IBMa-Java-TeSt" }; + AccessibilityCheckerPlaywrightTest.driver.navigate("https://altoromutual.12mc9fdq8fib.us-south.codeengine.appdomain.cloud/"); + ACReport report = AccessibilityChecker.getCompliance(driver, "Playwright_getComplianceTest"); + assertNotNull(report); + assertTrue(report.results.length > 0); + } + + @Test public void baselines() throws IOException { + Paths.get("baselines", "Playwright_getComplianceTest3.json").toFile().delete(); + AccessibilityCheckerPlaywrightTest.driver.navigate("https://altoromutual.12mc9fdq8fib.us-south.codeengine.appdomain.cloud/"); + ACReport report = AccessibilityChecker.getCompliance(driver, "Playwright_getComplianceTest2"); + assertEquals(eAssertResult.FAIL, AccessibilityChecker.assertCompliance(report)); + new File("baselines").mkdirs(); + Files.copy(Paths.get("results", "Playwright_getComplianceTest2.json").toFile(), Paths.get("baselines", "Playwright_getComplianceTest3.json").toFile()); + + report = AccessibilityChecker.getCompliance(driver, "Playwright_getComplianceTest3"); + assertEquals(eAssertResult.PASS, AccessibilityChecker.assertCompliance(report)); + Paths.get("baselines", "Playwright_getComplianceTest3.json").toFile().delete(); + } + + // @Test public void getComplianceLong() { + // AccessibilityCheckerTest.driver.navigate("https://openliberty.io/docs/latest/reference/javadoc/liberty-jakartaee8-javadoc.html?path=liberty-javaee8-javadoc/index-all.html"); + // ACReport report = AccessibilityChecker.getCompliance(driver, "Playwright_getComplianceLong"); + // assertNotNull(report); + // assertTrue(report.results.length > 0); + // } + + private void listFiles(File f, java.util.List retFiles) { + if (f.isFile() && f.exists() && (f.getName().endsWith("html") || f.getName().endsWith("htm"))) { + retFiles.add(f); + } else if (f.isDirectory()) { + for (File subF: f.listFiles((testFile, name) -> testFile.isDirectory() || name.endsWith(".htm") || name.endsWith(".html"))) { + listFiles(subF, retFiles); + } + } + + } + @Test public void getComplianceTestsuite() throws IOException { + ACConfigManager.resetConfig(); + File configFile = new File("achecker.json"); + try { + configFile.delete(); + FileWriter myWriter = new FileWriter("achecker.json"); + myWriter.write(""" +{ + "customRuleServer": true, + "rulePack": "https://localhost:9445/rules/archives/preview/js", + "ruleArchive": "preview", + "ignoreHTTPSErrors": true, + "policies": [ "IBM_Accessibility", "IBM_Accessibility_next"], + "failLevels": [ "violation", "potentialviolation" ], + "reportLevels": [ + "violation", + "potentialviolation", + "recommendation", + "potentialrecommendation", + "manual", + "pass" + ], + "outputFormat": [ "json" ], + "label": [ + "IBMa-Java-TeSt" + ] +} +"""); + myWriter.close(); + ACConfigManager.getConfig(); + + Gson gson = new Gson(); + File testRootDir = Paths.get(System.getProperty("user.dir"), "..","accessibility-checker-engine","test","v2","checker","accessibility","rules").toFile(); + ArrayList testFiles = new ArrayList<>(); + listFiles(testRootDir, testFiles); + + + // Skip test cases that don't work in this environment (e.g., can't disable meta refresh in chrome) + Set skipList = new HashSet<>(Arrays.asList(new File[] { + //not in karma conf file + Paths.get(testRootDir.getAbsolutePath(), "a_text_purpose_ruleunit", "A-hasTextEmbedded.html").toFile(), + // path.join(testRootDir, "a_text_purpose_ruleunit", "A-nonTabable.html"), + + // Meta refresh + Paths.get(testRootDir.getAbsolutePath(), "meta_refresh_delay_ruleunit", "Meta-invalidRefresh.html").toFile(), + Paths.get(testRootDir.getAbsolutePath(), "meta_refresh_delay_ruleunit", "Meta-validRefresh.html").toFile(), + Paths.get(testRootDir.getAbsolutePath(), "meta_redirect_optional_ruleunit", "Meta-RefreshZero.html").toFile(), + + // CSS test issues + Paths.get(testRootDir.getAbsolutePath(), "style_color_misuse_ruleunit","D543.html").toFile(), + Paths.get(testRootDir.getAbsolutePath(), "style_before_after_review_ruleunit","D100.html").toFile(), + + // Misc + // path.join(testRootDir, "aria_banner_label_unique_ruleunit", "validLandMarks-testCaseFromAnn.html"), + })); + + for (File testFile: testFiles) { + if (skipList.contains(testFile)) continue; + AccessibilityCheckerPlaywrightTest.driver.navigate("file://"+testFile.getAbsolutePath()); + ACReport report = AccessibilityChecker.getCompliance(driver, "Playwright_"+testFile.getAbsolutePath().substring(testRootDir.getAbsolutePath().length())); + String unitTestInfoStr = AccessibilityCheckerPlaywrightTest.driver.evaluate("() => JSON.stringify((typeof (window.UnitTest) !== 'undefined' && window.UnitTest))").toString(); + if (!"false".equals(unitTestInfoStr)) { + UnitTestInfo expectedInfo = gson.fromJson(unitTestInfoStr, UnitTestInfo.class); + List coveredRuleIds = Arrays.asList(expectedInfo.ruleIds); + if (expectedInfo != null && expectedInfo.ruleIds != null && expectedInfo.ruleIds.length > 0) { + System.out.println(testFile.getCanonicalPath()); + System.out.flush(); + List actualIssues = new LinkedList<>(Arrays.stream(report.results).filter(actualIssue -> coveredRuleIds.contains(actualIssue.ruleId)).toList()); + List expectedIssues = new LinkedList<>(Arrays.asList(expectedInfo.results)); + for (int idxActual=0; idxActual 0); } @Test public void baselines() throws IOException { - Paths.get("baselines", "getComplianceTest3.json").toFile().delete(); - AccessibilityCheckerTest.driver.get("https://altoromutual.12mc9fdq8fib.us-south.codeengine.appdomain.cloud/"); - ACReport report = AccessibilityChecker.getCompliance(driver, "getComplianceTest2"); + Paths.get("baselines", "Selenium_getComplianceTest3.json").toFile().delete(); + AccessibilityCheckerSeleniumTest.driver.get("https://altoromutual.12mc9fdq8fib.us-south.codeengine.appdomain.cloud/"); + ACReport report = AccessibilityChecker.getCompliance(driver, "Selenium_getComplianceTest2"); assertEquals(eAssertResult.FAIL, AccessibilityChecker.assertCompliance(report)); new File("baselines").mkdirs(); - Files.copy(Paths.get("results", "getComplianceTest2.json").toFile(), Paths.get("baselines", "getComplianceTest3.json").toFile()); + Files.copy(Paths.get("results", "Selenium_getComplianceTest2.json").toFile(), Paths.get("baselines", "Selenium_getComplianceTest3.json").toFile()); - report = AccessibilityChecker.getCompliance(driver, "getComplianceTest3"); + report = AccessibilityChecker.getCompliance(driver, "Selenium_getComplianceTest3"); assertEquals(eAssertResult.PASS, AccessibilityChecker.assertCompliance(report)); - Paths.get("baselines", "getComplianceTest3.json").toFile().delete(); + Paths.get("baselines", "Selenium_getComplianceTest3.json").toFile().delete(); } // @Test public void getComplianceLong() { // AccessibilityCheckerTest.driver.get("https://openliberty.io/docs/latest/reference/javadoc/liberty-jakartaee8-javadoc.html?path=liberty-javaee8-javadoc/index-all.html"); - // ACReport report = AccessibilityChecker.getCompliance(driver, "getComplianceLong"); + // ACReport report = AccessibilityChecker.getCompliance(driver, "Selenium_getComplianceLong"); // assertNotNull(report); // assertTrue(report.results.length > 0); // } @@ -208,9 +208,9 @@ private void listFiles(File f, java.util.List retFiles) { for (File testFile: testFiles) { if (skipList.contains(testFile)) continue; - AccessibilityCheckerTest.driver.get("file://"+testFile.getAbsolutePath()); - ACReport report = AccessibilityChecker.getCompliance(driver, testFile.getAbsolutePath().substring(testRootDir.getAbsolutePath().length())); - String unitTestInfoStr = AccessibilityCheckerTest.driver.executeScript("return JSON.stringify((typeof (window.UnitTest) !== 'undefined' && window.UnitTest))").toString(); + AccessibilityCheckerSeleniumTest.driver.get("file://"+testFile.getAbsolutePath()); + ACReport report = AccessibilityChecker.getCompliance(driver, "Selenium_"+testFile.getAbsolutePath().substring(testRootDir.getAbsolutePath().length())); + String unitTestInfoStr = AccessibilityCheckerSeleniumTest.driver.executeScript("return JSON.stringify((typeof (window.UnitTest) !== 'undefined' && window.UnitTest))").toString(); if (!"false".equals(unitTestInfoStr)) { UnitTestInfo expectedInfo = gson.fromJson(unitTestInfoStr, UnitTestInfo.class); List coveredRuleIds = Arrays.asList(expectedInfo.ruleIds);