feat(precheck): improve error reporting

precheck/precheck.py

@@ -3,6 +3,9 @@
 import logging
 import os
 import subprocess
+import time
+import traceback
+import xml.etree.ElementTree as ET
 
 import klayout.db as pya
 import klayout.rdb as rdb
@@ -18,6 +21,10 @@
     exit(1)
 
 
+class PrecheckFailure(Exception):
+    pass
+
+
 def magic_drc(gds: str, toplevel: str):
     logging.info(f"Running magic DRC on {gds} (module={toplevel})")
 
@@ -38,10 +45,7 @@ def magic_drc(gds: str, toplevel: str):
     )
 
     if magic.returncode != 0:
-        logging.error("Magic DRC failed")
-        return False
-
-    return True
+        raise PrecheckFailure("Magic DRC failed")
 
 
 def klayout_drc(gds: str, check: str):
@@ -62,19 +66,15 @@ def klayout_drc(gds: str, check: str):
         ],
     )
     if klayout.returncode != 0:
-        logging.error(f"Klayout {check} failed")
-        return False
+        raise PrecheckFailure(f"Klayout {check} failed")
 
     report = rdb.ReportDatabase("DRC")
     report.load(report_file)
 
     if report.num_items() > 0:
-        logging.error(
+        raise PrecheckFailure(
             f"Klayout {check} failed with {report.num_items()} DRC violations"
         )
-        return False
-
-    return True
 
 
 def klayout_checks(gds: str):
@@ -89,24 +89,19 @@ def klayout_checks(gds: str):
         "met5.label",
     ]
 
-    had_error = False
     for layer in forbidden_layers:
         layer_info = layers[layer]
         logging.info(f"* Checking {layer_info.name}")
         layer_index = layout.find_layer(layer_info.layer, layer_info.data_type)
         if layer_index is not None:
-            logging.error(f"Forbidden layer {layer} found in {gds}")
-            had_error = True
-
-    if had_error:
-        logging.error("Klayout checks failed")
-    return not had_error
+            raise PrecheckFailure(f"Forbidden layer {layer} found in {gds}")
 
 
 def main():
     parser = argparse.ArgumentParser()
     parser.add_argument("--gds", required=True)
     parser.add_argument("--top-module", required=False)
+    parser.add_argument("--markdown-report", required=False)
     args = parser.parse_args()
     logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
     logging.info(f"PDK_ROOT: {PDK_ROOT}")
@@ -116,15 +111,52 @@ def main():
     else:
         top_module = os.path.splitext(os.path.basename(args.gds))[0]
 
-    assert magic_drc(args.gds, top_module)
-
-    assert klayout_drc(args.gds, "feol")
-    assert klayout_drc(args.gds, "beol")
-    assert klayout_drc(args.gds, "offgrid")
-
-    assert klayout_checks(args.gds)
+    checks = [
+        ["Magic DRC", lambda: magic_drc(args.gds, top_module)],
+        ["KLayout FEOL", lambda: klayout_drc(args.gds, "feol")],
+        ["KLayout BEOL", lambda: klayout_drc(args.gds, "beol")],
+        ["KLayout offgrid", lambda: klayout_drc(args.gds, "offgrid")],
+        ["KLayout Checks", lambda: klayout_checks(args.gds)],
+    ]
 
-    logging.info(f"Precheck passed for {args.gds}! 🎉")
+    testsuite = ET.Element("testsuite", name="Tiny Tapeout Prechecks")
+    error_count = 0
+    markdown_table = "# Tiny Tapeout Precheck Results\n\n"
+    markdown_table += "| Check | Result |\n|-----------|--------|\n"
+    for [name, check] in checks:
+        start_time = time.time()
+        test_case = ET.SubElement(testsuite, "testcase", name=name)
+        try:
+            check()
+            elapsed_time = time.time() - start_time
+            markdown_table += f"| {name} | ✅ |\n"
+            test_case.set("time", str(round(elapsed_time, 2)))
+        except Exception as e:
+            error_count += 1
+            elapsed_time = time.time() - start_time
+            markdown_table += f"| {name} | Fail: {str(e)} |\n"
+            test_case.set("time", str(round(elapsed_time, 2)))
+            error = ET.SubElement(test_case, "error", message=str(e))
+            error.text = traceback.format_exc()
+    markdown_table += "\n"
+    markdown_table += "In case of failure, please reach out on [discord](https://tinytapeout.com/discord) for assistance."
+
+    testsuites = ET.Element("testsuites")
+    testsuites.append(testsuite)
+    xunit_report = ET.ElementTree(testsuites)
+    ET.indent(xunit_report, space="  ", level=0)
+    xunit_report.write(f"{REPORTS_PATH}/results.xml", encoding="unicode")
+
+    with open(f"{REPORTS_PATH}/results.md", "w") as f:
+        f.write(markdown_table)
+
+    if error_count > 0:
+        logging.error(f"Precheck failed for {args.gds}! 😭")
+        logging.error(f"See {REPORTS_PATH} for more details")
+        logging.error(f"Markdown report:\n{markdown_table}")
+        exit(1)
+    else:
+        logging.info(f"Precheck passed for {args.gds}! 🎉")
 
 
 if __name__ == "__main__":

precheck/reports/.gitignore

@@ -1,5 +1,7 @@
-magic_drc.mag
-magic_drc.txt
-drc_feol.xml
 drc_beol.xml
+drc_feol.xml
 drc_offgrid.xml
+magic_drc.mag
+magic_drc.txt
+results.md
+results.xml

precheck/test_precheck.py

@@ -68,40 +68,36 @@ def gds_fail_metal5_poly(tmp_path_factory: pytest.TempPathFactory):
 
 
 def test_magic_drc_pass(gds_empty: str):
-    result = precheck.magic_drc(gds_empty, "TEST_empty")
-    assert result is True
+    precheck.magic_drc(gds_empty, "TEST_empty")
 
 
 def test_magic_drc_fail(gds_fail_met1_poly: str):
-    result = precheck.magic_drc(gds_fail_met1_poly, "TEST_met1_error")
-    assert result is False
+    with pytest.raises(precheck.PrecheckFailure):
+        precheck.magic_drc(gds_fail_met1_poly, "TEST_met1_error")
 
 
 def test_klayout_feol_pass(gds_empty: str):
-    result = precheck.klayout_drc(gds_empty, "feol")
-    assert result is True
+    precheck.klayout_drc(gds_empty, "feol")
 
 
 def test_klayout_feol_fail(gds_fail_nwell_poly: str):
-    result = precheck.klayout_drc(gds_fail_nwell_poly, "feol")
-    assert result is False
+    with pytest.raises(precheck.PrecheckFailure):
+        precheck.klayout_drc(gds_fail_nwell_poly, "feol")
 
 
 def test_klayout_beol_pass(gds_empty: str):
-    result = precheck.klayout_drc(gds_empty, "beol")
-    assert result is True
+    precheck.klayout_drc(gds_empty, "beol")
 
 
 def test_klayout_beol_fail(gds_fail_met1_poly: str):
-    result = precheck.klayout_drc(gds_fail_met1_poly, "beol")
-    assert result is False
+    with pytest.raises(precheck.PrecheckFailure):
+        result = precheck.klayout_drc(gds_fail_met1_poly, "beol")
 
 
 def test_klayout_checks_pass(gds_empty: str):
-    result = precheck.klayout_checks(gds_empty)
-    assert result is True
+    precheck.klayout_checks(gds_empty)
 
 
 def test_klayout_checks_fail(gds_fail_metal5_poly: str):
-    result = precheck.klayout_checks(gds_fail_metal5_poly)
-    assert result is False
+    with pytest.raises(precheck.PrecheckFailure):
+        precheck.klayout_checks(gds_fail_metal5_poly)