Merge pull request #4 from joernio/andrei/owasp-downloader

[datasets] Added `OWASP`

.github/workflows/release.yaml

@@ -44,6 +44,8 @@ jobs:
           mv workspace/ichnaea.zip ichnaea.zip
           mv workspace/securibench-micro-JAVA.zip securibench-micro-JAVA.zip
           mv workspace/securibench-micro-JAVASRC.zip securibench-micro-JAVASRC.zip
+          mv workspace/OWASP-BenchmarkJava-JAVA.zip OWASP-BenchmarkJava-JAVA.zip
+          mv workspace/OWASP-BenchmarkJava-JAVASRC.zip OWASP-BenchmarkJava-JAVASRC.zip
       - name: Set next release version
         id: taggerFinal
         uses: anothrNick/[email protected]
@@ -57,4 +59,6 @@ jobs:
           files: |
             ichnaea.zip
             securibench-micro-JAVA.zip
-            securibench-micro-JAVASRC.zip
+            securibench-micro-JAVASRC.zip
+            OWASP-BenchmarkJava-JAVA.zip
+            OWASP-BenchmarkJava-JAVASRC.zip

src/main/scala/io/joern/benchmarks/datasets/BenchmarkDataset.scala

@@ -5,10 +5,8 @@ import io.joern.benchmarks.datasets.AvailableBenchmarks
 import io.joern.benchmarks.datasets.runner.{
   DatasetDownloader,
   IchnaeaDownloader,
+  OWASPJavaDownloader,
   SecuribenchMicroDownloader
-// TODO: Add when implementing
-
-//  OWASPJavaDownloader,
 }
 import org.slf4j.LoggerFactory
 import upickle.default.*
@@ -37,17 +35,22 @@ class BenchmarkDataset(config: BenchmarkDatasetConfig) {
 }
 
 object BenchmarkDataset {
-
   val benchmarkConstructors: Map[AvailableBenchmarks.Value, BenchmarkDatasetConfig => DatasetDownloader] = Map(
     // TODO: Add when implementing
-//    (AvailableBenchmarks.OWASP_JAVASRC, x => new OWASPJavaDownloader(x.datasetDir)),
-//    (AvailableBenchmarks.OWASP_JAVA, x => new OWASPJavaDownloader(x.datasetDir)),
+    (AvailableBenchmarks.OWASP_JAVASRC, x => new OWASPJavaDownloader(x.datasetDir, JavaCpgTypes.JAVASRC)),
+    (AvailableBenchmarks.OWASP_JAVA, x => new OWASPJavaDownloader(x.datasetDir, JavaCpgTypes.JAVA)),
     (
       AvailableBenchmarks.SECURIBENCH_MICRO_JAVASRC,
       x => new SecuribenchMicroDownloader(x.datasetDir, JavaCpgTypes.JAVASRC)
     ),
     (AvailableBenchmarks.SECURIBENCH_MICRO_JAVA, x => new SecuribenchMicroDownloader(x.datasetDir, JavaCpgTypes.JAVA)),
-    (AvailableBenchmarks.ICHNAEA_JSSRC, x => new IchnaeaDownloader(x.datasetDir))
+    (AvailableBenchmarks.ICHNAEA_JSSRC, x => new IchnaeaDownloader(x.datasetDir)),
+    (
+      AvailableBenchmarks.SECURIBENCH_MICRO_SEMGREP,
+      x => new SecuribenchMicroDownloader(x.datasetDir, JavaCpgTypes.SEMGREP)
+    ),
+    (AvailableBenchmarks.OWASP_SEMGREP, x => new OWASPJavaDownloader(x.datasetDir, JavaCpgTypes.SEMGREP)),
+    (AvailableBenchmarks.ICHNAEA_SEMGREP, x => new IchnaeaDownloader(x.datasetDir))
   )
 
 }

src/main/scala/io/joern/benchmarks/datasets/BenchmarkDatasetConfig.scala

@@ -8,17 +8,27 @@ case class BenchmarkDatasetConfig(
 )
 
 object AvailableBenchmarks extends Enumeration {
-  val ALL                       = Value
+  val ALL = Value
+
+  // Joern
   val OWASP_JAVASRC             = Value
   val OWASP_JAVA                = Value
   val SECURIBENCH_MICRO_JAVASRC = Value
   val SECURIBENCH_MICRO_JAVA    = Value
   val ICHNAEA_JSSRC             = Value
+  val THORAT_PYSRC              = Value
+
+  // Semgrep
+  val OWASP_SEMGREP             = Value
+  val SECURIBENCH_MICRO_SEMGREP = Value
+  val THORAT_SEMGREP            = Value
+  val ICHNAEA_SEMGREP           = Value
 }
 
 object JavaCpgTypes extends Enumeration {
   val JAVASRC = Value
   val JAVA    = Value
+  val SEMGREP = Value
 }
 
 object OutputFormat extends Enumeration {

src/main/scala/io/joern/benchmarks/datasets/runner/OWASPJavaDownloader.scala

@@ -0,0 +1,47 @@
+package io.joern.benchmarks.datasets.runner
+
+import better.files.File
+import io.joern.benchmarks.*
+import io.joern.benchmarks.datasets.JavaCpgTypes
+import org.slf4j.LoggerFactory
+
+import java.net.{URI, URL}
+import scala.util.{Failure, Success, Try}
+
+class OWASPJavaDownloader(datasetDir: File, cpgCreatorType: JavaCpgTypes.Value)
+    extends DatasetDownloader(datasetDir)
+    with SingleFileDownloader {
+
+  private val logger = LoggerFactory.getLogger(getClass)
+
+  override val benchmarkName = s"OWASP Java v1.2"
+
+  override protected val benchmarkUrl: URL = URI(
+    "https://github.com/OWASP-Benchmark/BenchmarkJava/archive/refs/tags/1.2beta.zip"
+  ).toURL
+  override protected val benchmarkFileName: String = "BenchmarkJava-1.2beta"
+  override protected val benchmarkBaseDir: File    = datasetDir / benchmarkFileName
+
+  private val apacheJdo = URI("https://repo1.maven.org/maven2/javax/jdo/jdo-api/3.1/jdo-api-3.1.jar").toURL
+
+  override def initialize(): Try[File] = Try {
+    downloadBenchmarkAndUnarchive(CompressionTypes.ZIP)
+
+    val datasetLabel =
+      if cpgCreatorType == JavaCpgTypes.JAVA then JavaCpgTypes.JAVA.toString
+      else JavaCpgTypes.JAVASRC.toString
+
+    compressBenchmark(
+      benchmarkBaseDir,
+      Option(File(s"${datasetDir.pathAsString}/OWASP-BenchmarkJava-$datasetLabel.zip"))
+    )
+  }
+
+  override def run(): Unit = {
+    initialize() match {
+      case Failure(exception) =>
+        logger.error(s"Unable to initialize benchmark '$getClass'", exception)
+      case Success(benchmarkDir) =>
+    }
+  }
+}

src/main/scala/io/joern/benchmarks/datasets/runner/SecuribenchMicroDownloader.scala

@@ -55,10 +55,11 @@ class SecuribenchMicroDownloader(datasetDir: File, cpgCreatorType: JavaCpgTypes.
       }
     }
 
-    compressBenchmark(
-      benchmarkBaseDir,
-      Option(File(s"${datasetDir.pathAsString}/securibench-micro-${cpgCreatorType.toString}.zip"))
-    )
+    val datasetLabel =
+      if cpgCreatorType == JavaCpgTypes.JAVA then JavaCpgTypes.JAVA.toString
+      else JavaCpgTypes.JAVASRC.toString
+
+    compressBenchmark(benchmarkBaseDir, Option(File(s"${datasetDir.pathAsString}/securibench-micro-$datasetLabel.zip")))
   }
 
   override def run(): Unit = {