Delta debugging integration into fuzzing

perun/deltadebugging/factory.py

@@ -5,56 +5,118 @@
 from __future__ import annotations
 
 import subprocess
+import tempfile
+import os
 from perun.utils.external import commands as external_commands
 from perun.utils import log
 from typing import Any, TYPE_CHECKING
 from pathlib import Path
 from perun.utils.common import common_kit
+from perun.fuzz.evaluate.by_perun import target_testing
+from typing import Iterable
+
+from perun.fuzz.structs import Mutation
 
 if TYPE_CHECKING:
-    from perun.utils.structs.common_structs import Executable
+    from perun.profile.factory import Profile
+    from perun.utils.structs.common_structs import Executable, MinorVersion, CollectStatus, Job
 
 
 def run_delta_debugging_for_command(
     executable: Executable,
-    input_sample: str,
     **kwargs: Any,
 ) -> None:
-
-    timeout: float = kwargs.get("timeout", 1.0)
     output_dir = Path(kwargs["output_dir"]).resolve()
-    read_input(input_sample)
-    inp = read_input(input_sample)
-    n = 2  # granularity
+    kwargs["is_fuzzing"] = False
+    debugged_input = delta_debugging_algorithm(executable, **kwargs)
+    log.minor_info("shortest failing input = " + debugged_input)
+    input_sample = kwargs["input_sample"]
+    create_debugging_file(output_dir, input_sample, debugged_input)
 
+
+def delta_debugging_algorithm(executable: Executable, is_fuzzing: bool, **kwargs: Any) -> str:
+    n = 2  # granularity
+    inp = read_input(is_fuzzing, **kwargs)
     while len(inp) >= 2:
         start = 0
         subset_length = int(len(inp) / n)
         program_fails = False
-
         while start < len(inp):
             complement = inp[: int(start)] + inp[int(start + subset_length) :]
-            try:
-                full_cmd = f"{executable} {complement}"
-                external_commands.run_safely_external_command(full_cmd, True, True, timeout)
-
-            except subprocess.TimeoutExpired:
+            if is_fuzzing:
+                program_fails = run_with_fuzzing(executable, complement, **kwargs)
+            else:
+                program_fails = run_command_with_input(executable, complement, **kwargs)
+            if program_fails:
                 inp = complement
                 n = max(n - 1, 2)
-                program_fails = True
                 break
             start += subset_length
 
         if not program_fails:
             if n == len(inp):
                 break
             n = min(n * 2, len(inp))
+    return inp
 
-    log.minor_info("shortest failing input = " + inp)
-    return create_debugging_file(output_dir, input_sample, inp)
 
+def run_with_fuzzing(
+    executable: Executable,
+    complement: str,
+    mutation: Mutation,
+    collector: str,
+    postprocessor: list[str],
+    minor_version_list: list[MinorVersion],
+    base_result: Iterable[tuple[CollectStatus, Profile, Job]],
+    **kwargs: Any,
+) -> bool:
+    with tempfile.NamedTemporaryFile(delete=False) as temp_file:
+        temp_file.write(complement.encode())
+        temp_file.flush()
+        input_variation = Mutation(temp_file.name, mutation.history, mutation.predecessor)
+        program_fails = target_testing(
+            executable,
+            input_variation,
+            collector,
+            postprocessor,
+            minor_version_list,
+            base_result,
+            **kwargs,
+        )
+        if program_fails:
+            os.remove(mutation.path)
+            mutation.path = temp_file.name
+        else:
+            os.remove(temp_file.name)
+        return program_fails
+
+
+def run_command_with_input(executable: Executable, complement: str, **kwargs: Any) -> bool:
+    timeout = kwargs.get("timeout", 1.0)
+    with tempfile.NamedTemporaryFile(delete=False) as temp_file:
+        temp_file.write(complement.encode())
+        temp_file.flush()
+
+    try:
+        full_cmd = f"{executable} {temp_file.name}"
+        external_commands.run_safely_external_command(full_cmd, True, True, timeout)
+        os.remove(temp_file.name)
+
+    except subprocess.TimeoutExpired:
+        return True
+
+    return False
+
+
+def read_input(is_fuzzing: bool, **kwargs: Any) -> str:
+    if is_fuzzing:
+        mutation = kwargs.get("mutation")
+        if mutation is None:
+            raise ValueError("Got unexpected mutation")
+        input_file = mutation.path
+    else:
+        input_file = kwargs.get("input_sample", "")
 
-def read_input(input_file: str) -> str:
     input_path = Path(input_file)
     if input_path.is_file():
         with open(input_path, "r") as file:

perun/fuzz/factory.py

@@ -12,7 +12,7 @@
 import threading
 import time
 from subprocess import CalledProcessError, TimeoutExpired
-from typing import Optional, Any, cast, TYPE_CHECKING
+from typing import Optional, Any, cast, TYPE_CHECKING, Iterable
 from uuid import uuid4
 
 # Third-Party Imports
@@ -33,11 +33,12 @@
 from perun.utils.exceptions import SuppressedExceptions
 import perun.fuzz.evaluate.by_perun as evaluate_workloads_by_perun
 import perun.fuzz.evaluate.by_coverage as evaluate_workloads_by_coverage
+from perun.deltadebugging.factory import delta_debugging_algorithm
 
 if TYPE_CHECKING:
     import types
-
-    from perun.utils.structs.common_structs import Executable, MinorVersion
+    from perun.profile.factory import Profile
+    from perun.utils.structs.common_structs import Executable, MinorVersion, CollectStatus, Job
 
 # to ignore numpy division warnings
 np.seterr(divide="ignore", invalid="ignore")
@@ -399,22 +400,40 @@ def teardown(
 
 
 def process_successful_mutation(
+    executable: Executable,
     mutation: Mutation,
+    collector: str,
+    postprocessor: list[str],
+    minor_version_list: list[MinorVersion],
     parents: list[Mutation],
     fuzz_progress: FuzzingProgress,
     rule_set: RuleSet,
     config: FuzzingConfiguration,
+    base_result_profile: Iterable[tuple[CollectStatus, Profile, Job]],
+    **kwargs: Any,
 ) -> None:
     """If the @p mutation is successful during the evaluation, we update the parent queue
     @p parents, as well as statistics for given rule in rule_set and stats stored in fuzzing
     progress.
 
+    :param executable: tested executable
+    :param collector: collector used to collect profiling data
+    :param postprocessor: list of postprocessors, which are run after collection
+    :param minor_version_list: list of minor version for which we are collecting
     :param mutation: successfully evaluated mutation
     :param fuzz_progress: collective state of fuzzing
     :param parents: list of parents, i.e. mutations which will be further mutated
     :param rule_set: set of applied rules
     :param config: configuration of the fuzzing
+    :param baseline_profile: baseline against which we are checking the degradation
+    :param kwargs: dictionary of additional params for postprocessor and collector
     """
+    kwargs["mutation"] = mutation
+    kwargs["collector"] = collector
+    kwargs["postprocessor"] = postprocessor
+    kwargs["minor_version_list"] = minor_version_list
+    kwargs["base_result"] = base_result_profile
+    delta_debugging_algorithm(executable, True, **kwargs)
     rule_set.hits[mutation.history[-1]] += 1
     rule_set.hits[-1] += 1
     # without cov testing we firstly rate the new parents here
@@ -623,7 +642,19 @@ def signal_handler(sig: int, _: Optional[types.FrameType]) -> None:
                     **kwargs,
                 )
                 if successful_result:
-                    process_successful_mutation(mutation, parents, fuzz_progress, rule_set, config)
+                    process_successful_mutation(
+                        executable,
+                        mutation,
+                        collector,
+                        postprocessor,
+                        minor_version_list,
+                        parents,
+                        fuzz_progress,
+                        rule_set,
+                        config,
+                        base_copy,
+                        **kwargs,
+                    )
 
             log.minor_status(f"{mutation.path}", status=f"{mutation.fitness}")
             # in case of testing with coverage, parent will not be removed but used for mutation

tests/sources/delta_debugging_examples/dd-minimal/main.c

@@ -1,9 +1,11 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <unistd.h>
+#include <stdlib.h>
 #include <string.h>
 
 #define MAGIC_NUMBER 100
+#define BUFFER_SIZE 256
 
 void magicLoop() {
     for (int i = 0; i < MAGIC_NUMBER; ++i) {
@@ -29,6 +31,21 @@ int main(int argc, char* argv[]) {
         return 1;
     }
 
-    checkInputString(argv[1]);
+    FILE * fp = fopen(argv[1],"r");
+
+    if (fp == NULL) {
+        printf("Error opening file\n");
+        return 1;
+    }
+
+    char fileContent[BUFFER_SIZE];
+
+    while (fscanf(fp, "%255s", fileContent) == 1) {
+
+        printf("read line: %s\n", fileContent);
+    }
+
+    fclose(fp);
+    checkInputString(fileContent);
     return 0;
 }