From f4fd86fc01c7ec17c711301065effb74ec77bef5 Mon Sep 17 00:00:00 2001 From: Ben Selwyn-Smith Date: Thu, 15 Aug 2024 08:57:47 +1000 Subject: [PATCH] chore: refactor provenance available check (#791) Signed-off-by: Ben Selwyn-Smith --- .../repo_finder/provenance_extractor.py | 18 +- src/macaron/repo_finder/provenance_finder.py | 194 ++++++- src/macaron/slsa_analyzer/analyze_context.py | 2 +- src/macaron/slsa_analyzer/analyzer.py | 182 ++++--- .../checks/provenance_available_check.py | 482 +----------------- .../checks/provenance_l3_check.py | 8 +- .../checks/provenance_verified_check.py | 31 +- .../slsa_analyzer/git_service/api_client.py | 22 +- .../slsa_analyzer/provenance/slsa/__init__.py | 1 - src/macaron/slsa_analyzer/specs/ci_spec.py | 10 +- tests/conftest.py | 5 +- .../check_results_policy.dl | 2 +- .../micronaut-test.dl | 6 +- .../micronaut_test_config.yaml | 2 +- .../integration/cases/sigstore_mock/policy.dl | 6 +- .../slsa-framework_slsa-verifier/policy.dl | 10 +- .../slsa-framework_slsa-verifier/test.yaml | 2 +- .../expectation/expectation.cue | 2 +- .../cases/urllib3_expectation_dir/policy.dl | 10 +- .../cases/urllib3_expectation_dir/test.yaml | 8 +- .../urllib3_expectation_file/expectation.cue | 2 +- .../cases/urllib3_expectation_file/policy.dl | 8 +- .../cases/urllib3_expectation_file/test.yaml | 8 +- .../urllib3_invalid_expectation/policy.dl | 9 +- .../urllib3_invalid_expectation/test.yaml | 8 +- .../cases/urllib3_no_tag/policy.dl | 11 + .../cases/urllib3_no_tag/test.yaml | 25 + .../output_reporter/test_jinja_extensions.py | 6 +- tests/output_reporter/test_reporter.py | 6 +- tests/repo_finder/test_provenance_finder.py | 224 ++++++++ .../checks/test_build_as_code_check.py | 10 +- .../checks/test_build_service_check.py | 6 +- .../checks/test_provenance_available_check.py | 176 +------ .../checks/test_provenance_l3_check.py | 12 +- .../test_provenance_l3_content_check.py | 6 +- .../test_provenance_repo_commit_checks.py | 2 +- .../checks/test_trusted_builder_l3_check.py | 2 +- tests/slsa_analyzer/checks/test_vcs_check.py | 2 +- .../resources/micronaut.intoto.jsonl | 1 + tests/slsa_analyzer/test_analyze_context.py | 8 +- 40 files changed, 675 insertions(+), 860 deletions(-) create mode 100644 tests/integration/cases/urllib3_no_tag/policy.dl create mode 100644 tests/integration/cases/urllib3_no_tag/test.yaml create mode 100644 tests/repo_finder/test_provenance_finder.py create mode 100644 tests/slsa_analyzer/provenance/resources/micronaut.intoto.jsonl diff --git a/src/macaron/repo_finder/provenance_extractor.py b/src/macaron/repo_finder/provenance_extractor.py index 2d32bead0..5c3307c58 100644 --- a/src/macaron/repo_finder/provenance_extractor.py +++ b/src/macaron/repo_finder/provenance_extractor.py @@ -243,11 +243,9 @@ def _clean_spdx(uri: str) -> str: return url -def check_if_input_repo_commit_provenance_conflict( +def check_if_input_repo_provenance_conflict( repo_path_input: str | None, - digest_input: str | None, provenance_repo_url: str | None, - provenance_commit_digest: str | None, ) -> bool: """Test if the input repo and commit match the contents of the provenance. @@ -255,12 +253,8 @@ def check_if_input_repo_commit_provenance_conflict( ---------- repo_path_input: str | None The repo URL from input. - digest_input: str | None - The digest from input. provenance_repo_url: str | None The repo URL from provenance. - provenance_commit_digest: str | None - The commit digest from provenance. Returns ------- @@ -277,16 +271,6 @@ def check_if_input_repo_commit_provenance_conflict( ) return True - # Check the provenance commit against the input commit. - if digest_input and provenance_commit_digest and digest_input != provenance_commit_digest: - logger.debug( - "The commit digest from input does not match what exists in the provenance. " - "Input Commit: %s, Provenance Commit: %s.", - digest_input, - provenance_commit_digest, - ) - return True - return False diff --git a/src/macaron/repo_finder/provenance_finder.py b/src/macaron/repo_finder/provenance_finder.py index c70693e5b..5f065900e 100644 --- a/src/macaron/repo_finder/provenance_finder.py +++ b/src/macaron/repo_finder/provenance_finder.py @@ -8,16 +8,22 @@ from functools import partial from packageurl import PackageURL +from pydriller import Git from macaron.config.defaults import defaults from macaron.repo_finder.commit_finder import AbstractPurlType, determine_abstract_purl_type +from macaron.slsa_analyzer.analyze_context import AnalyzeContext from macaron.slsa_analyzer.checks.provenance_available_check import ProvenanceAvailableException +from macaron.slsa_analyzer.ci_service import GitHubActions +from macaron.slsa_analyzer.ci_service.base_ci_service import NoneCIService from macaron.slsa_analyzer.package_registry import PACKAGE_REGISTRIES, JFrogMavenRegistry, NPMRegistry from macaron.slsa_analyzer.package_registry.npm_registry import NPMAttestationAsset from macaron.slsa_analyzer.provenance.intoto import InTotoPayload from macaron.slsa_analyzer.provenance.intoto.errors import LoadIntotoAttestationError from macaron.slsa_analyzer.provenance.loader import load_provenance_payload +from macaron.slsa_analyzer.provenance.slsa import SLSAProvenanceData from macaron.slsa_analyzer.provenance.witness import is_witness_provenance_payload, load_witness_verifier_config +from macaron.slsa_analyzer.specs.ci_spec import CIInfo logger: logging.Logger = logging.getLogger(__name__) @@ -49,6 +55,8 @@ def find_provenance(self, purl: PackageURL) -> list[InTotoPayload]: list[InTotoPayload] The provenance payload, or an empty list if not found. """ + logger.debug("Seeking provenance of: %s", purl) + if determine_abstract_purl_type(purl) == AbstractPurlType.REPOSITORY: # Do not perform default discovery for repository type targets. return [] @@ -331,7 +339,8 @@ def find_gav_provenance(purl: PackageURL, registry: JFrogMavenRegistry) -> list[ logger.error(msg) raise ProvenanceAvailableException(msg) - provenance_filepaths = [] + provenances = [] + witness_verifier_config = load_witness_verifier_config() try: with tempfile.TemporaryDirectory() as temp_dir: for provenance_asset in provenance_assets: @@ -342,24 +351,19 @@ def find_gav_provenance(purl: PackageURL, registry: JFrogMavenRegistry) -> list[ provenance_asset.name, ) continue - provenance_filepaths.append(provenance_filepath) - except OSError as error: - logger.error("Error while storing provenance in the temporary directory: %s", error) - - provenances = [] - witness_verifier_config = load_witness_verifier_config() - for provenance_filepath in provenance_filepaths: - try: - provenance_payload = load_provenance_payload(provenance_filepath) - except LoadIntotoAttestationError as error: - logger.error("Error while loading provenance: %s", error) - continue + try: + provenance_payload = load_provenance_payload(provenance_filepath) + except LoadIntotoAttestationError as load_error: + logger.error("Error while loading provenance: %s", load_error) + continue - if not is_witness_provenance_payload(provenance_payload, witness_verifier_config.predicate_types): - continue + if not is_witness_provenance_payload(provenance_payload, witness_verifier_config.predicate_types): + continue - provenances.append(provenance_payload) + provenances.append(provenance_payload) + except OSError as error: + logger.error("Error while storing provenance in the temporary directory: %s", error) if not provenances: logger.debug("No payloads found in provenance files.") @@ -367,3 +371,161 @@ def find_gav_provenance(purl: PackageURL, registry: JFrogMavenRegistry) -> list[ # We assume that there is only one provenance per GAV. return provenances[:1] + + +def find_provenance_from_ci(analyze_ctx: AnalyzeContext, git_obj: Git | None) -> InTotoPayload | None: + """Try to find provenance from CI services of the repository. + + Note that we stop going through the CI services once we encounter a CI service + that does host provenance assets. + + This method also loads the provenance payloads into the ``CIInfo`` object where + the provenance assets are found. + + Parameters + ---------- + analyze_ctx: AnalyzeContext + The contenxt of the ongoing analysis. + git_obj: Git | None + The Pydriller Git object representing the repository, if any. + + Returns + ------- + InTotoPayload | None + The provenance payload, or None if not found. + """ + provenance_extensions = defaults.get_list( + "slsa.verifier", + "provenance_extensions", + fallback=["intoto.jsonl"], + ) + component = analyze_ctx.component + ci_info_entries = analyze_ctx.dynamic_data["ci_services"] + + if not component.repository: + logger.debug("Unable to find a provenance because a repository was not found for %s.", component.purl) + return None + + repo_full_name = component.repository.full_name + for ci_info in ci_info_entries: + ci_service = ci_info["service"] + + if isinstance(ci_service, NoneCIService): + continue + + if isinstance(ci_service, GitHubActions): + # Find the release for the software component version being analyzed. + digest = component.repository.commit_sha + tag = None + if git_obj: + # Use the software component commit to find the tag. + if not digest: + logger.debug("Cannot retrieve asset provenance without commit digest.") + return None + tags = git_obj.repo.tags + for _tag in tags: + try: + tag_commit = str(_tag.commit) + except ValueError as error: + logger.debug("Commit of tag is a blob or tree: %s", error) + continue + if tag_commit and tag_commit == digest: + tag = str(_tag) + break + + if not tag: + logger.debug("Could not find the tag matching commit: %s", digest) + return None + + # Get the correct release using the tag. + release_payload = ci_service.api_client.get_release_by_tag(repo_full_name, tag) + if not release_payload: + logger.debug("Failed to find release matching tag: %s", tag) + return None + + # Store the release data for other checks. + ci_info["release"] = release_payload + + # Get the provenance assets. + for prov_ext in provenance_extensions: + provenance_assets = ci_service.api_client.fetch_assets( + release_payload, + ext=prov_ext, + ) + if not provenance_assets: + continue + + logger.info("Found the following provenance assets:") + for provenance_asset in provenance_assets: + logger.info("* %s", provenance_asset.url) + + # Store the provenance assets for other checks. + ci_info["provenance_assets"].extend(provenance_assets) + + # Download the provenance assets and load the provenance payloads. + download_provenances_from_github_actions_ci_service( + ci_info, + ) + + # TODO consider how to handle multiple payloads here. + return ci_info["provenances"][0].payload if ci_info["provenances"] else None + + else: + logger.debug("CI service not supported for provenance finding: %s", ci_service.name) + + return None + + +def download_provenances_from_github_actions_ci_service(ci_info: CIInfo) -> None: + """Download provenances from GitHub Actions. + + Parameters + ---------- + ci_info: CIInfo, + A ``CIInfo`` instance that holds a GitHub Actions git service object. + """ + ci_service = ci_info["service"] + prov_assets = ci_info["provenance_assets"] + + try: + with tempfile.TemporaryDirectory() as temp_path: + downloaded_provs = [] + for prov_asset in prov_assets: + # Check the size before downloading. + if prov_asset.size_in_bytes > defaults.getint( + "slsa.verifier", + "max_download_size", + fallback=1000000, + ): + logger.info( + "Skip verifying the provenance %s: asset size too large.", + prov_asset.name, + ) + continue + + provenance_filepath = os.path.join(temp_path, prov_asset.name) + + if not ci_service.api_client.download_asset( + prov_asset.url, + provenance_filepath, + ): + logger.debug( + "Could not download the provenance %s. Skip verifying...", + prov_asset.name, + ) + continue + + # Read the provenance. + try: + payload = load_provenance_payload(provenance_filepath) + except LoadIntotoAttestationError as error: + logger.error("Error logging provenance: %s", error) + continue + + # Add the provenance file. + downloaded_provs.append(SLSAProvenanceData(payload=payload, asset=prov_asset)) + + # Persist the provenance payloads into the CIInfo object. + ci_info["provenances"] = downloaded_provs + except OSError as error: + logger.error("Error while storing provenance in the temporary directory: %s", error) diff --git a/src/macaron/slsa_analyzer/analyze_context.py b/src/macaron/slsa_analyzer/analyze_context.py index e1efe9ed2..e54363f98 100644 --- a/src/macaron/slsa_analyzer/analyze_context.py +++ b/src/macaron/slsa_analyzer/analyze_context.py @@ -157,7 +157,7 @@ def provenances(self) -> dict[str, list[InTotoV01Statement | InTotoV1Statement]] result: dict[str, list[InTotoV01Statement | InTotoV1Statement]] = defaultdict(list) for ci_info in ci_services: result[ci_info["service"].name].extend( - prov_asset.payload.statement for prov_asset in ci_info["provenances"] + provenance.payload.statement for provenance in ci_info["provenances"] ) package_registry_entries = self.dynamic_data["package_registries"] for package_registry_entry in package_registry_entries: diff --git a/src/macaron/slsa_analyzer/analyzer.py b/src/macaron/slsa_analyzer/analyzer.py index 6cff9716a..904d68c14 100644 --- a/src/macaron/slsa_analyzer/analyzer.py +++ b/src/macaron/slsa_analyzer/analyzer.py @@ -38,10 +38,10 @@ from macaron.repo_finder.commit_finder import find_commit from macaron.repo_finder.provenance_extractor import ( check_if_input_purl_provenance_conflict, - check_if_input_repo_commit_provenance_conflict, + check_if_input_repo_provenance_conflict, extract_repo_and_commit_from_provenance, ) -from macaron.repo_finder.provenance_finder import ProvenanceFinder +from macaron.repo_finder.provenance_finder import ProvenanceFinder, find_provenance_from_ci from macaron.slsa_analyzer import git_url from macaron.slsa_analyzer.analyze_context import AnalyzeContext from macaron.slsa_analyzer.asset import VirtualReleaseAsset @@ -49,7 +49,6 @@ # To load all checks into the registry from macaron.slsa_analyzer.checks import * # pylint: disable=wildcard-import,unused-wildcard-import # noqa: F401,F403 -from macaron.slsa_analyzer.checks.check_result import CheckResult from macaron.slsa_analyzer.ci_service import CI_SERVICES from macaron.slsa_analyzer.database_store import store_analyze_context_to_db from macaron.slsa_analyzer.git_service import GIT_SERVICES, BaseGitService @@ -323,7 +322,7 @@ def run_single( ) provenance_is_verified = False - if not provenance_payload and parsed_purl and not config.get_value("path"): + if not provenance_payload and parsed_purl: # Try to find the provenance file for the parsed PURL. provenance_finder = ProvenanceFinder() provenances = provenance_finder.find_provenance(parsed_purl) @@ -344,13 +343,11 @@ def run_single( except ProvenanceError as error: logger.debug("Failed to extract repo or commit from provenance: %s", error) - # Try to validate the input repo and/or commit against provenance contents. - if (provenance_repo_url or provenance_commit_digest) and check_if_input_repo_commit_provenance_conflict( - repo_path_input, digest_input, provenance_repo_url, provenance_commit_digest - ): + # Try to validate the input repo against provenance contents. + if provenance_repo_url and check_if_input_repo_provenance_conflict(repo_path_input, provenance_repo_url): return Record( record_id=repo_id, - description="Input mismatch between repo/commit and provenance.", + description="Input mismatch between repo and provenance.", pre_config=config, status=SCMStatus.ANALYSIS_FAILED, ) @@ -433,6 +430,41 @@ def run_single( analyze_ctx.dynamic_data["expectation"] = self.expectations.get_expectation_for_target( analyze_ctx.component.purl.split("@")[0] ) + + git_service = self._determine_git_service(analyze_ctx) + self._determine_ci_services(analyze_ctx, git_service) + self._determine_build_tools(analyze_ctx, git_service) + self._determine_package_registries(analyze_ctx) + + if not provenance_payload: + # Look for provenance using the CI. + provenance_payload = find_provenance_from_ci(analyze_ctx, git_obj) + # If found, verify analysis target against new provenance + if provenance_payload: + # If repository URL was not provided as input, check the one found during analysis. + if not repo_path_input and component.repository: + repo_path_input = component.repository.remote_path + + # Extract the digest and repository URL from provenance. + provenance_repo_url = provenance_commit_digest = None + try: + provenance_repo_url, provenance_commit_digest = extract_repo_and_commit_from_provenance( + provenance_payload + ) + except ProvenanceError as error: + logger.debug("Failed to extract repo or commit from provenance: %s", error) + + # Try to validate the input repo against provenance contents. + if provenance_repo_url and check_if_input_repo_provenance_conflict( + repo_path_input, provenance_repo_url + ): + return Record( + record_id=repo_id, + description="Input mismatch between repo/commit and provenance.", + pre_config=config, + status=SCMStatus.ANALYSIS_FAILED, + ) + analyze_ctx.dynamic_data["provenance"] = provenance_payload if provenance_payload: analyze_ctx.dynamic_data["is_inferred_prov"] = False @@ -440,7 +472,7 @@ def run_single( analyze_ctx.dynamic_data["provenance_repo_url"] = provenance_repo_url analyze_ctx.dynamic_data["provenance_commit_digest"] = provenance_commit_digest - analyze_ctx.check_results = self.perform_checks(analyze_ctx) + analyze_ctx.check_results = registry.scan(analyze_ctx) return Record( record_id=repo_id, @@ -986,23 +1018,23 @@ def _resolve_local_path(start_dir: str, local_path: str) -> str: logger.error(error) return "" - def perform_checks(self, analyze_ctx: AnalyzeContext) -> dict[str, CheckResult]: - """Run the analysis on the target repo and return the results. + def _determine_git_service(self, analyze_ctx: AnalyzeContext) -> BaseGitService: + """Determine the Git service used by the software component.""" + remote_path = analyze_ctx.component.repository.remote_path if analyze_ctx.component.repository else None + git_service = self.get_git_service(remote_path) - Parameters - ---------- - analyze_ctx : AnalyzeContext - The object containing processed data for the target repo. + if isinstance(git_service, NoneGitService): + logger.info("Unable to find repository or unsupported git service for %s", analyze_ctx.component.purl) + else: + logger.info( + "Detected git service %s for %s.", git_service.name, analyze_ctx.component.repository.complete_name + ) + analyze_ctx.dynamic_data["git_service"] = git_service - Returns - ------- - dict[str, CheckResult] - The mapping between the check id and its result. - """ - # Determine the git service. - remote_path = analyze_ctx.component.repository.remote_path if analyze_ctx.component.repository else None + return git_service - # Load the build tools and determine the build tools that match the software component's PURL type. + def _determine_build_tools(self, analyze_ctx: AnalyzeContext, git_service: BaseGitService) -> None: + """Determine the build tools that match the software component's PURL type.""" for build_tool in BUILD_TOOLS: build_tool.load_defaults() if build_tool.purl_type == analyze_ctx.component.type: @@ -1011,67 +1043,70 @@ def perform_checks(self, analyze_ctx: AnalyzeContext) -> dict[str, CheckResult]: ) analyze_ctx.dynamic_data["build_spec"]["purl_tools"].append(build_tool) - git_service = self.get_git_service(remote_path) - if isinstance(git_service, NoneGitService): - logger.info("Unable to find repository or unsupported git service for %s", analyze_ctx.component.purl) - else: + if isinstance(git_service, NoneGitService): + continue + + if not analyze_ctx.component.repository: + continue + logger.info( - "Detected git service %s for %s.", git_service.name, analyze_ctx.component.repository.complete_name + "Checking if the repo %s uses build tool %s", + analyze_ctx.component.repository.complete_name, + build_tool.name, ) - analyze_ctx.dynamic_data["git_service"] = git_service - - # Detect the build tools by analyzing the repository. - for build_tool in BUILD_TOOLS: - logger.info( - "Checking if the repo %s uses build tool %s", - analyze_ctx.component.repository.complete_name, - build_tool.name, - ) - if build_tool.is_detected(analyze_ctx.component.repository.fs_path): - logger.info("The repo uses %s build tool.", build_tool.name) - analyze_ctx.dynamic_data["build_spec"]["tools"].append(build_tool) + if build_tool.is_detected(analyze_ctx.component.repository.fs_path): + logger.info("The repo uses %s build tool.", build_tool.name) + analyze_ctx.dynamic_data["build_spec"]["tools"].append(build_tool) - if not analyze_ctx.dynamic_data["build_spec"]["tools"]: + if not analyze_ctx.dynamic_data["build_spec"]["tools"]: + if analyze_ctx.component.repository: logger.info( "Unable to discover any build tools for repository %s or the build tools are not supported.", analyze_ctx.component.repository.complete_name, ) + else: + logger.info("Unable to discover build tools because repository is None.") - # Determine the CI services. - for ci_service in CI_SERVICES: - ci_service.load_defaults() - ci_service.set_api_client() + def _determine_ci_services(self, analyze_ctx: AnalyzeContext, git_service: BaseGitService) -> None: + """Determine the CI services used by the software component.""" + if isinstance(git_service, NoneGitService): + return - if ci_service.is_detected( - repo_path=analyze_ctx.component.repository.fs_path, - git_service=analyze_ctx.dynamic_data["git_service"], - ): - logger.info("The repo uses %s CI service.", ci_service.name) + # Determine the CI services. + for ci_service in CI_SERVICES: + ci_service.load_defaults() + ci_service.set_api_client() - # Parse configuration files and generate IRs. - # Add the bash commands to the context object to be used by other checks. - callgraph = ci_service.build_call_graph( - analyze_ctx.component.repository.fs_path, - os.path.relpath(analyze_ctx.component.repository.fs_path, analyze_ctx.output_dir), - ) - analyze_ctx.dynamic_data["ci_services"].append( - CIInfo( - service=ci_service, - callgraph=callgraph, - provenance_assets=[], - latest_release={}, - provenances=[ - SLSAProvenanceData( - payload=InTotoV01Payload(statement=Provenance().payload), - asset=VirtualReleaseAsset(name="No_ASSET", url="NO_URL", size_in_bytes=0), - ) - ], - ) + if ci_service.is_detected( + repo_path=analyze_ctx.component.repository.fs_path, + git_service=analyze_ctx.dynamic_data["git_service"], + ): + logger.info("The repo uses %s CI service.", ci_service.name) + + # Parse configuration files and generate IRs. + # Add the bash commands to the context object to be used by other checks. + callgraph = ci_service.build_call_graph( + analyze_ctx.component.repository.fs_path, + os.path.relpath(analyze_ctx.component.repository.fs_path, analyze_ctx.output_dir), + ) + analyze_ctx.dynamic_data["ci_services"].append( + CIInfo( + service=ci_service, + callgraph=callgraph, + provenance_assets=[], + release={}, + provenances=[ + SLSAProvenanceData( + payload=InTotoV01Payload(statement=Provenance().payload), + asset=VirtualReleaseAsset(name="No_ASSET", url="NO_URL", size_in_bytes=0), + ) + ], ) + ) - # Determine the package registries. - # We match the software component against package registries through build tools. + def _determine_package_registries(self, analyze_ctx: AnalyzeContext) -> None: + """Determine the package registries used by the software component based on its build tools.""" build_tools = ( analyze_ctx.dynamic_data["build_spec"]["tools"] or analyze_ctx.dynamic_data["build_spec"]["purl_tools"] ) @@ -1085,9 +1120,6 @@ def perform_checks(self, analyze_ctx: AnalyzeContext) -> dict[str, CheckResult]: ) ) - results = registry.scan(analyze_ctx) - return results - class DuplicateCmpError(DuplicateError): """This class is used for duplicated software component errors.""" diff --git a/src/macaron/slsa_analyzer/checks/provenance_available_check.py b/src/macaron/slsa_analyzer/checks/provenance_available_check.py index 81b895751..b67e5940d 100644 --- a/src/macaron/slsa_analyzer/checks/provenance_available_check.py +++ b/src/macaron/slsa_analyzer/checks/provenance_available_check.py @@ -4,45 +4,23 @@ """This module contains the implementation of the Provenance Available check.""" import logging -import os -import tempfile -from collections.abc import Sequence from sqlalchemy import ForeignKey from sqlalchemy.orm import Mapped, mapped_column from sqlalchemy.sql.sqltypes import String -from macaron.config.defaults import defaults -from macaron.database.table_definitions import CheckFacts, Component +from macaron.database.table_definitions import CheckFacts from macaron.errors import MacaronError from macaron.slsa_analyzer.analyze_context import AnalyzeContext -from macaron.slsa_analyzer.asset import AssetLocator -from macaron.slsa_analyzer.build_tool.gradle import Gradle -from macaron.slsa_analyzer.build_tool.npm import NPM -from macaron.slsa_analyzer.build_tool.yarn import Yarn from macaron.slsa_analyzer.checks.base_check import BaseCheck from macaron.slsa_analyzer.checks.check_result import CheckResultData, CheckResultType, Confidence, JustificationType -from macaron.slsa_analyzer.ci_service.base_ci_service import NoneCIService -from macaron.slsa_analyzer.ci_service.github_actions.github_actions_ci import GitHubActions -from macaron.slsa_analyzer.package_registry import JFrogMavenRegistry -from macaron.slsa_analyzer.package_registry.jfrog_maven_registry import JFrogMavenAsset -from macaron.slsa_analyzer.package_registry.npm_registry import NPMAttestationAsset, NPMRegistry -from macaron.slsa_analyzer.provenance.intoto import InTotoPayload -from macaron.slsa_analyzer.provenance.loader import LoadIntotoAttestationError, load_provenance_payload -from macaron.slsa_analyzer.provenance.slsa import SLSAProvenanceData -from macaron.slsa_analyzer.provenance.witness import ( - WitnessProvenanceData, - extract_repo_url, - is_witness_provenance_payload, - load_witness_verifier_config, -) from macaron.slsa_analyzer.registry import registry from macaron.slsa_analyzer.slsa_req import ReqName -from macaron.slsa_analyzer.specs.ci_spec import CIInfo -from macaron.slsa_analyzer.specs.package_registry_spec import PackageRegistryInfo logger: logging.Logger = logging.getLogger(__name__) +# TODO replace this check with the provenance verification check. + class ProvenanceAvailableException(MacaronError): """When there is an error while checking if a provenance is available.""" @@ -83,414 +61,6 @@ def __init__(self) -> None: ] super().__init__(check_id=check_id, description=description, depends_on=depends_on, eval_reqs=eval_reqs) - def find_provenance_assets_on_package_registries( - self, - component: Component, - package_registry_info_entries: list[PackageRegistryInfo], - provenance_extensions: list[str], - ) -> Sequence[AssetLocator]: - """Find provenance assets on package registries. - - Note that we stop going through package registries once we encounter a package - registry that does host provenance assets. - - Parameters - ---------- - component: Component - The target component under analysis. - package_registry_info_entries : list[PackageRegistryInfo] - A list of package registry info entries. - provenance_extensions : list[str] - A list of provenance extensions. Assets with these extensions are assumed - to be provenances. - - Returns - ------- - Sequence[AssetLocator] - A sequence of provenance assets found on one of the package registries. - This sequence is empty if there is no provenance assets found. - - Raises - ------ - ProvenanceAvailableException - If there is an error finding provenance assets that should result in failing - the check altogether. - """ - for package_registry_info_entry in package_registry_info_entries: - match package_registry_info_entry: - case PackageRegistryInfo( - build_tool=Gradle() as gradle, - package_registry=JFrogMavenRegistry() as jfrog_registry, - ) as info_entry: - # The current provenance discovery mechanism for JFrog Maven registry requires a - # repository to be available. Moreover, the repository path in Witness provenance - # contents are checked to match the target repository path. - # TODO: handle cases where a PURL string is provided for a software component but - # no repository is available. - if not component.repository: - logger.debug( - "Unable to find a provenance because a repository was not found for %s.", component.purl - ) - return [] - - # Triples of group id, artifact id, version. - gavs: list[tuple[str, str, str]] = [] - - group_ids = gradle.get_group_ids(component.repository.fs_path) - for group_id in group_ids: - artifact_ids = jfrog_registry.fetch_artifact_ids(group_id) - - for artifact_id in artifact_ids: - latest_version = jfrog_registry.fetch_latest_version( - group_id, - artifact_id, - ) - if not latest_version: - continue - logger.info( - "Found the latest version %s for Maven package %s:%s", - latest_version, - group_id, - artifact_id, - ) - gavs.append((group_id, artifact_id, latest_version)) - - provenance_assets = [] - for group_id, artifact_id, version in gavs: - provenance_assets.extend( - jfrog_registry.fetch_assets( - group_id=group_id, - artifact_id=artifact_id, - version=version, - extensions=set(provenance_extensions), - ) - ) - - if not provenance_assets: - continue - - # We check the size of the provenance against a max valid size. - # This is a prevention against malicious denial-of-service attacks when an - # adversary provides a super large malicious file. - - # TODO: refactor the size checking in this check and the `provenance_l3_check` - # so that we have consistent behavior when checking provenance size. - # The schema of the ini config also needs changing. - max_valid_provenance_size = defaults.getint( - "slsa.verifier", - "max_download_size", - fallback=1000000, - ) - - for provenance_asset in provenance_assets: - if provenance_asset.size_in_bytes > max_valid_provenance_size: - msg = ( - f"The provenance asset {provenance_asset.name} unexpectedly exceeds the " - f"max valid file size of {max_valid_provenance_size} (bytes). " - "The check will not proceed due to potential security risks." - ) - logger.error(msg) - raise ProvenanceAvailableException(msg) - - provenances = self.obtain_witness_provenances( - provenance_assets=provenance_assets, - repo_remote_path=component.repository.remote_path, - ) - - witness_provenance_assets = [] - - logger.info("Found the following provenance assets:") - for provenance in provenances: - logger.info("* %s", provenance.asset.url) - witness_provenance_assets.append(provenance.asset) - - # Persist the provenance assets in the package registry info entry. - info_entry.provenances.extend(provenances) - return provenance_assets - case PackageRegistryInfo( - build_tool=NPM() | Yarn(), - package_registry=NPMRegistry() as npm_registry, - ) as npm_info_entry: - if not component.version: - logger.debug( - "Unable to find provenance because artifact version is not available in %s.", component.purl - ) - return [] - - namespace = component.namespace - artifact_id = component.name - version = component.version - npm_provenance_assets = [] - - # The size of the asset (in bytes) is added to match the AssetLocator - # protocol and is not used because npm API registry does not provide it, so it is set to zero. - npm_provenance_asset = NPMAttestationAsset( - namespace=namespace, - artifact_id=artifact_id, - version=version, - npm_registry=npm_registry, - size_in_bytes=0, - ) - try: - with tempfile.TemporaryDirectory() as temp_dir: - download_path = os.path.join(temp_dir, f"{artifact_id}.intoto.jsonl") - if not npm_provenance_asset.download(download_path): - logger.debug("Unable to find an npm provenance for %s@%s", artifact_id, version) - return [] - try: - npm_provenance_payload = load_provenance_payload(download_path) - except LoadIntotoAttestationError as loadintotoerror: - logger.error("Error while loading provenance %s", loadintotoerror) - return [] - npm_info_entry.provenances.append( - SLSAProvenanceData(asset=npm_provenance_asset, payload=npm_provenance_payload) - ) - npm_provenance_assets.append(npm_provenance_asset) - except OSError as error: - logger.error("Error while storing provenance in the temporary directory: %s", error) - return npm_provenance_assets - return [] - - def obtain_witness_provenances( - self, - provenance_assets: Sequence[AssetLocator], - repo_remote_path: str, - ) -> list[WitnessProvenanceData]: - """Obtain the witness provenances produced from a repository. - - Parameters - ---------- - provenance_assets : Sequence[Asset] - A list of provenance assets, some of which can be witness provenances. - repo_remote_path : str - The remote path of the repo being analyzed. - - Returns - ------- - list[WitnessProvenance] - A list of witness provenances that are produced by the repo being analyzed. - """ - provenances = [] - witness_verifier_config = load_witness_verifier_config() - - try: - with tempfile.TemporaryDirectory() as temp_dir: - for provenance_asset in provenance_assets: - provenance_filepath = os.path.join(temp_dir, provenance_asset.name) - if not provenance_asset.download(provenance_filepath): - logger.debug( - "Could not download the provenance %s. Skip verifying...", - provenance_asset.name, - ) - continue - - try: - provenance_payload = load_provenance_payload(provenance_filepath) - except LoadIntotoAttestationError as error: - logger.error("Error while loading provenance: %s", error) - continue - - if not is_witness_provenance_payload( - provenance_payload, - witness_verifier_config.predicate_types, - ): - continue - - repo_url = extract_repo_url(provenance_payload) - if repo_url != repo_remote_path: - continue - - provenances.append( - WitnessProvenanceData( - asset=provenance_asset, - payload=provenance_payload, - ) - ) - except OSError as error: - logger.error("Error while storing provenance in the temporary directory: %s", error) - - return provenances - - def download_provenances_from_jfrog_maven_package_registry( - self, - download_dir: str, - provenance_assets: list[JFrogMavenAsset], - jfrog_maven_registry: JFrogMavenRegistry, - ) -> dict[str, InTotoPayload]: - """Download provenances from a JFrog Maven package registry. - - Parameters - ---------- - download_dir : str - The directory where provenance assets are downloaded to. - provenance_assets : list[JFrogMavenAsset] - The list of provenance assets. - jfrog_maven_registry : JFrogMavenRegistry - The JFrog Maven registry instance. - - Returns - ------- - dict[str, InTotoStatement] - The downloaded provenance payloads. Each key is the URL where the provenance - asset is hosted and each value is the corresponding provenance payload. - """ - # Note: In certain cases, Macaron can find the same provenance file in - # multiple different places on a package registry. - # - # We may consider de-duplicating this file, so that we do not run the same - # steps on the same file multiple times. - - # Download the provenance assets and load them into dictionaries. - provenances = {} - - for prov_asset in provenance_assets: - provenance_filepath = os.path.join(download_dir, prov_asset.name) - if not jfrog_maven_registry.download_asset(prov_asset.url, provenance_filepath): - logger.debug( - "Could not download the provenance %s. Skip verifying...", - prov_asset.name, - ) - continue - - try: - provenances[prov_asset.url] = load_provenance_payload( - provenance_filepath, - ) - except LoadIntotoAttestationError as error: - logger.error("Error while loading provenance: %s", error) - continue - - return provenances - - def find_provenance_assets_on_ci_services( - self, - component: Component, - ci_info_entries: list[CIInfo], - provenance_extensions: list[str], - ) -> Sequence[AssetLocator]: - """Find provenance assets on CI services. - - Note that we stop going through the CI services once we encounter a CI service - that does host provenance assets. - - This method also loads the provenance payloads into the ``CIInfo`` object where - the provenance assets are found. - - Parameters - ---------- - component: Component - The target component under analysis. - package_registry_info_entries : list[PackageRegistryInfo] - A list of package registry info entries. - provenance_extensions : list[str] - A list of provenance extensions. Assets with these extensions are assumed - to be provenances. - - Returns - ------- - Sequence[Asset] - A sequence of assets found on the given CI services. - """ - if not component.repository: - logger.debug("Unable to find a provenance because a repository was not found for %s.", component.purl) - return [] - - repo_full_name = component.repository.full_name - for ci_info in ci_info_entries: - ci_service = ci_info["service"] - - if isinstance(ci_service, NoneCIService): - continue - - if isinstance(ci_service, GitHubActions): - # Only get the latest release. - latest_release_payload = ci_service.api_client.get_latest_release(repo_full_name) - if not latest_release_payload: - logger.debug("Could not fetch the latest release payload from %s.", ci_service.name) - continue - - # Store the release data for other checks. - ci_info["latest_release"] = latest_release_payload - - # Get the provenance assets. - for prov_ext in provenance_extensions: - provenance_assets = ci_service.api_client.fetch_assets( - latest_release_payload, - ext=prov_ext, - ) - if not provenance_assets: - continue - - logger.info("Found the following provenance assets:") - for provenance_asset in provenance_assets: - logger.info("* %s", provenance_asset.url) - - # Store the provenance assets for other checks. - ci_info["provenance_assets"].extend(provenance_assets) - - # Download the provenance assets and load the provenance payloads. - self.download_provenances_from_github_actions_ci_service( - ci_info, - ) - - return ci_info["provenance_assets"] - - return [] - - def download_provenances_from_github_actions_ci_service(self, ci_info: CIInfo) -> None: - """Download provenances from GitHub Actions. - - Parameters - ---------- - ci_info: CIInfo, - A ``CIInfo`` instance that holds a GitHub Actions git service object. - """ - ci_service = ci_info["service"] - prov_assets = ci_info["provenance_assets"] - - try: - with tempfile.TemporaryDirectory() as temp_path: - downloaded_provs = [] - for prov_asset in prov_assets: - # Check the size before downloading. - if prov_asset.size_in_bytes > defaults.getint( - "slsa.verifier", - "max_download_size", - fallback=1000000, - ): - logger.info( - "Skip verifying the provenance %s: asset size too large.", - prov_asset.name, - ) - continue - - provenance_filepath = os.path.join(temp_path, prov_asset.name) - - if not ci_service.api_client.download_asset( - prov_asset.url, - provenance_filepath, - ): - logger.debug( - "Could not download the provenance %s. Skip verifying...", - prov_asset.name, - ) - continue - - # Read the provenance. - try: - payload = load_provenance_payload(provenance_filepath) - except LoadIntotoAttestationError as error: - logger.error("Error logging provenance: %s", error) - continue - - # Add the provenance file. - downloaded_provs.append(SLSAProvenanceData(payload=payload, asset=prov_asset)) - - # Persist the provenance payloads into the CIInfo object. - ci_info["provenances"] = downloaded_provs - except OSError as error: - logger.error("Error while storing provenance in the temporary directory: %s", error) - def run_check(self, ctx: AnalyzeContext) -> CheckResultData: """Implement the check in this method. @@ -504,45 +74,15 @@ def run_check(self, ctx: AnalyzeContext) -> CheckResultData: CheckResultData The result of the check. """ - if not ctx.dynamic_data["is_inferred_prov"] and ctx.dynamic_data["provenance"]: - return CheckResultData( - result_tables=[ProvenanceAvailableFacts(confidence=Confidence.HIGH)], - result_type=CheckResultType.PASSED, - ) - - provenance_extensions = defaults.get_list( - "slsa.verifier", - "provenance_extensions", - fallback=["intoto.jsonl"], + available = ctx.dynamic_data["provenance"] and not ctx.dynamic_data["is_inferred_prov"] + return CheckResultData( + result_tables=[ + ProvenanceAvailableFacts( + confidence=Confidence.HIGH, + ) + ], + result_type=CheckResultType.PASSED if available else CheckResultType.FAILED, ) - # We look for the provenances in the package registries first, then CI services. - # (Note the short-circuit evaluation with OR.) - try: - provenance_assets = self.find_provenance_assets_on_package_registries( - component=ctx.component, - package_registry_info_entries=ctx.dynamic_data["package_registries"], - provenance_extensions=provenance_extensions, - ) or self.find_provenance_assets_on_ci_services( - component=ctx.component, - ci_info_entries=ctx.dynamic_data["ci_services"], - provenance_extensions=provenance_extensions, - ) - except ProvenanceAvailableException as error: - logger.error(error) - return CheckResultData(result_tables=[], result_type=CheckResultType.FAILED) - - if provenance_assets: - ctx.dynamic_data["is_inferred_prov"] = False - - # We only write the result to the database when the check is PASSED. - result_tables: list[CheckFacts] = [ - ProvenanceAvailableFacts(asset_name=asset.name, asset_url=asset.url, confidence=Confidence.HIGH) - for asset in provenance_assets - ] - return CheckResultData(result_tables=result_tables, result_type=CheckResultType.PASSED) - - return CheckResultData(result_tables=[], result_type=CheckResultType.FAILED) - registry.register(ProvenanceAvailableCheck()) diff --git a/src/macaron/slsa_analyzer/checks/provenance_l3_check.py b/src/macaron/slsa_analyzer/checks/provenance_l3_check.py index e57a215a8..76029d461 100644 --- a/src/macaron/slsa_analyzer/checks/provenance_l3_check.py +++ b/src/macaron/slsa_analyzer/checks/provenance_l3_check.py @@ -1,7 +1,7 @@ # Copyright (c) 2022 - 2024, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. -"""This modules implements a check to verify a target repo has intoto provenance level 3.""" +"""This module implements a check to verify a target repo has intoto provenance level 3.""" import glob import hashlib @@ -307,7 +307,7 @@ class Feedback(NamedTuple): continue # Checking if we have found a release for the repo. - if not ci_info["latest_release"] or "assets" not in ci_info["latest_release"]: + if not ci_info["release"] or "assets" not in ci_info["release"]: logger.info("Could not find any release assets for the repository.") break @@ -317,7 +317,7 @@ class Feedback(NamedTuple): break prov_assets = ci_info["provenance_assets"] - all_assets = ci_info["latest_release"]["assets"] + all_assets = ci_info["release"]["assets"] # Download and verify the artifacts if they are not large. # Create a temporary directory and automatically remove it when we are done. @@ -356,7 +356,7 @@ class Feedback(NamedTuple): prov.version = "0.2" prov.release_commit_sha = "" prov.provenance_json = json.dumps(provenance_payload.statement) - prov.release_tag = ci_info["latest_release"]["tag_name"] + prov.release_tag = ci_info["release"]["tag_name"] prov.component = ctx.component # Iterate through the subjects and verify. diff --git a/src/macaron/slsa_analyzer/checks/provenance_verified_check.py b/src/macaron/slsa_analyzer/checks/provenance_verified_check.py index 1b1506dd2..4bcbc3a4c 100644 --- a/src/macaron/slsa_analyzer/checks/provenance_verified_check.py +++ b/src/macaron/slsa_analyzer/checks/provenance_verified_check.py @@ -79,37 +79,36 @@ def run_check(self, ctx: AnalyzeContext) -> CheckResultData: if predicate: build_type = json_extract(predicate, ["buildType"], str) - if build_type and build_type == "https://github.com/slsa-framework/slsa-github-generator/generic@v1": - # Provenance is created by the SLSA GitHub generator and therefore verified. + if not ctx.dynamic_data["provenance_verified"]: + # Provenance is not verified. return CheckResultData( result_tables=[ - ProvenanceVerifiedFacts(build_level=3, build_type=build_type, confidence=Confidence.HIGH) + ProvenanceVerifiedFacts( + build_level=1, + build_type=build_type, + confidence=Confidence.HIGH, + ) ], - result_type=CheckResultType.PASSED, + result_type=CheckResultType.FAILED, ) - if not ctx.dynamic_data["provenance_verified"]: - # Provenance is not verified. + if build_type != "https://github.com/slsa-framework/slsa-github-generator/generic@v1": + # Provenance is verified but the build service does not isolate generation in the control plane from the + # untrusted build process. return CheckResultData( result_tables=[ ProvenanceVerifiedFacts( - build_level=1, + build_level=2, build_type=build_type, confidence=Confidence.HIGH, ) ], - result_type=CheckResultType.FAILED, + result_type=CheckResultType.PASSED, ) - # Provenance is verified. + # Provenance is created by the SLSA GitHub generator and verified. return CheckResultData( - result_tables=[ - ProvenanceVerifiedFacts( - build_level=2, - build_type=build_type, - confidence=Confidence.HIGH, - ) - ], + result_tables=[ProvenanceVerifiedFacts(build_level=3, build_type=build_type, confidence=Confidence.HIGH)], result_type=CheckResultType.PASSED, ) diff --git a/src/macaron/slsa_analyzer/git_service/api_client.py b/src/macaron/slsa_analyzer/git_service/api_client.py index 44e4d6734..8e987e6ca 100644 --- a/src/macaron/slsa_analyzer/git_service/api_client.py +++ b/src/macaron/slsa_analyzer/git_service/api_client.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022 - 2023, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2022 - 2024, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """The module provides API clients for VCS services, such as GitHub.""" @@ -529,6 +529,26 @@ def get_relative_path_of_workflow(self, workflow_name: str) -> str: """ return f".github/workflows/{workflow_name}" + def get_release_by_tag(self, full_name: str, tag: str) -> dict | None: + """Return the release of the passed tag. + + Parameters + ---------- + full_name: str + The full name of the repo. + tag: str + The tag being analyzed. + + Returns + ------- + dict | None + The release object in JSON format, or None if not found. + """ + logger.debug("Get the release for '%s' using tag '%s'.", full_name, tag) + url = f"{GhAPIClient._REPO_END_POINT}/{full_name}/releases/tags/{tag}" + response_data = send_get_http(url, self.headers) + return response_data or None + def get_latest_release(self, full_name: str) -> dict: """Return the latest release for the repo. diff --git a/src/macaron/slsa_analyzer/provenance/slsa/__init__.py b/src/macaron/slsa_analyzer/provenance/slsa/__init__.py index cf9a9cfb7..b3418946f 100644 --- a/src/macaron/slsa_analyzer/provenance/slsa/__init__.py +++ b/src/macaron/slsa_analyzer/provenance/slsa/__init__.py @@ -2,7 +2,6 @@ # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. """This module implements SLSA provenance abstractions.""" - from typing import NamedTuple from macaron.slsa_analyzer.asset import AssetLocator diff --git a/src/macaron/slsa_analyzer/specs/ci_spec.py b/src/macaron/slsa_analyzer/specs/ci_spec.py index e5b98b375..7f4bef2a3 100644 --- a/src/macaron/slsa_analyzer/specs/ci_spec.py +++ b/src/macaron/slsa_analyzer/specs/ci_spec.py @@ -25,13 +25,13 @@ class CIInfo(TypedDict): """Release assets for provenances, e.g., asset for attestation.intoto.jsonl. For GitHub Actions, each asset is a member of the ``assets`` list in the GitHub - Actions latest release payload. - See: https://docs.github.com/en/rest/releases/releases?apiVersion=2022-11-28#get-the-latest-release. + Actions appropriate release payload. + See: https://docs.github.com/en/rest/releases/releases?apiVersion=2022-11-28#get-a-release-by-tag-name. """ - latest_release: dict - """The latest release. - Schema: https://docs.github.com/en/rest/releases/releases?apiVersion=2022-11-28#get-the-latest-release. + release: dict + """The appropriate release. + Schema: https://docs.github.com/en/rest/releases/releases?apiVersion=2022-11-28#get-a-release-by-tag-name """ provenances: Sequence[DownloadedProvenanceData] diff --git a/tests/conftest.py b/tests/conftest.py index 894f8db12..d6b83bd78 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -408,7 +408,10 @@ def __init__( purl=purl or "pkg:github.com/package-url/purl-spec@244fd47e07d1004f0aed9c", analysis=Analysis(), repository=Repository( - complete_name=complete_name or "github.com/package-url/purl-spec", fs_path=fs_path or "" + complete_name=complete_name or "github.com/package-url/purl-spec", + fs_path=fs_path or "", + # Must match test_provenance_finder.MockGit.MockTag.commit. + commit_sha="dig", ), ) super().__init__(component, *args, **kwargs) diff --git a/tests/integration/cases/micronaut-projects_micronaut-core/check_results_policy.dl b/tests/integration/cases/micronaut-projects_micronaut-core/check_results_policy.dl index 90410bdf4..0210abf11 100644 --- a/tests/integration/cases/micronaut-projects_micronaut-core/check_results_policy.dl +++ b/tests/integration/cases/micronaut-projects_micronaut-core/check_results_policy.dl @@ -8,8 +8,8 @@ Policy("test_policy", component_id, "") :- check_passed(component_id, "mcn_build_script_1"), check_passed(component_id, "mcn_build_service_1"), check_passed(component_id, "mcn_version_control_system_1"), + check_passed(component_id, "mcn_provenance_derived_repo_1"), check_failed(component_id, "mcn_provenance_derived_commit_1"), - check_failed(component_id, "mcn_provenance_derived_repo_1"), check_failed(component_id, "mcn_trusted_builder_level_three_1"), is_repo_url(component_id, "https://github.com/micronaut-projects/micronaut-core"). diff --git a/tests/integration/cases/micronaut-projects_micronaut-test/micronaut-test.dl b/tests/integration/cases/micronaut-projects_micronaut-test/micronaut-test.dl index 89eace2a1..f20692bcc 100644 --- a/tests/integration/cases/micronaut-projects_micronaut-test/micronaut-test.dl +++ b/tests/integration/cases/micronaut-projects_micronaut-test/micronaut-test.dl @@ -7,15 +7,15 @@ Policy("test_policy", component_id, "") :- check_passed(component_id, "mcn_build_as_code_1"), check_passed(component_id, "mcn_build_script_1"), check_passed(component_id, "mcn_build_service_1"), + check_passed(component_id, "mcn_version_control_system_1"), check_passed(component_id, "mcn_provenance_available_1"), + check_passed(component_id, "mcn_provenance_derived_repo_1"), check_passed(component_id, "mcn_provenance_level_three_1"), - check_passed(component_id, "mcn_version_control_system_1"), check_failed(component_id, "mcn_infer_artifact_pipeline_1"), check_failed(component_id, "mcn_provenance_derived_commit_1"), - check_failed(component_id, "mcn_provenance_derived_repo_1"), check_failed(component_id, "mcn_provenance_witness_level_one_1"), check_failed(component_id, "mcn_trusted_builder_level_three_1"), is_repo_url(component_id, "https://github.com/micronaut-projects/micronaut-test"). apply_policy_to("test_policy", component_id) :- - is_component(component_id, "pkg:github.com/micronaut-projects/micronaut-test@7679d10b4073a3b842b6c56877c35fa8cd10acff"). + is_component(component_id, "pkg:github.com/micronaut-projects/micronaut-test@5b81340f319a2287cb2e81ddec0154c0ea2510cf"). diff --git a/tests/integration/cases/micronaut-projects_micronaut-test/micronaut_test_config.yaml b/tests/integration/cases/micronaut-projects_micronaut-test/micronaut_test_config.yaml index 4f3d254f1..320e7f266 100644 --- a/tests/integration/cases/micronaut-projects_micronaut-test/micronaut_test_config.yaml +++ b/tests/integration/cases/micronaut-projects_micronaut-test/micronaut_test_config.yaml @@ -4,7 +4,7 @@ target: id: micronaut-test # https://github.com/micronaut-projects/micronaut-test/commit/7679d10b4073a3b842b6c56877c35fa8cd10acff - digest: 7679d10b4073a3b842b6c56877c35fa8cd10acff + digest: 5b81340f319a2287cb2e81ddec0154c0ea2510cf path: https://github.com/micronaut-projects/micronaut-test dependencies: diff --git a/tests/integration/cases/sigstore_mock/policy.dl b/tests/integration/cases/sigstore_mock/policy.dl index 6883c3256..c16d43f7c 100644 --- a/tests/integration/cases/sigstore_mock/policy.dl +++ b/tests/integration/cases/sigstore_mock/policy.dl @@ -9,13 +9,13 @@ Policy("test_policy", component_id, "") :- check_passed(component_id, "mcn_build_service_1"), check_passed(component_id, "mcn_provenance_available_1"), check_passed(component_id, "mcn_version_control_system_1"), + check_passed(component_id, "mcn_provenance_derived_commit_1"), + check_passed(component_id, "mcn_provenance_derived_repo_1"), + check_passed(component_id, "mcn_provenance_verified_1"), check_failed(component_id, "mcn_infer_artifact_pipeline_1"), - check_failed(component_id, "mcn_provenance_derived_commit_1"), - check_failed(component_id, "mcn_provenance_derived_repo_1"), check_failed(component_id, "mcn_provenance_level_three_1"), check_failed(component_id, "mcn_provenance_witness_level_one_1"), check_failed(component_id, "mcn_trusted_builder_level_three_1"), - check_failed(component_id, "mcn_provenance_verified_1"), is_repo_url(component_id, "https://github.com/sigstore/sigstore-js"). apply_policy_to("test_policy", component_id) :- diff --git a/tests/integration/cases/slsa-framework_slsa-verifier/policy.dl b/tests/integration/cases/slsa-framework_slsa-verifier/policy.dl index e27522f1a..67a171df2 100644 --- a/tests/integration/cases/slsa-framework_slsa-verifier/policy.dl +++ b/tests/integration/cases/slsa-framework_slsa-verifier/policy.dl @@ -7,15 +7,15 @@ Policy("test_policy", component_id, "") :- check_passed(component_id, "mcn_build_as_code_1"), check_passed(component_id, "mcn_build_script_1"), check_passed(component_id, "mcn_build_service_1"), - check_passed(component_id, "mcn_provenance_available_1"), - check_passed(component_id, "mcn_provenance_expectation_1"), check_passed(component_id, "mcn_trusted_builder_level_three_1"), check_passed(component_id, "mcn_version_control_system_1"), + check_passed(component_id, "mcn_provenance_available_1"), + check_passed(component_id, "mcn_provenance_derived_commit_1"), + check_passed(component_id, "mcn_provenance_derived_repo_1"), + check_passed(component_id, "mcn_provenance_expectation_1"), check_failed(component_id, "mcn_infer_artifact_pipeline_1"), - check_failed(component_id, "mcn_provenance_derived_commit_1"), - check_failed(component_id, "mcn_provenance_derived_repo_1"), check_failed(component_id, "mcn_provenance_witness_level_one_1"), is_repo_url(component_id, "https://github.com/slsa-framework/slsa-verifier"). apply_policy_to("test_policy", component_id) :- - is_component(component_id, "pkg:github.com/slsa-framework/slsa-verifier@fc50b662fcfeeeb0e97243554b47d9b20b14efac"). + is_component(component_id, "pkg:github.com/slsa-framework/slsa-verifier@e6428d7da594455a4c2b7f24907fec421a5e0e95"). diff --git a/tests/integration/cases/slsa-framework_slsa-verifier/test.yaml b/tests/integration/cases/slsa-framework_slsa-verifier/test.yaml index 2c3e0aaf9..37ba16095 100644 --- a/tests/integration/cases/slsa-framework_slsa-verifier/test.yaml +++ b/tests/integration/cases/slsa-framework_slsa-verifier/test.yaml @@ -20,7 +20,7 @@ steps: - -b - main - -d - - fc50b662fcfeeeb0e97243554b47d9b20b14efac + - e6428d7da594455a4c2b7f24907fec421a5e0e95 - --skip-deps - name: Run macaron verify-policy to verify passed/failed checks kind: verify diff --git a/tests/integration/cases/urllib3_expectation_dir/expectation/expectation.cue b/tests/integration/cases/urllib3_expectation_dir/expectation/expectation.cue index 8f42a812e..0866fac86 100644 --- a/tests/integration/cases/urllib3_expectation_dir/expectation/expectation.cue +++ b/tests/integration/cases/urllib3_expectation_dir/expectation/expectation.cue @@ -1,5 +1,5 @@ { - target: "pkg:github.com/urllib3/urllib3", + target: "pkg:pypi/urllib3", predicate: { invocation: { configSource: { diff --git a/tests/integration/cases/urllib3_expectation_dir/policy.dl b/tests/integration/cases/urllib3_expectation_dir/policy.dl index 141b722fa..048252508 100644 --- a/tests/integration/cases/urllib3_expectation_dir/policy.dl +++ b/tests/integration/cases/urllib3_expectation_dir/policy.dl @@ -7,16 +7,16 @@ Policy("test_policy", component_id, "") :- check_passed(component_id, "mcn_build_as_code_1"), check_passed(component_id, "mcn_build_script_1"), check_passed(component_id, "mcn_build_service_1"), + check_passed(component_id, "mcn_version_control_system_1"), check_passed(component_id, "mcn_provenance_available_1"), - check_passed(component_id, "mcn_provenance_expectation_1"), check_passed(component_id, "mcn_provenance_level_three_1"), - check_passed(component_id, "mcn_version_control_system_1"), + check_passed(component_id, "mcn_provenance_derived_commit_1"), + check_passed(component_id, "mcn_provenance_derived_repo_1"), + check_passed(component_id, "mcn_provenance_expectation_1"), check_failed(component_id, "mcn_infer_artifact_pipeline_1"), - check_failed(component_id, "mcn_provenance_derived_commit_1"), - check_failed(component_id, "mcn_provenance_derived_repo_1"), check_failed(component_id, "mcn_provenance_witness_level_one_1"), check_failed(component_id, "mcn_trusted_builder_level_three_1"), is_repo_url(component_id, "https://github.com/urllib3/urllib3"). apply_policy_to("test_policy", component_id) :- - is_component(component_id, "pkg:github.com/urllib3/urllib3@87a0ecee6e691fe5ff93cd000c0158deebef763b"). + is_component(component_id, "pkg:pypi/urllib3@2.0.0a1"). diff --git a/tests/integration/cases/urllib3_expectation_dir/test.yaml b/tests/integration/cases/urllib3_expectation_dir/test.yaml index a5f9698f7..2554fc7ff 100644 --- a/tests/integration/cases/urllib3_expectation_dir/test.yaml +++ b/tests/integration/cases/urllib3_expectation_dir/test.yaml @@ -14,12 +14,8 @@ steps: kind: analyze options: command_args: - - --repo-path - - https://github.com/urllib3/urllib3 - - --branch - - main - - --digest - - 87a0ecee6e691fe5ff93cd000c0158deebef763b + - -purl + - pkg:pypi/urllib3@2.0.0a1 - --provenance-expectation - expectation - --skip-deps diff --git a/tests/integration/cases/urllib3_expectation_file/expectation.cue b/tests/integration/cases/urllib3_expectation_file/expectation.cue index 8f42a812e..0866fac86 100644 --- a/tests/integration/cases/urllib3_expectation_file/expectation.cue +++ b/tests/integration/cases/urllib3_expectation_file/expectation.cue @@ -1,5 +1,5 @@ { - target: "pkg:github.com/urllib3/urllib3", + target: "pkg:pypi/urllib3", predicate: { invocation: { configSource: { diff --git a/tests/integration/cases/urllib3_expectation_file/policy.dl b/tests/integration/cases/urllib3_expectation_file/policy.dl index 141b722fa..79bfae7ee 100644 --- a/tests/integration/cases/urllib3_expectation_file/policy.dl +++ b/tests/integration/cases/urllib3_expectation_file/policy.dl @@ -7,16 +7,16 @@ Policy("test_policy", component_id, "") :- check_passed(component_id, "mcn_build_as_code_1"), check_passed(component_id, "mcn_build_script_1"), check_passed(component_id, "mcn_build_service_1"), + check_passed(component_id, "mcn_version_control_system_1"), check_passed(component_id, "mcn_provenance_available_1"), + check_passed(component_id, "mcn_provenance_derived_commit_1"), + check_passed(component_id, "mcn_provenance_derived_repo_1"), check_passed(component_id, "mcn_provenance_expectation_1"), check_passed(component_id, "mcn_provenance_level_three_1"), - check_passed(component_id, "mcn_version_control_system_1"), check_failed(component_id, "mcn_infer_artifact_pipeline_1"), - check_failed(component_id, "mcn_provenance_derived_commit_1"), - check_failed(component_id, "mcn_provenance_derived_repo_1"), check_failed(component_id, "mcn_provenance_witness_level_one_1"), check_failed(component_id, "mcn_trusted_builder_level_three_1"), is_repo_url(component_id, "https://github.com/urllib3/urllib3"). apply_policy_to("test_policy", component_id) :- - is_component(component_id, "pkg:github.com/urllib3/urllib3@87a0ecee6e691fe5ff93cd000c0158deebef763b"). + is_component(component_id, "pkg:pypi/urllib3@2.0.0a1"). diff --git a/tests/integration/cases/urllib3_expectation_file/test.yaml b/tests/integration/cases/urllib3_expectation_file/test.yaml index 8212f4bdd..84f22f20d 100644 --- a/tests/integration/cases/urllib3_expectation_file/test.yaml +++ b/tests/integration/cases/urllib3_expectation_file/test.yaml @@ -15,12 +15,8 @@ steps: options: expectation: expectation.cue command_args: - - --repo-path - - https://github.com/urllib3/urllib3 - - --branch - - main - - --digest - - 87a0ecee6e691fe5ff93cd000c0158deebef763b + - -purl + - pkg:pypi/urllib3@2.0.0a1 - --skip-deps - name: Run macaron verify-policy to verify passed/failed checks kind: verify diff --git a/tests/integration/cases/urllib3_invalid_expectation/policy.dl b/tests/integration/cases/urllib3_invalid_expectation/policy.dl index 2bcb5a8fb..e8a017826 100644 --- a/tests/integration/cases/urllib3_invalid_expectation/policy.dl +++ b/tests/integration/cases/urllib3_invalid_expectation/policy.dl @@ -7,15 +7,16 @@ Policy("test_policy", component_id, "") :- check_passed(component_id, "mcn_build_as_code_1"), check_passed(component_id, "mcn_build_script_1"), check_passed(component_id, "mcn_build_service_1"), + check_passed(component_id, "mcn_version_control_system_1"), check_passed(component_id, "mcn_provenance_available_1"), + check_passed(component_id, "mcn_provenance_derived_commit_1"), + check_passed(component_id, "mcn_provenance_derived_repo_1"), check_passed(component_id, "mcn_provenance_level_three_1"), - check_passed(component_id, "mcn_version_control_system_1"), check_failed(component_id, "mcn_infer_artifact_pipeline_1"), - check_failed(component_id, "mcn_provenance_derived_commit_1"), - check_failed(component_id, "mcn_provenance_derived_repo_1"), check_failed(component_id, "mcn_provenance_witness_level_one_1"), check_failed(component_id, "mcn_trusted_builder_level_three_1"), + check_failed(component_id, "mcn_provenance_expectation_1"), is_repo_url(component_id, "https://github.com/urllib3/urllib3"). apply_policy_to("test_policy", component_id) :- - is_component(component_id, "pkg:github.com/urllib3/urllib3@87a0ecee6e691fe5ff93cd000c0158deebef763b"). + is_component(component_id, "pkg:pypi/urllib3@2.0.0a1"). diff --git a/tests/integration/cases/urllib3_invalid_expectation/test.yaml b/tests/integration/cases/urllib3_invalid_expectation/test.yaml index 7be5e78da..710fac59c 100644 --- a/tests/integration/cases/urllib3_invalid_expectation/test.yaml +++ b/tests/integration/cases/urllib3_invalid_expectation/test.yaml @@ -15,12 +15,8 @@ steps: options: expectation: invalid_expectation.cue command_args: - - --repo-path - - https://github.com/urllib3/urllib3 - - --branch - - main - - --digest - - 87a0ecee6e691fe5ff93cd000c0158deebef763b + - -purl + - pkg:pypi/urllib3@2.0.0a1 - --skip-deps - name: Run macaron verify-policy to verify passed/failed checks kind: verify diff --git a/tests/integration/cases/urllib3_no_tag/policy.dl b/tests/integration/cases/urllib3_no_tag/policy.dl new file mode 100644 index 000000000..c6623b541 --- /dev/null +++ b/tests/integration/cases/urllib3_no_tag/policy.dl @@ -0,0 +1,11 @@ +/* Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. */ +/* Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. */ + +#include "prelude.dl" + +Policy("test_policy", component_id, "") :- + check_failed(component_id, "mcn_provenance_available_1"), + is_repo_url(component_id, "https://github.com/urllib3/urllib3"). + +apply_policy_to("test_policy", component_id) :- + is_component(component_id, "pkg:github.com/urllib3/urllib3@87a0ecee6e691fe5ff93cd000c0158deebef763b"). diff --git a/tests/integration/cases/urllib3_no_tag/test.yaml b/tests/integration/cases/urllib3_no_tag/test.yaml new file mode 100644 index 000000000..46cd232ed --- /dev/null +++ b/tests/integration/cases/urllib3_no_tag/test.yaml @@ -0,0 +1,25 @@ +# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +description: | + Testing the outcome of the provenance available check when the provided commit does not match a tag. + +tags: +- macaron-python-package + +steps: +- name: Run macaron analyze + kind: analyze + options: + command_args: + - --repo-path + - https://github.com/urllib3/urllib3 + - --branch + - main + - --digest + - 87a0ecee6e691fe5ff93cd000c0158deebef763b + - --skip-deps +- name: Run macaron verify-policy to verify failed check + kind: verify + options: + policy: policy.dl diff --git a/tests/output_reporter/test_jinja_extensions.py b/tests/output_reporter/test_jinja_extensions.py index 97b2636b1..8baaa528f 100644 --- a/tests/output_reporter/test_jinja_extensions.py +++ b/tests/output_reporter/test_jinja_extensions.py @@ -1,9 +1,7 @@ -# Copyright (c) 2022 - 2022, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2022 - 2024, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. -""" -This modules contains tests for the Jinja2 filter and test extensions. -""" +"""This module contains tests for the Jinja2 filter and test extensions.""" from hypothesis import given diff --git a/tests/output_reporter/test_reporter.py b/tests/output_reporter/test_reporter.py index e68c721de..df24dd5c3 100644 --- a/tests/output_reporter/test_reporter.py +++ b/tests/output_reporter/test_reporter.py @@ -1,9 +1,7 @@ -# Copyright (c) 2022 - 2023, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2022 - 2024, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. -""" -This modules contains tests for the JSON reporter. -""" +"""This module contains tests for the JSON reporter.""" import os from typing import Any diff --git a/tests/repo_finder/test_provenance_finder.py b/tests/repo_finder/test_provenance_finder.py new file mode 100644 index 000000000..3426fed0d --- /dev/null +++ b/tests/repo_finder/test_provenance_finder.py @@ -0,0 +1,224 @@ +# Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. +# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. + +"""This module tests the provenance finder.""" +import os +import shutil +import tempfile +from pathlib import Path +from types import SimpleNamespace + +import pytest +from git import InvalidGitRepositoryError +from packageurl import PackageURL +from pydriller import Git + +from macaron.code_analyzer.call_graph import BaseNode, CallGraph +from macaron.repo_finder.provenance_finder import find_gav_provenance, find_npm_provenance, find_provenance_from_ci +from macaron.slsa_analyzer.ci_service import BaseCIService, CircleCI, GitHubActions, GitLabCI, Jenkins, Travis +from macaron.slsa_analyzer.git_service.api_client import GhAPIClient +from macaron.slsa_analyzer.package_registry import JFrogMavenRegistry, NPMRegistry +from macaron.slsa_analyzer.package_registry.jfrog_maven_registry import JFrogMavenAsset, JFrogMavenAssetMetadata +from macaron.slsa_analyzer.specs.ci_spec import CIInfo +from tests.conftest import MockAnalyzeContext + + +class MockGitHubActions(GitHubActions): + """Mock the GitHubActions class.""" + + def has_latest_run_passed( + self, repo_full_name: str, branch_name: str | None, commit_sha: str, commit_date: str, workflow: str + ) -> str: + return "run_feedback" + + +class MockGhAPIClient(GhAPIClient): + """Mock GhAPIClient class.""" + + def __init__(self, profile: dict, resource_dir: str): + super().__init__(profile) + self.release = { + "assets": [ + {"name": "attestation.intoto.jsonl", "url": "URL", "size": 10}, + {"name": "artifact.txt", "url": "URL", "size": 10}, + ] + } + self.resource_dir = resource_dir + + def get_release_by_tag(self, full_name: str, tag: str) -> dict | None: + return self.release + + def download_asset(self, url: str, download_path: str) -> bool: + target = os.path.join( + self.resource_dir, + "slsa_analyzer", + "provenance", + "resources", + "valid_provenances", + "slsa-verifier-linux-amd64.intoto.jsonl", + ) + try: + shutil.copy2(target, download_path) + except shutil.Error: + return False + return True + + +class MockGit(Git): + """Mock Pydriller.Git class.""" + + def __init__(self) -> None: + # To safely create a Mock Git object we let instantiation occur and fail on an empty temporary directory. + try: + with tempfile.TemporaryDirectory() as temp: + super().__init__(temp) + except InvalidGitRepositoryError: + pass + + class MockTag: + """Mock Tag class.""" + + # Must match conftest.MockAnalyzeContext.Component.Repository.commit_sha. + commit = "dig" + + def __str__(self) -> str: + return self.commit + + repo = SimpleNamespace(tags=[MockTag()]) + + +class MockJFrogRegistry(JFrogMavenRegistry): + """Mock JFrogMavenRegistry class.""" + + def __init__(self, resource_dir: str): + self.resource_dir = resource_dir + super().__init__() + self.enabled = True + + def download_asset(self, url: str, dest: str) -> bool: + target = os.path.join(self.resource_dir, "slsa_analyzer", "provenance", "resources", "micronaut.intoto.jsonl") + try: + shutil.copy2(target, dest) + except shutil.Error: + return False + return True + + def fetch_assets( + self, + group_id: str, + artifact_id: str, + version: str, + extensions: set[str] | None = None, + ) -> list[JFrogMavenAsset]: + return [ + JFrogMavenAsset( + "micronaut.intoto.jsonl", + "io.micronaut", + "micronaut", + "1.0.0", + metadata=JFrogMavenAssetMetadata( + size_in_bytes=100, + sha256_digest="sha256", + download_uri="", + ), + jfrog_maven_registry=self, + ) + ] + + +class MockNPMRegistry(NPMRegistry): + """Mock NPMRegistry class.""" + + resource_valid_prov_dir: str + + def download_attestation_payload(self, url: str, download_path: str) -> bool: + src_path = os.path.join(self.resource_valid_prov_dir, "sigstore-mock.payload.json") + try: + shutil.copy2(src_path, download_path) + except shutil.Error: + return False + return True + + +@pytest.mark.parametrize( + "service", + [ + Jenkins(), + Travis(), + CircleCI(), + GitLabCI(), + ], +) +def test_provenance_on_unsupported_ci(macaron_path: Path, service: BaseCIService) -> None: + """Test the provenance finder on unsupported CI setups.""" + service.load_defaults() + + ci_info = CIInfo( + service=service, + callgraph=CallGraph(BaseNode(), ""), + provenance_assets=[], + release={}, + provenances=[], + ) + + # Set up the context object with provenances. + ctx = MockAnalyzeContext(macaron_path=macaron_path, output_dir="") + ctx.dynamic_data["ci_services"] = [ci_info] + + provenance = find_provenance_from_ci(ctx, None) + assert provenance is None + + +def test_provenance_on_supported_ci(macaron_path: Path, test_dir: Path) -> None: + """Test the provenance finder on supported CI setups.""" + github_actions = MockGitHubActions() + api_client = MockGhAPIClient({"headers": {}, "query": []}, str(test_dir)) + github_actions.api_client = api_client + github_actions.load_defaults() + + ci_info = CIInfo( + service=github_actions, + callgraph=CallGraph(BaseNode(), ""), + provenance_assets=[], + release={}, + provenances=[], + ) + + # Set up the context object with provenances. + ctx = MockAnalyzeContext(macaron_path=macaron_path, output_dir="") + ctx.dynamic_data["ci_services"] = [ci_info] + + # Test with a valid setup. + git_obj = MockGit() + provenance = find_provenance_from_ci(ctx, git_obj) + assert provenance + + # Test with a repo that doesn't have any accepted provenance. + api_client.release = {"assets": [{"name": "attestation.intoto", "url": "URL", "size": 10}]} + provenance = find_provenance_from_ci(ctx, MockGit()) + assert provenance is None + + +def test_provenance_available_on_npm_registry( + test_dir: Path, +) -> None: + """Test provenance published on npm registry.""" + purl = PackageURL.from_string("pkg:npm/@sigstore/mock@0.1.0") + npm_registry = MockNPMRegistry() + npm_registry.resource_valid_prov_dir = os.path.join( + test_dir, "slsa_analyzer", "provenance", "resources", "valid_provenances" + ) + provenance = find_npm_provenance(purl, npm_registry) + + assert provenance + + +def test_provenance_available_on_jfrog_registry( + test_dir: Path, +) -> None: + """Test provenance published on jfrog registry.""" + purl = PackageURL.from_string("pkg:/maven/io.micronaut/micronaut-core@4.2.3") + jfrog_registry = MockJFrogRegistry(str(test_dir)) + provenance = find_gav_provenance(purl, jfrog_registry) + + assert provenance diff --git a/tests/slsa_analyzer/checks/test_build_as_code_check.py b/tests/slsa_analyzer/checks/test_build_as_code_check.py index 5c425cff8..2cd4dd0eb 100644 --- a/tests/slsa_analyzer/checks/test_build_as_code_check.py +++ b/tests/slsa_analyzer/checks/test_build_as_code_check.py @@ -54,7 +54,7 @@ def test_build_as_code_check_no_callgraph( service=ci_services[ci_name], callgraph=CallGraph(BaseNode(), ""), provenance_assets=[], - latest_release={}, + release={}, provenances=[], ) use_build_tool = MockAnalyzeContext(macaron_path=macaron_path, output_dir="") @@ -104,7 +104,7 @@ def test_deploy_commands( service=github_actions_service, callgraph=build_github_actions_call_graph_for_commands(commands=commands), provenance_assets=[], - latest_release={}, + release={}, provenances=[], ) ci_info["service"] = github_actions_service @@ -141,7 +141,7 @@ def test_gha_workflow_deployment( service=github_actions_service, callgraph=CallGraph(BaseNode(), ""), provenance_assets=[], - latest_release={}, + release={}, provenances=[], ) @@ -186,7 +186,7 @@ def test_travis_ci_deploy( service=travis_service, callgraph=CallGraph(BaseNode(), ""), provenance_assets=[], - latest_release={}, + release={}, provenances=[], ) gradle_deploy = MockAnalyzeContext(macaron_path=macaron_path, output_dir="") @@ -206,7 +206,7 @@ def test_multibuild_facts_saved( service=github_actions_service, callgraph=build_github_actions_call_graph_for_commands(["./gradlew publishToSonatype", "mvn deploy"]), provenance_assets=[], - latest_release={}, + release={}, provenances=[], ) diff --git a/tests/slsa_analyzer/checks/test_build_service_check.py b/tests/slsa_analyzer/checks/test_build_service_check.py index 916f37cd1..74ee6e933 100644 --- a/tests/slsa_analyzer/checks/test_build_service_check.py +++ b/tests/slsa_analyzer/checks/test_build_service_check.py @@ -44,7 +44,7 @@ def test_build_service_check_no_callgraph( service=ci_services[ci_name], callgraph=CallGraph(BaseNode(), ""), provenance_assets=[], - latest_release={}, + release={}, provenances=[], ) use_build_tool = MockAnalyzeContext(macaron_path=macaron_path, output_dir="") @@ -94,7 +94,7 @@ def test_packaging_commands( service=github_actions_service, callgraph=build_github_actions_call_graph_for_commands(commands=commands), provenance_assets=[], - latest_release={}, + release={}, provenances=[], ) ci_info["service"] = github_actions_service @@ -112,7 +112,7 @@ def test_multibuild_facts_saved( service=github_actions_service, callgraph=build_github_actions_call_graph_for_commands(["./gradlew build", "mvn package"]), provenance_assets=[], - latest_release={}, + release={}, provenances=[], ) diff --git a/tests/slsa_analyzer/checks/test_provenance_available_check.py b/tests/slsa_analyzer/checks/test_provenance_available_check.py index 939eaca46..aebc28398 100644 --- a/tests/slsa_analyzer/checks/test_provenance_available_check.py +++ b/tests/slsa_analyzer/checks/test_provenance_available_check.py @@ -1,188 +1,20 @@ # Copyright (c) 2022 - 2024, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. -"""This modules contains tests for the provenance available check.""" +"""This module tests the provenance available check.""" - -import os -import shutil from pathlib import Path -import pytest - -from macaron.code_analyzer.call_graph import BaseNode, CallGraph -from macaron.database.table_definitions import Repository -from macaron.slsa_analyzer.build_tool.base_build_tool import BaseBuildTool from macaron.slsa_analyzer.checks.check_result import CheckResultType from macaron.slsa_analyzer.checks.provenance_available_check import ProvenanceAvailableCheck -from macaron.slsa_analyzer.ci_service.circleci import CircleCI -from macaron.slsa_analyzer.ci_service.github_actions.github_actions_ci import GitHubActions -from macaron.slsa_analyzer.ci_service.gitlab_ci import GitLabCI -from macaron.slsa_analyzer.ci_service.jenkins import Jenkins -from macaron.slsa_analyzer.ci_service.travis import Travis -from macaron.slsa_analyzer.git_service.api_client import GhAPIClient -from macaron.slsa_analyzer.package_registry.npm_registry import NPMRegistry -from macaron.slsa_analyzer.specs.ci_spec import CIInfo -from macaron.slsa_analyzer.specs.package_registry_spec import PackageRegistryInfo from tests.conftest import MockAnalyzeContext -class MockGitHubActions(GitHubActions): - """Mock the GitHubActions class.""" - - def has_latest_run_passed( - self, repo_full_name: str, branch_name: str | None, commit_sha: str, commit_date: str, workflow: str - ) -> str: - return "run_feedback" - - -class MockGhAPIClient(GhAPIClient): - """Mock GhAPIClient class.""" - - def __init__(self, profile: dict): - super().__init__(profile) - self.release = { - "assets": [ - {"name": "attestation.intoto.jsonl", "url": "URL", "size": 10}, - {"name": "artifact.txt", "url": "URL", "size": 10}, - ] - } - - def get_latest_release(self, full_name: str) -> dict: - return self.release - - def download_asset(self, url: str, download_path: str) -> bool: - return False - - -class MockNPMRegistry(NPMRegistry): - """Mock NPMRegistry class.""" - - resource_valid_prov_dir: str - - def download_attestation_payload(self, url: str, download_path: str) -> bool: - src_path = os.path.join(self.resource_valid_prov_dir, "sigstore-mock.payload.json") - try: - shutil.copy2(src_path, download_path) - except shutil.Error: - return False - return True - - -@pytest.mark.parametrize( - ("repository", "expected"), - [ - (None, CheckResultType.FAILED), - (Repository(complete_name="github.com/package-url/purl-spec", fs_path=""), CheckResultType.PASSED), - ], -) -def test_provenance_available_check_with_repos(macaron_path: Path, repository: Repository, expected: str) -> None: - """Test the provenance available check on different types of repositories.""" - check = ProvenanceAvailableCheck() - github_actions = MockGitHubActions() - api_client = MockGhAPIClient({"headers": {}, "query": []}) - github_actions.api_client = api_client - github_actions.load_defaults() - - ci_info = CIInfo( - service=github_actions, - callgraph=CallGraph(BaseNode(), ""), - provenance_assets=[], - latest_release={}, - provenances=[], - ) - - # Set up the context object with provenances. - ctx = MockAnalyzeContext(macaron_path=macaron_path, output_dir="") - ctx.component.repository = repository - ctx.dynamic_data["ci_services"] = [ci_info] - assert check.run_check(ctx).result_type == expected - - -def test_provenance_available_check_on_ci(macaron_path: Path) -> None: - """Test the provenance available check on different types of CI services.""" - check = ProvenanceAvailableCheck() - github_actions = MockGitHubActions() - api_client = MockGhAPIClient({"headers": {}, "query": []}) - github_actions.api_client = api_client - github_actions.load_defaults() - jenkins = Jenkins() - jenkins.load_defaults() - travis = Travis() - travis.load_defaults() - circle_ci = CircleCI() - circle_ci.load_defaults() - gitlab_ci = GitLabCI() - gitlab_ci.load_defaults() - - ci_info = CIInfo( - service=github_actions, - callgraph=CallGraph(BaseNode(), ""), - provenance_assets=[], - latest_release={}, - provenances=[], - ) - - # Set up the context object with provenances. - ctx = MockAnalyzeContext(macaron_path=macaron_path, output_dir="") - ctx.dynamic_data["ci_services"] = [ci_info] - - # Repo doesn't have a provenance. - api_client.release = {"assets": [{"name": "attestation.intoto", "url": "URL", "size": 10}]} - assert check.run_check(ctx).result_type == CheckResultType.FAILED - - api_client.release = {"assets": [{"name": "attestation.intoto.jsonl", "url": "URL", "size": 10}]} - - # Test Jenkins. - ci_info["service"] = jenkins - assert check.run_check(ctx).result_type == CheckResultType.FAILED - - # Test Travis. - ci_info["service"] = travis - assert check.run_check(ctx).result_type == CheckResultType.FAILED - - # Test Circle CI. - ci_info["service"] = circle_ci - assert check.run_check(ctx).result_type == CheckResultType.FAILED - - # Test GitLab CI. - ci_info["service"] = gitlab_ci - assert check.run_check(ctx).result_type == CheckResultType.FAILED - - -@pytest.mark.parametrize( - ( - "build_tool_name", - "expected", - ), - [ - ("npm", CheckResultType.PASSED), - ("yarn", CheckResultType.PASSED), - ("go", CheckResultType.FAILED), - ("maven", CheckResultType.FAILED), - ], -) -def test_provenance_available_check_on_npm_registry( +def test_provenance_available_check_( macaron_path: Path, - test_dir: Path, - build_tool_name: str, - expected: CheckResultType, - build_tools: dict[str, BaseBuildTool], ) -> None: - """Test npm provenances published on npm registry.""" + """Test provenance available check.""" check = ProvenanceAvailableCheck() ctx = MockAnalyzeContext(macaron_path=macaron_path, output_dir="") - ctx.component.purl = "pkg:npm/@sigstore/mock@0.1.0" - npm_registry = MockNPMRegistry() - npm_registry.resource_valid_prov_dir = os.path.join( - test_dir, "slsa_analyzer", "provenance", "resources", "valid_provenances" - ) - npm_registry.load_defaults() - ctx.dynamic_data["package_registries"] = [ - PackageRegistryInfo( - build_tool=build_tools[build_tool_name], - package_registry=npm_registry, - ) - ] - assert check.run_check(ctx).result_type == expected + assert check.run_check(ctx).result_type == CheckResultType.FAILED diff --git a/tests/slsa_analyzer/checks/test_provenance_l3_check.py b/tests/slsa_analyzer/checks/test_provenance_l3_check.py index d0f96466d..6f6220051 100644 --- a/tests/slsa_analyzer/checks/test_provenance_l3_check.py +++ b/tests/slsa_analyzer/checks/test_provenance_l3_check.py @@ -1,7 +1,7 @@ # Copyright (c) 2022 - 2024, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. -"""This modules contains tests for the provenance l3 check.""" +"""This module contains tests for the provenance l3 check.""" from macaron.code_analyzer.call_graph import BaseNode, CallGraph @@ -70,7 +70,7 @@ def test_provenance_l3_check(self) -> None: service=github_actions, callgraph=CallGraph(BaseNode(), ""), provenance_assets=[], - latest_release={}, + release={}, provenances=[], ) @@ -86,7 +86,7 @@ def test_provenance_l3_check(self) -> None: ) ] ) - ci_info["latest_release"] = { + ci_info["release"] = { "assets": [ {"name": "attestation.intoto.jsonl", "url": "URL", "size": 10}, {"name": "artifact.txt", "url": "URL", "size": 10}, @@ -108,7 +108,7 @@ def test_provenance_l3_check(self) -> None: ) ] ) - ci_info["latest_release"] = { + ci_info["release"] = { "assets": [ {"name": "attestation.intoto.jsonl", "url": "URL", "size": 100_000_000}, {"name": "artifact.txt", "url": "URL", "size": 10}, @@ -118,7 +118,7 @@ def test_provenance_l3_check(self) -> None: # No provenance available. ci_info["provenance_assets"] = [] - ci_info["latest_release"] = { + ci_info["release"] = { "assets": [ {"name": "attestation.intoto.jsonl", "url": "URL", "size": 10}, {"name": "artifact.txt", "url": "URL", "size": 10}, @@ -138,7 +138,7 @@ def test_provenance_l3_check(self) -> None: ) ] ) - ci_info["latest_release"] = {} + ci_info["release"] = {} assert check.run_check(ctx).result_type == CheckResultType.FAILED # Test Jenkins. diff --git a/tests/slsa_analyzer/checks/test_provenance_l3_content_check.py b/tests/slsa_analyzer/checks/test_provenance_l3_content_check.py index cb07b6b13..c04eaf3fe 100644 --- a/tests/slsa_analyzer/checks/test_provenance_l3_content_check.py +++ b/tests/slsa_analyzer/checks/test_provenance_l3_content_check.py @@ -1,7 +1,7 @@ # Copyright (c) 2023 - 2024, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. -"""This modules contains tests for the expectation check.""" +"""This module contains tests for the expectation check.""" import os @@ -82,7 +82,7 @@ def test_expectation_check(self) -> None: service=github_actions, callgraph=CallGraph(BaseNode(), ""), provenance_assets=[], - latest_release={}, + release={}, provenances=[], ) ctx.dynamic_data["ci_services"] = [ci_info] @@ -143,7 +143,7 @@ def test_expectation_check(self) -> None: SLSAProvenanceData( asset=VirtualReleaseAsset(name="No_ASSET", url="NO_URL", size_in_bytes=0), payload=load_provenance_payload(os.path.join(prov_dir, "slsa-verifier-linux-amd64.intoto.jsonl.gz")), - ), + ) ] ctx.dynamic_data["expectation"] = CUEExpectation.make_expectation( os.path.join(expectation_dir, "valid_expectations", "slsa_verifier_PASS.cue") diff --git a/tests/slsa_analyzer/checks/test_provenance_repo_commit_checks.py b/tests/slsa_analyzer/checks/test_provenance_repo_commit_checks.py index 8b9adf46d..57f60875e 100644 --- a/tests/slsa_analyzer/checks/test_provenance_repo_commit_checks.py +++ b/tests/slsa_analyzer/checks/test_provenance_repo_commit_checks.py @@ -1,7 +1,7 @@ # Copyright (c) 2024 - 2024, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. -"""This modules contains tests for the provenance available check.""" +"""This module contains tests for the provenance available check.""" from pathlib import Path from typing import TypeVar diff --git a/tests/slsa_analyzer/checks/test_trusted_builder_l3_check.py b/tests/slsa_analyzer/checks/test_trusted_builder_l3_check.py index 95b1bfc0e..88f2ec841 100644 --- a/tests/slsa_analyzer/checks/test_trusted_builder_l3_check.py +++ b/tests/slsa_analyzer/checks/test_trusted_builder_l3_check.py @@ -47,7 +47,7 @@ def test_trusted_builder_l3_check( service=github_actions_service, callgraph=CallGraph(BaseNode(), ""), provenance_assets=[], - latest_release={}, + release={}, provenances=[], ) diff --git a/tests/slsa_analyzer/checks/test_vcs_check.py b/tests/slsa_analyzer/checks/test_vcs_check.py index d3c5d82d4..e7454b3d0 100644 --- a/tests/slsa_analyzer/checks/test_vcs_check.py +++ b/tests/slsa_analyzer/checks/test_vcs_check.py @@ -1,7 +1,7 @@ # Copyright (c) 2022 - 2024, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. -"""This modules contains tests for the provenance available check.""" +"""This module contains tests for the provenance available check.""" import os from pathlib import Path diff --git a/tests/slsa_analyzer/provenance/resources/micronaut.intoto.jsonl b/tests/slsa_analyzer/provenance/resources/micronaut.intoto.jsonl new file mode 100644 index 000000000..ebe01447c --- /dev/null +++ b/tests/slsa_analyzer/provenance/resources/micronaut.intoto.jsonl @@ -0,0 +1 @@ +{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YwLjEiLAogICJzdWJqZWN0IjogWwogICAgewogICAgICAibmFtZSI6ICJtaWNyb25hdXQiLAogICAgICAiZGlnZXN0IjogeyAKICAgICAgICAic2hhMjU2IjoiYmY5NjY0ODE2OWJhODljMjg0YjNlOTQxMDgwNzRjN2Q1ZTU4MDZjN2I5NDk4MDMxYWNlZGVkNWNhMTM5ZWQ2OSIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly93aXRuZXNzLnRlc3RpZnlzZWMuY29tL2F0dGVzdGF0aW9uLWNvbGxlY3Rpb24vdjAuMSIsCiAgInByZWRpY2F0ZSI6IHsKICAgICJuYW1lIjogIm1pY3JvIiwKICAgICJhdHRlc3RhdGlvbnMiOiBbXQogIH0KfQo=","payloadType":"application/vnd.in-toto+json","signatures":[{"keyid":"1","sig":"2"}]} diff --git a/tests/slsa_analyzer/test_analyze_context.py b/tests/slsa_analyzer/test_analyze_context.py index dd33bda50..7328b862e 100644 --- a/tests/slsa_analyzer/test_analyze_context.py +++ b/tests/slsa_analyzer/test_analyze_context.py @@ -1,9 +1,7 @@ # Copyright (c) 2022 - 2024, Oracle and/or its affiliates. All rights reserved. # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/. -""" -This modules contains tests for the AnalyzeContext module -""" +"""This module contains tests for the AnalyzeContext module.""" from unittest import TestCase from unittest.mock import MagicMock @@ -96,11 +94,11 @@ def test_provenances(self) -> None: service=gh_actions, callgraph=CallGraph(BaseNode(), ""), provenance_assets=[], - latest_release={}, + release={}, provenances=[ SLSAProvenanceData( payload=expected_payload, asset=VirtualReleaseAsset(name="No_ASSET", url="NO_URL", size_in_bytes=0) - ) + ), ], )