From 071c2c380b9f047b49f4a587fe2a6acf8ea99f37 Mon Sep 17 00:00:00 2001 From: Nathan Leach Date: Fri, 12 Apr 2024 09:08:12 -0500 Subject: [PATCH] API bug fixes and security issue resolution (#1) * auth bug fix * added release notes * security updates * security fix * ignore mitmproxy CA * support for custom CA * support for custom CA * redact auth token from logs --- .gitignore | 2 +- Dockerfile | 14 ++-- README.md | 19 ++++++ RELEASE_NOTES.md | 13 ++++ cxone_api/__init__.py | 155 +++++++++++++++++++++++++++++++++--------- entrypoint.sh | 6 +- logic/__init__.py | 5 +- scanner.py | 4 +- scheduler.py | 8 ++- 9 files changed, 183 insertions(+), 43 deletions(-) create mode 100644 RELEASE_NOTES.md diff --git a/.gitignore b/.gitignore index 1e6057a..770f3ef 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,7 @@ run/* **/cxone_oauth_client_secret **/cxone_tenant env - +mitm* # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/Dockerfile b/Dockerfile index 8146740..f84f484 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,14 @@ -FROM python:3.12 +FROM ubuntu:24.04 LABEL org.opencontainers.image.source https://github.com/checkmarx-ts/cxone-scan-scheduler LABEL org.opencontainers.image.vendor Checkmarx Professional Services LABEL org.opencontainers.image.title Checkmarx One Scan Scheduler LABEL org.opencontainers.image.description Schedules scans for projects in Checkmarx One +USER root -RUN apt-get update && apt-get install -y cron && apt-get clean && \ +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends tzdata && \ + apt-get install -y cron python3.12 python3-pip python3-debugpy bash && \ usermod -s /bin/bash nobody && \ mkdir -p /opt/cxone && \ mkfifo /opt/cxone/logfifo && \ @@ -14,8 +17,12 @@ RUN apt-get update && apt-get install -y cron && apt-get clean && \ WORKDIR /opt/cxone COPY *.txt /opt/cxone -RUN pip install debugpy && pip install -r requirements.txt +RUN pip install -r requirements.txt --no-cache-dir --break-system-packages && \ + apt-get remove -y perl && \ + apt-get autoremove -y && \ + apt-get clean && \ + dpkg --purge $(dpkg --get-selections | grep deinstall | cut -f1) COPY cxone_api /opt/cxone/cxone_api COPY logic /opt/cxone/logic @@ -28,6 +35,5 @@ COPY *.json /opt/cxone RUN ln -s scheduler.py scheduler && \ ln -s scheduler.py audit -# ENTRYPOINT ["python", "-Xfrozen_modules=off", "-m", "debugpy", "--listen", "0.0.0.0:5678", "--wait-for-client"] CMD ["scheduler"] ENTRYPOINT ["/opt/cxone/entrypoint.sh"] \ No newline at end of file diff --git a/README.md b/README.md index 007ea60..85b122e 100644 --- a/README.md +++ b/README.md @@ -144,6 +144,25 @@ The Scan Scheduler runs as a container. At startup, it crawls the tenant's proj checks periodically for any schedule changes and updates the scan schedules accordingly. +### Add Optional Trusted CA Certificates + +While the CheckmarxOne system uses TLS certificates signed by a public CA, it is possible that corporate +proxies use certificates signed by a private CA. If so, it is possible to import custom CA certificates +when the scheduler starts. + +The custom certificates must meet the following criteria: + +* Must be in the PEM format. +* Must be in a file ending with the extension `.crt`. +* Only one certificate is in the file. +* Must be mapped to the container path `/usr/local/share/ca-certificates`. + +As an example, if using Docker, it is possible to map a local file to a file in the container with this +mapping option added to the container execution command line: + +`-v $(pwd)/custom-ca.pem:/usr/local/share/ca-certificates/custom-ca.crt` + + ### Required Secrets Docker secrets are used to securely store secrets needed during runtime. diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md new file mode 100644 index 0000000..da30ceb --- /dev/null +++ b/RELEASE_NOTES.md @@ -0,0 +1,13 @@ +# Release Notes + +## v1.1 + +* Bug fix for auth token not refreshing properly. +* Added log output to indicate number of scheduled scans on start and schedule update. +* Added support for loading custom CA certificates at startup. + +## v1.0 + +Initial release + + diff --git a/cxone_api/__init__.py b/cxone_api/__init__.py index 8dcbb0f..4a0415d 100644 --- a/cxone_api/__init__.py +++ b/cxone_api/__init__.py @@ -1,4 +1,4 @@ -import asyncio, uuid, requests, urllib, datetime +import asyncio, uuid, requests, urllib, datetime, re from requests.compat import urljoin from pathlib import Path @@ -8,8 +8,25 @@ class AuthException(BaseException): pass class CommunicationException(BaseException): - pass + @staticmethod + def __clean(content): + if type(content) is list: + return [CommunicationException.__clean(x) for x in content] + elif type(content) is tuple: + return (CommunicationException.__clean(x) for x in content) + elif type(content) is dict: + return {k:CommunicationException.__clean(v) for k,v in content.items()} + elif type(content) is str: + if re.match("^Bearer.*", content): + return "REDACTED" + else: + return content + else: + return content + + def __init__(self, op, *args, **kwargs): + BaseException.__init__(self, f"Operation: {op.__name__} args: [{CommunicationException.__clean(args)}] kwargs: [{CommunicationException.__clean(kwargs)}]") class CxOneAuthEndpoint: @@ -172,20 +189,17 @@ async def paged_api(coro, array_element, offset_field='offset', **kwargs): yield buf.pop() - - class CxOneClient: __AGENT_NAME = 'CxOne PyClient' - def __init__(self, oauth_id, oauth_secret, agent_name, agent_version, tenant_auth_endpoint, api_endpoint, timeout=60, retries=3, proxy=None, ssl_verify=True): + def __common__init(self, agent_name, agent_version, tenant_auth_endpoint, api_endpoint, timeout, retries, proxy, ssl_verify): with open(Path(__file__).parent / "version.txt", "rt") as version: self.__version = version.readline().rstrip() self.__agent = f"{agent_name}/{agent_version}/({CxOneClient.__AGENT_NAME}/{self.__version})" self.__proxy = proxy self.__ssl_verify = ssl_verify - self.__auth_lock = asyncio.Lock() self.__corelation_id = str(uuid.uuid4()) @@ -193,16 +207,43 @@ def __init__(self, oauth_id, oauth_secret, agent_name, agent_version, tenant_aut self.__api_endpoint = api_endpoint self.__timeout = timeout self.__retries = retries - - - self.__auth_content = urllib.parse.urlencode( { + + self.__auth_endpoint = tenant_auth_endpoint + self.__api_endpoint = api_endpoint + self.__timeout = timeout + self.__retries = retries + + self.__auth_result = None + + + + + + @staticmethod + def create_with_oauth(oauth_id, oauth_secret, agent_name, agent_version, tenant_auth_endpoint, api_endpoint, timeout=60, retries=3, proxy=None, ssl_verify=True): + inst = CxOneClient() + inst.__common__init(agent_name, agent_version, tenant_auth_endpoint, api_endpoint, timeout, retries, proxy, ssl_verify) + + inst.__auth_content = urllib.parse.urlencode( { "grant_type" : "client_credentials", "client_id" : oauth_id, "client_secret" : oauth_secret }) - - self.__auth_result = None + return inst + + @staticmethod + def create_with_api_key(api_key, agent_name, agent_version, tenant_auth_endpoint, api_endpoint, timeout=60, retries=3, proxy=None, ssl_verify=True): + inst = CxOneClient() + inst.__common__init(agent_name, agent_version, tenant_auth_endpoint, api_endpoint, timeout, retries, proxy, ssl_verify) + + inst.__auth_content = urllib.parse.urlencode( { + "grant_type" : "refresh_token", + "client_id" : "ast-app", + "refresh_token" : api_key + }) + + return inst @property def auth_endpoint(self): @@ -259,6 +300,14 @@ async def __exec_request(self, op, *args, **kwargs): kwargs['verify'] = self.__ssl_verify for _ in range(0, self.__retries): + auth_headers = await self.__get_request_headers() + + if 'headers' in kwargs.keys(): + for h in auth_headers.keys(): + kwargs['headers'][h] = auth_headers[h] + else: + kwargs['headers'] = auth_headers + response = await asyncio.to_thread(op, *args, **kwargs) if response.status_code == 401: @@ -266,7 +315,7 @@ async def __exec_request(self, op, *args, **kwargs): else: return response - raise CommunicationException(f"{str(op)}{str(args)}{str(kwargs)}") + raise CommunicationException(op, *args, **kwargs) @staticmethod @@ -288,47 +337,91 @@ def __join_query_dict(url, querydict): return urljoin(url, f"?{'&'.join(query)}" if len(query) > 0 else '') - + @dashargs("tags-keys", "tags-values") + async def get_applications(self, **kwargs): + url = urljoin(self.api_endpoint, "applications") + url = CxOneClient.__join_query_dict(url, kwargs) + return await self.__exec_request(requests.get, url) + + async def get_application(self, id, **kwargs): + url = urljoin(self.api_endpoint, f"applications/{id}") + url = CxOneClient.__join_query_dict(url, kwargs) + return await self.__exec_request(requests.get, url) + @dashargs("repo-url", "name-regex", "tags-keys", "tags-values") async def get_projects(self, **kwargs): url = urljoin(self.api_endpoint, "projects") - url = CxOneClient.__join_query_dict(url, kwargs) + return await self.__exec_request(requests.get, url) + - return await self.__exec_request(requests.get, url, headers=await self.__get_request_headers() ) + async def get_project(self, projectid): + url = urljoin(self.api_endpoint, f"projects/{projectid}") + return await self.__exec_request(requests.get, url) - async def get_project(self, id): - url = urljoin(self.api_endpoint, f"projects/{id}") - return await self.__exec_request(requests.get, url, headers=await self.__get_request_headers() ) + async def get_project_configuration(self, projectid): + url = urljoin(self.api_endpoint, f"configuration/project?project-id={projectid}") + return await self.__exec_request(requests.get, url) - async def get_project_configuration(self, id): - url = urljoin(self.api_endpoint, f"configuration/project?project-id={id}") - return await self.__exec_request(requests.get, url, headers=await self.__get_request_headers() ) + async def get_tenant_configuration(self): + url = urljoin(self.api_endpoint, f"configuration/tenant") + return await self.__exec_request(requests.get, url) @dashargs("from-date", "project-id", "project-ids", "scan-ids", "project-names", "source-origin", "source-type", "tags-keys", "tags-values", "to-date") async def get_scans(self, **kwargs): url = urljoin(self.api_endpoint, "scans") - url = CxOneClient.__join_query_dict(url, kwargs) - - return await self.__exec_request(requests.get, url, headers=await self.__get_request_headers() ) + return await self.__exec_request(requests.get, url) + @dashargs("scan-ids") + async def get_sast_scans_metadata(self, **kwargs): + url = urljoin(self.api_endpoint, "sast-metadata") + url = CxOneClient.__join_query_dict(url, kwargs) + return await self.__exec_request(requests.get, url) - async def execute_scan(self, payload, **kwargs): + async def get_sast_scan_metadata(self, scanid, **kwargs): + url = urljoin(self.api_endpoint, f"sast-metadata/{scanid}") + url = CxOneClient.__join_query_dict(url, kwargs) + return await self.__exec_request(requests.get, url) + + @dashargs("source-node-operation", "source-node", "source-line-operation", "source-line", "source-file-operation", "source-file", \ + "sink-node-operation", "sink-node", "sink-line-operation", "sink-line", "sink-file-operation", \ + "sink-file", "result-ids", "preset-id", "number-of-nodes-operation", "number-of-nodes", "notes-operation", \ + "first-found-at-operation", "first-found-at", "apply-predicates") + async def get_sast_scan_aggregate_results(self, scanid, groupby_field=['SEVERITY'], **kwargs): + url = urljoin(self.api_endpoint, f"sast-scan-summary/aggregate") + url = CxOneClient.__join_query_dict(url, kwargs | { + 'scan-id' : scanid, + 'group-by-field' : groupby_field + }) + return await self.__exec_request(requests.get, url) + async def execute_scan(self, payload, **kwargs): url = urljoin(self.api_endpoint, "scans") url = CxOneClient.__join_query_dict(url, kwargs) + return await self.__exec_request(requests.post, url, json=payload) - return await self.__exec_request(requests.post, url, json=payload, headers=await self.__get_request_headers() ) - - async def get_sast_scan_log(self, scanid, stream=False): url = urljoin(self.api_endpoint, f"logs/{scanid}/sast") - return await self.__exec_request(requests.get, url, stream=stream, headers=await self.__get_request_headers() ) + response = await self.__exec_request(requests.get, url) + + if response.ok and response.status_code == 307: + response = await self.__exec_request(requests.get, response.headers['Location'], stream=stream) + + return response + + async def get_groups(self, **kwargs): + url = CxOneClient.__join_query_dict(urljoin(self.admin_endpoint, "groups"), kwargs) + return await self.__exec_request(requests.get, url) async def get_groups(self, **kwargs): url = CxOneClient.__join_query_dict(urljoin(self.admin_endpoint, "groups"), kwargs) - return await self.__exec_request(requests.get, url, headers=await self.__get_request_headers() ) + return await self.__exec_request(requests.get, url) + + async def get_scan_workflow(self, scanid, **kwargs): + url = urljoin(self.api_endpoint, f"scans/{scanid}/workflow") + url = CxOneClient.__join_query_dict(url, kwargs) + return await self.__exec_request(requests.get, url) class ProjectRepoConfig: @@ -360,4 +453,4 @@ async def primary_branch(self): @property async def repo_url(self): url = await self.__get_logical_repo_url() - return url if len(url) > 0 else None + return url if len(url) > 0 else None \ No newline at end of file diff --git a/entrypoint.sh b/entrypoint.sh index df99d63..1c6b077 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -1,5 +1,9 @@ #!/bin/bash +update-ca-certificates +export REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt +echo "export REQUESTS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt" >> /etc/environment + [ -n "$CXONE_REGION" ] && echo "export CXONE_REGION=$CXONE_REGION" >> /etc/environment [ -n "$SINGLE_TENANT_AUTH" ] && echo "export SINGLE_TENANT_AUTH=$SINGLE_TENANT_AUTH" >> /etc/environment [ -n "$SINGLE_TENANT_API" ] && echo "export SINGLE_TENANT_API=$SINGLE_TENANT_API" >> /etc/environment @@ -13,4 +17,4 @@ fi service cron start > /dev/null 2>&1 -python $@ +python3 $@ diff --git a/logic/__init__.py b/logic/__init__.py index 9ab2dea..45991e4 100644 --- a/logic/__init__.py +++ b/logic/__init__.py @@ -1,7 +1,6 @@ import logging, asyncio, utils from cxone_api import paged_api, ProjectRepoConfig - class Scheduler: __log = logging.getLogger("Scheduler") @@ -177,6 +176,10 @@ async def refresh_schedule(self): return len(new_scheduled_projects), len(removed_projects), len(changed_schedule.keys()) + @property + def scheduled_scans(self): + return len(self.__the_schedule.keys()) + async def __load_schedule(self, bad_cb = None): tagged, grouped = await asyncio.gather(self.__get_tagged_project_schedule(bad_cb), self.__get_untagged_project_schedule(bad_cb)) diff --git a/scanner.py b/scanner.py index 9f03dba..b26969a 100755 --- a/scanner.py +++ b/scanner.py @@ -1,4 +1,4 @@ -#!/usr/local/bin/python +#!/usr/bin/python3 import logging, argparse, utils, asyncio from cxone_api import CxOneClient from posix_ipc import Semaphore, BusyError, O_CREAT @@ -38,7 +38,7 @@ async def main(): with open("version.txt", "rt") as ver: version = ver.readline().strip() - client = CxOneClient(oauth_id, oauth_secret, agent, version, auth_endpoint, + client = CxOneClient.create_with_oauth(oauth_id, oauth_secret, agent, version, auth_endpoint, api_endpoint, ssl_verify=ssl_verify, proxy=proxy) diff --git a/scheduler.py b/scheduler.py index da7a85a..307e742 100755 --- a/scheduler.py +++ b/scheduler.py @@ -1,4 +1,4 @@ -#!/usr/local/bin/python +#!/usr/bin/python3 import sys, os, logging, utils if sys.argv[0].lower().startswith("audit"): @@ -35,7 +35,7 @@ with open("version.txt", "rt") as ver: version = ver.readline().strip() - client = CxOneClient(oauth_id, oauth_secret, agent, version, auth_endpoint, + client = CxOneClient.create_with_oauth(oauth_id, oauth_secret, agent, version, auth_endpoint, api_endpoint, ssl_verify=ssl_verify, proxy=proxy) @@ -73,6 +73,7 @@ async def scheduler(): __log.info("Scheduler loop started") short_delay = False while True: + __log.info(f"Projects with scheduled scans: {the_scheduler.scheduled_scans}") await asyncio.sleep(update_delay if not short_delay else 90) __log.info("Updating schedule...") try: @@ -93,7 +94,8 @@ def skipped_entry_cb(project_id, reason): for entry in (await Scheduler.audit(client, default_schedule, group_schedules, policies, skipped_entry_cb)).values(): for sched in entry: - print(f'"{sched.project_id}","SCHEDULED","{str(sched).replace("'", "")}"') + clean_sched = str(sched).replace("'", "") + print(f'"{sched.project_id}","SCHEDULED","{clean_sched}"') if is_audit: asyncio.run(audit())