From f55f6da839addc65e7936e5d2412367ea6212bf2 Mon Sep 17 00:00:00 2001 From: Jose Javier Merchante Date: Wed, 13 Nov 2024 10:25:42 +0100 Subject: [PATCH] Fix tasks without required SortingHat client MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tasks that didn’t require the SortingHat client were failing or receiving incorrect arguments because the client was being passed as the second argument. Since the client is required for all tasks, it has been added as an argument in the subclass constructors. Signed-off-by: Jose Javier Merchante --- .../tasks-not-requiring-sortinghat-fixed.yml | 9 ++++ sirmordred/task_collection.py | 4 +- sirmordred/task_identities.py | 4 +- sirmordred/task_panels.py | 8 ++-- sirmordred/utils/micro.py | 48 ++++++++++++++++--- 5 files changed, 59 insertions(+), 14 deletions(-) create mode 100644 releases/unreleased/tasks-not-requiring-sortinghat-fixed.yml diff --git a/releases/unreleased/tasks-not-requiring-sortinghat-fixed.yml b/releases/unreleased/tasks-not-requiring-sortinghat-fixed.yml new file mode 100644 index 00000000..0c4ec157 --- /dev/null +++ b/releases/unreleased/tasks-not-requiring-sortinghat-fixed.yml @@ -0,0 +1,9 @@ +--- +title: Panels import bug and Micro Mordred failure +category: fixed +author: Jose Javier Merchante +issue: null +notes: > + A bug was causing Panels to fail to import in Kibiter + and preventing micro Mordred from working. The issue + was in tasks that didn’t require a SortingHat client to run. diff --git a/sirmordred/task_collection.py b/sirmordred/task_collection.py index 47eb875f..d123494e 100644 --- a/sirmordred/task_collection.py +++ b/sirmordred/task_collection.py @@ -41,8 +41,8 @@ class TaskRawDataCollection(Task): """ Basic class shared by all collection tasks """ - def __init__(self, config, backend_section=None, allowed_repos=None): - super().__init__(config) + def __init__(self, config, sortinghat_client=None, backend_section=None, allowed_repos=None): + super().__init__(config, sortinghat_client) self.backend_section = backend_section self.allowed_repos = set(allowed_repos) if allowed_repos else None diff --git a/sirmordred/task_identities.py b/sirmordred/task_identities.py index 7436f328..90d144c2 100644 --- a/sirmordred/task_identities.py +++ b/sirmordred/task_identities.py @@ -38,8 +38,8 @@ class TaskIdentitiesMerge(Task): """ Task for processing identities in SortingHat """ - def __init__(self, conf, soringhat_client): - super().__init__(conf, soringhat_client) + def __init__(self, conf, sortinghat_client): + super().__init__(conf, sortinghat_client) self.last_autorefresh = datetime.utcnow() # Last autorefresh date def is_backend_task(self): diff --git a/sirmordred/task_panels.py b/sirmordred/task_panels.py index 00928860..24bdf30b 100644 --- a/sirmordred/task_panels.py +++ b/sirmordred/task_panels.py @@ -264,8 +264,8 @@ class TaskPanels(Task): # Panels to be uploaded always, no matter the data sources configured panels_common = panels_multi_ds + ["panels/json/about.json"] - def __init__(self, conf): - super().__init__(conf) + def __init__(self, conf, sortinghat_client=None): + super().__init__(conf, sortinghat_client) # Read panels and menu description from yaml file with open(self.conf['general']['menu_file'], 'r') as f: try: @@ -473,8 +473,8 @@ class TaskPanelsMenu(Task): } } - def __init__(self, conf): - super().__init__(conf) + def __init__(self, conf, sortinghat_client=None): + super().__init__(conf, sortinghat_client) # Read panels and menu description from yaml file """ with open(self.conf['general']['menu_file'], 'r') as f: try: diff --git a/sirmordred/utils/micro.py b/sirmordred/utils/micro.py index 87a4d680..69a7055b 100644 --- a/sirmordred/utils/micro.py +++ b/sirmordred/utils/micro.py @@ -33,6 +33,7 @@ from sirmordred.task_enrich import TaskEnrich from sirmordred.task_panels import TaskPanels, TaskPanelsMenu from sirmordred.task_projects import TaskProjects +from sortinghat.cli.client import SortingHatClient COLOR_LOG_FORMAT_SUFFIX = "\033[1m %(log_color)s " LOG_COLORS = {'DEBUG': 'white', 'INFO': 'cyan', 'WARNING': 'yellow', 'ERROR': 'red', 'CRITICAL': 'red,bg_white'} @@ -50,6 +51,34 @@ def main(): args.panels) +def create_sortinghat_client(config): + """Create a SortingHat client""" + + conf = config.get_conf() + + sortinghat = conf.get('sortinghat', None) + if not sortinghat: + return None + + db_user = sortinghat['user'] if sortinghat else None + db_password = sortinghat['password'] if sortinghat else None + db_host = sortinghat['host'] if sortinghat else '127.0.0.1' + db_path = sortinghat.get('path', None) if sortinghat else None + db_port = sortinghat.get('port', None) if sortinghat else None + db_ssl = sortinghat.get('ssl', False) if sortinghat else False + db_verify_ssl = sortinghat.get('verify_ssl', True) if sortinghat else True + db_tenant = sortinghat.get('tenant', True) if sortinghat else None + + client = SortingHatClient(host=db_host, port=db_port, + path=db_path, ssl=db_ssl, + user=db_user, password=db_password, + verify_ssl=db_verify_ssl, + tenant=db_tenant) + client.connect() + + return client + + def micro_mordred(config, backend_sections, repos_to_check, raw, identities_merge, enrich, panels): """Execute the Mordred tasks using the configuration file. @@ -66,12 +95,17 @@ def micro_mordred(config, backend_sections, repos_to_check, raw, identities_merg for backend in backend_sections: get_raw(config, backend, repos_to_check) + if identities_merge or enrich: + sortinghat_client = create_sortinghat_client(config) + else: + sortinghat_client = None + if identities_merge: - get_identities_merge(config) + get_identities_merge(config, sortinghat_client) if enrich: for backend in backend_sections: - get_enrich(config, backend, repos_to_check) + get_enrich(config, sortinghat_client, backend, repos_to_check) if panels: get_panels(config) @@ -97,29 +131,31 @@ def get_raw(config, backend_section, repos_to_check=None): sys.exit(-1) -def get_identities_merge(config): +def get_identities_merge(config, sortinghat_client): """Execute the merge identities phase + :param sortinghat_client: a SortingHat client :param config: a Mordred config object """ TaskProjects(config).execute() - task = TaskIdentitiesMerge(config) + task = TaskIdentitiesMerge(config, sortinghat_client) task.execute() logging.info("Merging identities finished!") -def get_enrich(config, backend_section, repos_to_check=None): +def get_enrich(config, sortinghat_client, backend_section, repos_to_check=None): """Execute the enrich phase for a given backend section Repos are only checked if they are in BOTH `repos_to_check` and the `projects.json` :param config: a Mordred config object + :param sortinghat_client: a SortingHat client :param backend_section: the backend section where the enrich phase is executed :param repos_to_check: A list of repo URLs to check, or None to check all repos """ TaskProjects(config).execute() - task = TaskEnrich(config, backend_section=backend_section, allowed_repos=repos_to_check) + task = TaskEnrich(config, sortinghat_client, backend_section=backend_section, allowed_repos=repos_to_check) try: task.execute() logging.info("Loading enriched data finished!")