diff --git a/releases/unreleased/tasks-not-requiring-sortinghat-fixed.yml b/releases/unreleased/tasks-not-requiring-sortinghat-fixed.yml new file mode 100644 index 00000000..0c4ec157 --- /dev/null +++ b/releases/unreleased/tasks-not-requiring-sortinghat-fixed.yml @@ -0,0 +1,9 @@ +--- +title: Panels import bug and Micro Mordred failure +category: fixed +author: Jose Javier Merchante +issue: null +notes: > + A bug was causing Panels to fail to import in Kibiter + and preventing micro Mordred from working. The issue + was in tasks that didn’t require a SortingHat client to run. diff --git a/sirmordred/task_collection.py b/sirmordred/task_collection.py index 47eb875f..d123494e 100644 --- a/sirmordred/task_collection.py +++ b/sirmordred/task_collection.py @@ -41,8 +41,8 @@ class TaskRawDataCollection(Task): """ Basic class shared by all collection tasks """ - def __init__(self, config, backend_section=None, allowed_repos=None): - super().__init__(config) + def __init__(self, config, sortinghat_client=None, backend_section=None, allowed_repos=None): + super().__init__(config, sortinghat_client) self.backend_section = backend_section self.allowed_repos = set(allowed_repos) if allowed_repos else None diff --git a/sirmordred/task_identities.py b/sirmordred/task_identities.py index 7436f328..90d144c2 100644 --- a/sirmordred/task_identities.py +++ b/sirmordred/task_identities.py @@ -38,8 +38,8 @@ class TaskIdentitiesMerge(Task): """ Task for processing identities in SortingHat """ - def __init__(self, conf, soringhat_client): - super().__init__(conf, soringhat_client) + def __init__(self, conf, sortinghat_client): + super().__init__(conf, sortinghat_client) self.last_autorefresh = datetime.utcnow() # Last autorefresh date def is_backend_task(self): diff --git a/sirmordred/task_panels.py b/sirmordred/task_panels.py index 00928860..24bdf30b 100644 --- a/sirmordred/task_panels.py +++ b/sirmordred/task_panels.py @@ -264,8 +264,8 @@ class TaskPanels(Task): # Panels to be uploaded always, no matter the data sources configured panels_common = panels_multi_ds + ["panels/json/about.json"] - def __init__(self, conf): - super().__init__(conf) + def __init__(self, conf, sortinghat_client=None): + super().__init__(conf, sortinghat_client) # Read panels and menu description from yaml file with open(self.conf['general']['menu_file'], 'r') as f: try: @@ -473,8 +473,8 @@ class TaskPanelsMenu(Task): } } - def __init__(self, conf): - super().__init__(conf) + def __init__(self, conf, sortinghat_client=None): + super().__init__(conf, sortinghat_client) # Read panels and menu description from yaml file """ with open(self.conf['general']['menu_file'], 'r') as f: try: diff --git a/sirmordred/utils/micro.py b/sirmordred/utils/micro.py index 87a4d680..69a7055b 100644 --- a/sirmordred/utils/micro.py +++ b/sirmordred/utils/micro.py @@ -33,6 +33,7 @@ from sirmordred.task_enrich import TaskEnrich from sirmordred.task_panels import TaskPanels, TaskPanelsMenu from sirmordred.task_projects import TaskProjects +from sortinghat.cli.client import SortingHatClient COLOR_LOG_FORMAT_SUFFIX = "\033[1m %(log_color)s " LOG_COLORS = {'DEBUG': 'white', 'INFO': 'cyan', 'WARNING': 'yellow', 'ERROR': 'red', 'CRITICAL': 'red,bg_white'} @@ -50,6 +51,34 @@ def main(): args.panels) +def create_sortinghat_client(config): + """Create a SortingHat client""" + + conf = config.get_conf() + + sortinghat = conf.get('sortinghat', None) + if not sortinghat: + return None + + db_user = sortinghat['user'] if sortinghat else None + db_password = sortinghat['password'] if sortinghat else None + db_host = sortinghat['host'] if sortinghat else '127.0.0.1' + db_path = sortinghat.get('path', None) if sortinghat else None + db_port = sortinghat.get('port', None) if sortinghat else None + db_ssl = sortinghat.get('ssl', False) if sortinghat else False + db_verify_ssl = sortinghat.get('verify_ssl', True) if sortinghat else True + db_tenant = sortinghat.get('tenant', True) if sortinghat else None + + client = SortingHatClient(host=db_host, port=db_port, + path=db_path, ssl=db_ssl, + user=db_user, password=db_password, + verify_ssl=db_verify_ssl, + tenant=db_tenant) + client.connect() + + return client + + def micro_mordred(config, backend_sections, repos_to_check, raw, identities_merge, enrich, panels): """Execute the Mordred tasks using the configuration file. @@ -66,12 +95,17 @@ def micro_mordred(config, backend_sections, repos_to_check, raw, identities_merg for backend in backend_sections: get_raw(config, backend, repos_to_check) + if identities_merge or enrich: + sortinghat_client = create_sortinghat_client(config) + else: + sortinghat_client = None + if identities_merge: - get_identities_merge(config) + get_identities_merge(config, sortinghat_client) if enrich: for backend in backend_sections: - get_enrich(config, backend, repos_to_check) + get_enrich(config, sortinghat_client, backend, repos_to_check) if panels: get_panels(config) @@ -97,29 +131,31 @@ def get_raw(config, backend_section, repos_to_check=None): sys.exit(-1) -def get_identities_merge(config): +def get_identities_merge(config, sortinghat_client): """Execute the merge identities phase + :param sortinghat_client: a SortingHat client :param config: a Mordred config object """ TaskProjects(config).execute() - task = TaskIdentitiesMerge(config) + task = TaskIdentitiesMerge(config, sortinghat_client) task.execute() logging.info("Merging identities finished!") -def get_enrich(config, backend_section, repos_to_check=None): +def get_enrich(config, sortinghat_client, backend_section, repos_to_check=None): """Execute the enrich phase for a given backend section Repos are only checked if they are in BOTH `repos_to_check` and the `projects.json` :param config: a Mordred config object + :param sortinghat_client: a SortingHat client :param backend_section: the backend section where the enrich phase is executed :param repos_to_check: A list of repo URLs to check, or None to check all repos """ TaskProjects(config).execute() - task = TaskEnrich(config, backend_section=backend_section, allowed_repos=repos_to_check) + task = TaskEnrich(config, sortinghat_client, backend_section=backend_section, allowed_repos=repos_to_check) try: task.execute() logging.info("Loading enriched data finished!")