From 2206d66b429c8d03340117926036664f7d2efcd0 Mon Sep 17 00:00:00 2001 From: "Mark A. Grondona" Date: Mon, 22 Jan 2024 09:18:14 -0800 Subject: [PATCH 1/6] python: add constraint parameter to flux.job.job_list Problem: There is no way to pass a constraint to flux.job.job_list() Add an optional constraint parameter to job_list() and job_list_inactive() --- src/bindings/python/flux/job/list.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/bindings/python/flux/job/list.py b/src/bindings/python/flux/job/list.py index d8172146eefb..af3a2cebfbb3 100644 --- a/src/bindings/python/flux/job/list.py +++ b/src/bindings/python/flux/job/list.py @@ -51,9 +51,15 @@ def job_list( since=0.0, name=None, queue=None, + constraint=None, ): - # N.B. an "and" operation with no values returns everything - constraint = {"and": []} + if constraint is None: + # N.B. an "and" operation with no values returns everything + constraint = {"and": []} + else: + # O/w, a provided constraint will be anded with other parameters below: + constraint = {"and": [constraint]} + if userid != flux.constants.FLUX_USERID_UNKNOWN: constraint["and"].append({"userid": [userid]}) if name: @@ -83,7 +89,13 @@ def job_list( def job_list_inactive( - flux_handle, since=0.0, max_entries=1000, attrs=["all"], name=None, queue=None + flux_handle, + since=0.0, + max_entries=1000, + attrs=["all"], + name=None, + queue=None, + constraint=None, ): """Same as ``flux.job.list.job_list``, but lists only inactive jobs.""" return job_list( @@ -95,6 +107,7 @@ def job_list_inactive( since=since, name=name, queue=queue, + constraint=constraint, ) From 954e67d3326acd7f6f5267097d2a6cd31a742b67 Mon Sep 17 00:00:00 2001 From: "Mark A. Grondona" Date: Thu, 25 Jan 2024 16:18:34 -0800 Subject: [PATCH 2/6] python: add optional constraint parameter to JobList Problem: There is no way to pass extra constraints via the JobList interface. Add an optional constraint parameter to the JobList initializer. This parameter takes a string in constraint query syntax which is parsed by an instance of JobListConstraintParser. The result is then joined with ``and`` to any other constraints before being passed to the job-list module. --- src/bindings/python/flux/job/list.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/bindings/python/flux/job/list.py b/src/bindings/python/flux/job/list.py index af3a2cebfbb3..73f342f40bbf 100644 --- a/src/bindings/python/flux/job/list.py +++ b/src/bindings/python/flux/job/list.py @@ -220,6 +220,10 @@ class JobList: :since: Limit jobs to those that have been active since a given timestamp. :name: Limit jobs to those with a specific name. :queue: Limit jobs to those submitted to a specific queue or queues + :constraint: An RFC 31 Constraint object describing a job-list constraint + as documented in RFC 43 Constraint Operators section. This constraint + may then be joined with other constraints provided by above parameters + via the ``and`` operator. """ # pylint: disable=too-many-instance-attributes @@ -253,6 +257,7 @@ def __init__( since=0.0, name=None, queue=None, + constraint=None, ): self.handle = flux_handle self.attrs = list(attrs) @@ -268,6 +273,7 @@ def __init__( for x in fname.split(","): self.add_filter(x) self.set_user(user) + self.constraint = constraint def set_user(self, user): """Only return jobs for user (may be a username or userid)""" @@ -319,6 +325,7 @@ def fetch_jobs(self): since=self.since, name=self.name, queue=self.queue, + constraint=self.constraint, ) def jobs(self): From fc83f32a6a37e078b188549f7ebbb0eb56a72d56 Mon Sep 17 00:00:00 2001 From: "Mark A. Grondona" Date: Fri, 9 Aug 2024 21:04:21 +0000 Subject: [PATCH 3/6] flux-jobs: add -i, --include=HOSTS|RANKS option Problem: There is no easy way to restrict the output of flux-jobs(1) based on a set of hosts or ranks, but this is basic and useful functionality. Add a new `-i, --include=` optiont that takes a hostlist or idset of targets, and adds a constraint to the JobList query that restricts matching jobs to those that were allocated the provided targets. Fixes #6202 --- src/cmd/flux-jobs.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/cmd/flux-jobs.py b/src/cmd/flux-jobs.py index 8080cd2de924..91b66914c4a9 100755 --- a/src/cmd/flux-jobs.py +++ b/src/cmd/flux-jobs.py @@ -17,6 +17,8 @@ import sys import flux.constants +from flux.hostlist import Hostlist +from flux.idset import IDset from flux.job import JobID, JobInfo, JobInfoFormat, JobList, job_fields_to_attrs from flux.job.stats import JobStats from flux.util import ( @@ -156,6 +158,16 @@ def fetch_jobs_flux(args, fields, flux_handle=None): if not args.filter: args.filter = {"pending", "running"} + constraint = None + if args.include: + try: + constraint = {"ranks": [IDset(args.include).encode()]} + except ValueError: + try: + constraint = {"hostlist": [Hostlist(args.include).encode()]} + except ValueError: + raise ValueError(f"-i/--include: invalid targets: {args.include}") + jobs_rpc = JobList( flux_handle, ids=args.jobids, @@ -166,6 +178,7 @@ def fetch_jobs_flux(args, fields, flux_handle=None): since=since, name=args.name, queue=args.queue, + constraint=constraint, ) jobs = jobs_rpc.jobs() @@ -270,6 +283,14 @@ def parse_args(): metavar="QUEUE,...", help="Limit output to specific queue or queues", ) + parser.add_argument( + "-i", + "--include", + type=str, + metavar="HOSTS|RANKS", + help="Limit output to jobs that were allocated to the specified " + + "HOSTS or RANKS provided as a hostlist or idset.", + ) parser.add_argument( "-o", "--format", From e764720ddee4237bc2e0e8681d35bc68a59ef08b Mon Sep 17 00:00:00 2001 From: "Mark A. Grondona" Date: Fri, 9 Aug 2024 22:27:21 +0000 Subject: [PATCH 4/6] testsuite: add Python job_list* constraint tests Problem: None of the Python tests ensure the job_list and job_list_inactive optional constraint parameters work. Add a couple simple tests to t0013-job-list.py that ensure the constraint parameters to these functions do something. --- t/python/t0013-job-list.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/t/python/t0013-job-list.py b/t/python/t0013-job-list.py index 3cd5399bbb0b..849ba8d57fb1 100755 --- a/t/python/t0013-job-list.py +++ b/t/python/t0013-job-list.py @@ -320,6 +320,18 @@ def test_18_list_id_fail(self): with self.assertRaises(FileNotFoundError): rpc_handle.get_jobinfo() + def test_19_list_inactive_constraints(self): + for job in flux.job.job_list_inactive( + self.fh, constraint={"name": ["sleep"]} + ).get_jobinfos(): + self.assertEqual(job.name, "sleep") + + def test_20_list_constraints(self): + for job in flux.job.job_list( + self.fh, constraint={"name": ["sleep"]} + ).get_jobinfos(): + self.assertEqual(job.name, "sleep") + if __name__ == "__main__": from subflux import rerun_under_flux From c34103bb93befbc3e02605e71cf96e84175424a4 Mon Sep 17 00:00:00 2001 From: "Mark A. Grondona" Date: Fri, 9 Aug 2024 23:07:59 +0000 Subject: [PATCH 5/6] testsuite: add tests for flux-jobs -i, --include option Problem: There are no tests of the flux-jobs -i, --include option. Add some simple tests of this option to t2800-jobs-cmd.t. --- t/t2800-jobs-cmd.t | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/t/t2800-jobs-cmd.t b/t/t2800-jobs-cmd.t index 7280f507d618..adcd8447f7bc 100755 --- a/t/t2800-jobs-cmd.t +++ b/t/t2800-jobs-cmd.t @@ -412,6 +412,28 @@ test_expect_success 'flux-jobs --count works' ' test $count -eq 8 ' +# +# -i, --include tests +# +test_expect_success 'flux-jobs -i, --include works with ranks' ' + for rank in $(flux jobs -ai 0 -no {ranks}); do + test $rank -eq 0 + done +' +test_expect_success 'flux-jobs -i, --include works with ranks' ' + for rank in $(flux jobs -ai 0,3 -no {ranks}); do + test $rank -eq 0 -o $rank -eq 3 + done +' +test_expect_success 'flux jobs -i, --include works with hosts' ' + for host in $(flux jobs -ai $(hostname) -no {nodelist}); do + test $host = $(hostname) + done +' +test_expect_success 'flux jobs -i, --include fails with bad idset/hostlist' ' + test_must_fail flux jobs -ai "foo[" +' + # # test specific IDs # From b2868cc23733081e94c0a0cad6565d581be411f0 Mon Sep 17 00:00:00 2001 From: "Mark A. Grondona" Date: Sat, 10 Aug 2024 01:19:59 +0000 Subject: [PATCH 6/6] doc: document -i, --include in flux-jobs(1) Problem: The -i, --include option is not documented in flux-jobs(1). Document it. --- doc/man1/flux-jobs.rst | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/doc/man1/flux-jobs.rst b/doc/man1/flux-jobs.rst index a009ec7313f9..90e2dd1408c3 100644 --- a/doc/man1/flux-jobs.rst +++ b/doc/man1/flux-jobs.rst @@ -51,6 +51,13 @@ OPTIONS List jobs in a specific queue or queues. Multiple queues may be separated by a comma or by using the :option:`-q, --queue` option multiple times. +.. option:: -i, --include=HOSTS|RANKS + + List only jobs where the assigned resources intersect with the supplied + argument, which may be specified either as an RFC 22 idset of broker ranks + or an RFC 29 hostlist of host names. It is not an error to specify ranks or + hosts which do not exist. + .. option:: -c, --count=N Limit output to N jobs (default 1000) @@ -618,6 +625,12 @@ RESOURCES .. include:: common/resources.rst +FLUX RFC +======== + +| :doc:`rfc:spec_22` +| :doc:`rfc:spec_29` + SEE ALSO ========