From b71915e0f51836e25c9163a6375a99f916523996 Mon Sep 17 00:00:00 2001 From: Jeff Ohrstrom Date: Tue, 2 Jul 2024 10:07:16 -0400 Subject: [PATCH 1/5] force utf-8 encoding --- lib/ood_core/job/adapters/slurm.rb | 1 + .../output/slurm/non_utf8_job_name.txt | 2 ++ spec/job/adapters/slurm_spec.rb | 23 +++++++++++++++++++ 3 files changed, 26 insertions(+) create mode 100644 spec/fixtures/output/slurm/non_utf8_job_name.txt diff --git a/lib/ood_core/job/adapters/slurm.rb b/lib/ood_core/job/adapters/slurm.rb index fa7d6cc79..ed4264265 100644 --- a/lib/ood_core/job/adapters/slurm.rb +++ b/lib/ood_core/job/adapters/slurm.rb @@ -169,6 +169,7 @@ def get_jobs(id: "", owner: nil, attrs: nil) # jobs << job # # assuming keys and values are same length! if not we have an error! + line = line.force_encoding('UTF-8') values = line.chomp(RECORD_SEPARATOR).strip.split(UNIT_SEPARATOR) jobs << Hash[fields.keys.zip(values)] unless values.empty? end diff --git a/spec/fixtures/output/slurm/non_utf8_job_name.txt b/spec/fixtures/output/slurm/non_utf8_job_name.txt new file mode 100644 index 000000000..71a72bc2f --- /dev/null +++ b/spec/fixtures/output/slurm/non_utf8_job_name.txt @@ -0,0 +1,2 @@ +ACCOUNTJOBIDEXEC_HOSTMIN_CPUSCPUSMIN_TMP_DISKNODESEND_TIMEDEPENDENCYFEATURESARRAY_JOB_IDGROUPGROUPOVER_SUBSCRIBESOCKETS_PER_NODEJOBIDCORES_PER_SOCKETNAMETHREADS_PER_CORECOMMENTARRAY_TASK_IDTIME_LIMITTIME_LEFTMIN_MEMORYTIMEREQ_NODESNODELISTCOMMANDCONTIGUOUSQOSPARTITIONPRIORITYREASONSTART_TIMESTSTATEUSERUIDRESERVATIONSUBMIT_TIMEWCKEYLICENSESEXC_NODESCORE_SPECNICESCHEDNODESS:C:TWORK_DIRTRES_PER_NODE +pas1960123p0220140012024-06-26T14:19:40(null)(null)29471683PAS06544067OK*29471683_9*â non-utf8*stdout=/tmp/job.output930:0026:554556M3:05p0220/tmp/job.sh0pitzer-defaultcondo-ccapp-backfill-serial100072289None2024-06-26T14:16:35CDCOMPLETEDannie.oakley23327(null)2024-06-26T14:01:13(null)(null)N/A0(null)*:*:*/tmpgres/gpfs:ess:1 diff --git a/spec/job/adapters/slurm_spec.rb b/spec/job/adapters/slurm_spec.rb index 4bdde1e7b..e9804598c 100644 --- a/spec/job/adapters/slurm_spec.rb +++ b/spec/job/adapters/slurm_spec.rb @@ -770,6 +770,29 @@ def job_info(opts = {}) is_expected.to eql(job_info) end end + + context "when job name has non utf8 characters" do + + let(:squeue_args) {[ + "squeue", + "--all", + "--states=all", + "--noconvert", + "-o", + "\u001E%a\u001F%A\u001F%B\u001F%c\u001F%C\u001F%d\u001F%D\u001F%e\u001F%E\u001F%f\u001F%F\u001F%g\u001F%G\u001F%h\u001F%H\u001F%i\u001F%I\u001F%j\u001F%J\u001F%k\u001F%K\u001F%l\u001F%L\u001F%m\u001F%M\u001F%n\u001F%N\u001F%o\u001F%O\u001F%q\u001F%P\u001F%Q\u001F%r\u001F%S\u001F%t\u001F%T\u001F%u\u001F%U\u001F%v\u001F%V\u001F%w\u001F%W\u001F%x\u001F%X\u001F%y\u001F%Y\u001F%z\u001F%Z\u001F%b", + "-j", + "123" + ]} + + it "correctly handles non utf8 characters" do + stdout = File.read('spec/fixtures/output/slurm/non_utf8_job_name.txt') + stdout.force_encoding(Encoding::ASCII) + allow(Open3).to receive(:capture3).with({}, *squeue_args, stdin_data: "").and_return([stdout, '', double("success?" => true)]) + job = OodCore::Job::Factory.build_slurm({}).info('123') + expect(job.job_owner).to eq('annie.oakley') + expect(job.job_name).to eq('â non-utf8') + end + end end describe "#status" do From 1f24ad2af7a8a9657de2cad3f20162df272369fa Mon Sep 17 00:00:00 2001 From: Jeff Ohrstrom Date: Tue, 2 Jul 2024 12:16:03 -0400 Subject: [PATCH 2/5] add invalid and undef keywords --- lib/ood_core/job/adapters/slurm.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ood_core/job/adapters/slurm.rb b/lib/ood_core/job/adapters/slurm.rb index ed4264265..d54edc764 100644 --- a/lib/ood_core/job/adapters/slurm.rb +++ b/lib/ood_core/job/adapters/slurm.rb @@ -169,7 +169,7 @@ def get_jobs(id: "", owner: nil, attrs: nil) # jobs << job # # assuming keys and values are same length! if not we have an error! - line = line.force_encoding('UTF-8') + line = line.force_encoding('UTF-8', invalid: :replace, undef: :replace, ) values = line.chomp(RECORD_SEPARATOR).strip.split(UNIT_SEPARATOR) jobs << Hash[fields.keys.zip(values)] unless values.empty? end From 3559f789c3165fc36dee1c8995bf0f65fc00fff5 Mon Sep 17 00:00:00 2001 From: Jeff Ohrstrom Date: Tue, 2 Jul 2024 12:16:36 -0400 Subject: [PATCH 3/5] fix syntax here --- lib/ood_core/job/adapters/slurm.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ood_core/job/adapters/slurm.rb b/lib/ood_core/job/adapters/slurm.rb index d54edc764..82a18d716 100644 --- a/lib/ood_core/job/adapters/slurm.rb +++ b/lib/ood_core/job/adapters/slurm.rb @@ -169,7 +169,7 @@ def get_jobs(id: "", owner: nil, attrs: nil) # jobs << job # # assuming keys and values are same length! if not we have an error! - line = line.force_encoding('UTF-8', invalid: :replace, undef: :replace, ) + line = line.force_encoding('UTF-8', invalid: :replace, undef: :replace) values = line.chomp(RECORD_SEPARATOR).strip.split(UNIT_SEPARATOR) jobs << Hash[fields.keys.zip(values)] unless values.empty? end From f3e57e1538f331e033659c4e42a8adce6a6979f8 Mon Sep 17 00:00:00 2001 From: Jeff Ohrstrom Date: Mon, 8 Jul 2024 11:17:53 -0400 Subject: [PATCH 4/5] should be encode not force_encoding --- lib/ood_core/job/adapters/slurm.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ood_core/job/adapters/slurm.rb b/lib/ood_core/job/adapters/slurm.rb index 82a18d716..72f111210 100644 --- a/lib/ood_core/job/adapters/slurm.rb +++ b/lib/ood_core/job/adapters/slurm.rb @@ -169,7 +169,7 @@ def get_jobs(id: "", owner: nil, attrs: nil) # jobs << job # # assuming keys and values are same length! if not we have an error! - line = line.force_encoding('UTF-8', invalid: :replace, undef: :replace) + line = line.encode('UTF-8', invalid: :replace, undef: :replace) values = line.chomp(RECORD_SEPARATOR).strip.split(UNIT_SEPARATOR) jobs << Hash[fields.keys.zip(values)] unless values.empty? end From dd99a777ce52008d494229bbd8f961c18f9052fb Mon Sep 17 00:00:00 2001 From: Jeff Ohrstrom Date: Wed, 10 Jul 2024 10:12:03 -0400 Subject: [PATCH 5/5] correct this test case --- spec/fixtures/output/slurm/non_utf8_job_name.txt | 2 +- spec/job/adapters/slurm_spec.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/fixtures/output/slurm/non_utf8_job_name.txt b/spec/fixtures/output/slurm/non_utf8_job_name.txt index 71a72bc2f..53e4e5064 100644 --- a/spec/fixtures/output/slurm/non_utf8_job_name.txt +++ b/spec/fixtures/output/slurm/non_utf8_job_name.txt @@ -1,2 +1,2 @@ ACCOUNTJOBIDEXEC_HOSTMIN_CPUSCPUSMIN_TMP_DISKNODESEND_TIMEDEPENDENCYFEATURESARRAY_JOB_IDGROUPGROUPOVER_SUBSCRIBESOCKETS_PER_NODEJOBIDCORES_PER_SOCKETNAMETHREADS_PER_CORECOMMENTARRAY_TASK_IDTIME_LIMITTIME_LEFTMIN_MEMORYTIMEREQ_NODESNODELISTCOMMANDCONTIGUOUSQOSPARTITIONPRIORITYREASONSTART_TIMESTSTATEUSERUIDRESERVATIONSUBMIT_TIMEWCKEYLICENSESEXC_NODESCORE_SPECNICESCHEDNODESS:C:TWORK_DIRTRES_PER_NODE -pas1960123p0220140012024-06-26T14:19:40(null)(null)29471683PAS06544067OK*29471683_9*â non-utf8*stdout=/tmp/job.output930:0026:554556M3:05p0220/tmp/job.sh0pitzer-defaultcondo-ccapp-backfill-serial100072289None2024-06-26T14:16:35CDCOMPLETEDannie.oakley23327(null)2024-06-26T14:01:13(null)(null)N/A0(null)*:*:*/tmpgres/gpfs:ess:1 +pas1960123p0220140012024-06-26T14:19:40(null)(null)29471683PAS06544067OK*29471683_9*� non-utf8*stdout=/tmp/job.output930:0026:554556M3:05p0220/tmp/job.sh0pitzer-defaultcondo-ccapp-backfill-serial100072289None2024-06-26T14:16:35CDCOMPLETEDannie.oakley23327(null)2024-06-26T14:01:13(null)(null)N/A0(null)*:*:*/tmpgres/gpfs:ess:1 diff --git a/spec/job/adapters/slurm_spec.rb b/spec/job/adapters/slurm_spec.rb index e9804598c..d4c707fd9 100644 --- a/spec/job/adapters/slurm_spec.rb +++ b/spec/job/adapters/slurm_spec.rb @@ -790,7 +790,7 @@ def job_info(opts = {}) allow(Open3).to receive(:capture3).with({}, *squeue_args, stdin_data: "").and_return([stdout, '', double("success?" => true)]) job = OodCore::Job::Factory.build_slurm({}).info('123') expect(job.job_owner).to eq('annie.oakley') - expect(job.job_name).to eq('â non-utf8') + expect(job.job_name).to eq('��� non-utf8') end end end