LHCb Open Data Curation scripts

lhcb-YYYY-open-data-curation/README.md

@@ -0,0 +1,26 @@
+# How to prepare the metadata for an LHCb OD release
+
+## Configuration: 
+    Various parameters are configurable from `config.yaml`. Uncomment the relevant paths and streams, change values of the parameters.  
+  - `directory` + `stream` directive is used for DIRAC related activities, to construct a bookkeeping path.
+  - `stripping_version` is used for documentation pages (the paths here are slightly different from those on DIRAC). 
+  - `release_dir` - output directory.
+  - `stripping_input_dir` - stripping pages input directory. 
+
+
+## Steps:
+- Make sure DIRAC is available on your platform (either cvmfs is mounted or work on lxplus)
+- GRID proxy is required. Run `lhcb-proxy-init`, enter the password for your GRID certificate. 
+- To write metadata for a single DIRAC Bookkeeping path, run: 
+  ``` lb-dirac python MetadataWriter.py --BK="<your bookkeepinng path>"```
+  - Available flags are:
+    - `--verbose` - provides various interim output while running the script.
+    - `--staging` which writes out a file with `pfns` used to stage the files on open data portal. 
+  This will create a `.JSON` file with metadata for the specified path conforming to OpenData Portal schema. 
+- To write metadata for all paths being released, run the script `make_release.py`.
+
+## Converting the Stripping pages into Markdown files
+- Download the html files from `/eos/project/l/lhcbwebsites/www/projects/stripping/config/strippingXXX`
+- Set the `stripping_input_dir` to the path where the files are downloaded.  
+  - This can be done manually with `xrdcp`, `scp`, `eos cp` commands.
+- Run `snakemake --cores all` in the `scripts/stripping_pages` directory. 

lhcb-YYYY-open-data-curation/config.yaml

@@ -0,0 +1,49 @@
+directory:
+ - "/LHCb/Collision11/Beam3500GeV-VeloClosed-MagDown/Real Data/Reco14/Stripping21r1/90000000/"
+ - "/LHCb/Collision11/Beam3500GeV-VeloClosed-MagUp/Real Data/Reco14/Stripping21r1/90000000/"
+ - "/LHCb/Collision12/Beam4000GeV-VeloClosed-MagDown/Real Data/Reco14/Stripping21/90000000/"
+ - "/LHCb/Collision12/Beam4000GeV-VeloClosed-MagUp/Real Data/Reco14/Stripping21/90000000/"
+ - "/LHCb/Collision11/Beam3500GeV-VeloClosed-MagDown/Real Data/Reco14/Stripping21r1p1a/90000000/"
+ - "/LHCb/Collision11/Beam3500GeV-VeloClosed-MagUp/Real Data/Reco14/Stripping21r1p1a/90000000/"
+ - "/LHCb/Collision11/Beam3500GeV-VeloClosed-MagDown/Real Data/Reco14/Stripping21r1p2/90000000/"
+ - "/LHCb/Collision11/Beam3500GeV-VeloClosed-MagUp/Real Data/Reco14/Stripping21r1p2/90000000/"
+ - "/LHCb/Collision12/Beam4000GeV-VeloClosed-MagDown/Real Data/Reco14/Stripping21r0p1a/90000000/"
+ - "/LHCb/Collision12/Beam4000GeV-VeloClosed-MagUp/Real Data/Reco14/Stripping21r0p1a/90000000/"
+ - "/LHCb/Collision12/Beam4000GeV-VeloClosed-MagDown/Real Data/Reco14/Stripping21r0p2/90000000/"
+ - "/LHCb/Collision12/Beam4000GeV-VeloClosed-MagUp/Real Data/Reco14/Stripping21r0p2/90000000/"
+
+stream: 
+  - "EW.DST"
+  - "LEPTONIC.MDST"
+  - "RADIATIVE.DST"
+  - "BHADRON.MDST"
+  - "BHADRONCOMPLETEEVENT.DST"
+  - "CHARM.MDST"
+  - "CHARMCOMPLETEEVENT.DST"
+  - "DIMUON.DST"
+  - "SEMILEPTONIC.DST"
+
+stripping_version:
+  - "stripping21"
+  - "stripping21r0p1" # Doesn't have RADIATIVE stream 
+  - "stripping21r0p2" # Doesn't have RADIATIVE stream 
+  - "stripping21r1"
+  - "stripping21r1p1"
+  - "stripping21r1p2"
+
+stripping_versions_year:
+    "2011":
+      - "stripping21r1"
+      - "stripping21r1p1"
+      - "stripping21r1p2"
+    "2012":
+      - "stripping21"
+      - "stripping21r0p1"
+      - "stripping21r0p2"
+
+release_dir: "../../release/"
+stripping_input_dir: "/Users/mindaugassarpis/Work/StrippingPages/"
+
+# Staging
+eos_existing: "./existing.txt"
+stage_frac: 0.16

lhcb-YYYY-open-data-curation/scripts/file_staging/file_renamer.py

@@ -0,0 +1,57 @@
+import json
+
+stagingindexefiles = [
+    "FilesToStage_Beam3500GeV-VeloClosed-MagDown__RealData_Reco14_Stripping21r1_EW.DST.txt",
+    "FilesToStage_Beam3500GeV-VeloClosed-MagDown__RealData_Reco14_Stripping21r1_LEPTONIC.MDST.txt",
+    "FilesToStage_Beam3500GeV-VeloClosed-MagDown__RealData_Reco14_Stripping21r1p1a_EW.DST.txt",
+    "FilesToStage_Beam3500GeV-VeloClosed-MagDown__RealData_Reco14_Stripping21r1p1a_LEPTONIC.MDST.txt",
+    "FilesToStage_Beam3500GeV-VeloClosed-MagDown__RealData_Reco14_Stripping21r1p2_EW.DST.txt",
+    "FilesToStage_Beam3500GeV-VeloClosed-MagDown__RealData_Reco14_Stripping21r1p2_LEPTONIC.MDST.txt",
+    "FilesToStage_Beam3500GeV-VeloClosed-MagDown__RealData_Reco14_Stripping21r1_RADIATIVE.DST.txt",
+    "FilesToStage_Beam3500GeV-VeloClosed-MagUp__RealData_Reco14_Stripping21r1_EW.DST.txt",
+    "FilesToStage_Beam3500GeV-VeloClosed-MagUp__RealData_Reco14_Stripping21r1_LEPTONIC.MDST.txt",
+    "FilesToStage_Beam3500GeV-VeloClosed-MagUp__RealData_Reco14_Stripping21r1p1a_EW.DST.txt",
+    "FilesToStage_Beam3500GeV-VeloClosed-MagUp__RealData_Reco14_Stripping21r1p1a_LEPTONIC.MDST.txt",
+    "FilesToStage_Beam3500GeV-VeloClosed-MagUp__RealData_Reco14_Stripping21r1p2_EW.DST.txt",
+    "FilesToStage_Beam3500GeV-VeloClosed-MagUp__RealData_Reco14_Stripping21r1p2_LEPTONIC.MDST.txt",
+    "FilesToStage_Beam4000GeV-VeloClosed-MagDown__RealData_Reco14_Stripping21_EW.DST.txt",
+    "FilesToStage_Beam4000GeV-VeloClosed-MagDown__RealData_Reco14_Stripping21_LEPTONIC.MDST.txt",
+    "FilesToStage_Beam4000GeV-VeloClosed-MagDown__RealData_Reco14_Stripping21r0p1a_EW.DST.txt",
+    "FilesToStage_Beam4000GeV-VeloClosed-MagDown__RealData_Reco14_Stripping21r0p1a_LEPTONIC.MDST.txt",
+    "FilesToStage_Beam4000GeV-VeloClosed-MagDown__RealData_Reco14_Stripping21r0p2_EW.DST.txt",
+    "FilesToStage_Beam4000GeV-VeloClosed-MagDown__RealData_Reco14_Stripping21r0p2_LEPTONIC.MDST.txt",
+    "FilesToStage_Beam4000GeV-VeloClosed-MagDown__RealData_Reco14_Stripping21_RADIATIVE.DST.txt",
+    "FilesToStage_Beam4000GeV-VeloClosed-MagUp__RealData_Reco14_Stripping21_EW.DST.txt",
+    "FilesToStage_Beam4000GeV-VeloClosed-MagUp__RealData_Reco14_Stripping21_LEPTONIC.MDST.txt",
+    "FilesToStage_Beam4000GeV-VeloClosed-MagUp__RealData_Reco14_Stripping21r0p1a_EW.DST.txt",
+    "FilesToStage_Beam4000GeV-VeloClosed-MagUp__RealData_Reco14_Stripping21r0p1a_LEPTONIC.MDST.txt",
+    "FilesToStage_Beam4000GeV-VeloClosed-MagUp__RealData_Reco14_Stripping21r0p2_EW.DST.txt",
+    "FilesToStage_Beam4000GeV-VeloClosed-MagUp__RealData_Reco14_Stripping21r0p2_LEPTONIC.MDST.txt",
+    "FilesToStage_Beam4000GeV-VeloClosed-MagUp__RealData_Reco14_Stripping21_RADIATIVE.DST.txt",
+]
+
+newdict = {}
+
+for indfile in stagingindexefiles:
+    with open(f"FTSReady/{indfile}") as f:
+        filesdict = json.load(f)
+
+    if "MDST" in indfile:
+        newfilename = indfile.removeprefix("FilesToStage_").replace("-","_").replace("__","_").replace(".MDST.txt","_MDST")
+    else:
+        newfilename = indfile.removeprefix("FilesToStage_").replace("-","_").replace("__","_").replace(".DST.txt","_DST")
+
+    for file in filesdict["files"]:
+
+        if "MDST" in indfile:
+            newdict[file["destinations"][0]] = file["destinations"][0].split(".MDST")[0] + "/"+newfilename + file["destinations"][0].split(".MDST")[1]
+        else:
+            newdict[file["destinations"][0]] = file["destinations"][0].split(".DST")[0] + "/"+newfilename + file["destinations"][0].split(".DST")[1]
+
+    print(json.dumps(newdict, indent=4, sort_keys=True))
+
+with open("eosrenamedfiles.json", "w") as fl:
+
+    json.dump(newdict, fl, indent=2)
+
+

lhcb-YYYY-open-data-curation/scripts/file_staging/file_stager.py

@@ -0,0 +1,108 @@
+import os
+
+Files = [
+# "FilesToStage_Beam3500GeV-VeloClosed-MagDown__RealData_Reco14_Stripping21r1_EW.DST.txt",
+# "FilesToStage_Beam3500GeV-VeloClosed-MagDown__RealData_Reco14_Stripping21r1_LEPTONIC.MDST.txt",
+# "FilesToStage_Beam3500GeV-VeloClosed-MagDown__RealData_Reco14_Stripping21r1p1a_EW.DST.txt",
+# "FilesToStage_Beam3500GeV-VeloClosed-MagDown__RealData_Reco14_Stripping21r1p1a_LEPTONIC.MDST.txt",
+# "FilesToStage_Beam3500GeV-VeloClosed-MagDown__RealData_Reco14_Stripping21r1p2_EW.DST.txt",
+# "FilesToStage_Beam3500GeV-VeloClosed-MagDown__RealData_Reco14_Stripping21r1p2_LEPTONIC.MDST.txt",
+# "FilesToStage_Beam3500GeV-VeloClosed-MagDown__RealData_Reco14_Stripping21r1_RADIATIVE.DST.txt",
+# "FilesToStage_Beam3500GeV-VeloClosed-MagUp__RealData_Reco14_Stripping21r1_EW.DST.txt",
+# "FilesToStage_Beam3500GeV-VeloClosed-MagUp__RealData_Reco14_Stripping21r1_LEPTONIC.MDST.txt",
+# "FilesToStage_Beam3500GeV-VeloClosed-MagUp__RealData_Reco14_Stripping21r1p1a_EW.DST.txt",
+# "FilesToStage_Beam3500GeV-VeloClosed-MagUp__RealData_Reco14_Stripping21r1p1a_LEPTONIC.MDST.txt",
+# "FilesToStage_Beam3500GeV-VeloClosed-MagUp__RealData_Reco14_Stripping21r1p2_EW.DST.txt",
+# "FilesToStage_Beam3500GeV-VeloClosed-MagUp__RealData_Reco14_Stripping21r1p2_LEPTONIC.MDST.txt",
+# "FilesToStage_Beam4000GeV-VeloClosed-MagDown__RealData_Reco14_Stripping21_EW.DST.txt",
+# "FilesToStage_Beam4000GeV-VeloClosed-MagDown__RealData_Reco14_Stripping21_LEPTONIC.MDST.txt",
+# "FilesToStage_Beam4000GeV-VeloClosed-MagDown__RealData_Reco14_Stripping21r0p1a_EW.DST.txt",
+# "FilesToStage_Beam4000GeV-VeloClosed-MagDown__RealData_Reco14_Stripping21r0p1a_LEPTONIC.MDST.txt",
+# "FilesToStage_Beam4000GeV-VeloClosed-MagDown__RealData_Reco14_Stripping21r0p2_EW.DST.txt",
+# "FilesToStage_Beam4000GeV-VeloClosed-MagDown__RealData_Reco14_Stripping21r0p2_LEPTONIC.MDST.txt",
+# "FilesToStage_Beam4000GeV-VeloClosed-MagDown__RealData_Reco14_Stripping21_RADIATIVE.DST.txt",
+# "FilesToStage_Beam4000GeV-VeloClosed-MagUp__RealData_Reco14_Stripping21_EW.DST.txt",
+# "FilesToStage_Beam4000GeV-VeloClosed-MagUp__RealData_Reco14_Stripping21_LEPTONIC.MDST.txt",
+# "FilesToStage_Beam4000GeV-VeloClosed-MagUp__RealData_Reco14_Stripping21r0p1a_EW.DST.txt",
+# "FilesToStage_Beam4000GeV-VeloClosed-MagUp__RealData_Reco14_Stripping21r0p1a_LEPTONIC.MDST.txt",
+# "FilesToStage_Beam4000GeV-VeloClosed-MagUp__RealData_Reco14_Stripping21r0p2_EW.DST.txt",
+# "FilesToStage_Beam4000GeV-VeloClosed-MagUp__RealData_Reco14_Stripping21r0p2_LEPTONIC.MDST.txt",
+# "FilesToStage_Beam4000GeV-VeloClosed-MagUp__RealData_Reco14_Stripping21_RADIATIVE.DST.txt",
+"LHCb_2011_Beam3500GeV_VeloClosed_MagDown_RealData_Reco14_Stripping21r1_BHADRONCOMPLETEEVENT_DST_FilesToStage.txt",
+# "LHCb_2011_Beam3500GeV_VeloClosed_MagDown_RealData_Reco14_Stripping21r1_BHADRON_MDST_FilesToStage.txt",
+# "LHCb_2011_Beam3500GeV_VeloClosed_MagDown_RealData_Reco14_Stripping21r1_CHARMCOMPLETEEVENT_DST_FilesToStage.txt",
+# "LHCb_2011_Beam3500GeV_VeloClosed_MagDown_RealData_Reco14_Stripping21r1_CHARM_MDST_FilesToStage.txt",
+# "LHCb_2011_Beam3500GeV_VeloClosed_MagDown_RealData_Reco14_Stripping21r1_DIMUON_DST_FilesToStage.txt",
+# "LHCb_2011_Beam3500GeV_VeloClosed_MagDown_RealData_Reco14_Stripping21r1p1a_BHADRONCOMPLETEEVENT_DST_FilesToStage.txt",
+# "LHCb_2011_Beam3500GeV_VeloClosed_MagDown_RealData_Reco14_Stripping21r1p1a_BHADRON_MDST_FilesToStage.txt",
+# "LHCb_2011_Beam3500GeV_VeloClosed_MagDown_RealData_Reco14_Stripping21r1p1a_CHARMCOMPLETEEVENT_DST_FilesToStage.txt",
+# "LHCb_2011_Beam3500GeV_VeloClosed_MagDown_RealData_Reco14_Stripping21r1p1a_CHARM_MDST_FilesToStage.txt",
+# "LHCb_2011_Beam3500GeV_VeloClosed_MagDown_RealData_Reco14_Stripping21r1p1a_DIMUON_DST_FilesToStage.txt",
+# "LHCb_2011_Beam3500GeV_VeloClosed_MagDown_RealData_Reco14_Stripping21r1p1a_SEMILEPTONIC_DST_FilesToStage.txt",
+# "LHCb_2011_Beam3500GeV_VeloClosed_MagDown_RealData_Reco14_Stripping21r1p2_BHADRONCOMPLETEEVENT_DST_FilesToStage.txt",
+# "LHCb_2011_Beam3500GeV_VeloClosed_MagDown_RealData_Reco14_Stripping21r1p2_BHADRON_MDST_FilesToStage.txt",
+# "LHCb_2011_Beam3500GeV_VeloClosed_MagDown_RealData_Reco14_Stripping21r1p2_CHARMCOMPLETEEVENT_DST_FilesToStage.txt",
+# "LHCb_2011_Beam3500GeV_VeloClosed_MagDown_RealData_Reco14_Stripping21r1p2_CHARM_MDST_FilesToStage.txt",
+# "LHCb_2011_Beam3500GeV_VeloClosed_MagDown_RealData_Reco14_Stripping21r1p2_DIMUON_DST_FilesToStage.txt",
+# "LHCb_2011_Beam3500GeV_VeloClosed_MagDown_RealData_Reco14_Stripping21r1p2_SEMILEPTONIC_DST_FilesToStage.txt",
+# "LHCb_2011_Beam3500GeV_VeloClosed_MagDown_RealData_Reco14_Stripping21r1_SEMILEPTONIC_DST_FilesToStage.txt",
+# "LHCb_2011_Beam3500GeV_VeloClosed_MagUp_RealData_Reco14_Stripping21r1_BHADRONCOMPLETEEVENT_DST_FilesToStage.txt",
+# "LHCb_2011_Beam3500GeV_VeloClosed_MagUp_RealData_Reco14_Stripping21r1_BHADRON_MDST_FilesToStage.txt",
+# "LHCb_2011_Beam3500GeV_VeloClosed_MagUp_RealData_Reco14_Stripping21r1_CHARMCOMPLETEEVENT_DST_FilesToStage.txt",
+# "LHCb_2011_Beam3500GeV_VeloClosed_MagUp_RealData_Reco14_Stripping21r1_CHARM_MDST_FilesToStage.txt",
+# "LHCb_2011_Beam3500GeV_VeloClosed_MagUp_RealData_Reco14_Stripping21r1_DIMUON_DST_FilesToStage.txt",
+# "LHCb_2011_Beam3500GeV_VeloClosed_MagUp_RealData_Reco14_Stripping21r1p1a_BHADRONCOMPLETEEVENT_DST_FilesToStage.txt",
+# "LHCb_2011_Beam3500GeV_VeloClosed_MagUp_RealData_Reco14_Stripping21r1p1a_BHADRON_MDST_FilesToStage.txt",
+# "LHCb_2011_Beam3500GeV_VeloClosed_MagUp_RealData_Reco14_Stripping21r1p1a_CHARMCOMPLETEEVENT_DST_FilesToStage.txt",
+# "LHCb_2011_Beam3500GeV_VeloClosed_MagUp_RealData_Reco14_Stripping21r1p1a_CHARM_MDST_FilesToStage.txt",
+# "LHCb_2011_Beam3500GeV_VeloClosed_MagUp_RealData_Reco14_Stripping21r1p1a_DIMUON_DST_FilesToStage.txt",
+# "LHCb_2011_Beam3500GeV_VeloClosed_MagUp_RealData_Reco14_Stripping21r1p1a_SEMILEPTONIC_DST_FilesToStage.txt",
+# "LHCb_2011_Beam3500GeV_VeloClosed_MagUp_RealData_Reco14_Stripping21r1p2_BHADRONCOMPLETEEVENT_DST_FilesToStage.txt",
+# "LHCb_2011_Beam3500GeV_VeloClosed_MagUp_RealData_Reco14_Stripping21r1p2_BHADRON_MDST_FilesToStage.txt",
+# "LHCb_2011_Beam3500GeV_VeloClosed_MagUp_RealData_Reco14_Stripping21r1p2_CHARMCOMPLETEEVENT_DST_FilesToStage.txt",
+# "LHCb_2011_Beam3500GeV_VeloClosed_MagUp_RealData_Reco14_Stripping21r1p2_CHARM_MDST_FilesToStage.txt",
+# "LHCb_2011_Beam3500GeV_VeloClosed_MagUp_RealData_Reco14_Stripping21r1p2_DIMUON_DST_FilesToStage.txt",
+# "LHCb_2011_Beam3500GeV_VeloClosed_MagUp_RealData_Reco14_Stripping21r1p2_SEMILEPTONIC_DST_FilesToStage.txt",
+# "LHCb_2011_Beam3500GeV_VeloClosed_MagUp_RealData_Reco14_Stripping21r1_SEMILEPTONIC_DST_FilesToStage.txt",
+# "LHCb_2012_Beam4000GeV_VeloClosed_MagDown_RealData_Reco14_Stripping21_BHADRONCOMPLETEEVENT_DST_FilesToStage.txt",
+# "LHCb_2012_Beam4000GeV_VeloClosed_MagDown_RealData_Reco14_Stripping21_BHADRON_MDST_FilesToStage.txt",
+# "LHCb_2012_Beam4000GeV_VeloClosed_MagDown_RealData_Reco14_Stripping21_CHARMCOMPLETEEVENT_DST_FilesToStage.txt",
+# "LHCb_2012_Beam4000GeV_VeloClosed_MagDown_RealData_Reco14_Stripping21_CHARM_MDST_FilesToStage.txt",
+# "LHCb_2012_Beam4000GeV_VeloClosed_MagDown_RealData_Reco14_Stripping21_DIMUON_DST_FilesToStage.txt",
+# "LHCb_2012_Beam4000GeV_VeloClosed_MagDown_RealData_Reco14_Stripping21r0p1a_BHADRONCOMPLETEEVENT_DST_FilesToStage.txt",
+# "LHCb_2012_Beam4000GeV_VeloClosed_MagDown_RealData_Reco14_Stripping21r0p1a_BHADRON_MDST_FilesToStage.txt",
+# "LHCb_2012_Beam4000GeV_VeloClosed_MagDown_RealData_Reco14_Stripping21r0p1a_CHARMCOMPLETEEVENT_DST_FilesToStage.txt",
+# "LHCb_2012_Beam4000GeV_VeloClosed_MagDown_RealData_Reco14_Stripping21r0p1a_CHARM_MDST_FilesToStage.txt",
+# "LHCb_2012_Beam4000GeV_VeloClosed_MagDown_RealData_Reco14_Stripping21r0p1a_DIMUON_DST_FilesToStage.txt",
+# "LHCb_2012_Beam4000GeV_VeloClosed_MagDown_RealData_Reco14_Stripping21r0p1a_SEMILEPTONIC_DST_FilesToStage.txt",
+# "LHCb_2012_Beam4000GeV_VeloClosed_MagDown_RealData_Reco14_Stripping21r0p2_BHADRONCOMPLETEEVENT_DST_FilesToStage.txt",
+# "LHCb_2012_Beam4000GeV_VeloClosed_MagDown_RealData_Reco14_Stripping21r0p2_BHADRON_MDST_FilesToStage.txt",
+# "LHCb_2012_Beam4000GeV_VeloClosed_MagDown_RealData_Reco14_Stripping21r0p2_CHARMCOMPLETEEVENT_DST_FilesToStage.txt",
+# "LHCb_2012_Beam4000GeV_VeloClosed_MagDown_RealData_Reco14_Stripping21r0p2_CHARM_MDST_FilesToStage.txt",
+# "LHCb_2012_Beam4000GeV_VeloClosed_MagDown_RealData_Reco14_Stripping21r0p2_DIMUON_DST_FilesToStage.txt",
+# "LHCb_2012_Beam4000GeV_VeloClosed_MagDown_RealData_Reco14_Stripping21r0p2_SEMILEPTONIC_DST_FilesToStage.txt",
+# "LHCb_2012_Beam4000GeV_VeloClosed_MagDown_RealData_Reco14_Stripping21_SEMILEPTONIC_DST_FilesToStage.txt",
+# "LHCb_2012_Beam4000GeV_VeloClosed_MagUp_RealData_Reco14_Stripping21_BHADRONCOMPLETEEVENT_DST_FilesToStage.txt",
+# "LHCb_2012_Beam4000GeV_VeloClosed_MagUp_RealData_Reco14_Stripping21_BHADRON_MDST_FilesToStage.txt",
+# "LHCb_2012_Beam4000GeV_VeloClosed_MagUp_RealData_Reco14_Stripping21_CHARMCOMPLETEEVENT_DST_FilesToStage.txt",
+# "LHCb_2012_Beam4000GeV_VeloClosed_MagUp_RealData_Reco14_Stripping21_CHARM_MDST_FilesToStage.txt",
+# "LHCb_2012_Beam4000GeV_VeloClosed_MagUp_RealData_Reco14_Stripping21_DIMUON_DST_FilesToStage.txt",
+# "LHCb_2012_Beam4000GeV_VeloClosed_MagUp_RealData_Reco14_Stripping21r0p1a_BHADRONCOMPLETEEVENT_DST_FilesToStage.txt",
+# "LHCb_2012_Beam4000GeV_VeloClosed_MagUp_RealData_Reco14_Stripping21r0p1a_BHADRON_MDST_FilesToStage.txt",
+# "LHCb_2012_Beam4000GeV_VeloClosed_MagUp_RealData_Reco14_Stripping21r0p1a_CHARMCOMPLETEEVENT_DST_FilesToStage.txt",
+# "LHCb_2012_Beam4000GeV_VeloClosed_MagUp_RealData_Reco14_Stripping21r0p1a_CHARM_MDST_FilesToStage.txt",
+# "LHCb_2012_Beam4000GeV_VeloClosed_MagUp_RealData_Reco14_Stripping21r0p1a_DIMUON_DST_FilesToStage.txt",
+# "LHCb_2012_Beam4000GeV_VeloClosed_MagUp_RealData_Reco14_Stripping21r0p1a_SEMILEPTONIC_DST_FilesToStage.txt",
+# "LHCb_2012_Beam4000GeV_VeloClosed_MagUp_RealData_Reco14_Stripping21r0p2_BHADRONCOMPLETEEVENT_DST_FilesToStage.txt",
+# "LHCb_2012_Beam4000GeV_VeloClosed_MagUp_RealData_Reco14_Stripping21r0p2_BHADRON_MDST_FilesToStage.txt",
+# "LHCb_2012_Beam4000GeV_VeloClosed_MagUp_RealData_Reco14_Stripping21r0p2_CHARMCOMPLETEEVENT_DST_FilesToStage.txt",
+# "LHCb_2012_Beam4000GeV_VeloClosed_MagUp_RealData_Reco14_Stripping21r0p2_CHARM_MDST_FilesToStage.txt",
+# "LHCb_2012_Beam4000GeV_VeloClosed_MagUp_RealData_Reco14_Stripping21r0p2_DIMUON_DST_FilesToStage.txt",
+# "LHCb_2012_Beam4000GeV_VeloClosed_MagUp_RealData_Reco14_Stripping21r0p2_SEMILEPTONIC_DST_FilesToStage.txt",
+# "LHCb_2012_Beam4000GeV_VeloClosed_MagUp_RealData_Reco14_Stripping21_SEMILEPTONIC_DST_FilesToStage.txt",
+]
+
+for file in Files:
+    string = f"lb-dirac fts-rest-transfer-submit -s https://fts3-pilot.cern.ch:8446 -f {file} &"
+    print(string)
+    os.system(string)

lhcb-YYYY-open-data-curation/scripts/glossary/parse_glossary.py

@@ -0,0 +1,30 @@
+import json
+import html2text 
+
+gloss = []
+i=0
+with open("./loki.json", 'r') as origfile:
+    origdict = json.load(origfile)
+
+    for key in origdict.keys():
+
+        if "<" or ">" in origdict[key]["documentation"].split("</p>")[0].removeprefix("<div class=\"memdoc\">\n<p>"):
+            shortdef = ""
+        else:
+            shortdef = html2text.html2text(origdict[key]["documentation"].split("</p>")[0].removeprefix("<div class=\"memdoc\">\n<p>").strip())
+
+        gloss.append({
+            "anchor": f"{key}",
+            "category": "generic",
+            "definition" : origdict[key]["documentation"],
+            "short_definition" : shortdef,
+            "term" : [f"{key}"],
+            "type": {
+            "primary": "Glossary"
+            }            
+        },)
+        i += 1
+
+    print(i)
+    with open("glossfromnwizz.json", "w") as f:
+        json.dump(gloss, f, indent=2, sort_keys=True)

lhcb-YYYY-open-data-curation/scripts/make_release.py

@@ -0,0 +1,17 @@
+import yaml
+import subprocess 
+
+# Read in the configuration file 
+with open("../config.yaml", "r") as f: 
+    conf = yaml.safe_load(f)
+
+# Run the metadata_writer on each directory+stream combination 
+for dir in conf["directory"]:
+	for stream in conf["stream"]:
+		    processes = [subprocess.Popen(f'lb-dirac python metadata_writer.py --BK=\"{dir+stream}\" --staging &', shell=True)]
+
+# Wait for all processes to finish
+for p in processes:
+	p.wait()
+
+print(f'Metadata to be written to {conf["release_dir"]}')