merged changes

.github/workflows/documentation.yml

@@ -0,0 +1,27 @@
+name: documentation
+
+on: [push]
+
+permissions:
+  contents: write
+
+jobs:
+  docs:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v3
+      - name: Install dependencies
+        run: |
+          pip install sphinx sphinx_rtd_theme myst_parser renku-sphinx-theme
+      - name: Sphinx build
+        run: |
+          sphinx-build docs build
+      - name: Deploy to GitHub Pages
+        uses: peaceiris/actions-gh-pages@v3
+        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
+        with:
+          publish_branch: gh-pages
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: build/
+          force_orphan: true

docs/.buildinfo

@@ -0,0 +1,4 @@
+# Sphinx build info version 1
+# This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
+config: bc337f8532e856cf6f060ca59d0e6fab
+tags: 645f666f9bcd5a90fca523b33c5a78b7

docs/FAQ.rst

@@ -0,0 +1,2 @@
+FAQ
+===

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
+SOURCEDIR     = source
+BUILDDIR      = build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/conf.py

@@ -0,0 +1,28 @@
+# Configuration file for the Sphinx documentation builder.
+#
+# For the full list of built-in configuration values, see the documentation:
+# https://www.sphinx-doc.org/en/master/usage/configuration.html
+
+# -- Project information -----------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information
+
+project = 'Fre-Cli'
+copyright = '2024, Bennett Chang'
+author = 'Bennett Chang'
+release = '1.0'
+
+# -- General configuration ---------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration
+
+extensions = []
+
+templates_path = ['_templates']
+exclude_patterns = []
+
+
+
+# -- Options for HTML output -------------------------------------------------
+# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
+
+html_theme = 'renku'
+html_static_path = ['_static']

docs/index.rst

@@ -0,0 +1,23 @@
+.. Fre-Cli documentation master file, created by
+   sphinx-quickstart on Wed Mar  6 22:28:21 2024.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to Fre-Cli's documentation!
+===================================
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+   setup
+   usage
+   subtools
+   FAQ
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`

docs/setup.rst

@@ -0,0 +1,32 @@
+Setup
+=====
+
+Need to set up Conda environment first and foremost
+
+If on workstation:
+module load conda
+
+Create new Conda environment
+conda create -n [environmentName]
+
+Append necessary channels
+conda config --append channels noaa-gfdl
+conda config --append channels conda-forge
+
+Run conda install on needed dependencies
+conda install noaa-gfdl::fre-cli should install the CLI package located at https://anaconda.org/NOAA-GFDL/fre-cli created from the meta.yaml file
+
+All other dependencies used by the tools are installed along with this install (configured inside the meta.yaml), with the exception of local modules
+setup.py file allows fre.py to be ran with fre as the entry point on the command line instead of python fre.py
+
+Enter commands and follow --help messages for guidance (brief rundown of commands also provided below)
+
+If the user just runs fre, it will list all the command groups following fre, such as run, make, pp, etc. and once the user specifies a command group, the list of available subcommands for that group will be shown
+
+Commands that require arguments to run will alert user about missing arguments, and will also list the rest of the optional parameters if --help is executed
+
+Argument flags are not positional, can be specified in any order as long as they are specified
+
+Can run directly from any directory, no need to clone repository
+
+May need to deactivate environment and reactivate it in order for changes to apply

docs/subtools.rst

@@ -0,0 +1,20 @@
+Subtools
+========
+
+frecheck
+--------
+
+frelist
+-------
+
+fremake
+-------
+
+frepp
+-----
+
+frerun
+------
+
+fretest
+-------

docs/usage.rst

@@ -0,0 +1,156 @@
+Usage
+=====
+
+User Usage
+----------
+
+**Conda environment set up**
+
+Load Conda
+
+.. code-block::console
+ module load conda
+
+Create new Conda environment
+
+.. code-block::console
+ conda create -n [environmentName]
+
+Append necessary channels
+
+.. code-block::console
+ conda config --append channels noaa-gfdl
+ conda config --append channels conda-forge
+
+Install needed dependencies
+
+.. code-block::console
+ conda install noaa-gfdl::fre-cli
+ 
+setup.py file allows fre.py to be ran with fre as the entry point on the command line instead of python fre.py
+
+Enter commands and follow *--help* messages for guidance (brief rundown of commands also provided below)
+
+If the user just runs *fre*, it will list all the command groups following *fre*, such as *run*, *make*, *pp*, etc. and once the user specifies a command group, the list of available subcommands for that group will be shown
+
+Commands that require arguments to run will alert user about missing arguments, and will also list the rest of the optional parameters if *--help* is executed
+
+Argument flags are not positional, can be specified in any order as long as they are specified
+
+Can run directly from any directory, no need to clone repository
+
+May need to deactivate environment and reactivate it in order for changes to apply
+
+
+Tools
+-----
+
+A few subtools are currently in development:
+
+**fre pp**
+
+1. configure 
+
+* Postprocessing yaml configuration
+* Minimal Syntax: *fre pp configure -y [user-edit yaml file]*
+* Module(s) needed: n/a
+* Example: *fre pp configure -y /home/$user/pp/ue2/user-edits/edits.yaml*
+
+2. checkout
+
+* Checkout template file and clone gitlab.gfdl.noaa.gov/fre2/workflows/postprocessing.git repository
+* Minimal Syntax: *fre pp checkout -e [experiment name] -p [platform name] -t [target name]*
+* Module(s) needed: n/a
+* Example: *fre pp checkout -e c96L65_am5f4b4r0_amip -p gfdl.ncrc5-deploy -t prod-openmp*
+
+
+**fre catalog**
+
+1. buildCatalog1
+* Builds json and csv format catalogs from user input directory path
+* Minimal Syntax: *fre catalog buildCatalog -i [input path] -o [output path]*
+* Module(s) needed: n/a
+* Example: *fre catalog buildCatalog -i /archive/am5/am5/am5f3b1r0/c96L65_am5f3b1r0_pdclim1850F/gfdl.ncrc5-deploy-prod-openmp/pp -o ~/output --overwrite*
+
+**To be developed:**
+
+#. fre check
+#. fre list
+#. fre make
+#. fre run
+#. fre test
+#. fre yamltools
+
+
+Usage (Developers)
+------------------
+
+Developers are free to use the user guide above to familiarize with the CLI and save time from having to install any dependencies, but development within a Conda environment is heavily recommended regardless
+
+Gain access to the repository with *git clone [email protected]:NOAA-GFDL/fre-cli.git* or your fork's link (recommended) and an SSH RSA key
+
+Once inside the repository, developers can test local changes by running a *pip install .* inside of the root directory to install the fre-cli package locally with the newest local changes
+
+Test as a normal user would use the CLI
+
+**Adding New Tools - Checklist**
+
+If there is *no* subdirectory created for the new tool you are trying to develop, there are a few steps to follow:
+
+1. Create a subdirectory for the tool group inside the /fre folder; i.e. /fre/fre(subTool)
+
+2. Add an *__init__.py* inside of the new subdirectory
+
+* This will contain one line, *from fre.fre(subTool) import **
+* The purpose of this line is to allow the subTool module to include all the scripts and functions within it when invoked by fre
+
+3. Add a file named *fre(subTool).py*. This will serve as the main file to house all of the tool's related subcommands
+
+4. Add a Click group named after the subTool within *fre(subTool).py*
+
+* This group will contain all of the subcommands
+
+5. Create separate files to house the code for each different subcommand; do not code out the full implemetation of a function inside of a Click command within *fre(subTool).py*
+
+6. Be sure to import the contents of the needed subcommand scripts inside of fre(subTool).py
+
+* i.e. from fre.fre(subTool).subCommandScript import *
+
+7. At this point, you can copy and paste the parts of your main Click subcommand from its script into *fre(subTool).py* when implementing the function reflective of the subcommand function
+
+* Everything will remain the same; i.e. arguments, options, etc.
+
+* However, this new function within *fre(subTool).py* must a new line after the arguments, options, and other command components; *@click.pass_context*
+
+* Along with this, a new argument "context" must now be added to the parameters of the command (preferably at the beginning, but it won't break it if it's not)
+
+8. From here, all that needs to be added after defining the command with a name is *context.forward(mainFunctionOfSubcommand)*, and done!
+
+9. After this step, it is important to add *from fre.fre(subTool) import* to the *__init__.py* within the /fre folder
+
+10. The last step is to replicate the subcommand in the same way as done in *fre(subTool).py* inside of *fre.py*, but make sure to add *from fre import fre(subTool)* and *from fre.fre(subTool).fre(subTool) import **
+
+Please refer to this issue when encountering naming issues: `NOAA-GFDL#31 <https://github.com/NOAA-GFDL/fre-cli/issues/31>`_
+
+**Adding Tools From Other Repositories**
+
+Currently, the solution to this task is to approach it using Conda packages. The tool that is being added must reside within a repository that contains a meta.yaml that includes Conda dependencies like the one in this repository and ideally a setup.py (may be subject to change due to deprecation) that may include any potentially needed pip dependencies
+
+* Once published as a Conda package, ideally on the NOAA-GFDL channel at https://anaconda.org/NOAA-GFDL, an addition can be made to the "run" section under the "requirements" category in the meta.yaml of the fre-cli following the syntax channel::package
+
+* On pushes to the main branch, the package located at https://anaconda.org/NOAA-GFDL/fre-cli will automatically be updated using the workflow file
+
+**MANIFEST.in**
+
+In the case where non-python files like templates, examples, and outputs are to be included in the fre-cli package, MANIFEST.in can provide the solution. Ensure that the file exists within the correct folder, and add a line to the MANIFEST.in file saying something like *include fre/fre(subTool)/fileName.fileExtension*
+
+* For more efficiency, if there are multiple files of the same type needed, the MANIFEST.in addition can be something like *recursive-include fre/fre(subTool) *.fileExtension* which would recursively include every file matching that fileExtension within the specified directory and its respective subdirectories.
+
+**Example /fre Directory Structure**
+.
+├── __init__.py
+├── fre.py
+├── fre(subTool)
+│   ├── __init__.py
+│   ├── subCommandScript.py
+│   └── fre(subTool).py

fre/__init__.py

@@ -5,3 +5,5 @@
 from .run import *
 from .catalog import *
 from .yamltools import *
+from .app import *
+from .cmor import *

fre/app/__init__.py

@@ -0,0 +1,3 @@
+from .maskAtmosPlevel import maskAtmosPlevel_subtool
+
+__all__ = ["maskAtmosPlevel_subtool"]