Merge pull request #22 from willem0boone/dev

Dev

README.md

@@ -1,3 +1,4 @@
+[![DOI](https://sandbox.zenodo.org/badge/851485026.svg)](https://handle.stage.datacite.org/10.5072/zenodo.109216)
 # Edito Resampling datasets
 
 ## About

codemeta.json

@@ -15,7 +15,7 @@
     "@type": "ComputerLanguage",
     "name": "Python",
     "url": "https://www.python.org/"
-    },
+    }
    ],
   "operatingSystem": "Windows 11",
   "runtimePlatform": ["Python 3.12.1"],
@@ -180,7 +180,7 @@
       "name": "xarray_datatree",
       "version": "0.0.14"
     }
-  }
+  },
   "keywords": ["resampling", "zarr", "EDITO-INFRA"],
   "readme": "https://github.com/willem0boone/Edito_resampling_datasets/blob/main/README.md"
 }

docs/source/API.rst

@@ -1,55 +1,55 @@
-resampling package
+API
 ==================
 
-resampling.down\_scale
+object_store
 -----------------------------
 
-.. automodule:: resampling.down_scale
+.. automodule:: resampling.object_store
    :members:
    :undoc-members:
    :show-inheritance:
 
-resampling.extract
--------------------------
+my_store
+-----------------------------
 
-.. automodule:: resampling.extract
+.. automodule:: resampling.my_store
    :members:
    :undoc-members:
    :show-inheritance:
 
-resampling.load
-----------------------
+define_windows
+-----------------------------
 
-.. automodule:: resampling.load
+.. automodule:: resampling.define_windows
    :members:
    :undoc-members:
    :show-inheritance:
 
-resampling.loggers
--------------------------
+down\_scale
+-----------------------------
 
-.. automodule:: resampling.loggers
+.. automodule:: resampling.down_scale
    :members:
    :undoc-members:
    :show-inheritance:
 
-resampling.plot\_logs
+plot\_logs
 ----------------------------
 
 .. automodule:: resampling.plot_logs
    :members:
    :undoc-members:
    :show-inheritance:
 
-resampling.plot\_zarr
+plot\_zarr
 ----------------------------
 
 .. automodule:: resampling.plot_zarr
    :members:
    :undoc-members:
    :show-inheritance:
 
-resampling.transform
+transform
 ---------------------------
 
 .. automodule:: resampling.transform

docs/source/conf.py

@@ -1,13 +1,37 @@
+import json
 # Configuration file for the Sphinx documentation builder.
 
 # -- Project information
 
-project = 'Edito_resampling_datasets'
+project = 'Resampling datasets'
 copyright = '2024, Willem Boone'
 author = 'Willem Boone'
 
-release = '0.1'
-version = '0.1.0'
+
+def extract_version_from_file(file_path):
+    try:
+        # Open and read the JSON file
+        with open(file_path, 'r') as file:
+            data = json.load(file)
+
+        # Extract and return the version
+        return data.get('version', 'Version not found')
+
+    except FileNotFoundError:
+        return f"Error: The file '{file_path}' was not found."
+
+    except json.JSONDecodeError:
+        return "Error: The file could not be decoded as JSON."
+
+    except Exception as e:
+        return f"An unexpected error occurred: {e}"
+
+
+# release = '0.1'
+# version = '0.1.0'
+
+codemeta = "../codemeta.json"
+version = extract_version_from_file(codemeta)
 
 # -- General configuration
 
@@ -34,3 +58,11 @@
 
 # -- Options for EPUB output
 epub_show_urls = 'footnote'
+
+html_context = {
+    "display_github": True, # Integrate GitHub
+    "github_user": "willem0boone", # Username
+    "github_repo": "Edito_resampling_datasets", # Repo name
+    "github_version": "master", # Version
+    "conf_py_path": "/docs/source/", # Path in the checkout to the docs root
+}

docs/source/index.rst

@@ -1,12 +1,34 @@
-Welcome to documentation
+Resampling datasets
 ========================
-To be created.
+
+
+About
+------
+
+This project is part of EDITO-INFRA (`Grant agreement ID: 101101473 <https://doi.org/10.3030/101101473>`_):
+
+    *T7.3: End-to-end demonstrator for aquaculture and maritime industry*
+
+Check out `this section <https://github.com/willem0boone/Edito_resampling_datasets/blob/main/README.md#goal>`_ for more information about the project.
+
+Developer:
+
+* **Author**: Willem Boone
+* **Contact**: `[email protected] <mailto:[email protected]>`_
+* **GitHub**: `Edito_resampling_datasets <https://github.com/willem0boone/Edito_resampling_datasets>`_
 
 
 Contents
 --------
 
 .. toctree::
+   :maxdepth: 1
+   :titlesonly:
+
+   installation
    usage
    API
 
+
+
+

docs/source/installation.rst

@@ -0,0 +1,63 @@
+Installation
+============
+
+Installation
+------------
+
+Using pip install:
+
+.. code-block:: console
+
+    pip install git+https://github.com/willem0boone/Edito_resampling_datasets
+
+
+Configuration
+-------------
+
+For IO with your S3 storage, the :class:`resampling.object_store.ObjectStore`
+class is used. The initiation of this class requires your S3 credentials.
+
+You can configure this package by storing your S3 credentials in a config file.
+Doing so, you can use :func:`resampling.my_store.get_my_store` to  create an
+instance of :class:`resampling.object_store.ObjectStore`.
+
+:func:`resampling.my_store.get_my_store` will look by default at a
+config file stored at:
+
+    resampling/config/config.toml
+
+.. code-block:: python
+
+    my_store = get_my_store()
+
+However, you can also provide your own config.toml file.
+
+.. code-block:: python
+
+    my_store = get_my_store('path/to/my_config.toml')
+
+
+The content of the file should look like this:
+
+.. code-block:: python
+
+    endpoint_url=''
+    bucket=''
+    aws_access_key_id=''
+    aws_secret_access_key=''
+    aws_session_token=''
+
+
+Alternatively, you can initiate :class:`resampling.object_store.ObjectStore`
+manually as follows:
+
+.. code-block:: python
+
+    my_object_store = ObjectStore(
+        endpoint_url='str',
+        aws_access_key_id='str',
+        aws_secret_access_key='str',
+        aws_session_token='str',
+        bucket='str',
+    )
+

docs/source/pyramids.rst

@@ -0,0 +1,120 @@
+Make Pyramid
+============
+
+Implement `ndpyramid <https://github.com/carbonplan/ndpyramid/tree/main>`_
+(Credit: Carbonplan) to create a pyramid datatree.datatree 'pyramid' in which
+each level has a higher resolution. Such pyramids are compatible with several
+Carbonplan webviewer and can be tested in
+`ncview <https://ncview-js.staging.carbonplan.org/>`_.
+
+1. Imports
+----------
+.. code-block:: python
+
+    import sys
+    from pathlib import Path
+    import numpy as np
+    import xarray as xr
+    import pandas as pd
+    import matplotlib.pyplot as plt
+
+Import Resampling modules
+
+.. code-block:: python
+
+    parent_dir = Path().resolve().parent
+    sys.path.append(str(parent_dir))
+    from resampling.my_store import get_my_store
+    from resampling.transform import make_pyramid
+    from resampling.transform import expand_to_global_coverage
+
+2. Load datasets
+----------------
+
+.. code-block:: python
+
+    def extract_all_ds():
+        my_dataset = xr.Dataset()
+
+        bathy = "https://minio.lab.dive.edito.eu/oidc-willemboone/EDITO_DUC/EDITO_DUC_bathymetry.zarr"
+        saly = "https://minio.lab.dive.edito.eu/oidc-willemboone/EDITO_DUC/EDITO_DUC_SALINITY_baseline.zarr"
+        temp = "https://minio.lab.dive.edito.eu/oidc-willemboone/EDITO_DUC/EDITO_DUC_SST_baseline.zarr"
+
+        ds_bathy = xr.open_zarr(bathy)
+        ds_saly = xr.open_zarr(saly).sel(time="2010-01-01")
+        ds_temp = xr.open_zarr(temp).sel(time="2010-01-01")
+
+        my_dataset["saly"] = xr.DataArray(ds_saly["average_sea_water_practical_salinity_biooracle_baseline"].values,
+                                     dims=("latitude", "longitude"),
+                                     coords={"latitude": ds_saly.latitude.values,
+                                             "longitude": ds_saly.longitude.values}
+                                    )
+        my_dataset["SST"] = xr.DataArray(ds_temp["average_sea_water_temperature_biooracle_baseline"].values,
+                                         dims=("latitude", "longitude"),
+                                         coords={"latitude": ds_temp.latitude.values,
+                                                 "longitude": ds_temp.longitude.values}
+                                        )
+        my_dataset["elevation"] = xr.DataArray(ds_bathy["elevation"].values,
+                                               dims=("latitude", "longitude"),
+                                              )
+
+        return my_dataset
+
+.. code-block:: python
+
+    ds = extract_all_ds()
+
+3. Convert datasets
+-------------------
+
+Ensure global coverage
+^^^^^^^^^^^^^^^^^^^^^^
+
+Expand to global coverage, make sure the lat & lon step are conform the lat/lon
+resolution of the dataset. Carbonplan maps requires the datasets to have a
+global extend. Therefore the function *expand_to_global_coverage* can be used.
+
+.. code-block:: python
+
+    global_ds = expand_to_global_coverage(ds, step_lon=0.1, step_lat=0.1)
+    print(global_ds)
+
+Notice that matplotlib & xarray don't always understands the meaning of
+latitude/longitude as geographical coordinates. On the y axis, coordinates go
+from 1800 (north pole> to 900 (equator> to 0 (south pole>. Notice that this is
+high to low while the plot uses low to high. so the plotting of the map is
+mirrored but it doesn't affect the dataset and pyramid building.
+Create pyramid
+^^^^^^^^^^^^^^
+
+.. code-block:: python
+
+    merged_pyramid = make_pyramid(
+        ds=ds,
+        pixels_per_tile=128,
+        version=0.17,
+        levels=6,
+    )
+
+4. Extract
+----------
+
+.. code-block:: python
+
+    my_store = get_my_store()
+    my_store.write_zarr(merged_pyramid, name="tutorial_pyramid.zarr")
+
+5. Make data public accessible
+------------------------------
+
+Run this command in a terminal in the datalab to make your dataset public
+available:
+.. code-block:: console
+
+    mc anonymous set public "s3/oidc-willemboone/tutorial_pyramid.zarr"
+
+Copy the link of your dataset:
+
+    https://minio.lab.dive.edito.eu/oidc-willemboone/EDITO_DUC/tutorial_pyramid.zarr
+
+And visualise is in this webviewer: https://ncview-js.staging.carbonplan.org/