Merge pull request #926 from OpenSourceBrain/release/0.7.6

Release/0.7.6
OpenSourceBrain · Apr 26, 2024 · b293f8c · b293f8c
2 parents 7321230 + 14a58db
commit b293f8c
Show file tree

Hide file tree

Showing 116 changed files with 712,121 additions and 1,744 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,11 +1,14 @@
 cloud-harness
 .idea
-**/helm
+deployment/helm
 /build
 .vscode
 skaffold.yaml
 /deployment.yaml
 .pytest_cache
 /.overrides
 .hypothesis
-skaffold-overrides.yaml
+/libraries/client/__pycache__
+/libraries/client/github.auth
+skaffold-overrides.yaml
+/libraries/client/netpyne-web/
diff --git a/applications/accounts-api/backend/README.md b/applications/accounts-api/backend/README.md
@@ -8,7 +8,7 @@ is an example of building a OpenAPI-enabled Flask server.
 This example uses the [Connexion](https://github.com/zalando/connexion) library on top of Flask.
 
 ## Requirements
-Python 3.5.2+
+Python 3.9+
 
 ## Usage
 To run the server, please execute the following from the root directory:

diff --git a/applications/accounts-api/backend/accounts_api/services/user_service.py b/applications/accounts-api/backend/accounts_api/services/user_service.py
@@ -95,6 +95,7 @@ def update_user(userid, user: User):
         updated_user = {
             'firstName': user.first_name or current_user['firstName'],
             'lastName': user.last_name or current_user['lastName'],
+            'email': user.email or current_user['email'],
             'attributes': {
                 **(current_user.get('attributes') or {}),
                 **({('profile--' + k): user.profiles[k] for k in user.profiles} if user.profiles else {}),

diff --git a/applications/accounts/deploy/values.yaml b/applications/accounts/deploy/values.yaml
@@ -0,0 +1,6 @@
+harness:
+  database:
+    resources:
+      requests:
+        memory: 128Mi
+        cpu: 10m
diff --git a/applications/accounts/themes/custom/login/login.ftl b/applications/accounts/themes/custom/login/login.ftl
@@ -5,7 +5,7 @@
             <span>${msg("doLogIn")} </span>
             <#if realm.password && realm.registrationAllowed && !registrationDisabled??>
                 <div id="kc-registration" class="kc-register">
-                    <span>${msg("noAccount")} <a tabindex="6" href="${url.registrationUrl}">${msg("doRegister")}</a></span>
+                    <span><a tabindex="6" href="${url.registrationUrl}">${msg("noAccount")} ${msg("doRegister")}</a></span>
                 </div>
             </#if>
        </div>

diff --git a/applications/accounts/themes/custom/login/register.ftl b/applications/accounts/themes/custom/login/register.ftl
@@ -5,7 +5,7 @@
         <span>${msg("registerTitle")}</span>
         <div id="kc-form-options">
             <div class="${properties.kcFormOptionsWrapperClass!}">
-                <span>Already have an account? <a href="${url.loginUrl}">${kcSanitize(msg("Log In"))?no_esc}</a></span>
+                <span><a href="${url.loginUrl}">Already have an account? ${kcSanitize(msg("Log In"))?no_esc}</a></span>
             </div>
         </div>
         </div>

diff --git a/applications/jupyterhub/deploy/values-prod.yaml b/applications/jupyterhub/deploy/values-prod.yaml
@@ -23,9 +23,9 @@ singleuser:
     capacity: 2Gi
   cpu:
     limit: 1
-    guarantee: 0.2
+    guarantee: 0.02
   memory:
-    limit: 1.5G
-    guarantee: 0.5G
+    limit: 0.5G
+    guarantee: 0.1G
 legacyusermax: 445
 legacyworkspacemax: 465
diff --git a/applications/jupyterhub/deploy/values.yaml b/applications/jupyterhub/deploy/values.yaml
@@ -46,7 +46,7 @@ singleuser:
   storage:
     type: dynamic
     capacity: 2Mi
-
+    startTimeout: 3000
     dynamic:
       pvcNameTemplate: osb-user-{userid}
       volumeNameTemplate: osb-user-{userid}
@@ -57,10 +57,10 @@ singleuser:
     extraLabels: {}
   cpu:
     limit: 1
-    guarantee: 0.2
+    guarantee: 0.02
   memory:
-    limit: 1G
-    guarantee: 0.5G
+    limit: 0.5G
+    guarantee: 0.1G
 prePuller:
 
   # hook relates to the hook-image-awaiter Job and hook-image-puller DaemonSet
@@ -69,3 +69,8 @@ prePuller:
   continuous:
     enabled: false
   pullProfileListImages: false
+scheduling:
+  userPods:
+    nodeAffinity:
+      matchNodePurpose: require
+
diff --git a/applications/jupyterhub/src/osb_jupyter/osb_jupyter/osb_jupyterhub.py b/applications/jupyterhub/src/osb_jupyter/osb_jupyter/osb_jupyterhub.py
@@ -8,6 +8,8 @@
 from cloudharness.auth.exceptions import UserNotFound
 from urllib.parse import parse_qs, urlparse
 
+from harness_jupyter.jupyterhub import set_key_value
+
 allowed_chars = set(
     "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
 
@@ -33,6 +35,7 @@ class CookieNotFound(Exception):
     pass
 
 
+
 def change_pod_manifest(self: KubeSpawner):
     """
     Application Hook to change the manifest of the notebook image
@@ -135,8 +138,30 @@ def workspace_volume_is_legacy(workspace_id):
         self.storage_pvc_ensure = False
         self.volumes = []
         self.volume_mounts = []
-        print("Starting anonymoous session with no volumes")
 
+        print("Setting user quota cpu/mem usage")
+        from cloudharness.applications import get_current_configuration, get_configuration
+        try:
+            subdomain = self.handler.request.host.split(
+                str(self.config['domain']))[0][0:-1]
+            appname = next(app["name"] for app in self.config['apps'].values(
+            ) if app["harness"]["subdomain"] == subdomain)
+            app_conf = get_configuration(appname).to_dict()
+            cpu_conf = app_conf.get("singleuser", {}).get("cpu", {})
+            mem_conf = app_conf.get("singleuser", {}).get("memory", {})
+        except StopIteration:
+            cpu_conf = {}
+            mem_conf = {}
+
+        jh_conf = get_configuration('jupyterhub')
+        cpu_conf = {**jh_conf.singleuser["cpu"].to_dict(), **cpu_conf}
+        mem_conf = {**jh_conf.singleuser["memory"].to_dict(), **mem_conf}
+        set_key_value(self, key="cpu_guarantee", value=cpu_conf["guarantee"])
+        set_key_value(self, key="cpu_limit", value=cpu_conf["limit"])
+        set_key_value(self, key="mem_guarantee",
+                      value=mem_conf["guarantee"])
+        set_key_value(self, key="mem_limit", value=mem_conf["limit"])
+        print("Starting anonymous session with no volumes")
     except Exception as e:
         log.error('Change pod manifest failed due to an error.', exc_info=True)
 
@@ -159,8 +184,6 @@ def workspace_volume_is_legacy(workspace_id):
     })
 
 
-
-
 def get_app_user(user: User):
     auth_client = AuthClient()
     kc_user = auth_client.get_user(user.name)

diff --git a/applications/jupyterhub/theming/page.html b/applications/jupyterhub/theming/page.html
@@ -131,6 +131,11 @@ <h4 class="modal-title" id="{{key}}-label">{{title}}</h4>
     margin-bottom: 57px;
   }
 
+  #warning-message {
+    color: #ffcc00;
+    margin: 10px;
+  }
+
   .spawn-container {
     display: flex;
     align-items: center;

diff --git a/applications/jupyterhub/theming/spawn_pending.html b/applications/jupyterhub/theming/spawn_pending.html
@@ -8,6 +8,7 @@
       {% block message %}
       <p>We are loading the workspace…</p>
       {% endblock %}
+      <p id="warning-message"></p>
       <p id="progress-message"></p>
       <div class="progress">
         <div id="progress-bar" class="progress-bar" role="progressbar" aria-valuenow="0" aria-valuemin="0"
@@ -56,9 +57,13 @@
       var html_message;
       if (evt.html_message !== undefined) {
         progressMessage.html(evt.html_message);
+
         html_message = evt.html_message;
       } else if (evt.message !== undefined) {
         progressMessage.text(evt.message);
+        if(evt.message.includes("anonymous-")) {
+          $("#warning-message").html(`You are running as an anonymous user. The current session will have limited resources and might be deleted at any time. Please <a target="_blank" href="//${window.location.host.split(".").splice(1).join(".")}">log in as an OSBv2 user</a> to get more resources and persisted workspaces.`);
+        }
         html_message = progressMessage.html();
       }
       if (html_message) {

diff --git a/applications/jupyterlab-minimal/deploy/values.yaml b/applications/jupyterlab-minimal/deploy/values.yaml
@@ -22,3 +22,9 @@ harness:
 
         c.JupyterHub.tornado_settings = { "headers": { "Content-Security-Policy": "frame-ancestors *"}}
 
+singleuser:
+  storage:
+    capacity: 2Mi
+  cpu:
+    limit: 3
+    guarantee: 0.13
diff --git a/applications/jupyterlab/Dockerfile b/applications/jupyterlab/Dockerfile
@@ -31,7 +31,6 @@ RUN pip install -r requirements.txt --upgrade --no-cache-dir
 RUN jupyter labextension install plotlywidget
 
 
-
 ##############
 
 USER root
@@ -44,6 +43,10 @@ RUN echo -e '#!/bin/bash\n#Reusing the jNeuroML jar from the pip installed pyNeu
 RUN chmod +x /opt/conda/bin/jnml
 ENV JNML_HOME=/opt/conda/bin
 
+### Set up lems, reusing pynml jar
+RUN echo -e '#!/bin/bash\n#Reusing the jNeuroML jar from the pip installed pyNeuroML for the lems command\n\njava -classpath /opt/conda/lib/python3.9/site-packages/pyneuroml/lib/jNeuroML-*-jar-with-dependencies.jar org.lemsml.jlems.viz.VizMain $@' >> /opt/conda/bin/lems
+RUN chmod +x /opt/conda/bin/lems
+
 RUN cat ~/.bashrc
 
 
@@ -84,13 +87,34 @@ RUN apt-get update && apt-get install libopenmpi-dev -y
 #  Octave etc.
 RUN apt-get update && apt-get install octave octave-statistics -y
 
+#  NEST 
+ENV NEST_VER=3.5
+ENV NEST_HOME=/opt/conda/nest
+RUN cd /tmp && \
+    wget -nv https://github.com/nest/nest-simulator/archive/v$NEST_VER.tar.gz && \
+    tar xvzf v$NEST_VER.tar.gz && \
+    mv nest-simulator-$NEST_VER nest && \
+    cd nest && \
+    mkdir $NEST_HOME && \
+    apt-get install libgsl-dev -y && \
+    cmake -DCMAKE_INSTALL_PREFIX:PATH=$NEST_HOME -DPYTHON_EXECUTABLE:FILEPATH=/opt/conda/bin/python -DPYTHON_INCLUDE_DIR=/opt/conda/include/python3.9 . && \
+    make -j7 && \
+    make install
+ENV PYTHONPATH=$NEST_HOME/lib/python3.9/site-packages
+ENV PATH=$PATH:$NEST_HOME/bin
+
 USER jovyan
 #### Install Python packages
 RUN pip install -r requirements.txt --upgrade --no-cache-dir
 
+#  Compile NEURON mod files for PyNN
+RUN cd /opt/conda/lib/python3.9/site-packages/pyNN/neuron/nmodl && nrnivmodl
+# See https://github.com/OpenSourceBrain/osb-model-validation/issues/91 for status of this
+RUN pip install 'numpy<=1.23.0' # temp fix for https://levelup.gitconnected.com/fix-attributeerror-module-numpy-has-no-attribute-float-d7d68c5a4971
+
 #########################################################################
 # fix for https://github.com/jupyter/notebook/issues/7048
-
 RUN pip install traitlets==5.9.0
+
 COPY --chown=jovyan:users overrides/* /opt/conda/share/jupyter/lab/static/
 WORKDIR /opt/workspace
diff --git a/applications/jupyterlab/deploy/values.yaml b/applications/jupyterlab/deploy/values.yaml
@@ -22,3 +22,10 @@ harness:
 
         c.JupyterHub.tornado_settings = { "headers": { "Content-Security-Policy": "frame-ancestors 'self' localhost:3000 *.osb.local osb.local localhost *.metacell.us *.opensourcebrain.org "}}
 
+singleuser:
+  cpu:
+    limit: 0.1
+    guarantee: 0.02
+  memory:
+    limit: 0.5G
+    guarantee: 0.1G
diff --git a/applications/jupyterlab/requirements.txt b/applications/jupyterlab/requirements.txt
@@ -11,7 +11,7 @@ git+https://github.com/Neurosim-lab/netpyne.git@osbv2#egg=netpyne
 ####  Other simulators
 
 # Arbor
-arbor==0.6.0
+arbor==0.9.0
 
 # EDEN
 eden-simulator==0.2.1
@@ -42,10 +42,10 @@ seaborn
 
 # Note: this just installs pytorch for cpu, the default install adds a v large cuda lib
 -f https://download.pytorch.org/whl/torch_stable.html
-torch==1.11.0+cpu
+torch==2.1.2+cpu
 
 # For MDF
-modeci_mdf==0.4.5 # big jump in size of image...
+modeci_mdf==0.4.8 # big jump in size of image...
 
 scikit-learn # Required for some Neuromatch Academy material
 fasttext # Required for some Neuromatch Academy material
@@ -57,8 +57,18 @@ tvb-data
 ipympl
 
 
+####  SBML/COMBINE packages
+python-libsbml
+tellurium
+
+
 ####  General Python packages
 plotly
 
+
+#### Install OMV
+git+https://github.com/OpenSourceBrain/[email protected]
+
+
 ####  Final updates
 numpy  # Removes some issues with LFPy...
diff --git a/applications/netpyne/Dockerfile b/applications/netpyne/Dockerfile
@@ -1,4 +1,4 @@
-FROM gcr.io/metacellllc/netpyne-ui:1.0.0-osb
+FROM gcr.io/metacellllc/netpyne-ui:1.1.0
 
 COPY overrides/requirements.txt overrides/requirements.txt
 RUN --mount=type=cache,target=/root/.cache python -m pip install --upgrade pip &&\

diff --git a/applications/netpyne/deploy/values.yaml b/applications/netpyne/deploy/values.yaml
@@ -18,4 +18,11 @@ harness:
         c.JupyterHub.tornado_settings = { "headers": { "Content-Security-Policy": "frame-ancestors 'self' localhost:3000 *.osb.local localhost *.metacell.us *.opensourcebrain.org "}}
   dependencies:
     hard:
-    - jupyterhub
+    - jupyterhub
+singleuser:
+  cpu:
+    limit: 1
+    guarantee: 0.02
+  memory:
+    limit: 1G
+    guarantee: 0.5G
diff --git a/applications/nfsserver/deploy/values.yaml b/applications/nfsserver/deploy/values.yaml
@@ -1,4 +1,3 @@
-
 # nfs server pvc disk size (/exports)
 server:
-  diskSize: 90Gi
+  diskSize: 120Gi
diff --git a/applications/nwb-explorer/.dockerignore b/applications/nwb-explorer/.dockerignore
@@ -1 +1,2 @@
-/deploy
+/deploy
+.git