2023-12 refresh

.github/workflows/docs.yaml

@@ -14,7 +14,7 @@ jobs:
     if: github.repository_owner == 'cesium-ml'
 
     steps:
-      - uses: actions/cache@v2
+      - uses: actions/cache@v3
         with:
           path: |
             ~/.cache/pip
@@ -24,11 +24,11 @@ jobs:
             ~/.npm
           key: ${{ runner.os }}-${{ hashFiles('**/lockfiles') }}
 
-      - uses: actions/setup-python@v1
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.8"
 
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
 
       - name: Install dependencies
         run: |

.github/workflows/lint.yaml

@@ -10,6 +10,6 @@ jobs:
     name: pre-commit-hooks
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
       - uses: pre-commit/action@646c83fcd040023954eafda54b4db0192ce70507 # v3.0.0

.github/workflows/test.yaml

@@ -32,25 +32,29 @@ jobs:
           --health-retries 5
 
     steps:
-      - uses: actions/setup-python@v1
+      - uses: actions/setup-python@v5
         with:
           python-version: "3.8"
 
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 20
+
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Install template app
         run: |
           git clone https://github.com/cesium-ml/baselayer_template_app
           cp -rf baselayer baselayer_template_app/
 
-      - uses: actions/cache@v2
+      - uses: actions/cache@v3
         with:
           path: |
             ~/.npm
           key: ${{ runner.os }}-npm-${{ hashFiles('baselayer_template_app/package.json') }}
 
-      - uses: actions/cache@v2
+      - uses: actions/cache@v3
         with:
           path: |
             ~/.cache/pip
@@ -60,8 +64,7 @@ jobs:
         run: |
           cd baselayer
 
-          curl -sL https://deb.nodesource.com/setup_18.x | sudo -E bash -
-
+          sudo apt update -y
           sudo apt install -y wget nodejs unzip firefox nginx
 
           pip install --upgrade pip
@@ -90,7 +93,7 @@ jobs:
           fi
           sudo tar -xzf ${CACHED_DOWNLOAD_DIR}/geckodriver-v${GECKO_VER}-linux64.tar.gz -C /usr/local/bin
           geckodriver --version
-          pip install --upgrade selenium
+          pip install selenium==4.8.3
           python -c "import selenium; print(f'Selenium {selenium.__version__}')"
 
       - name: Write configuration & build DB

.pre-commit-config.yaml

@@ -1,12 +1,12 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v2.5.0
+    rev: v4.5.0
     hooks:
       - id: check-yaml
       - id: end-of-file-fixer
       - id: trailing-whitespace
   - repo: https://github.com/asottile/pyupgrade
-    rev: v2.29.1
+    rev: v3.15.0
     hooks:
       - id: pyupgrade
         args: [--py38-plus]
@@ -17,24 +17,24 @@ repos:
         name: isort (python)
         args: ["--profile", "black"]
   - repo: https://github.com/python/black
-    rev: 22.3.0
+    rev: 23.12.0
     hooks:
       - id: black
         pass_filenames: true
         exclude: baselayer|node_modules|static
   - repo: https://github.com/pycqa/flake8
-    rev: 3.8.4
+    rev: 6.1.0
     hooks:
       - id: flake8
         pass_filenames: true
         exclude: baselayer|node_modules|static|__init__.py
   - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: "v2.3.0" # Use the sha or tag you want to point at
+    rev: v3.1.0 # Use the sha or tag you want to point at
     hooks:
       - id: prettier
         pass_filenames: true
   - repo: https://github.com/pre-commit/mirrors-eslint
-    rev: 5d73b3501b5fd211a62693736b2ab83816855ebe # v8.33.0
+    rev: 4bfe10ad902d472399bcacf83cba41500542718a # v8.55.0
     hooks:
       - id: eslint
         additional_dependencies:

Makefile

@@ -41,7 +41,7 @@ help:
 	@python ./baselayer/tools/makefile_to_help.py $(MAKEFILE_LIST)
 
 dependencies: README.md
-	@cd baselayer && ./tools/check_app_environment.py
+	@baselayer/tools/check_app_environment.py
 	@PYTHONPATH=. python baselayer/tools/pip_install_requirements.py baselayer/requirements.txt requirements.txt
 	@./baselayer/tools/silent_monitor.py baselayer/tools/check_js_deps.sh
 
@@ -67,7 +67,7 @@ paths:
 	@mkdir -p ./log/sv_child
 
 fill_conf_values:
-	@find . -name '[^.]*.template' | grep -v "node_modules" | PYTHONPATH=. xargs ./baselayer/tools/fill_conf_values.py $(FLAGS)
+	@find -L . -name '[^.]*.template' | grep -v "node_modules" | PYTHONPATH=. xargs ./baselayer/tools/fill_conf_values.py $(FLAGS)
 
 system_setup: | paths dependencies fill_conf_values service_setup
 

app/config.py

@@ -1,4 +1,4 @@
-import collections.abc as collections
+import collections
 import os
 from pathlib import Path
 
@@ -12,7 +12,7 @@
 def recursive_update(d, u):
     # Based on https://stackoverflow.com/a/3233356/214686
     for k, v in u.items():
-        if isinstance(v, collections.Mapping):
+        if isinstance(v, collections.abc.Mapping):
             r = recursive_update(d.get(k, {}) or {}, v)
             d[k] = r
         else:
@@ -109,7 +109,7 @@ def load_config(config_files=[]):
         Path(basedir / "config.yaml.defaults"),
         Path(basedir / "../config.yaml.defaults"),
     ] + config_files
-    all_configs = [cfg for cfg in all_configs if os.path.isfile(cfg)]
+    all_configs = [cfg for cfg in all_configs if os.path.exists(os.path.normpath(cfg))]
     all_configs = [os.path.abspath(Path(c).absolute()) for c in all_configs]
 
     cfg = Config(all_configs)

app/models.py

@@ -747,7 +747,6 @@ def query_accessible_rows(self, cls, user_or_token, columns=None):
         # table against accessible related rows via their relationships
         # to the target table
         for prop in self.properties_and_modes:
-
             # get the kind of access required on the relationship
             mode = self.properties_and_modes[prop]
             relationship = sa.inspect(cls).mapper.relationships[prop]
@@ -810,7 +809,6 @@ def select_accessible_rows(self, cls, user_or_token, columns=None):
         # table against accessible related rows via their relationships
         # to the target table
         for prop in self.properties_and_modes:
-
             # get the kind of access required on the relationship
             mode = self.properties_and_modes[prop]
             relationship = sa.inspect(cls).mapper.relationships[prop]
@@ -931,7 +929,6 @@ def query_accessible_rows(self, cls, user_or_token, columns=None):
         accessible_id_cols = []
 
         for access_control in self.access_controls:
-
             # Just ignore public ACLs
             if isinstance(access_control, Public):
                 continue
@@ -1003,7 +1000,6 @@ def select_accessible_rows(self, cls, user_or_token, columns=None):
         accessible_id_cols = []
 
         for access_control in self.access_controls:
-
             # Just ignore public ACLs
             if isinstance(access_control, Public):
                 continue
@@ -1263,7 +1259,6 @@ def select_accessible_rows(self, cls, user_or_token, columns=None):
 
 
 class BaseMixin:
-
     # permission control logic
     create = read = public
     update = delete = restricted

app/test_util.py

@@ -146,7 +146,7 @@ def click_css(self, css, timeout=10, scroll_parent=False):
 def driver(request):
     from selenium import webdriver
 
-    options = webdriver.firefox.options.Options()
+    options = webdriver.FirefoxOptions()
     if "BASELAYER_TEST_HEADLESS" in os.environ:
         options.headless = True
     options.set_preference("devtools.console.stdout.content", True)

config.yaml.defaults

@@ -87,8 +87,8 @@ services:
       - ./baselayer/services
       - ./services
 
-    # By default, all services are enabled.  Use this list to add
-    # services if 'disabled' is set to '*'
+    # By default, all services other than dask are enabled.
+    # Use this list to add services if 'disabled' is set to '*'
     enabled:
 
     # A list of service names, or '*'

doc/setup.md

@@ -39,16 +39,18 @@ See [below](#configuration) for more information on modifying the baselayer conf
 - Using `apt-get`:
   `sudo apt-get install nginx supervisor postgresql libpq-dev npm nodejs-legacy`
 - It may be necessary to configure your database permissions: at
-  the end of your `pg_hba.conf` (typically in `/etc/postgresql/13.3/main`),
+  the end of your `pg_hba.conf` (typically in `/etc/postgresql/13.3/main` or `/var/lib/pgsql/data`),
   add the following lines and restart PostgreSQL
-  (`sudo service postgresql restart`):
+  (`sudo service postgresql restart` or `systemctl reload postgresql`):
 
   ```
-  local all postgres peer
-  local baselayer baselayer trust
-  local baselayer_test baselayer trust
+  # CONNECTION DATABASE USER ADDRESS METHOD
+  host template_app template_app localhost trust
+  host all postgres localhost trust
   ```
 
+  Substitute the correct database name and user, as defined in your `config.yaml`.
+
 ## Building the baselayer database
 
 - Initialize the database with `make db_init` (also tests that your

services/dask/supervisor.conf.template

@@ -1,11 +1,11 @@
 [program:dask_scheduler]
-command=dask-scheduler --host 127.0.0.1 --port={{ ports.dask }} --no-bokeh
+command=dask-scheduler --host 127.0.0.1 --port={{ ports.dask }} --no-dashboard
 environment=PYTHONPATH=".",PYTHONUNBUFFERED="1"
 stdout_logfile=log/dask_scheduler.log
 redirect_stderr=true
 
 [program:dask_worker]
-command=dask-worker --nthreads=1 --nprocs=4 127.0.0.1:{{ ports.dask }}
+command=dask-worker --nthreads=1 --nworkers=4 127.0.0.1:{{ ports.dask }}
 environment=PYTHONPATH=".",PYTHONUNBUFFERED="1"
 stdout_logfile=log/dask_workers.log
 redirect_stderr=true

services/external_logging/external_logging.py

@@ -42,7 +42,7 @@ def check_config(config, service):
             ),
         ]
 
-    for (cond, msg) in conditions:
+    for cond, msg in conditions:
         if cond:
             log(msg)
 

services/status_server/status_server.py

@@ -10,6 +10,10 @@ class MainHandler(tornado.web.RequestHandler):
     def get(self):
         self.set_status(503)
         self.write(f"<h2>{cfg['app.title']} is being provisioned</h2>")
+        self.write(
+            "<p>Sysadmins can run <code>make monitor</code> on the server to see how that is progressing."
+        )
+        self.write("<p>System logs are in <code>./log/app_*.log</code></p>")
 
 
 class MainAPIHandler(tornado.web.RequestHandler):

services/websocket_server/websocket_server.py

@@ -160,7 +160,6 @@ def broadcast(cls, data):
                 socket.write_message(payload)
 
         else:
-
             for socket in cls.sockets[user_id]:
                 log(f"Forwarding message to user {user_id}")
 

static/js/API.js

@@ -30,16 +30,16 @@ function API(endpoint, actionType, method = "GET", body = {}, otherArgs = {}) {
       return dispatch(
         showNotification(
           "API invocation error: no actionType specified",
-          "error"
-        )
+          "error",
+        ),
       );
     }
     dispatch({ type: API_CALL, parameters });
     try {
       const response = await fetch(endpoint, fetchInit);
       if (response.status !== 200) {
         throw new Error(
-          `Could not fetch data from server (${response.status})`
+          `Could not fetch data from server (${response.status})`,
         );
       }
 

static/js/components/Notifications.jsx

@@ -79,7 +79,7 @@ export function showNotification(
   note,
   type = "info",
   duration = 3000,
-  tag = "default"
+  tag = "default",
 ) {
   const thisId = nextNotificationId;
   nextNotificationId += 1;

static/js/components/WebSocket.jsx

@@ -56,7 +56,7 @@ function getAuthToken(auth_url) {
 
 function showWebsocketNotification(dispatch, msg, tag) {
   dispatch(hideNotificationByTag(tag)).then(
-    dispatch(showNotification(msg, "warning", 50 * MS_PER_YEAR, tag))
+    dispatch(showNotification(msg, "warning", 50 * MS_PER_YEAR, tag)),
   );
 }
 
@@ -85,7 +85,7 @@ class WebSocket extends React.Component {
       showWebsocketNotification(
         dispatch,
         "No WebSocket connection: limited functionality may be available",
-        tag
+        tag,
       );
     };
 
@@ -110,7 +110,7 @@ class WebSocket extends React.Component {
           showWebsocketNotification(
             dispatch,
             "WebSocket connection authentication failed: limited functionality may be available",
-            tag
+            tag,
           );
           break;
         case "AUTH OK":
@@ -130,7 +130,7 @@ class WebSocket extends React.Component {
       showWebsocketNotification(
         dispatch,
         "No WebSocket connection: limited functionality may be available",
-        tag
+        tag,
       );
     };
   }

static/js/reconnecting-websocket.js

@@ -241,7 +241,7 @@
           console.debug(
             "ReconnectingWebSocket",
             "connection-timeout",
-            self.url
+            self.url,
           );
         }
         timedOut = true;
@@ -293,7 +293,7 @@
             },
             timeout > self.maxReconnectInterval
               ? self.maxReconnectInterval
-              : timeout
+              : timeout,
           );
         }
       };
@@ -303,7 +303,7 @@
             "ReconnectingWebSocket",
             "onmessage",
             self.url,
-            event.data
+            event.data,
           );
         }
         var e = generateEvent("message");