NGINX: support for brotli compression (#379)

* support for brotli compression in NGINX if installed on the machine.
* Provide documentation on how to install the nginx brotli module on both MacOS and debian-based linux.

.github/workflows/test.yaml

@@ -71,7 +71,15 @@ jobs:
           sudo add-apt-repository ppa:mozillateam/ppa
           printf 'Package: *\nPin: release o=LP-PPA-mozillateam\nPin-Priority: 1001' | sudo tee /etc/apt/preferences.d/mozilla-firefox
 
-          sudo apt install -y wget nodejs unzip firefox nginx
+          sudo apt install -y wget nodejs unzip firefox
+
+          # if nginx is already installed, remove it
+          sudo apt remove -y nginx nginx-common nginx-core
+
+          sudo add-apt-repository ppa:ondrej/nginx-mainline -y
+          sudo apt update -y
+          sudo apt install -y nginx libnginx-mod-brotli
+
 
           pip install --upgrade pip
           pip install wheel

doc/setup.md

@@ -13,7 +13,8 @@ Clone that application, and then proceed with the following instructions.
 
 ### On macOS
 
-- Using [Homebrew](http://brew.sh/): `brew install supervisor nginx postgresql node`
+- Using [Homebrew](http://brew.sh/): `brew install supervisor postgresql node`
+  - If you want to use [brotli compression](https://en.wikipedia.org/wiki/Brotli) with NGINX (better compression rates for the frontend), you can install NGINX with the `ngx_brotli` module with this command: `brew tap denji/nginx && brew install nginx-full --with-brotli`. _If you already had NGINX installed, you may need to uninstall it first with `brew unlink nginx`._ Otherwise, you can install NGINX normally with `brew install nginx`.
   - Start the postgresql server:
     - to start automatically at login: `brew services start postgresql`
     - to start manually: `pg_ctl -D /usr/local/var/postgres start`
@@ -37,7 +38,19 @@ See [below](#configuration) for more information on modifying the baselayer conf
 ### On Linux
 
 - Using `apt-get`:
-  `sudo apt-get install nginx supervisor postgresql libpq-dev npm nodejs-legacy`
+  `sudo apt-get install supervisor postgresql libpq-dev npm nodejs-legacy`
+
+  If you want to use [brotli compression](https://en.wikipedia.org/wiki/Brotli) with NGINX (better compression rates for the frontend), you have to install NGINX and the brotli module from another source with:
+
+  ```
+  sudo apt remove -y nginx nginx-common nginx-core
+  sudo add-apt-repository ppa:ondrej/nginx-mainline -y
+  sudo apt update -y
+  sudo apt install -y nginx libnginx-mod-brotli
+  ```
+
+  Otherwise, you can install NGINX normally with `sudo apt-get install nginx`.
+
 - It may be necessary to configure your database permissions: at
   the end of your `pg_hba.conf` (typically in `/etc/postgresql/13.3/main` or `/var/lib/pgsql/data`),
   add the following lines and restart PostgreSQL

services/nginx/mime.types

@@ -57,6 +57,7 @@ application/geopackage+sqlite3			gpkg;
 application/gltf-buffer				glbin glbuf;
 application/gml+xml				gml;
 application/gzip				gz tgz;
+application/br                  br;
 application/hyperstudio				stk;
 application/inkml+xml				ink inkml;
 application/ipfix				ipfix;

services/nginx/nginx.conf.template

@@ -1,6 +1,12 @@
 error_log log/error.log error;
 pid run/nginx.pid;
 
+{% if fill_config_feature.nginx_brotli.dynamic and fill_config_feature.nginx_brotli.modules_path %}
+load_module {{ fill_config_feature.nginx_brotli.modules_path }}/ngx_http_brotli_filter_module.so; # for compressing responses on-the-fly
+load_module {{ fill_config_feature.nginx_brotli.modules_path }}/ngx_http_brotli_static_module.so; # for serving pre-compressed files
+{% endif %}
+
+
 # Choose number of NGINX worker processes based on number of CPUs
 worker_processes auto;
 
@@ -21,6 +27,19 @@ http {
              text/javascript
              application/javascript;
 
+  {% if fill_config_feature.nginx_brotli.installed %}
+  # also enable brotli compression
+  brotli on;
+  brotli_comp_level 6;
+  brotli_types text/plain
+               text/css
+               application/json
+               application/x-javascript
+               application/xml
+               text/javascript
+               application/javascript;
+  {% endif %}
+
   # Only retry if there was a communication error, not a timeout
   # on the Tornado server (to avoid propagating "queries of death"
   # to all frontends)

tools/fill_conf_values.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 
 import os
+import subprocess
 
 import jinja2
 from status import status
@@ -36,12 +37,89 @@ def hash_filter(string, htype):
     return h.hexdigest()
 
 
+def nginx_brotli_installed():
+    """Check if the nginx brotli module is installed
+
+    Returns
+    -------
+    installed : bool
+        True if the nginx brotli module is installed, False otherwise
+    dynamic : bool
+        True if the module is dynamically loaded, False otherwise
+    modules_path : str
+        The directory where the nginx modules are located if dynamic loading is used
+    """
+
+    installed = False
+    dynamic = False
+    modules_path = None
+
+    try:
+        output = subprocess.check_output(
+            ["nginx", "-V"], stderr=subprocess.STDOUT
+        ).decode("utf-8")
+        # Option 1: installed at compilation: always loaded
+        if (
+            "--add-module" in output
+            and "brotli" in output.split("--add-module")[1].strip()
+        ):
+            installed = True
+        # Option 2: installed dynamically at compilation or later: has to be loaded
+        else:
+            # a. find the modules path
+            config_path = (
+                str(output.split("--conf-path=")[1].split(" ")[0]).strip()
+                if "--conf-path" in output
+                else None
+            )
+            modules_path = (
+                str(output.split("--modules-path=")[1].split(" ")[0]).strip()
+                if "--modules-path" in output
+                else None
+            )
+            # if there's no modules path, try to guess it from the config path
+            if config_path and not modules_path:
+                modules_path = os.path.dirname(config_path).replace(
+                    "nginx.conf", "modules"
+                )
+                if not modules_path or not os.path.isdir(modules_path):
+                    modules_path = None
+
+            # b. check if there is a brotli module in the modules path
+            if modules_path:
+                modules_path = modules_path.rstrip("/")
+                if all(
+                    os.path.isfile(os.path.join(modules_path, f))
+                    for f in [
+                        "ngx_http_brotli_filter_module.so",
+                        "ngx_http_brotli_static_module.so",
+                    ]
+                ):
+                    installed = True
+                    dynamic = True
+                else:
+                    installed = False
+                    dynamic = False
+                    modules_path = None
+    except subprocess.CalledProcessError:
+        pass
+    return installed, dynamic, modules_path
+
+
 custom_filters = {"md5sum": md5sum, "version": version, "hash": hash_filter}
 
 
 def fill_config_file_values(template_paths):
     log("Compiling configuration templates")
     env, cfg = load_env()
+    installed, dynamic, modules_path = nginx_brotli_installed()
+    cfg["fill_config_feature"] = {
+        "nginx_brotli": {
+            "installed": installed,
+            "dynamic": dynamic,
+            "modules_path": modules_path,
+        }
+    }
 
     for template_path in template_paths:
         with status(template_path):