From 459a8bd4e44eb55da5b30b5fce142e7770ad97ca Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?S=C3=A9bastien=20Besson?= <sbesson@glencoesoftware.com>
Date: Fri, 2 Feb 2024 08:21:46 +0000
Subject: [PATCH] Remove deployment of EFK stack on IDR systems

Unlike the Prometheus/Grafana stack, this experimental stack seems
to be largely unused at the moment. Additionally, the versions of the
different components are out-of-date. Rather than looking into their
upgrade with the migration to Rocky Linux 9, this proposes to remove
the EFK stack from the monitoring playbook.
---
 ansible/files/fluentd-server/fluent.conf.j2 | 182 ------------------
 ansible/files/fluentd/forward-conf.j2       |  39 ----
 ansible/files/fluentd/nginx-proxy.conf      |  31 ---
 ansible/files/fluentd/omero.conf            | 120 ------------
 ansible/group_vars/omero-hosts.yml          |   7 -
 ansible/group_vars/proxy-hosts.yml          |   6 -
 ansible/idr-09-monitoring.yml               |   1 -
 ansible/management-fluentd.yml              | 198 --------------------
 ansible/requirements.yml                    |   3 -
 docs/deployment.md                          |   2 +-
 docs/monitoring.md                          |  20 --
 11 files changed, 1 insertion(+), 608 deletions(-)
 delete mode 100644 ansible/files/fluentd-server/fluent.conf.j2
 delete mode 100644 ansible/files/fluentd/forward-conf.j2
 delete mode 100644 ansible/files/fluentd/nginx-proxy.conf
 delete mode 100644 ansible/files/fluentd/omero.conf
 delete mode 100644 ansible/management-fluentd.yml

diff --git a/ansible/files/fluentd-server/fluent.conf.j2 b/ansible/files/fluentd-server/fluent.conf.j2
deleted file mode 100644
index 50a56a7d..00000000
--- a/ansible/files/fluentd-server/fluent.conf.j2
+++ /dev/null
@@ -1,182 +0,0 @@
-###########################################################################
-# Receive TCP logs.
-###########################################################################
-
-# TODO: Secure with client/server SSL certs
-<source>
-  @type forward
-  @id   input1
-  port  24224
-  <security>
-    self_hostname {{ ansible_hostname }}
-    shared_key {{ fluentd_shared_key }}
-  </security>
-</source>
-
-# For debugging only
-#<filter **>
-#  @type stdout
-#</filter>
-
-
-###########################################################################
-# Store logs
-# WARNING: OMERO logs will take up a lot of space!
-###########################################################################
-
-<label @STORE_LOGS>
-  <match **>
-    @type file
-    path /data/fluentd/aggregated
-    symlink_path /data/fluentd/current.log
-    append true
-    compress gzip
-    <buffer>
-      # Aggregated logs rotated daily
-      timekey 1d
-      timekey_use_utc true
-    </buffer>
-  </match>
-</label>
-
-
-###########################################################################
-# Send to Elasticsearch
-###########################################################################
-
-<label @ELASTICSEARCH>
-  <match **>
-    @type elasticsearch
-    logstash_format true
-    host "{{ fluentd_elasticsearch_host }}"
-    port 9200
-    include_tag_key true
-    tag_key @log_name
-    flush_interval 10s
-  </match>
-</label>
-
-
-###########################################################################
-# nginx-proxy
-###########################################################################
-
-# In future more log routes can be added
-<match nginx.**>
-  @type copy
-  <store>
-    @type relabel
-    @label @STORE_LOGS
-  </store>
-  <store>
-    @type relabel
-    @label @ELASTICSEARCH
-  </store>
-  <store>
-    @type relabel
-    @label @NGINX_NOTIFICATION
-  </store>
-</match>
-
-# Notify 5XX and all error logs apart from known warnings
-<label @NGINX_NOTIFICATION>
-  <filter nginx.access>
-    @type grep
-    <regexp>
-      key code
-      pattern ^5\d\d$
-    </regexp>
-  </filter>
-  <filter nginx.access>
-    @type record_transformer
-    enable_ruby true
-    <record>
-      message ${record.to_json}
-    </record>
-  </filter>
-  <filter nginx.error>
-    @type grep
-    <exclude>
-      key message
-      pattern (buffered to a temporary file)
-    </exclude>
-  </filter>
-
-  <match nginx.*>
-    @type slack
-    token "{{ fluentd_slack_token }}"
-    username fluentd-nginx
-    icon_emoji :warning:
-    channel "{{ fluentd_slack_channel }}"
-    message ":warning: *%s* %s _%s_ @here
-```
-%s
-```
-"
-    message_keys tag,hostname,time,message
-    flush_interval 5s
-  </match>
-</label>
-
-
-###########################################################################
-# omero
-###########################################################################
-
-# In future more log routes can be added
-<match omero.**>
-  @type copy
-  <store>
-    @type relabel
-    @label @STORE_LOGS
-  </store>
-  <store>
-    @type relabel
-    @label @ELASTICSEARCH
-  </store>
-  <store>
-    @type relabel
-    @label @OMERO_NOTIFICATION
-  </store>
-</match>
-
-# Send slack notifications for error and critical
-# Exclude known errors
-<label @OMERO_NOTIFICATION>
-  <filter omero.**>
-    @type grep
-    <regexp>
-      key level
-      pattern ^(error|critical)
-    </regexp>
-    <exclude>
-      key message
-      pattern ^(Failed during repository takeover|Method interface .+ invocation took)
-    </exclude>
-  </filter>
-
-  <filter omero.**>
-    @type record_transformer
-    enable_ruby
-    <record>
-      # Limit message length
-      message "${record['message'][0, 1024] + (record['message'].length > 1024 ? ' ...' : '')}"
-    </record>
-  </filter>
-
-  <match omero.**>
-    @type slack
-    token "{{ fluentd_slack_token }}"
-    username fluentd-omero
-    icon_emoji :warning:
-    channel "{{ fluentd_slack_channel }}"
-    message ":warning: *%s* %s %s @here
-```
-%s
-```
-"
-    message_keys tag,hostname,time,message
-    time_format %F %T.%L
-    flush_interval 5s
-  </match>
-</label>
diff --git a/ansible/files/fluentd/forward-conf.j2 b/ansible/files/fluentd/forward-conf.j2
deleted file mode 100644
index 82f64646..00000000
--- a/ansible/files/fluentd/forward-conf.j2
+++ /dev/null
@@ -1,39 +0,0 @@
-# Forward all logs labelled FORWARD to a central server
-# https://docs.fluentd.org/v1.0/articles/out_forward
-# TODO: Add multiple central servers for HA
-
-<label @FORWARD>
-  # Add hostname to all records
-  <filter **>
-    @type record_transformer
-    <record>
-      hostname "#{Socket.gethostname[/^[^.]+/]}"
-    </record>
-  </filter>
-
-  <match **>
-    @type forward
-    send_timeout 60s
-    recover_wait 10s
-    heartbeat_interval 1s
-    phi_threshold 16
-    hard_timeout 60s
-
-    <server>
-      host {{ fluentd_server_address }}
-      port 24224
-      weight 60
-    </server>
-
-    <security>
-      self_hostname {{ ansible_hostname }}
-      shared_key {{ fluentd_shared_key }}
-    </security>
-
-    # Causes a warning: secondary type should be same with primary one
-    <secondary>
-      @type file
-      path /var/log/td-agent/forward-failed.log
-    </secondary>
-  </match>
-</label>
diff --git a/ansible/files/fluentd/nginx-proxy.conf b/ansible/files/fluentd/nginx-proxy.conf
deleted file mode 100644
index 7604444a..00000000
--- a/ansible/files/fluentd/nginx-proxy.conf
+++ /dev/null
@@ -1,31 +0,0 @@
-# Slack alerts for nginx access.log 5XX and all error.log
-
-# Log format: main_timed_cache_upstream
-# https://github.com/openmicroscopy/ansible-role-nginx-proxy/blob/1.6.0/templates/nginx-conf.j2#L39
-<source>
-  @type tail
-  path /var/log/nginx/access.log
-  pos_file /var/log/td-agent/nginx-access.log.pos
-  tag nginx.access
-
-  format /^(?<host>[^ ]*) - (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")? "(?<forwarded>[^\"]*)" (?<reqtime>[^ ]*) (?<cachestatus>[^ ]*) (?<upstream>[^ ]*)/
-  time_format %d/%b/%Y:%H:%M:%S %z
-</source>
-
-# In some situations error.log may be multiline
-<source>
-  @type tail
-  path /var/log/nginx/error.log
-  pos_file /var/log/td-agent/nginx-error.log.pos
-  tag nginx.error
-
-  format multiline
-  format_firstline /^\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2} \[\w+\] (?<pid>\d+).(?<tid>\d+): /
-  format1 /^(?<time>\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}) \[(?<log_level>\w+)\] (?<pid>\d+).(?<tid>\d+): (?<message>.*)/
-  multiline_flush_interval 3s
-</source>
-
-<match nginx.*>
-  @type relabel
-  @label @FORWARD
-</match>
diff --git a/ansible/files/fluentd/omero.conf b/ansible/files/fluentd/omero.conf
deleted file mode 100644
index 2b7b9d67..00000000
--- a/ansible/files/fluentd/omero.conf
+++ /dev/null
@@ -1,120 +0,0 @@
-# http://fluentular.herokuapp.com/
-
-
-######################################################################
-# Parse multiline log messages of the form
-# YYYY-MM-DD hh:mm:ss.mss level [class] (thread) ...
-######################################################################
-<source>
-  @type tail
-  path /opt/omero/server/OMERO.server/var/log/*.log,/opt/omero/web/OMERO.web/var/log/*.log
-
-  pos_file /var/log/td-agent/omero.log.pos
-  tag raw.omero.*
-
-  format multiline
-  format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}\s+\w+/
-  format1 /^(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3})\s*(?<level>\w+)\s*\[\s*(?<class>[^\]]+)\]\s*\((?<thread>[^\)]+)\)\s*(?<message>.*)/
-  time_format %Y-%m-%d %H:%M:%S,%L
-  multiline_flush_interval 5s
-</source>
-
-# Add filename
-# Standardise level: info, warn, error, critical
-# Truncate message at 4KB
-<filter raw.omero.**>
-  @type record_transformer
-  enable_ruby
-  <record>
-    file ${tag_parts[-2]}.${tag_parts[-1]}
-    level "${record['level'].downcase.gsub(/^inf.*/i, 'info').downcase.gsub(/^war.*/i, 'warn').gsub(/^err.*/i, 'error').gsub(/^cri.*/i, 'critical').gsub(/^fat.*/i, 'critical')}"
-    message "${record['message'][0, 4096] + (record['message'].length > 4096 ? ' ...' : '')}"
-  </record>
-</filter>
-
-# Add server component to tag
-<match raw.omero.**>
-  @type rewrite_tag_filter
-  <rule>
-    key file
-    pattern ^(\w+)(-\w+)?\.log$
-    tag raw1.omero.$1
-  </rule>
-</match>
-
-
-######################################################################
-# *.{err,out} logs follow a mix of formats, a datetime is not
-# guaranteed
-# Assume any line that doesn't start with whitespace is the start of a
-# multiline log message
-######################################################################
-<source>
-  @type tail
-  path /opt/omero/server/OMERO.server/var/log/*.err,/opt/omero/server/OMERO.server/var/log/*.out
-
-  pos_file /var/log/td-agent/omero.node.pos
-  tag raw.omeronode.*
-
-  format multiline
-  format_firstline /^\S/
-  format1 /^(?<message>.+)/
-  multiline_flush_interval 5s
-</source>
-
-# Add filename
-# Add level: out:info, err:error
-# Truncate message at 4KB
-<filter raw.omeronode.**>
-  @type record_transformer
-  enable_ruby
-  <record>
-    file ${tag_parts[-2]}.${tag_parts[-1]}
-    level "${tag_parts[-1].downcase.gsub(/^out.*/i, 'info').downcase.gsub(/^err.*/i, 'error')}"
-    message "${record['message'][0, 4096] + (record['message'].length > 4096 ? ' ...' : '')}"
-  </record>
-</filter>
-
-# Add server node to tag
-<match raw.omeronode.**>
-  @type rewrite_tag_filter
-  <rule>
-    key file
-    pattern ^(\w+)\.\w+$
-    tag raw1.omero.$1
-  </rule>
-</match>
-
-
-######################################################################
-# Common OMERO log processing
-######################################################################
-
-# For debugging only
-#<filter **>
-#  @type stdout
-#</filter>
-
-# Add log level to tag
-<match raw1.omero.**>
-  @type rewrite_tag_filter
-  <rule>
-    key level
-    pattern (.+)
-    tag omero.${tag_parts[2]}.$1
-  </rule>
-</match>
-
-# Discard debug logs
-<filter omero.**>
-  @type grep
-  <regexp>
-    key level
-    pattern ^(info|warn|error|critical)
-  </regexp>
-</filter>
-
-<match omero.**>
-  @type relabel
-  @label @FORWARD
-</match>
diff --git a/ansible/group_vars/omero-hosts.yml b/ansible/group_vars/omero-hosts.yml
index 99819b4a..27ee318b 100644
--- a/ansible/group_vars/omero-hosts.yml
+++ b/ansible/group_vars/omero-hosts.yml
@@ -28,13 +28,6 @@ ice_install_python: false
 # This is needed to ensure the client version matches the server
 postgresql_version: "13"
 
-
-######################################################################
-# openmicroscopy.fluentd vars
-fluentd_source_configs:
-- files/fluentd/omero.conf
-
-
 ######################################################################
 # openmicroscopy.omero-server role vars
 
diff --git a/ansible/group_vars/proxy-hosts.yml b/ansible/group_vars/proxy-hosts.yml
index 63ecb95c..52d39773 100644
--- a/ansible/group_vars/proxy-hosts.yml
+++ b/ansible/group_vars/proxy-hosts.yml
@@ -357,9 +357,3 @@ idr_backend_reserved_offset: 2
 idr_haproxy_frontend_omero_offset: 14060
 idr_haproxy_frontend_omero_host: idr.openmicroscopy.org
 
-######################################################################
-# openmicroscopy.fluentd vars
-fluentd_groups:
-- adm
-fluentd_source_configs:
-- files/fluentd/nginx-proxy.conf
diff --git a/ansible/idr-09-monitoring.yml b/ansible/idr-09-monitoring.yml
index e0681269..bbf7f7f2 100644
--- a/ansible/idr-09-monitoring.yml
+++ b/ansible/idr-09-monitoring.yml
@@ -4,4 +4,3 @@
 - import_playbook: management.yml
 - import_playbook: management-prometheus.yml
 - import_playbook: management-grafana.yml
-- import_playbook: management-fluentd.yml
diff --git a/ansible/management-fluentd.yml b/ansible/management-fluentd.yml
deleted file mode 100644
index 73e04180..00000000
--- a/ansible/management-fluentd.yml
+++ /dev/null
@@ -1,198 +0,0 @@
-# Fluentd log handling
-# + Elasticsearch + Kibana in Docker
-# https://www.fluentd.org/guides/recipes/elasticsearch-and-s3
-
-# This assumes docker has already been installed
-# (by management-prometheus.yml)
-- hosts: "{{ idr_environment | default('idr') }}-management-hosts"
-
-  tasks:
-
-  - name: Create fluentd server configuration directory
-    become: yes
-    file:
-      path: /etc/fluentd
-      recurse: yes
-      state: directory
-
-  - name: Create data top level directory
-    become: yes
-    file:
-      path: /data
-      state: directory
-      owner: root
-      group: root
-
-  - name: Create fluentd aggregated logs directory
-    become: yes
-    file:
-      path: /data/fluentd
-      state: directory
-      owner: "{{ fluentd_uid }}"
-      group: "{{ fluentd_uid }}"
-
-  - name: Copy fluentd server configuration files
-    become: yes
-    template:
-      src: files/fluentd-server/fluent.conf.j2
-      dest: /etc/fluentd/fluent.conf
-    register: fluent_conf_status
-
-  - name: Create elasticsearch directory
-    become: yes
-    file:
-      path: /data/elasticsearch
-      state: directory
-      # User id in elasticsearch Docker image
-      owner: 1000
-      group: 1000
-
-  - name: Create docker network
-    become: yes
-    docker_network:
-      name: fluent-es-kb
-      state: present
-
-  - name: Run docker elasticsearch
-    become: yes
-    docker_container:
-      image: "{{ elasticsearch_docker_image }}"
-      name: elasticsearch
-      cleanup: True
-      env:
-        discovery.type: single-node
-        ES_JAVA_OPTS: "-Xmx4096m"
-      networks:
-      - name: fluent-es-kb
-      published_ports:
-      - "9200:9200"
-      #- "9300:9300"
-      state: started
-      restart_policy: always
-      volumes:
-      - /data/elasticsearch:/usr/share/elasticsearch/data
-
-  - name: Run docker fluentd
-    become: yes
-    docker_container:
-      image: "{{ fluentd_docker_image }}"
-      name: fluentd
-      cleanup: True
-      env:
-        FLUENT_UID: "{{ fluentd_uid }}"
-      networks:
-      - name: fluent-es-kb
-      published_ports:
-      - "24224:24224/udp"
-      - "24224:24224"
-      restart: "{{ fluent_conf_status is changed }}"
-      state: started
-      restart_policy: always
-      volumes:
-      - /etc/fluentd/fluent.conf:/fluentd/etc/fluent.conf:ro
-      - /data/fluentd:/data/fluentd
-
-  - name: Run docker kibana
-    become: yes
-    docker_container:
-      image: "{{ kibana_docker_image }}"
-      name: kibana
-      cleanup: True
-      networks:
-      - name: fluent-es-kb
-      published_ports:
-      - "5601:5601"
-      state: started
-      restart_policy: always
-
-  - name: Run elasticsearch curator
-    become: yes
-    docker_container:
-      image: "{{ elasticsearch_curator_docker_image }}"
-      name: elasticsearch-curator
-      cleanup: True
-      env:
-        OLDER_THAN_IN_DAYS: "{{ elasticsearch_expire_logs_days }}"
-        INTERVAL_IN_HOURS: "24"
-      networks:
-      - name: fluent-es-kb
-      state: started
-      restart_policy: always
-
-  vars:
-    fluentd_shared_key: "{{ idr_secret_fluentd_shared_key | default('fluentd') }}"
-    fluentd_slack_token: "{{ idr_secret_management_slack_token | default(None) }}"
-    fluentd_slack_channel: "{{ idr_logs_slack_channel | default('idr-logs') }}"
-    fluentd_elasticsearch_host: elasticsearch
-    fluentd_uid: "1010"
-    elasticsearch_docker_image: "docker.elastic.co/elasticsearch/elasticsearch-oss:6.1.1"
-    elasticsearch_curator_docker_image: "openmicroscopy/elasticsearch-curator:5.4.1"
-    elasticsearch_expire_logs_days: "14"
-    fluentd_docker_image: "openmicroscopy/fluentd:0.1.0"
-    kibana_docker_image: "docker.elastic.co/kibana/kibana-oss:6.1.1"
-
-
-# Load hostvars for management server
-- hosts: >-
-    {{ idr_environment | default('idr') }}-management-hosts
-    {{ idr_parent_environment | default('idr') }}-management-hosts
-
-
-- hosts: >
-    {{ idr_environment | default('idr') }}-proxy-hosts
-    {{ idr_environment | default('idr') }}-omero-hosts
-
-  # TODO: Ideally we'd use the `validate:` option to check these config
-  # files, but it's not possible to validate only a fragment.
-  # Instead we need to create these files before applying the role
-  # However, we can only trigger the restart handler if the agent is
-  # already installed.
-  pre_tasks:
-
-  - name: Check if td-agent already installed
-    stat:
-      path: /etc/init.d/td-agent
-    register: _td_agent_st
-
-  - name: Create fluentd conf.d
-    become: yes
-    file:
-      path: /etc/td-agent/conf.d
-      recurse: yes
-      state: directory
-
-  - name: Configure fluentd forwarding
-    become: yes
-    template:
-      src: files/fluentd/forward-conf.j2
-      dest: /etc/td-agent/conf.d/forward.conf
-    notify:
-    - restart fluentd if installed
-
-  - name: Copy fluentd configuration
-    become: yes
-    copy:
-      src: "{{ item }}"
-      dest: "/etc/td-agent/conf.d/{{ item | basename }}"
-    with_items:
-    # These are specifc to each host-group
-    - "{{ fluentd_source_configs }}"
-    notify:
-    - restart fluentd if installed
-
-  handlers:
-  - name: restart fluentd if installed
-    become: yes
-    systemd:
-      daemon_reload: yes
-      name: td-agent
-      state: restarted
-    when: _td_agent_st.stat.exists
-
-  roles:
-  - role: ome.fluentd
-
-  vars:
-    _monitoring_idr_environment: "{{ idr_parent_environment | default(idr_environment | default('idr')) + '-management-hosts' }}"
-    fluentd_server_address: "{{ hostvars[groups[_monitoring_idr_environment][0]]['ansible_' + (idr_net_iface | default('eth0'))]['ipv4']['address'] }}"
-    fluentd_shared_key: "{{ idr_secret_fluentd_shared_key | default('fluentd') }}"
diff --git a/ansible/requirements.yml b/ansible/requirements.yml
index 3dda9e44..c9483c05 100644
--- a/ansible/requirements.yml
+++ b/ansible/requirements.yml
@@ -152,9 +152,6 @@
 ######################################################################
 # External development roles
 
-- src: ome.fluentd
-  version: 0.2.3
-
 - src: ome.omero_prometheus_exporter
   version: 0.3.6
 
diff --git a/docs/deployment.md b/docs/deployment.md
index 2deca38e..d9ef5d39 100644
--- a/docs/deployment.md
+++ b/docs/deployment.md
@@ -27,7 +27,7 @@ This playbook is *not* idempotent.
 
 ### [`idr-09-monitoring.yml`](../ansible/idr-09-monitoring.yml)
 This is an optional playbook to set up monitoring of the IDR and OMERO.server.
-This includes configuring all servers with [Prometheus](hhttps://prometheus.io/), setting up centralised logging with [fluentd](https://www.fluentd.org/)/, and enabling Slack notifications errors.
+This includes configuring all servers with [Prometheus](hhttps://prometheus.io/), and enabling Slack notifications errors.
 You will need to provide a secret Slack token for Slack notifications.
 
 
diff --git a/docs/monitoring.md b/docs/monitoring.md
index 6a741b46..6fe7cd8a 100644
--- a/docs/monitoring.md
+++ b/docs/monitoring.md
@@ -22,20 +22,6 @@ If you want to query the raw metrics directly you can connect to prometheus:
 
 Open http://localhost:9090/ and you can interactively query the metrics using [PromQL (Prometheus Query Language)](https://prometheus.io/docs/prometheus/latest/querying/basics/).
 
-
-## Centralised logging
-
-The IDR has a experimental centralised logging collector using Fluentd, ElasticSearch and Kibana:
-
-    ssh idr-proxy -L 5601:management:5601
-
-Open http://localhost:5601/.
-Limitations:
-- An index pattern `logstash-*` may need to be create on the first access
-- The ElasticSearch server is not clustered and has limited querying capacity. It can be easily overloaded.
-- Logs are kept for two weeks.
-
-
 ## IDR Pilot Monitoring
 
 All IDR pilot test instances are currently monitored by a single monitoring server.
@@ -68,9 +54,3 @@ This playbook also deploys all the required [Prometheus exporters](https://prome
 
 Grafana runs in Docker and is [deployed by a playbook](../ansible/management-grafana.yml).
 The playbook automatically configures Grafana and uploads some pre-created dashboards using Grafana's REST API.
-
-
-### ElasticSearch, Kibana, Fluentd
-
-ElasticSearch, Kibana and the Fluentd server are run in Docker and [deployed by a playbook](../ansible/management-fluentd.yml).
-This playbook also deploys the Fluentd logging agents that collect the logs on the OMERO and proxy servers, and forward them to the central Fluentd server.