diff --git a/ansible/files/fluentd-server/fluent.conf.j2 b/ansible/files/fluentd-server/fluent.conf.j2
deleted file mode 100644
index 50a56a7d..00000000
--- a/ansible/files/fluentd-server/fluent.conf.j2
+++ /dev/null
@@ -1,182 +0,0 @@
-###########################################################################
-# Receive TCP logs.
-###########################################################################
-
-# TODO: Secure with client/server SSL certs
-
- @type forward
- @id input1
- port 24224
-
- self_hostname {{ ansible_hostname }}
- shared_key {{ fluentd_shared_key }}
-
-
-
-# For debugging only
-#
-# @type stdout
-#
-
-
-###########################################################################
-# Store logs
-# WARNING: OMERO logs will take up a lot of space!
-###########################################################################
-
-
-
-
-###########################################################################
-# Send to Elasticsearch
-###########################################################################
-
-
-
-
-###########################################################################
-# nginx-proxy
-###########################################################################
-
-# In future more log routes can be added
-
- @type copy
-
- @type relabel
- @label @STORE_LOGS
-
-
- @type relabel
- @label @ELASTICSEARCH
-
-
- @type relabel
- @label @NGINX_NOTIFICATION
-
-
-
-# Notify 5XX and all error logs apart from known warnings
-
-
-
-###########################################################################
-# omero
-###########################################################################
-
-# In future more log routes can be added
-
- @type copy
-
- @type relabel
- @label @STORE_LOGS
-
-
- @type relabel
- @label @ELASTICSEARCH
-
-
- @type relabel
- @label @OMERO_NOTIFICATION
-
-
-
-# Send slack notifications for error and critical
-# Exclude known errors
-
diff --git a/ansible/files/fluentd/forward-conf.j2 b/ansible/files/fluentd/forward-conf.j2
deleted file mode 100644
index 82f64646..00000000
--- a/ansible/files/fluentd/forward-conf.j2
+++ /dev/null
@@ -1,39 +0,0 @@
-# Forward all logs labelled FORWARD to a central server
-# https://docs.fluentd.org/v1.0/articles/out_forward
-# TODO: Add multiple central servers for HA
-
-
diff --git a/ansible/files/fluentd/nginx-proxy.conf b/ansible/files/fluentd/nginx-proxy.conf
deleted file mode 100644
index 7604444a..00000000
--- a/ansible/files/fluentd/nginx-proxy.conf
+++ /dev/null
@@ -1,31 +0,0 @@
-# Slack alerts for nginx access.log 5XX and all error.log
-
-# Log format: main_timed_cache_upstream
-# https://github.com/openmicroscopy/ansible-role-nginx-proxy/blob/1.6.0/templates/nginx-conf.j2#L39
-
- @type tail
- path /var/log/nginx/access.log
- pos_file /var/log/td-agent/nginx-access.log.pos
- tag nginx.access
-
- format /^(?[^ ]*) - (?[^ ]*) \[(?
-
-# In some situations error.log may be multiline
-
- @type tail
- path /var/log/nginx/error.log
- pos_file /var/log/td-agent/nginx-error.log.pos
- tag nginx.error
-
- format multiline
- format_firstline /^\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2} \[\w+\] (?\d+).(?\d+): /
- format1 /^(?
-
-
- @type relabel
- @label @FORWARD
-
diff --git a/ansible/files/fluentd/omero.conf b/ansible/files/fluentd/omero.conf
deleted file mode 100644
index 2b7b9d67..00000000
--- a/ansible/files/fluentd/omero.conf
+++ /dev/null
@@ -1,120 +0,0 @@
-# http://fluentular.herokuapp.com/
-
-
-######################################################################
-# Parse multiline log messages of the form
-# YYYY-MM-DD hh:mm:ss.mss level [class] (thread) ...
-######################################################################
-
- @type tail
- path /opt/omero/server/OMERO.server/var/log/*.log,/opt/omero/web/OMERO.web/var/log/*.log
-
- pos_file /var/log/td-agent/omero.log.pos
- tag raw.omero.*
-
- format multiline
- format_firstline /^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3}\s+\w+/
- format1 /^(?
-
-# Add filename
-# Standardise level: info, warn, error, critical
-# Truncate message at 4KB
-
- @type record_transformer
- enable_ruby
-
- file ${tag_parts[-2]}.${tag_parts[-1]}
- level "${record['level'].downcase.gsub(/^inf.*/i, 'info').downcase.gsub(/^war.*/i, 'warn').gsub(/^err.*/i, 'error').gsub(/^cri.*/i, 'critical').gsub(/^fat.*/i, 'critical')}"
- message "${record['message'][0, 4096] + (record['message'].length > 4096 ? ' ...' : '')}"
-
-
-
-# Add server component to tag
-
- @type rewrite_tag_filter
-
- key file
- pattern ^(\w+)(-\w+)?\.log$
- tag raw1.omero.$1
-
-
-
-
-######################################################################
-# *.{err,out} logs follow a mix of formats, a datetime is not
-# guaranteed
-# Assume any line that doesn't start with whitespace is the start of a
-# multiline log message
-######################################################################
-
- @type tail
- path /opt/omero/server/OMERO.server/var/log/*.err,/opt/omero/server/OMERO.server/var/log/*.out
-
- pos_file /var/log/td-agent/omero.node.pos
- tag raw.omeronode.*
-
- format multiline
- format_firstline /^\S/
- format1 /^(?.+)/
- multiline_flush_interval 5s
-
-
-# Add filename
-# Add level: out:info, err:error
-# Truncate message at 4KB
-
- @type record_transformer
- enable_ruby
-
- file ${tag_parts[-2]}.${tag_parts[-1]}
- level "${tag_parts[-1].downcase.gsub(/^out.*/i, 'info').downcase.gsub(/^err.*/i, 'error')}"
- message "${record['message'][0, 4096] + (record['message'].length > 4096 ? ' ...' : '')}"
-
-
-
-# Add server node to tag
-
- @type rewrite_tag_filter
-
- key file
- pattern ^(\w+)\.\w+$
- tag raw1.omero.$1
-
-
-
-
-######################################################################
-# Common OMERO log processing
-######################################################################
-
-# For debugging only
-#
-# @type stdout
-#
-
-# Add log level to tag
-
- @type rewrite_tag_filter
-
- key level
- pattern (.+)
- tag omero.${tag_parts[2]}.$1
-
-
-
-# Discard debug logs
-
- @type grep
-
- key level
- pattern ^(info|warn|error|critical)
-
-
-
-
- @type relabel
- @label @FORWARD
-
diff --git a/ansible/group_vars/omero-hosts.yml b/ansible/group_vars/omero-hosts.yml
index 99819b4a..27ee318b 100644
--- a/ansible/group_vars/omero-hosts.yml
+++ b/ansible/group_vars/omero-hosts.yml
@@ -28,13 +28,6 @@ ice_install_python: false
# This is needed to ensure the client version matches the server
postgresql_version: "13"
-
-######################################################################
-# openmicroscopy.fluentd vars
-fluentd_source_configs:
-- files/fluentd/omero.conf
-
-
######################################################################
# openmicroscopy.omero-server role vars
diff --git a/ansible/group_vars/proxy-hosts.yml b/ansible/group_vars/proxy-hosts.yml
index 63ecb95c..52d39773 100644
--- a/ansible/group_vars/proxy-hosts.yml
+++ b/ansible/group_vars/proxy-hosts.yml
@@ -357,9 +357,3 @@ idr_backend_reserved_offset: 2
idr_haproxy_frontend_omero_offset: 14060
idr_haproxy_frontend_omero_host: idr.openmicroscopy.org
-######################################################################
-# openmicroscopy.fluentd vars
-fluentd_groups:
-- adm
-fluentd_source_configs:
-- files/fluentd/nginx-proxy.conf
diff --git a/ansible/idr-09-monitoring.yml b/ansible/idr-09-monitoring.yml
index e0681269..bbf7f7f2 100644
--- a/ansible/idr-09-monitoring.yml
+++ b/ansible/idr-09-monitoring.yml
@@ -4,4 +4,3 @@
- import_playbook: management.yml
- import_playbook: management-prometheus.yml
- import_playbook: management-grafana.yml
-- import_playbook: management-fluentd.yml
diff --git a/ansible/management-fluentd.yml b/ansible/management-fluentd.yml
deleted file mode 100644
index 73e04180..00000000
--- a/ansible/management-fluentd.yml
+++ /dev/null
@@ -1,198 +0,0 @@
-# Fluentd log handling
-# + Elasticsearch + Kibana in Docker
-# https://www.fluentd.org/guides/recipes/elasticsearch-and-s3
-
-# This assumes docker has already been installed
-# (by management-prometheus.yml)
-- hosts: "{{ idr_environment | default('idr') }}-management-hosts"
-
- tasks:
-
- - name: Create fluentd server configuration directory
- become: yes
- file:
- path: /etc/fluentd
- recurse: yes
- state: directory
-
- - name: Create data top level directory
- become: yes
- file:
- path: /data
- state: directory
- owner: root
- group: root
-
- - name: Create fluentd aggregated logs directory
- become: yes
- file:
- path: /data/fluentd
- state: directory
- owner: "{{ fluentd_uid }}"
- group: "{{ fluentd_uid }}"
-
- - name: Copy fluentd server configuration files
- become: yes
- template:
- src: files/fluentd-server/fluent.conf.j2
- dest: /etc/fluentd/fluent.conf
- register: fluent_conf_status
-
- - name: Create elasticsearch directory
- become: yes
- file:
- path: /data/elasticsearch
- state: directory
- # User id in elasticsearch Docker image
- owner: 1000
- group: 1000
-
- - name: Create docker network
- become: yes
- docker_network:
- name: fluent-es-kb
- state: present
-
- - name: Run docker elasticsearch
- become: yes
- docker_container:
- image: "{{ elasticsearch_docker_image }}"
- name: elasticsearch
- cleanup: True
- env:
- discovery.type: single-node
- ES_JAVA_OPTS: "-Xmx4096m"
- networks:
- - name: fluent-es-kb
- published_ports:
- - "9200:9200"
- #- "9300:9300"
- state: started
- restart_policy: always
- volumes:
- - /data/elasticsearch:/usr/share/elasticsearch/data
-
- - name: Run docker fluentd
- become: yes
- docker_container:
- image: "{{ fluentd_docker_image }}"
- name: fluentd
- cleanup: True
- env:
- FLUENT_UID: "{{ fluentd_uid }}"
- networks:
- - name: fluent-es-kb
- published_ports:
- - "24224:24224/udp"
- - "24224:24224"
- restart: "{{ fluent_conf_status is changed }}"
- state: started
- restart_policy: always
- volumes:
- - /etc/fluentd/fluent.conf:/fluentd/etc/fluent.conf:ro
- - /data/fluentd:/data/fluentd
-
- - name: Run docker kibana
- become: yes
- docker_container:
- image: "{{ kibana_docker_image }}"
- name: kibana
- cleanup: True
- networks:
- - name: fluent-es-kb
- published_ports:
- - "5601:5601"
- state: started
- restart_policy: always
-
- - name: Run elasticsearch curator
- become: yes
- docker_container:
- image: "{{ elasticsearch_curator_docker_image }}"
- name: elasticsearch-curator
- cleanup: True
- env:
- OLDER_THAN_IN_DAYS: "{{ elasticsearch_expire_logs_days }}"
- INTERVAL_IN_HOURS: "24"
- networks:
- - name: fluent-es-kb
- state: started
- restart_policy: always
-
- vars:
- fluentd_shared_key: "{{ idr_secret_fluentd_shared_key | default('fluentd') }}"
- fluentd_slack_token: "{{ idr_secret_management_slack_token | default(None) }}"
- fluentd_slack_channel: "{{ idr_logs_slack_channel | default('idr-logs') }}"
- fluentd_elasticsearch_host: elasticsearch
- fluentd_uid: "1010"
- elasticsearch_docker_image: "docker.elastic.co/elasticsearch/elasticsearch-oss:6.1.1"
- elasticsearch_curator_docker_image: "openmicroscopy/elasticsearch-curator:5.4.1"
- elasticsearch_expire_logs_days: "14"
- fluentd_docker_image: "openmicroscopy/fluentd:0.1.0"
- kibana_docker_image: "docker.elastic.co/kibana/kibana-oss:6.1.1"
-
-
-# Load hostvars for management server
-- hosts: >-
- {{ idr_environment | default('idr') }}-management-hosts
- {{ idr_parent_environment | default('idr') }}-management-hosts
-
-
-- hosts: >
- {{ idr_environment | default('idr') }}-proxy-hosts
- {{ idr_environment | default('idr') }}-omero-hosts
-
- # TODO: Ideally we'd use the `validate:` option to check these config
- # files, but it's not possible to validate only a fragment.
- # Instead we need to create these files before applying the role
- # However, we can only trigger the restart handler if the agent is
- # already installed.
- pre_tasks:
-
- - name: Check if td-agent already installed
- stat:
- path: /etc/init.d/td-agent
- register: _td_agent_st
-
- - name: Create fluentd conf.d
- become: yes
- file:
- path: /etc/td-agent/conf.d
- recurse: yes
- state: directory
-
- - name: Configure fluentd forwarding
- become: yes
- template:
- src: files/fluentd/forward-conf.j2
- dest: /etc/td-agent/conf.d/forward.conf
- notify:
- - restart fluentd if installed
-
- - name: Copy fluentd configuration
- become: yes
- copy:
- src: "{{ item }}"
- dest: "/etc/td-agent/conf.d/{{ item | basename }}"
- with_items:
- # These are specifc to each host-group
- - "{{ fluentd_source_configs }}"
- notify:
- - restart fluentd if installed
-
- handlers:
- - name: restart fluentd if installed
- become: yes
- systemd:
- daemon_reload: yes
- name: td-agent
- state: restarted
- when: _td_agent_st.stat.exists
-
- roles:
- - role: ome.fluentd
-
- vars:
- _monitoring_idr_environment: "{{ idr_parent_environment | default(idr_environment | default('idr')) + '-management-hosts' }}"
- fluentd_server_address: "{{ hostvars[groups[_monitoring_idr_environment][0]]['ansible_' + (idr_net_iface | default('eth0'))]['ipv4']['address'] }}"
- fluentd_shared_key: "{{ idr_secret_fluentd_shared_key | default('fluentd') }}"
diff --git a/ansible/requirements.yml b/ansible/requirements.yml
index 3dda9e44..c9483c05 100644
--- a/ansible/requirements.yml
+++ b/ansible/requirements.yml
@@ -152,9 +152,6 @@
######################################################################
# External development roles
-- src: ome.fluentd
- version: 0.2.3
-
- src: ome.omero_prometheus_exporter
version: 0.3.6
diff --git a/docs/deployment.md b/docs/deployment.md
index 2deca38e..d9ef5d39 100644
--- a/docs/deployment.md
+++ b/docs/deployment.md
@@ -27,7 +27,7 @@ This playbook is *not* idempotent.
### [`idr-09-monitoring.yml`](../ansible/idr-09-monitoring.yml)
This is an optional playbook to set up monitoring of the IDR and OMERO.server.
-This includes configuring all servers with [Prometheus](hhttps://prometheus.io/), setting up centralised logging with [fluentd](https://www.fluentd.org/)/, and enabling Slack notifications errors.
+This includes configuring all servers with [Prometheus](hhttps://prometheus.io/), and enabling Slack notifications errors.
You will need to provide a secret Slack token for Slack notifications.
diff --git a/docs/monitoring.md b/docs/monitoring.md
index 6a741b46..6fe7cd8a 100644
--- a/docs/monitoring.md
+++ b/docs/monitoring.md
@@ -22,20 +22,6 @@ If you want to query the raw metrics directly you can connect to prometheus:
Open http://localhost:9090/ and you can interactively query the metrics using [PromQL (Prometheus Query Language)](https://prometheus.io/docs/prometheus/latest/querying/basics/).
-
-## Centralised logging
-
-The IDR has a experimental centralised logging collector using Fluentd, ElasticSearch and Kibana:
-
- ssh idr-proxy -L 5601:management:5601
-
-Open http://localhost:5601/.
-Limitations:
-- An index pattern `logstash-*` may need to be create on the first access
-- The ElasticSearch server is not clustered and has limited querying capacity. It can be easily overloaded.
-- Logs are kept for two weeks.
-
-
## IDR Pilot Monitoring
All IDR pilot test instances are currently monitored by a single monitoring server.
@@ -68,9 +54,3 @@ This playbook also deploys all the required [Prometheus exporters](https://prome
Grafana runs in Docker and is [deployed by a playbook](../ansible/management-grafana.yml).
The playbook automatically configures Grafana and uploads some pre-created dashboards using Grafana's REST API.
-
-
-### ElasticSearch, Kibana, Fluentd
-
-ElasticSearch, Kibana and the Fluentd server are run in Docker and [deployed by a playbook](../ansible/management-fluentd.yml).
-This playbook also deploys the Fluentd logging agents that collect the logs on the OMERO and proxy servers, and forward them to the central Fluentd server.