several documentation updates

This cleans up the formatting of some of the `description` fields so that the asciidoc version preserves the formatting. There are several other minor formatting changes. Most of the previously undocumented fields now have at least some minimal documentation. This simplifies the code that ensures utf-8 correctness. This should work with both python2 and python3.
ViaQ · Jun 9, 2017 · 6eaf0cf · 6eaf0cf
1 parent 4d3bb80
commit 6eaf0cf
Show file tree

Hide file tree

Showing 9 changed files with 144 additions and 120 deletions.
diff --git a/namespaces/_default_.yml b/namespaces/_default_.yml
@@ -8,16 +8,16 @@ field_defaults:
 _default_:
   type: group
   name: "Default"
-  description: >
-    Contains common fields and generic pattern-matched field descriptions; this
-    is what populates the actual mappings of _default_ in the template's mapping
-    section.
+  description: |
+    The top level fields are common to every application, and may be present in every record.
+    For the Elasticsearch template, this is what populates the actual mappings
+    of _default_ in the template's mapping section.
   fields:
     - name: "@timestamp"
       type: date
       format: yyyy-MM-dd HH:mm:ss,SSSZ||yyyy-MM-dd'T'HH:mm:ss.SSSSSSZ||yyyy-MM-dd'T'HH:mm:ssZ||dateOptionalTime
       example: 2015-01-24T14:06:05.071Z
-      description: >
+      description: |
         UTC value marking when the log payload was created, or when log payload was first collected if the creation time is not known;
         this is the log processing pipeline’s “best effort” determination of when the log payload was generated
         FYI: the “@” prefix convention to note a field as being reserved for a particular use; in this case, most tools by default look for “@timestamp” with ElasticSearch
@@ -39,8 +39,10 @@ _default_:
     - name: hostname
       type: string
       description: >
-        FQDN of the entity generating the original payload
-        This field is a best effort attempt to derive this context; sometimes the entity generating it knows it; other times that entity has a restricted namespace itself, and the collector or normalizer knows that
+        FQDN of the entity generating the original payload.  This field is a
+        best effort attempt to derive this context; sometimes the entity
+        generating it knows it; other times that entity has a restricted
+        namespace itself, and the collector or normalizer knows that.
 
     - name: ipaddr4
       type: ip
@@ -61,19 +63,21 @@ _default_:
     - name: level
       type: string
       example: info
-      description: >
+      description: |
         Logging level as provided by: rsyslog(severitytext property), python's
         logging module, etc.
-        Possible values are as listed here: http://sourceware.org/git/?p=glibc.git;a=blob;f=misc/sys/syslog.h;h=ee01478c4b19a954426a96448577c5a76e6647c0;hb=HEAD#l74 plus trace
-        That is: alert crit debug emerg err info notice trace warning
+        Possible values are as listed here: http://sourceware.org/git/?p=glibc.git;a=blob;f=misc/sys/syslog.h;h=ee01478c4b19a954426a96448577c5a76e6647c0;hb=HEAD#l74 plus `trace` and `unknown`
+        That is: alert crit debug emerg err info notice trace unknown warning
         Note that `trace` isn't in the syslog.h list but many applications use it
-        `trace` should be considered the highest (most verbose) level, higher than `debug`
+        `unknown` is only used when the logging system gets a value it doesn't understand
+        `unknown` is the highest level
+        `trace` should be considered as higher (more verbose) than `debug`
         `error` should be converted to `err`
         `panic` should be converted to `emerg`
         `warn` should be converted to `warning`
         Numeric values from syslog/journal PRIORITY can usually be mapped using the priority values as listed here:
         http://sourceware.org/git/?p=glibc.git;a=blob;f=misc/sys/syslog.h;h=ee01478c4b19a954426a96448577c5a76e6647c0;hb=HEAD#l51
-        That is, 0 -> emerg, 1 -> alert, ..., 7 -> debug
+        That is, 0 -> emerg, 1 -> alert, ..., 7 -> debug, 8 -> trace, 9 -> unknown
         Log levels/priorities from other logging systems should be mapped to the nearest match
         For example, from python logging: https://docs.python.org/2.7/library/logging.html#logging-levels
         CRITICAL -> crit, ERROR -> err, ...., DEBUG -> debug
@@ -140,8 +144,9 @@ _default_:
       example: my-cool-project-in-lab04
       doc_values: false
       index: not_analyzed
-      description: >
-        format [a-zA-Z][a-zA-Z0-9-]{0,61}[a-zA-Z0-9]
+      description: |
+        format: [a-zA-Z][a-zA-Z0-9-]{0,61}[a-zA-Z0-9]
+
         Associate this record with the namespace with this name.
         This value will not be stored.  It is only used to associate the
         record with the appropriate namespace for access control and
@@ -155,8 +160,9 @@ _default_:
     - name: namespace_uuid
       type: string
       example: 82f13a8e-882a-4344-b103-f0a6f30fd218
-      description: >
-        format [a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}
+      description: |
+        format: [a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}
+
         The uuid associated with the namespace_name.
         This value will not be stored.  It is only used to associate the
         record with the appropriate namespace for access control and

diff --git a/namespaces/aushape.yml b/namespaces/aushape.yml
@@ -3,8 +3,7 @@ namespace:
   name: aushape
   type: group
   description: >
-    Audit events converted with aushape
-    For more information please see
+    Audit events converted with aushape.  For more information please see
     https://github.com/Scribery/aushape
   fields:
   - name: serial

diff --git a/namespaces/kubernetes.yml b/namespaces/kubernetes.yml
@@ -48,5 +48,15 @@ namespace:
     description: >
       Kubernetes node name
 
+  - name: master_url
+    type: string
+    description: >
+      Kubernetes Master URL
+
+  - name: annotations
+    type: group
+    description: >
+      Annotations associated with the OpenShift object
+
 doc_sections:
   - ["pod_log", "Openshift logging metadata"]
diff --git a/namespaces/systemd.yml b/namespaces/systemd.yml
@@ -3,9 +3,12 @@ namespace:
   name: systemd
   type: group
   description: >
-    Contains common fields specific to systemd journal.
-    For more information please see
+    Contains common fields specific to systemd journal.  For more information
+    please see
     https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html
+    Applications may write their own fields to the journal.  These will be
+    available under the `systemd.u` namespace.  `RESULT` and `UNIT` are two
+    such fields.
   fields:
   - name: k
     type: group
@@ -15,182 +18,183 @@ namespace:
     - name: KERNEL_DEVICE
       type: string
       description: >
-        TODO
+        https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html#_KERNEL_DEVICE=
 
     - name: KERNEL_SUBSYSTEM
       type: string
       description: >
-        TODO
+        https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html#_KERNEL_SUBSYSTEM=
 
     - name: UDEV_DEVLINK
       type: string
       description: >
-        TODO
+        https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html#_UDEV_DEVLINK=
 
     - name: UDEV_DEVNODE
       type: string
       description: >
-        TODO
+        https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html#_UDEV_DEVNODE=
 
     - name: UDEV_SYSNAME
       type: string
       description: >
-         TODO
+         https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html#_UDEV_SYSNAME=
 
   - name: t
     type: group
     description: >
-      TODO.
+      trusted journal fields, fields that are implicitly added by the journal and cannot be altered by client code.
     fields:
     - name: AUDIT_LOGINUID
       type: string
       description: >
-        TODO
+        https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html#_AUDIT_SESSION=
 
     - name: BOOT_ID
       type: string
       description: >
-        TODO
+        https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html#_BOOT_ID=
 
     - name: AUDIT_SESSION
       type: string
       description: >
-        TODO
+        https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html#_AUDIT_SESSION=
 
     - name: CAP_EFFECTIVE
       type: string
       description: >
-        TODO
+        https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html#_CAP_EFFECTIVE=
 
     - name: CMDLINE
       type: string
       description: >
-        TODO
+        https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html#_COMM=
       norms:
         enabled: false
 
     - name: COMM
       type: string
       description: >
-        TODO
+        https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html#_COMM=
 
     - name: EXE
       type: string
       description: >
-        TODO
+        https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html#_COMM=
       norms:
         enabled: false
 
     - name: GID
       type: string
       description: >
-        TODO
+        https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html#_PID=
 
     - name: HOSTNAME
       type: string
       description: >
-        TODO
+        https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html#_HOSTNAME=
 
     - name: MACHINE_ID
       type: string
       description: >
-        TODO
+        https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html#_MACHINE_ID=
 
     - name: PID
       type: string
       description: >
-        TODO
+        https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html#_PID=
 
     - name: SELINUX_CONTEXT
       type: string
       description: >
-        TODO
+        https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html#_SELINUX_CONTEXT=
 
     - name: SOURCE_REALTIME_TIMESTAMP
       type: string
       description: >
-        TODO
+        https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html#_SOURCE_REALTIME_TIMESTAMP=
+        This is converted to RFC 3339 NS format
 
     - name: SYSTEMD_CGROUP
       type: string
       description: >
-        TODO
+        https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html#_SYSTEMD_CGROUP=
 
     - name: SYSTEMD_OWNER_UID
       type: string
       description: >
-        TODO
+        https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html#_SYSTEMD_CGROUP=
 
     - name: SYSTEMD_SESSION
       type: string
       description: >
-        TODO
+        https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html#_SYSTEMD_CGROUP=
 
     - name: SYSTEMD_SLICE
       type: string
       description: >
-        TODO
+        https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html#_SYSTEMD_CGROUP=
 
     - name: SYSTEMD_UNIT
       type: string
       description: >
-        TODO
+        https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html#_SYSTEMD_CGROUP=
 
     - name: SYSTEMD_USER_UNIT
       type: string
       description: >
-        TODO
+        https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html#_SYSTEMD_CGROUP=
 
     - name: TRANSPORT
       type: string
       description: >
-        TODO
+        https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html#_TRANSPORT=
 
     - name: UID
       type: string
       description: >
-        TODO
+        https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html#_PID=
 
   - name: u
     type: group
     description: >
-      TODO.
+      fields that are directly passed from clients and stored in the journal.
     fields:
     - name: CODE_FILE
       type: string
       norms:
         enabled: false
       description: >
-        TODO
+        https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html#CODE_FILE=
 
     - name: CODE_FUNCTION
       type: string
       description: >
-        TODO
+        https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html#CODE_FILE=
 
     - name: CODE_LINE
       type: string
       description: >
-        TODO
+        https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html#CODE_FILE=
 
     - name: ERRNO
       type: string
       description: >
-        TODO
+        https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html#ERRNO=
 
     - name: MESSAGE_ID
       type: string
       description: >
-        TODO
+        https://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html#MESSAGE_ID=
 
     - name: RESULT
       type: string
       norms:
         enabled: false
       description: >
-        TODO
+        For private use only.
 
     - name: UNIT
       type: string
       description: >
-        TODO
+        For private use only.
diff --git a/namespaces/tlog.yml b/namespaces/tlog.yml
@@ -3,8 +3,7 @@ namespace:
   name: tlog
   type: group
   description: >
-    Tlog terminal I/O recording messages
-    For more information please see
+    Tlog terminal I/O recording messages.  For more information please see
     https://github.com/Scribery/tlog
   fields:
   - name: ver