Merge branch 'rc-v1.36.0' into 'master'

chore: v1.36.0 release See merge request automation-toolchain/f5-telemetry!732
F5Networks · Aug 14, 2024 · 0245330 · 0245330
2 parents 3dd0c45 + c267990
commit 0245330
Show file tree

Hide file tree

Showing 366 changed files with 54,894 additions and 29,951 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -26,7 +26,12 @@ variables:
       - "false"
       - "true"
     description: "Force docs to be build and published to GitLab on non-docs branches"
-
+  SKIP_UNIT_TESTS:
+    value: "false"
+    options:
+      - "false"
+      - "true"
+    description: "Force CI/CD to skip 'test_nodeX' jobs on feature branches"
 
 ##############################################################
 #                                                            #
@@ -42,6 +47,12 @@ variables:
     not_docs:
         rules:
             - if: $CI_COMMIT_BRANCH !~ /^docs/
+    skip_unit_feat_brach:
+        rules:
+            - if: $CI_COMMIT_BRANCH == 'master' || $CI_COMMIT_BRANCH == 'develop'
+            - if: $SKIP_UNIT_TESTS == 'true'
+              when: never
+            - !reference [.rules_config, not_docs, rules]
 
 
 .run_unittest_cmd: &run_unittest_cmd
@@ -52,9 +63,7 @@ variables:
 - echo "**** Node.js version - $nodeFullVer (bundled with npm@$(npm --version)) ****"
 - ver=$(echo $nodeFullVer | cut -d. -f 1)
 - installNPM=no
-- if [ "$ver" = "v4" ]; then
--   installNPM=5
-- elif [ "$ver" = "v8" ]; then
+- if [ "$ver" = "v8" ]; then
 -   installNPM=6
 - elif [ "$ver" = "v10" ] || [ "$ver" = "v12" ] || [ "$ver" = "v14" ]; then
 -   installNPM=7
@@ -72,7 +81,6 @@ variables:
     rules:
         - !reference [.rules_config, not_docs, rules]
 
-
 .test_job_definition: &test_job_definition
     extends:
         - .job_definition
@@ -175,43 +183,13 @@ lint:
         - *install_unittest_packages_cmd
         - npm run lint
 
-# BIG-IP 13.x and BIG-IP 14.0
-test_node4:
-    extends:
-        - .run_unittest
-    image: ${ARTIFACTORY_DOCKER_HUB}/node:4.8.0
-
 # BIG-IP 14.1+
 test_node8:
     extends:
         - .run_unittest
     image: ${ARTIFACTORY_DOCKER_HUB}/node:8.11.1
-
-test_node10:
-    extends:
-        - .run_unittest
-    image: ${ARTIFACTORY_DOCKER_HUB}/node:10
-
-test_node12:
-    extends:
-        - .run_unittest
-    image: ${ARTIFACTORY_DOCKER_HUB}/node:12
-
-test_node14:
-    extends:
-        - .run_unittest
-    image: ${ARTIFACTORY_DOCKER_HUB}/node:14
-
-test_node16:
-    extends:
-        - .run_unittest
-    image: ${ARTIFACTORY_DOCKER_HUB}/node:16
-
-# disabled for now, failing unit tests
-.test_node_latest:
-    extends:
-        - .run_unittest
-    image: ${ARTIFACTORY_DOCKER_HUB}/node:latest
+    rules:
+        - !reference [.rules_config, skip_unit_feat_brach, rules]
 
 # packages audit
 npm_audit:
@@ -246,6 +224,8 @@ coverage:
         name: ${CI_COMMIT_REF_NAME}_unittests_coverage
         paths:
             - coverage
+    rules:
+        - !reference [.rules_config, skip_unit_feat_brach, rules]
 
 build_rpm:
     image: ${ATG_ARTIFACTORY_PUBLISH_URL}/${ATG_ARTIFACTORY_DOCKER_REPO}/f5-telemetry-streaming-rpm-builder-image:v1.2
@@ -326,7 +306,10 @@ test_functional:
             # enable this job
             - $RUN_FUNCTIONAL_TESTS == "true"
     script:
-        - export TEST_HARNESS_FILE=${CI_PROJECT_DIR}/harness_facts_flat.json
+        - if [ -z "${TEST_HARNESS_FILE}" ]; then
+        -     export TEST_HARNESS_FILE=${CI_PROJECT_DIR}/harness_facts_flat.json
+        - fi
+        - echo "Harness data - ${TEST_HARNESS_FILE}"
         - ls ./dist -ls
         # really only need dev dependencies
         - *install_unittest_packages_cmd
@@ -347,8 +330,6 @@ teardown_env:
 ##############################################################
 # END VIO                                                    #
 ##############################################################
-
-
 ##############################################################
 # BEGIN CLOUD                                                #
 ##############################################################
@@ -380,7 +361,6 @@ teardown_env_azure:
     script:
         - source ./scripts/functional_testing/azure/teardown_env.sh
 
-
 # Azure Gov
 deploy_env_azure_gov:
     extends:
@@ -402,14 +382,12 @@ test_functional_azure_gov:
         - build_rpm
         - deploy_env_azure_gov
 
-
 teardown_env_azure_gov:
     extends:
         - .teardown_azure
     script:
         - source ./scripts/functional_testing/azure/teardown_env.sh 'gov'
 
-
 deploy_env_aws:
     extends:
         - .base_aws
@@ -469,7 +447,6 @@ pages:
     rules:
         - !reference [.rules_config, docs_only, rules]
 
-
 # Publish docs to clouddocs.f5.com
 publish_docs_to_production:
     image: ${CONTAINTHEDOCS_IMAGE}

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,6 +1,33 @@
 # Changelog
 Changes to this project are documented in this file. More detail and links can be found in the Telemetry Streaming [Document Revision History](https://clouddocs.f5.com/products/extensions/f5-telemetry-streaming/latest/revision-history.html).
 
+## 1.Y.Z
+### Added
+### Fixed
+### Changed
+### Removed
+
+## 1.36.0
+**Important**: Starting from BIG-IP Telemetry Streaming version 1.36.0, BIG-IP Telemetry Streaming no longer supports BIG-IP 13.1 to 15.0.x. However, if you are still using the BIG-IP 13.1 to 15.0.x versions, you can use BIG-IP Telemetry Streaming 1.35.0 or earlier.
+### Added
+- New "httpTimeout" option to configure HTTP timeout value for incoming REST API requests to Pull Consumers
+- MBIPMP-37620: EndpointLoader support pagination
+- MBIPMP-37621: Support for additional system poller settings: workers and httpAgentOpts
+### Fixed
+- Event Listener throws an uncaught error when the buffer pointer was set to a wrong position
+- Event Listener allocates more memory than required for chunked data
+- MBIPMP-41257: Update iHealth to use new authentication API
+### Changed
+- MBIPMP-37253: Update memoryMonitor.provisionedMemory limits (should allow to provision up to runtime.maxHeapSize value)
+- MBIPMP-37255: Removed node 4 testing due to the end of support for BIG-IP 13.1 : ([Supported BIG-IP Versions](https://my.f5.com/manage/s/article/K5903)).
+- MBIPMP-37374: Update Kafka consumer to support multi-host, formatting and custom opts
+- MBIPMP-37631: Update Splunk consumer to use data queues, HTTP agent options, events batching
+- Update Telemetry_System to require `passphrase` when `username` defined.
+- Set default logging level to "info" instead of "debug".
+- Allow to set single-digit minute value for iHealth polling interval
+### Removed
+- MBIPMP-37374: Kafka consumer support for ZooKeepeer
+
 ## 1.35.0
 ### Added
 - NEXTACC-414: Add Resource Monitor

diff --git a/SUPPORT.md b/SUPPORT.md
@@ -24,8 +24,7 @@ Currently supported versions:
 | Software Version | Release Type           | First Customer Ship | End of Support   |
 |------------------|------------------------|---------------------|------------------|
 | TS 1.33.0        | LTS                    |  22-Mar-2023        | Maintenance mode |
-| TS 1.34.0        | Feature                |  16-Jan-2024        | 16-Apr-2024      |
-| TS 1.35.0        | Feature                |  23-Feb-2024        | 23-May-2024      |
+| TS 1.36.0        | Feature                |  14-Aug-2024        | 14-Nov-2024      |
 
 Versions no longer supported:
 
@@ -65,5 +64,7 @@ Versions no longer supported:
 | TS 1.30.0        | Feature                |  15-Jul-2022        | 15-Oct-2022     |
 | TS 1.32.0        | Feature                |  04-Oct-2022        | 04-Jan-2023     |
 | TS 1.27.1        | LTS                    |  19-Apr-2022        | 19-Apr-2023     |
+| TS 1.34.0        | Feature                |  16-Jan-2024        | 16-Apr-2024     |
+| TS 1.35.0        | Feature                |  23-Feb-2024        | 23-May-2024     |
 
 See the [Release notes](https://github.com/F5Networks/f5-telemetry-streaming/releases) and [Telemetry Streaming documentation](https://clouddocs.f5.com/products/extensions/f5-telemetry-streaming/latest/revision-history.html) for new features and issues resolved for each release.
diff --git a/contributing/README.md b/contributing/README.md
@@ -108,7 +108,7 @@ How does the project handle a typical `POST` request?
             "trace": false,
             "format": "default"
         },
-        "schemaVersion": "1.35.0"
+        "schemaVersion": "1.36.0"
     }
 }
 ```
@@ -121,13 +121,13 @@ What happens in the system internals between request and response?
 - LX worker receives request which validates URI, etc.
     - ref: [restWorker.js](../src/nodejs/restWorker.js)
 - The appropriate handler processes the request
-    - ref: [router.js](../src/lib/requestHandlers/router.js)
+    - ref: [REST API](../src/lib/restAPI)
 - Request is validated using JSON schema and AJV, config event fires
     - ref: [config.js](../src/lib/config.js)
 - System poller, event listener, etc. configures system resources
-    - ref: [systemPoller.js](../src/lib/systemPoller.js), [eventListener.js](../src/lib/eventListener.js), etc.
+    - ref: [systemPoller.js](../src/lib/systemPoller), [eventListener](../src/lib/eventListener/), etc.
 - Client response sent with validated config
-    - ref: [declareHandler](../src/lib/requestHandlers/declareHandler.js)
+    - ref: [declareHandler](../src/lib/restAPI/handlers/declare.js)
     ```javascript
         return promise.then((config) => {
             this.code = 200;
@@ -160,16 +160,14 @@ All core modules are included inside `../src/lib/`
     - Purpose: Hook for incoming HTTP requests
 - [config.js](../src/lib/config.js)
     - Purpose: Handle configuration actions... such as validation, persistent storage, etc.
-- [systemPoller.js](../src/lib/systemPoller.js)
+- [systemPoller](../src/lib/systemPoller/)
     - Purpose: Handles CRUD-like actions for any system pollers required based on client configuration
-    - Related: See [iHealthPoller.js](../src/lib/iHealthPoller.js)
-- [eventListener.js](../src/lib/eventListener/index.js)
+    - Related: See [iHealthPoller.js](../src/lib/ihealth/)
+- [eventListener](../src/lib/eventListener/)
     - Purpose: Handles CRUD-like actions for any event listeners required based on client configuration.
-- [systemStats.js](../src/lib/systemStats.js)
-    - Purpose: Called by system poller to create stats object based on the static JSON configuration files available in `config/` directory such as [properties.json](../src/lib/properties.json)
-- [consumers.js](../src/lib/consumers.js)
+- [consumers](../src/lib/consumers/)
     - Purpose: Handles load/unload actions for any consumers required based on client configuration. Consumers must exist in `consumers` directory, see [Adding a New Consumer](#adding-a-new-consumer)
-- [forwarder.js](../src/lib/forwarder.js)
+- [dataPipeline](../src/lib/dataPipeline/)
     - Purpose: Handles calling each loaded consumer when an event is ready for forwarding (system poller event, event listener event, etc.)
 
 ---
@@ -198,13 +196,17 @@ Collect the raw data from the device by adding a new endpoint to the paths confi
     "path": "/mgmt/tm/sys/someEndpoint", // REST endpoint
     "includeStats": true, // Certain data is only available via /mgmt/tm/sys/someEndpoint as opposed to /mgmt/tm/sys/someEndpoint/stats, this property accommodates for this by making call to /stats (for each item) and adding that data to the original object
     "expandReferences": { "membersReference": { "endpointSuffix": "/stats" } }, // Certain data requires getting a list of objects and then in each object expanding/following references to a child object.  'membersReference' is the name of that key (currently looking under 'items' in the data returned) and will result in self link data being retrieved and 'membersReference' key being replaced with that data. If 'endpointSuffix' is supplied, a suffix is added to each self link prior to retrieval, otherwise, the value of self link as is will be used. In cases like gslb where both config and stats are needed, both the `link` and `link/stats` need to be fetched, hence, the resulting config is "expandReferences": { "membersReference": { "includeStats": true } }, which is equivalent to "expandReferences": { "membersReference": { "endpointSuffix": "", "includeStats": true } }. TODO: revisit keywords/ naming here to consolidate and avoid confusion
-    "endpointFields": [ "name", "fullPath", "selfLink", "ipProtocol", "mask" ], // Will collect only these fields from the endpoint. Useful when using includeStats and the same property exists in both endpoints. Also can be used instead of a large exclude/include statement in properties.json
     "body": "{ \"command\": \"run\", \"utilCmdArgs\": \"-c \\\"/bin/df -P | /usr/bin/tr -s ' ' ','\\\"\" }", // Certain information may require using POST instead of GET and require an HTTP body, if body is defined that gets used along with a POST. Body can be either string or object
     "name": "someStatRef", // Alternate name to reference in properties.json, default is to use the endpoint
     "ignoreCached": true // Invalidate cached response of previous request to endpoint
 }
 ```
 
+*Response caching:* When endpoint validates against at least one of the following constraints then response will not be cached:
+- **pagination** property set to `true`
+- **body** property used to specify a body for HTTP POST request
+- **ignoreCached** property omitted or not equals to `false`
+
 ---
 #### Adding System Poller Stats - Properties.json
 

diff --git a/contributing/docs_release.md b/contributing/docs_release.md
@@ -31,6 +31,7 @@ If you need to update docs for the most recent publicly available release:
 
 If you need to update docs for the LST release:
 - create `docs` (any name allowed) branch off `docs-X.Y.Z`, where X.Y.Z is the LTS release version
+- update CI file to look like the one from `docs-1.27.1` branch (do not forget to change the target version in the file)
 - do docs changes
 - once the work done, create MR to merge `docs` branch to `docs-X.Y.Z-staging`
 - review your changes once deployed to GitLab Pages

diff --git a/contributing/process_release.md b/contributing/process_release.md
@@ -67,6 +67,8 @@
     * 1.33.0 - 22.1 MB
     * 1.34.0 - 18.4 MB
     * 1.35.0 - 18.4 MB
+    * 1.36.0 - 20.6 MB (NOTE: inclusion of new version of Kafka library)
+    * 1.37.0 - ???? MB
   * Install build to BIG-IP, navigate to folder `/var/config/rest/iapps/f5-telemetry/` and check following:
     * Run `du -sh` and check that folder's size (shouldn't be much greater than previous versions):
       * 1.4.0 - 65 MB
@@ -100,6 +102,8 @@
       * 1.32.0 - 154 MB
       * 1.33.0 - 164 MB
       * 1.35.0 - 164 MB
+      * 1.36.0 - 154 MB
+      * 1.37.0 - ??? MB
     * Check `node_modules` folder - if you see `eslint`, `mocha` or something else from [package.json](package.json) `devDependencies` section - something wrong with build process. Probably some `npm` flags are work as not expected and it MUST BE FIXED before publishing.
 * Ensure that all tests (unit tests and functional tests passed)
 * Optional: Ensure that your local tags match remote. If not, remove all and re-fetch: