Merge branch 'rc-1.34.0' into 'master'

chore: release 1.34.0

See merge request automation-toolchain/f5-telemetry!687

.gitlab-ci.yml

@@ -1,4 +1,4 @@
-image: ${ARTIFACTORY_DOCKER_HUB}/node:8
+image: ${ARTIFACTORY_DOCKER_HUB}/node:16
 
 stages:
     - lint
@@ -25,31 +25,29 @@ include:
 # Jobs and commands templates                                #
 #                                                            #
 ##############################################################
-.install_unittest_packages_cmd: &install_unittest_packages_cmd
-- nodeFullVer=$(node --version)
-- echo "Node.js version - $nodeFullVer"
-- echo "NPM origin version - $(npm --version)"
-- ver=$(echo $nodeFullVer | head -c 3)
-- if [[ "$ver" == "v4." ]]; then
--   npm install --global [email protected]
--   npm run install-test
--   npm install [email protected] --no-optional --no-save
--   echo "**** Using mocha 5.2.0 ****"
-- elif [[ "$ver" == "v6." ]] || [[ "$nodeFullVer" == "v8.11.1" ]]; then
-# workaround for npm upgrade, otherwise it missing basic modules
--   npm i -g npm3 && npm3 -g uninstall npm
--   npm3 i -g [email protected]
--   npm run install-test
--   npm install [email protected] --no-optional -- no-save
--   echo "**** Using mocha 6.2.2 ****"
-- else
--   npm run install-test
-- fi
-- echo "NPM updated version - $(npm --version)"
 
 .run_unittest_cmd: &run_unittest_cmd
 - npm run test-only
 
+.install_unittest_packages_cmd: &install_unittest_packages_cmd
+- nodeFullVer=$(node --version)
+- echo "**** Node.js version - $nodeFullVer (bundled with npm@$(npm --version)) ****"
+- ver=$(echo $nodeFullVer | cut -d. -f 1)
+- installNPM=no
+- if [ "$ver" = "v4" ]; then
+-   installNPM=5
+- elif [ "$ver" = "v8" ]; then
+-   installNPM=6
+- elif [ "$ver" = "v10" ] || [ "$ver" = "v12" ] || [ "$ver" = "v14" ]; then
+-   installNPM=7
+- fi
+- if [ "$installNPM" != "no" ]; then
+-   echo "Updating npm to '${installNPM}'"
+-   npm install --global npm@"${installNPM}"
+-   echo "**** Installed npm@$(npm --version)) ****"
+- fi
+- npm run install-test
+
 .job_definition: &job_definition
     tags:
         - docker-executor
@@ -150,66 +148,53 @@ include:
 lint:
     extends:
         - .test_job_definition
-    image: ${ARTIFACTORY_DOCKER_HUB}/node:12
+    image: ${ARTIFACTORY_DOCKER_HUB}/node:14
     stage: lint
     script:
         - *install_unittest_packages_cmd
         - npm run lint
 
-# BIG-IP 13.x and BIG-IP 14.0, unittests only (without coverage check)
+# BIG-IP 13.x and BIG-IP 14.0
 test_node4:
     extends:
         - .run_unittest
     image: ${ARTIFACTORY_DOCKER_HUB}/node:4.8.0
 
-# just in case, unittests only (without coverage check)
-test_node6:
-    extends:
-        - .run_unittest
-    image: ${ARTIFACTORY_DOCKER_HUB}/node:6
-
-# BIG-IP 14.1+, unittests only (without coverage check)
+# BIG-IP 14.1+
 test_node8:
     extends:
         - .run_unittest
     image: ${ARTIFACTORY_DOCKER_HUB}/node:8.11.1
 
-# Node 10 LTS (active, maintenance)
 test_node10:
     extends:
         - .run_unittest
     image: ${ARTIFACTORY_DOCKER_HUB}/node:10
 
-# Node 12 LTS (active, maintenance)
 test_node12:
     extends:
         - .run_unittest
     image: ${ARTIFACTORY_DOCKER_HUB}/node:12
 
-# Node 14 LTS (active, maintenance)
 test_node14:
     extends:
         - .run_unittest
     image: ${ARTIFACTORY_DOCKER_HUB}/node:14
 
-# Node 16 LTS (active, maintenance)
-# Pinned to 16.14.0, as later versions include versions of npm that are incompatible with our package-lock.json file version
 test_node16:
     extends:
         - .run_unittest
-    image: ${ARTIFACTORY_DOCKER_HUB}/node:16.14.0
+    image: ${ARTIFACTORY_DOCKER_HUB}/node:16
 
-# mostly for containers, unittests only (without coverage check)
-# disabled for now, npm has one issue with DNS (IPv6), probably
-# docker image issue. Node version 17.x, npm version 8.1.0
+# disabled for now, failing unit tests
 .test_node_latest:
     extends:
         - .run_unittest
     image: ${ARTIFACTORY_DOCKER_HUB}/node:latest
 
 # packages audit
 npm_audit:
-    image: ${ARTIFACTORY_DOCKER_HUB}/node:14
+    image: ${ARTIFACTORY_DOCKER_HUB}/node:16
     extends:
         - .test_job_definition
     allow_failure: true
@@ -221,32 +206,30 @@ npm_audit:
         - *install_unittest_packages_cmd
         # npm list - ignore errors, interested in audit errors only
         - npm list --json || echo ""
+        - npm outdated || echo ""
         # npm audit - install includes audit, but perform specific check and fail if needed
         - audit_report=$(npm audit --production --json) || echo ""
         - echo "$audit_report"
-        - actions=$(echo $audit_report | jq .actions | jq length)
-        - if [ $actions -ne 0 ]; then echo 'ERROR! vulnerabilities exist'; exit 1; fi
+        - vulnerabilities=$(echo $audit_report | jq .vulnerabilities | jq length)
+        - if [ $vulnerabilities -ne 0 ]; then echo 'ERROR! vulnerabilities exist'; exit 1; fi
 
-# run tests and check code coverage
 coverage:
+    # bind to 14.1+ version to test @grppc-js and OpenTelemetry
     image: ${ARTIFACTORY_DOCKER_HUB}/node:8.11.1
     extends:
         - .test_job_definition
     script:
         - *install_unittest_packages_cmd
-        # run tests with coverage report
         - npm test
     artifacts:
         name: ${CI_COMMIT_REF_NAME}_unittests_coverage
         paths:
             - coverage
 
 build_rpm:
-    image: ${ARTIFACTORY_DOCKER_HUB}/node:12
+    image: ${ATG_ARTIFACTORY_PUBLISH_URL}/${ATG_ARTIFACTORY_DOCKER_REPO}/f5-telemetry-streaming-rpm-builder-image:v1.2
     stage: build
     script:
-        - echo 'CI BUILD'
-        - apt-get update && apt-get install -y rpm --no-install-recommends
         - npm run build
     tags:
         - cm-official-docker-executor
@@ -321,7 +304,6 @@ test_functional:
         - ls ./dist -ls
         # really only need dev dependencies
         - *install_unittest_packages_cmd
-        - npm install [email protected] --no-save
         - npm run test-functional
 
 # should be executed manually to remove the harness
@@ -361,7 +343,6 @@ test_functional_azure:
         - echo $AZURE_VM_HOSTNAME
         - echo $AZURE_VM_IP
         - *install_unittest_packages_cmd
-        - npm install [email protected] --no-save
         - npm run test-functional-cloud-azure
     needs:
         - build_rpm
@@ -390,7 +371,6 @@ test_functional_azure_gov:
         - echo $AZURE_VM_HOSTNAME
         - echo $AZURE_VM_IP
         - *install_unittest_packages_cmd
-        - npm install [email protected] --no-save
         - npm run test-functional-cloud-azure
     needs:
         - build_rpm
@@ -423,7 +403,6 @@ test_functional_aws:
     script:
         - source ./env_metadata/aws.sh
         - *install_unittest_packages_cmd
-        - npm install [email protected] --no-save
         - npm run test-functional-cloud-aws
 
 teardown_env_aws:

CHANGELOG.md

@@ -1,6 +1,16 @@
 # Changelog
 Changes to this project are documented in this file. More detail and links can be found in the Telemetry Streaming [Document Revision History](https://clouddocs.f5.com/products/extensions/f5-telemetry-streaming/latest/revision-history.html).
 
+## 1.34.0
+### Added
+- NEXTACC-387: Update Event Listener to utilize memory and CPU more efficient
+- NEXTACC-396: Add more "build" related metadata to /info endpoint
+- NEXTACC-402: Add 'verbose' logging to reduce amount of unnecessary log messages
+### Fixed
+### Changed
+- TS-752: Update npm packages (@f5devcentral/f5-teem from 1.5.0 to 1.6.1, @grpc/grpc-js from 1.7.1 to 1.8.18, @grpc/proto-loader from 0.7.3 to 0.7.8, @opentelemetry/api from 1.2.0 to 1.4.1, @opentelemetry/exporter-metrics-otlp-grpc from 0.33.0 to 0.41.0, @opentelemetry/exporter-metrics-otlp-http from 0.33.0 to 0.41.0, @opentelemetry/exporter-metrics-otlp-proto from 0.33.0 to 0.41.0, @opentelemetry/sdk-metrics from 0.33.0 to 1.15.0)
+### Removed
+
 ## 1.33.0
 ### Added
 - TS-697: [GitHub #213](https://github.com/F5Networks/f5-telemetry-streaming/issues/213) Allow user provided endpoints for the Azure consumers

README.md

@@ -26,17 +26,17 @@ If you come across a bug or other issue when using BIG-IP Telemetry Streaming, u
 
 **Important**: Github Issues are consistently monitored by F5 staff, but should be considered as best effort only and you should not expect to receive the same level of response as provided by F5 Support. Please open a case as described below with F5 if this is a critical issue.
 
-Because BIG-IP Telemetry Streaming has been created and fully tested by F5 Networks, it is fully supported by F5. This means you can get assistance if necessary from [F5 Technical Support](https://support.f5.com/csp/article/K25327565).
+Because BIG-IP Telemetry Streaming has been created and fully tested by F5, it is fully supported by F5. This means you can get assistance if necessary from [F5 Technical Support](https://support.f5.com/csp/article/K25327565).
 
 Be sure to see the [Support page](SUPPORT.md) in this repo for more details and supported versions of BIG-IP Telemetry Streaming.
 
 ## Copyright
 
-Copyright 2014-2022 F5 Networks Inc.
+Copyright 2014-2024 F5, Inc.
 
-### F5 Networks Contributor License Agreement
+### F5 Contributor License Agreement
 
-Before you start contributing to any project sponsored by F5 Networks, Inc. (F5) on GitHub, you will need to sign a Contributor License Agreement (CLA).  
+Before you start contributing to any project sponsored by F5, Inc. (F5) on GitHub, you will need to sign a Contributor License Agreement (CLA).  
 
 If you are signing as an individual, we recommend that you talk to your employer (if applicable) before signing the CLA since some employment agreements may have restrictions on your contributions to other projects. Otherwise by submitting a CLA you represent that you are legally entitled to grant the licenses recited therein.  
 

SUPPORT.md

@@ -23,8 +23,8 @@ Currently supported versions:
 
 | Software Version | Release Type           | First Customer Ship | End of Support   |
 |------------------|------------------------|---------------------|------------------|
-| TS 1.27.1        | LTS                    |  19-Apr-2022        | 19-Apr-2023      |
 | TS 1.33.0        | LTS                    |  22-Mar-2023        | Maintenance mode |
+| TS 1.34.0        | Feature                |  16-Jan-2024        | 16-Apr-2024      |
 
 Versions no longer supported:
 
@@ -63,5 +63,6 @@ Versions no longer supported:
 | TS 1.31.0        | Feature                |  23-Aug-2022        | 23-Nov-2022     |
 | TS 1.30.0        | Feature                |  15-Jul-2022        | 15-Oct-2022     |
 | TS 1.32.0        | Feature                |  04-Oct-2022        | 04-Jan-2023     |
+| TS 1.27.1        | LTS                    |  19-Apr-2022        | 19-Apr-2023     |
 
 See the [Release notes](https://github.com/F5Networks/f5-telemetry-streaming/releases) and [Telemetry Streaming documentation](https://clouddocs.f5.com/products/extensions/f5-telemetry-streaming/latest/revision-history.html) for new features and issues resolved for each release.

contributing/README.md

@@ -108,7 +108,7 @@ How does the project handle a typical `POST` request?
             "trace": false,
             "format": "default"
         },
-        "schemaVersion": "1.33.0"
+        "schemaVersion": "1.34.0"
     }
 }
 ```
@@ -163,7 +163,7 @@ All core modules are included inside `../src/lib/`
 - [systemPoller.js](../src/lib/systemPoller.js)
     - Purpose: Handles CRUD-like actions for any system pollers required based on client configuration
     - Related: See [iHealthPoller.js](../src/lib/iHealthPoller.js)
-- [eventListener.js](../src/lib/eventListener.js)
+- [eventListener.js](../src/lib/eventListener/index.js)
     - Purpose: Handles CRUD-like actions for any event listeners required based on client configuration.
 - [systemStats.js](../src/lib/systemStats.js)
     - Purpose: Called by system poller to create stats object based on the static JSON configuration files available in `config/` directory such as [properties.json](../src/lib/properties.json)
@@ -349,9 +349,8 @@ Additional information about the testing methodology can be found in the [test r
 ---
 ### Release methodology
 
-Build/publish makes heavy use of GitLab and [.gitlab-ci.yml](../.gitlab-ci.yml).  Check out CI file and GitLab documentation for more details.
+Build/publish makes heavy use of GitLab and [.gitlab-ci.yml](../.gitlab-ci.yml). Check out CI file and GitLab documentation for more details.
 
-- Add *new* RPM to `dist/` directory (from build artifact on mainline development branch)
 - Publish to artifactory (automated on new tags)
 - Push to GitLab (mainline release branch)
 - Push to GitHub (mainline release branch)
@@ -375,3 +374,19 @@ The current process involves adding a `doc` label to an issue to note it require
 See the [examples](../examples/declarations) directory for curated artifacts such as declaration examples, output examples, AS3 declaration example, etc.
 
 See the [INTERNAL_README.md](../INTERNAL_README.md) for an internal explanation of most features.
+
+### Development
+
+Local environment requirements:
+- NodeJS - any version
+- npm - preferable is latest release for v7 because it uses lockfile version 2 that is backward compatible with version 1 (important for node 4.8.0 + npm@5 and node 8.11.1 + npm@6). But if you not going to add new packages while working on a task then any npm version can be used and then you should use `npm run install-test` command to install packages and prepare dev env.
+
+Note: to make everyday live easier you may start to use tools like `volta` or `nvm`.
+
+### Package updates
+
+Some packages has newer versions that dropped support for node@4 and node@8 - TS should use older version (despite the type of package - dev or prod). In this case those packages may have vulnerabilities that will be never fixed. We can try to contribute to the corresponding project/package but it is unlikle the fix will be accepted for such old version. In this case add note(s) to package.json file - see **comments** section for examples.
+
+Some `devDependencies`, e.g. **mocha**, **nyc**, intentionally pinned to older versions to avoid additional manipulations at CI/CD time or local envrionment.
+
+`devDependencies` that are unique for `functional` testing only, e.g. **ssh2**, may use the version that applicable for `functional` testing evnrionment only.

contributing/process_release.md

@@ -69,6 +69,7 @@
     * 1.31.0 - 20.5 MB (NOTE: inclusion of OpenTelemetry and grpc-js libraries)
     * 1.32.0 - 20.5 MB
     * 1.33.0 - 22.1 MB
+    * 1.34.0 - 18.4 MB
   * Install build to BIG-IP, navigate to folder `/var/config/rest/iapps/f5-telemetry/` and check following:
     * Run `du -sh` and check that folder's size (shouldn't be much greater than previous versions):
       * 1.4.0 - 65 MB
@@ -101,6 +102,7 @@
       * 1.31.0 - 153 MB (NOTE: inclusion of OpenTelemetry and grpc-js libraries)
       * 1.32.0 - 154 MB
       * 1.33.0 - 164 MB
+      * 1.34.0 - 136 MB
     * Check `node_modules` folder - if you see `eslint`, `mocha` or something else from [package.json](package.json) `devDependencies` section - something wrong with build process. Probably some `npm` flags are work as not expected and it MUST BE FIXED before publishing.
 * Ensure that all tests (unit tests and functional tests passed)
 * Optional: Ensure that your local tags match remote. If not, remove all and re-fetch:

docs/conf.py

@@ -69,7 +69,7 @@
 
 # General information about the project.
 project = u'F5 BIG-IP Telemetry Streaming'
-copyright = u'2022 F5 Networks Inc'
+copyright = u'2024 F5, Inc.'
 author = u'F5 Networks'
 
 # The version info for the project you're documenting, acts as replacement for
@@ -79,7 +79,7 @@
 # The short X.Y version.
 version = u''
 # The full version, including alpha/beta/rc tags.
-release = u'1.33.0'
+release = u'1.34.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/data-modification.rst

@@ -2,6 +2,13 @@ Data Modification
 =================
 This section details how you can manipulate the data to better meet your Telemetry goals.  This includes the new :ref:`Actions Chain<actions>`, which can contain the :ref:`includeData<include>` and :ref:`excludeData<exclude>` options introduced in F5 BIG-IP Telemetry Streaming 1.8.0, and the previously introduced :ref:`setTag<tagproperty>` property.
 
+Filtering
+---------
+If a configuration contains multiple items, filtering is not recommended as filtering *disables* endpoints. 
+For example, if your configuration contains 250 pools, TS will take all 250 pools at the same time then applies filtering by name.
+However, filtering is useful if you want to exclude specific data such as *pools*.  
+In this case TS would not make an HTTP request to a REST API endpoint, reducing the load on BIG-IP.
+
 .. _actions: 
 
 Actions chain